[v4,3/7] example: instrum: use low level API to retrieve performance counters

Message ID 1517911222-3896-4-git-send-email-odpbot@yandex.ru
State New
Headers show
Series
  • Code instrumentation with PAPI library
Related show

Commit Message

Github ODP bot Feb. 6, 2018, 10 a.m.
From: Bogdan Pricope <bogdan.pricope@linaro.org>


Use low level PAPI API to get performance counters.
Exemplify on some ODP APIs.

Signed-off-by: Bogdan Pricope <bogdan.pricope@linaro.org>

---
/** Email created from pull request 443 (bogdanPricope:master_benchmark_pr)
 ** https://github.com/Linaro/odp/pull/443
 ** Patch: https://github.com/Linaro/odp/pull/443.patch
 ** Base sha: 27a7923236030fed0272cc9072f0cde62496e91d
 ** Merge commit sha: 34bd4730244cef8aa4ef2778135e469012c66768
 **/
 example/instrum/Makefile.am      |  14 +++-
 example/instrum/init.c           |  54 +++++++++++++++
 example/instrum/init.h           |  19 ++++++
 example/instrum/instrum.c        |  19 +++++-
 example/instrum/instrum_common.h |  34 ++++++++++
 example/instrum/papi_cnt.c       | 143 +++++++++++++++++++++++++++++++++++++++
 example/instrum/papi_cnt.h       |  27 ++++++++
 example/instrum/pktio_direct.c   |  67 ++++++++++++++++++
 example/instrum/pktio_direct.h   |  19 ++++++
 example/instrum/sample.h         |  28 ++++++++
 example/instrum/sched.c          |  43 ++++++++++++
 example/instrum/sched.h          |  19 ++++++
 example/instrum/store.c          | 134 ++++++++++++++++++++++++++++++++++++
 example/instrum/store.h          |  36 ++++++++++
 14 files changed, 654 insertions(+), 2 deletions(-)
 create mode 100644 example/instrum/init.c
 create mode 100644 example/instrum/init.h
 create mode 100644 example/instrum/instrum_common.h
 create mode 100644 example/instrum/papi_cnt.c
 create mode 100644 example/instrum/papi_cnt.h
 create mode 100644 example/instrum/pktio_direct.c
 create mode 100644 example/instrum/pktio_direct.h
 create mode 100644 example/instrum/sample.h
 create mode 100644 example/instrum/sched.c
 create mode 100644 example/instrum/sched.h
 create mode 100644 example/instrum/store.c
 create mode 100644 example/instrum/store.h

Patch

diff --git a/example/instrum/Makefile.am b/example/instrum/Makefile.am
index ff4a04060..852bdc77f 100644
--- a/example/instrum/Makefile.am
+++ b/example/instrum/Makefile.am
@@ -12,4 +12,16 @@  AM_LDFLAGS = $(PAPI_LIBS)
 lib_LTLIBRARIES = $(LIB)/libinstrum.la
 
 __LIB__libinstrum_la_SOURCES = \
-		instrum.c
+		instrum.c \
+		store.c \
+		papi_cnt.c \
+		init.c \
+		pktio_direct.c\
+		sched.c \
+		instrum_common.h \
+		sample.h \
+		store.h \
+		papi_cnt.h \
+		init.h \
+		pktio_direct.h \
+		sched.h
diff --git a/example/instrum/init.c b/example/instrum/init.c
new file mode 100644
index 000000000..8691ff978
--- /dev/null
+++ b/example/instrum/init.c
@@ -0,0 +1,54 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <odp_api.h>
+#include <instrum_common.h>
+#include <init.h>
+#include <store.h>
+
+static int (*instr_odp_init_local)(odp_instance_t instance,
+				   odp_thread_type_t thr_type);
+
+static int (*instr_odp_term_local)(void);
+
+int instr_odpinit_init(void)
+{
+	INSTR_FUNCTION(odp_init_local);
+
+	if (!instr_odp_init_local) {
+		printf("odp_init_local: Not Found\n");
+		return -1;
+	}
+
+	INSTR_FUNCTION(odp_term_local);
+
+	if (!instr_odp_term_local) {
+		printf("odp_term_local: Not Found\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type)
+{
+	int ret;
+
+	ret = (*instr_odp_init_local)(instance, thr_type);
+
+	instr_store_init_local();
+
+	return ret;
+}
+
+int odp_term_local(void)
+{
+	instr_store_term_local();
+
+	return (*instr_odp_term_local)();
+}
diff --git a/example/instrum/init.h b/example/instrum/init.h
new file mode 100644
index 000000000..b92e9d4a3
--- /dev/null
+++ b/example/instrum/init.h
@@ -0,0 +1,19 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_INIT_H__
+#define __INSTRUM_INIT_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int instr_odpinit_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /*__INSTRUM_INIT_H__*/
diff --git a/example/instrum/instrum.c b/example/instrum/instrum.c
index 88580c58f..605675154 100644
--- a/example/instrum/instrum.c
+++ b/example/instrum/instrum.c
@@ -1,4 +1,4 @@ 
-/* Copyright (c) 2017, Linaro Limited
+/* Copyright (c) 2018, Linaro Limited
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -6,13 +6,30 @@ 
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <store.h>
+#include <init.h>
+#include <sched.h>
+#include <pktio_direct.h>
 
 static __attribute__((constructor)) void setup_wrappers(void)
 {
 	printf("Setup Wrappers\n");
+
+	if (instr_store_init())
+		return;
+
+	if (instr_odpinit_init())
+		return;
+
+	if (instr_odpsched_init())
+		return;
+
+	if (instr_odppktio_direct_init())
+		return;
 }
 
 static __attribute__((destructor)) void teardown_wrappers(void)
 {
 	printf("Teardown Wrappers\n");
+	instr_store_term();
 }
diff --git a/example/instrum/instrum_common.h b/example/instrum/instrum_common.h
new file mode 100644
index 000000000..594423e1f
--- /dev/null
+++ b/example/instrum/instrum_common.h
@@ -0,0 +1,34 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_COMMON_H__
+#define __INSTRUM_COMMON_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef RTLD_NEXT
+/*#define __GNU_SOURCE*/
+#define __USE_GNU
+#endif
+
+#include <dlfcn.h>
+#include <errno.h>
+
+#define INSTR_FUNCTION(func) do {			\
+		instr_##func = dlsym(RTLD_NEXT, #func);	\
+		if (dlerror()) {			\
+			errno = EACCES;			\
+			instr_##func = NULL;	\
+		}					\
+	} while (0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __INSTRUM_COMMON_H__ */
diff --git a/example/instrum/papi_cnt.c b/example/instrum/papi_cnt.c
new file mode 100644
index 000000000..c38c856a0
--- /dev/null
+++ b/example/instrum/papi_cnt.c
@@ -0,0 +1,143 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <papi.h>
+#include <papi_cnt.h>
+
+static int papi_event_tab[SAMPLE_COUNTER_TAB_SIZE] = {PAPI_BR_CN, PAPI_L2_DCM};
+
+static __thread int event_set = PAPI_NULL;
+
+int papi_init(void)
+{
+	int retval, i;
+
+	retval = PAPI_library_init(PAPI_VER_CURRENT);
+	if (retval != PAPI_VER_CURRENT) {
+		printf("PAPI Library initialization error!\n");
+		return -1;
+	}
+
+	retval = PAPI_thread_init((unsigned long(*)(void))(pthread_self));
+	if (retval != PAPI_OK) {
+		printf("PAPI_thread_init error!\n");
+		goto err_shutdown;
+	}
+
+	if (PAPI_set_granularity(PAPI_GRN_THR) != PAPI_OK) {
+		printf("PAPI_set_granularity error!\n");
+		goto err_shutdown;
+	}
+
+	for (i = 0; i < SAMPLE_COUNTER_TAB_SIZE; i++) {
+		retval = PAPI_query_event(papi_event_tab[i]);
+		if (retval != PAPI_OK) {
+			printf("PAPI_query_event %d - error\n", i);
+			goto err_shutdown;
+		}
+	}
+
+	return 0;
+
+err_shutdown:
+	PAPI_shutdown();
+
+	return -1;
+}
+
+void papi_term(void)
+{
+	PAPI_shutdown();
+}
+
+int papi_init_local(void)
+{
+	int retval;
+
+	retval = PAPI_register_thread();
+	if (retval != PAPI_OK) {
+		printf("PAPI_register_thread failed - %d\n", retval);
+		return -1;
+	}
+
+	/* Create LL event set */
+	event_set = PAPI_NULL;
+	retval = PAPI_create_eventset(&event_set);
+	if (retval != PAPI_OK) {
+		printf("PAPI_create_eventset error: %d\n", retval);
+		return -1;
+	}
+
+	retval = PAPI_add_events(event_set, papi_event_tab,
+				 SAMPLE_COUNTER_TAB_SIZE);
+	if (retval != PAPI_OK) {
+		printf("PAPI_add_events error: %d\n", retval);
+		goto err_clean_evset;
+	}
+
+	retval = PAPI_start(event_set);
+	if (retval != PAPI_OK) {
+		printf("PAPI_start error: %d\n", retval);
+		goto err_clean_evset;
+	}
+
+	return 0;
+
+err_clean_evset:
+	PAPI_cleanup_eventset(event_set);
+	PAPI_destroy_eventset(&event_set);
+
+	return -1;
+}
+
+int papi_term_local(void)
+{
+	long long last_counters[SAMPLE_COUNTER_TAB_SIZE];
+
+	if (PAPI_stop(event_set, last_counters) == PAPI_OK) {
+		int i;
+
+		for (i = 0; i < SAMPLE_COUNTER_TAB_SIZE; i++)
+			printf("Counter[%d] = %lld\n", i, last_counters[i]);
+	}
+
+	PAPI_cleanup_eventset(event_set);
+	PAPI_destroy_eventset(&event_set);
+
+	return 0;
+}
+
+int papi_sample_start(profiling_sample_t *spl)
+{
+	spl->timestamp_ns = PAPI_get_real_nsec();
+	if (PAPI_read_ts(event_set, spl->counters, &spl->diff_cyc) != PAPI_OK) {
+		fprintf(stderr, "PAPI_read_counters - FAILED\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int papi_sample_end(profiling_sample_t *spl)
+{
+	long long end_counters[SAMPLE_COUNTER_TAB_SIZE], end_cyc;
+	int i;
+
+	if (PAPI_read_ts(event_set, end_counters, &end_cyc) != PAPI_OK) {
+		fprintf(stderr, "PAPI_read_counters - FAILED\n");
+		return -1;
+	}
+
+	for (i = 0; i < SAMPLE_COUNTER_TAB_SIZE; i++)
+		spl->counters[i] = end_counters[i] - spl->counters[i];
+
+	spl->diff_cyc = end_cyc - spl->diff_cyc;
+
+	return 0;
+}
diff --git a/example/instrum/papi_cnt.h b/example/instrum/papi_cnt.h
new file mode 100644
index 000000000..a4546c954
--- /dev/null
+++ b/example/instrum/papi_cnt.h
@@ -0,0 +1,27 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_PAPI_COUNTERS_H__
+#define __INSTRUM_PAPI_COUNTERS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sample.h>
+
+int papi_init(void);
+void papi_term(void);
+int papi_init_local(void);
+int papi_term_local(void);
+
+int papi_sample_start(profiling_sample_t *spl);
+int papi_sample_end(profiling_sample_t *spl);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_PAPI_COUNTERS_H__ */
diff --git a/example/instrum/pktio_direct.c b/example/instrum/pktio_direct.c
new file mode 100644
index 000000000..79cda85ac
--- /dev/null
+++ b/example/instrum/pktio_direct.c
@@ -0,0 +1,67 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <odp_api.h>
+#include <instrum_common.h>
+#include <pktio_direct.h>
+#include <store.h>
+
+static int (*instr_odp_pktout_send)(odp_pktout_queue_t queue,
+				    const odp_packet_t packets[],
+				    int num);
+
+static int (*instr_odp_pktin_recv_tmo)(odp_pktin_queue_t queue,
+				       odp_packet_t packets[],
+				       int num, uint64_t wait);
+
+int instr_odppktio_direct_init(void)
+{
+	INSTR_FUNCTION(odp_pktout_send);
+
+	if (!instr_odp_pktout_send) {
+		printf("odp_pktout_send: Not Found\n");
+		return -1;
+	}
+
+	INSTR_FUNCTION(odp_pktin_recv_tmo);
+
+	if (!instr_odp_pktin_recv_tmo) {
+		printf("odp_pktin_recv_tmo: Not Found\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int odp_pktout_send(odp_pktout_queue_t queue, const odp_packet_t packets[],
+		    int num)
+{
+	int ret;
+
+	STORE_SAMPLE_INIT;
+
+	STORE_SAMPLE_START;
+	ret = (*instr_odp_pktout_send)(queue, packets, num);
+	STORE_SAMPLE_END;
+
+	return ret;
+}
+
+int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[],
+		       int num, uint64_t wait)
+{
+	int ret;
+
+	STORE_SAMPLE_INIT;
+
+	STORE_SAMPLE_START;
+	ret = (*instr_odp_pktin_recv_tmo)(queue, packets, num, wait);
+	STORE_SAMPLE_END;
+
+	return ret;
+}
diff --git a/example/instrum/pktio_direct.h b/example/instrum/pktio_direct.h
new file mode 100644
index 000000000..1bc544dd2
--- /dev/null
+++ b/example/instrum/pktio_direct.h
@@ -0,0 +1,19 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_PKTIO_DIRECT_H__
+#define __INSTRUM_PKTIO_DIRECT_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int instr_odppktio_direct_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_PKTIO_DIRECT_H__ */
diff --git a/example/instrum/sample.h b/example/instrum/sample.h
new file mode 100644
index 000000000..090308348
--- /dev/null
+++ b/example/instrum/sample.h
@@ -0,0 +1,28 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_SAMPLE_H__
+#define __INSTRUM_SAMPLE_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SAMPLE_NAME_SIZE_MAX 20
+#define SAMPLE_COUNTER_TAB_SIZE 2
+
+typedef struct {
+	char name[SAMPLE_NAME_SIZE_MAX];
+	long long timestamp_ns;
+	long long diff_cyc;
+
+	long long counters[SAMPLE_COUNTER_TAB_SIZE];
+} profiling_sample_t;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_SAMPLE_H__ */
diff --git a/example/instrum/sched.c b/example/instrum/sched.c
new file mode 100644
index 000000000..3e4a03bf1
--- /dev/null
+++ b/example/instrum/sched.c
@@ -0,0 +1,43 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <odp_api.h>
+#include <instrum_common.h>
+#include <sched.h>
+#include <store.h>
+
+static int (*instr_odp_schedule_multi)(odp_queue_t *from,
+				       uint64_t wait,
+				       odp_event_t events[],
+				       int num);
+
+int instr_odpsched_init(void)
+{
+	INSTR_FUNCTION(odp_schedule_multi);
+
+	if (!instr_odp_schedule_multi) {
+		printf("odp_schedule_multi: Not Found\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[],
+		       int num)
+{
+	int ret;
+
+	STORE_SAMPLE_INIT;
+
+	STORE_SAMPLE_START;
+	ret = (*instr_odp_schedule_multi)(from, wait, events, num);
+	STORE_SAMPLE_END;
+
+	return ret;
+}
diff --git a/example/instrum/sched.h b/example/instrum/sched.h
new file mode 100644
index 000000000..2a092b0ed
--- /dev/null
+++ b/example/instrum/sched.h
@@ -0,0 +1,19 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_SCHED_H__
+#define __INSTRUM_SCHED_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int instr_odpsched_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_SCHED_H__ */
diff --git a/example/instrum/store.c b/example/instrum/store.c
new file mode 100644
index 000000000..fd9cc512c
--- /dev/null
+++ b/example/instrum/store.c
@@ -0,0 +1,134 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <odp_api.h>
+#include <store.h>
+#include <sample.h>
+#include <papi_cnt.h>
+
+#define SAMPLE_TAB_SIZE 50000
+
+static __thread profiling_sample_t profile_sample_tab[SAMPLE_TAB_SIZE];
+static __thread uint64_t profile_sample_idx;
+static __thread uint64_t profile_sample_ovf;
+
+#define STORE_DIR_ENV "ODP_INSTRUM_STORE_DIR"
+#define STORE_DIR_NAME_DFLT "/tmp"
+#define STORE_DIR_NAME_SIZE_MAX 250
+#define STORE_FILE_NAME_SIZE_MAX 250
+
+static char store_dir[STORE_DIR_NAME_SIZE_MAX];
+
+static void store_dump(int last)
+{
+	FILE *f = NULL;
+	char file_name[STORE_DIR_NAME_SIZE_MAX + STORE_FILE_NAME_SIZE_MAX];
+	char smpl[250], smpl_tmp[250];
+	int i, j, dump_size = SAMPLE_TAB_SIZE;
+
+	if (last)
+		dump_size = profile_sample_idx;
+
+	sprintf(file_name, "%s/profile_%d_%ju.csv",
+		store_dir, odp_thread_id(),
+		profile_sample_ovf);
+
+	f = fopen(file_name, "w");
+	if (f == NULL) {
+		printf("Failed to create profiling file %s\n", file_name);
+		return;
+	}
+
+	for (i = 0; i < dump_size; i++) {
+		sprintf(smpl, "%lld,%lld,%s",
+			profile_sample_tab[i].timestamp_ns,
+			profile_sample_tab[i].diff_cyc,
+			profile_sample_tab[i].name);
+		for (j = 0; j < SAMPLE_COUNTER_TAB_SIZE; j++) {
+			sprintf(smpl_tmp, ",%lld",
+				profile_sample_tab[i].counters[j]);
+			strcat(smpl, smpl_tmp);
+		}
+		fprintf(f, "%s\n", smpl);
+	}
+
+	fclose(f);
+}
+
+int instr_store_init(void)
+{
+	const char *store_dir_env = NULL;
+
+	store_dir_env = getenv(STORE_DIR_ENV);
+	if (!store_dir_env)
+		store_dir_env = STORE_DIR_NAME_DFLT;
+
+	strncpy(store_dir, store_dir_env, STORE_DIR_NAME_SIZE_MAX);
+	store_dir[STORE_DIR_NAME_SIZE_MAX - 1] = '\0';
+
+	if (papi_init())
+		return -1;
+
+	return 0;
+}
+
+void instr_store_term(void)
+{
+	papi_term();
+}
+
+int instr_store_init_local(void)
+{
+	return papi_init_local();
+}
+
+int instr_store_term_local(void)
+{
+	int ret = papi_term_local();
+
+	store_dump(1);
+
+	return ret;
+}
+
+instr_profiling_sample_t store_sample_start(const char *func)
+{
+	profiling_sample_t *spl = NULL;
+
+	if (profile_sample_idx == SAMPLE_TAB_SIZE)
+		return NULL;
+
+	spl = &profile_sample_tab[profile_sample_idx];
+
+	strncpy(spl->name, func, SAMPLE_NAME_SIZE_MAX);
+	spl->name[SAMPLE_NAME_SIZE_MAX - 1] = '\0';
+
+	if (papi_sample_start(spl))
+		return NULL;
+
+	profile_sample_idx++;
+	return spl;
+}
+
+void store_sample_end(instr_profiling_sample_t _spl)
+{
+	profiling_sample_t *spl = _spl;
+
+	if (!spl) /* failed sample - on start */
+		return;
+
+	if (papi_sample_end(spl))
+		spl->name[0] = 0; /* failed sample - on end*/
+
+	if (profile_sample_idx == SAMPLE_TAB_SIZE) {
+		store_dump(0);
+		profile_sample_idx = 0;
+		profile_sample_ovf++;
+	}
+}
diff --git a/example/instrum/store.h b/example/instrum/store.h
new file mode 100644
index 000000000..1d671fbfd
--- /dev/null
+++ b/example/instrum/store.h
@@ -0,0 +1,36 @@ 
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_STORE_H__
+#define __INSTRUM_STORE_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void *instr_profiling_sample_t;
+
+#define STORE_SAMPLE_INIT \
+	instr_profiling_sample_t _spl
+
+#define STORE_SAMPLE_START \
+	(_spl = store_sample_start(__func__))
+
+#define STORE_SAMPLE_END \
+	store_sample_end(_spl)
+
+int instr_store_init(void);
+void instr_store_term(void);
+int instr_store_init_local(void);
+int instr_store_term_local(void);
+
+instr_profiling_sample_t store_sample_start(const char *);
+void store_sample_end(instr_profiling_sample_t);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_STORE_H__ */