[v3] linux-dpdk: shm: use native dpdk memzones

Message ID 1496310538-12727-1-git-send-email-matias.elo@nokia.com
State New
Headers show

Commit Message

Matias Elo June 1, 2017, 9:48 a.m.
Replace odp-linux shared memory implementation with native DPDK memzones.

Signed-off-by: Matias Elo <matias.elo@nokia.com>
---
V3:
- Change handle_is_valid() and mz_name_used() return values to odp_bool_t

 platform/linux-dpdk/Makefile.am                |  10 +-
 platform/linux-dpdk/include/odp_shm_internal.h |  32 ++
 platform/linux-dpdk/odp_init.c                 |  31 +-
 platform/linux-dpdk/odp_shared_memory.c        | 437 +++++++++++++++++++++++++
 4 files changed, 481 insertions(+), 29 deletions(-)
 create mode 100644 platform/linux-dpdk/include/odp_shm_internal.h
 create mode 100644 platform/linux-dpdk/odp_shared_memory.c

Patch hide | download patch | download mbox

diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index 9ffd6fd..2876582 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -120,9 +120,6 @@  odpdrvinclude_HEADERS = \
 		  $(srcdir)/include/odp/drv/compiler.h
 
 noinst_HEADERS = \
-		  ${top_srcdir}/platform/linux-generic/include/_fdserver_internal.h \
-		  ${top_srcdir}/platform/linux-generic/include/_ishm_internal.h \
-		  ${top_srcdir}/platform/linux-generic/include/_ishmphy_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_align_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_atomic_internal.h \
 		  ${srcdir}/include/odp_buffer_inlines.h \
@@ -153,7 +150,7 @@  noinst_HEADERS = \
 		  ${top_srcdir}/platform/linux-generic/include/odp_ring_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_schedule_if.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_sorted_list_internal.h \
-		  ${top_srcdir}/platform/linux-generic/include/odp_shm_internal.h \
+		  ${srcdir}/include/odp_shm_internal.h \
 		  ${srcdir}/include/odp_time_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_timer_internal.h \
 		  ${top_srcdir}/platform/linux-generic/include/odp_timer_wheel_internal.h \
@@ -166,9 +163,6 @@  noinst_HEADERS = \
 		  ${srcdir}/Makefile.inc
 
 __LIB__libodp_dpdk_la_SOURCES = \
-			   ../linux-generic/_fdserver.c \
-			   ../linux-generic/_ishm.c \
-			   ../linux-generic/_ishmphy.c \
 			   ../linux-generic/odp_atomic.c \
 			   ../linux-generic/odp_barrier.c \
 			   ../linux-generic/odp_bitmap.c \
@@ -198,7 +192,7 @@  __LIB__libodp_dpdk_la_SOURCES = \
 			   ../linux-generic/odp_schedule.c \
 			   ../linux-generic/odp_schedule_if.c \
 			   ../linux-generic/odp_schedule_iquery.c \
-			   ../linux-generic/odp_shared_memory.c \
+			   odp_shared_memory.c \
 			   ../linux-generic/odp_sorted_list.c \
 			   ../linux-generic/odp_spinlock.c \
 			   ../linux-generic/odp_spinlock_recursive.c \
diff --git a/platform/linux-dpdk/include/odp_shm_internal.h b/platform/linux-dpdk/include/odp_shm_internal.h
new file mode 100644
index 0000000..072a4f1
--- /dev/null
+++ b/platform/linux-dpdk/include/odp_shm_internal.h
@@ -0,0 +1,32 @@ 
+/* Copyright (c) 2017, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP shared memory - implementation internal
+ */
+
+#ifndef ODP_SHM_INTERNAL_H_
+#define ODP_SHM_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int _odp_shm_init_global(void);
+
+int _odp_shm_init_local(void);
+
+int _odp_shm_term_global(void);
+
+int _odp_shm_term_local(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c
index 6cea393..fdd9d43 100644
--- a/platform/linux-dpdk/odp_init.c
+++ b/platform/linux-dpdk/odp_init.c
@@ -12,6 +12,7 @@ 
 #include <unistd.h>
 #include <odp_internal.h>
 #include <odp_schedule_if.h>
+#include <odp_shm_internal.h>
 #include <string.h>
 #include <stdio.h>
 #include <linux/limits.h>
@@ -238,14 +239,8 @@  int odp_init_global(odp_instance_t *instance,
 		cleanup_files(hpdir, odp_global_data.main_pid);
 	stage = SYSINFO_INIT;
 
-	if (_odp_fdserver_init_global()) {
-		ODP_ERR("ODP fdserver init failed.\n");
-		goto init_failed;
-	}
-	stage = FDSERVER_INIT;
-
-	if (_odp_ishm_init_global()) {
-		ODP_ERR("ODP ishm init failed.\n");
+	if (_odp_shm_init_global()) {
+		ODP_ERR("ODP shm init failed.\n");
 		goto init_failed;
 	}
 	stage = ISHM_INIT;
@@ -405,19 +400,13 @@  int _odp_term_global(enum init_stage stage)
 		/* Fall through */
 
 	case ISHM_INIT:
-		if (_odp_ishm_term_global()) {
-			ODP_ERR("ODP ishm term failed.\n");
+		if (_odp_shm_term_global()) {
+			ODP_ERR("ODP shm term failed.\n");
 			rc = -1;
 		}
 		/* Fall through */
-
+	/* Needed to prevent compiler warning */
 	case FDSERVER_INIT:
-		if (_odp_fdserver_term_global()) {
-			ODP_ERR("ODP fdserver term failed.\n");
-			rc = -1;
-		}
-		/* Fall through */
-
 	case SYSINFO_INIT:
 		if (odp_system_info_term()) {
 			ODP_ERR("ODP system info term failed.\n");
@@ -455,8 +444,8 @@  int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type)
 		goto init_fail;
 	}
 
-	if (_odp_ishm_init_local()) {
-		ODP_ERR("ODP ishm local init failed.\n");
+	if (_odp_shm_init_local()) {
+		ODP_ERR("ODP shm local init failed.\n");
 		goto init_fail;
 	}
 	stage = ISHM_INIT;
@@ -531,8 +520,8 @@  int _odp_term_local(enum init_stage stage)
 		/* Fall through */
 
 	case ISHM_INIT:
-		if (_odp_ishm_term_local()) {
-			ODP_ERR("ODP ishm local term failed.\n");
+		if (_odp_shm_term_local()) {
+			ODP_ERR("ODP shm local term failed.\n");
 			rc = -1;
 		}
 		/* Fall through */
diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c
new file mode 100644
index 0000000..88d184a
--- /dev/null
+++ b/platform/linux-dpdk/odp_shared_memory.c
@@ -0,0 +1,437 @@ 
+/* Copyright (c) 2017, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <odp_align_internal.h>
+#include <odp_config_internal.h>
+#include <odp/api/debug.h>
+#include <odp_debug_internal.h>
+#include <odp/api/shared_memory.h>
+#include <odp/api/spinlock.h>
+#include <odp/api/plat/strong_types.h>
+#include <odp_shm_internal.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <inttypes.h>
+
+#include <rte_lcore.h>
+#include <rte_memzone.h>
+
+#define SHM_MAX_ALIGN (0x80000000)
+#define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s"
+
+ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS,
+		  "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS");
+
+ODP_STATIC_ASSERT(ODP_SHM_NAME_LEN >= RTE_MEMZONE_NAMESIZE,
+		  "ODP_SHM_NAME_LEN < RTE_MEMZONE_NAMESIZE");
+
+typedef enum {
+	SHM_TYPE_LOCAL = 0,
+	SHM_TYPE_REMOTE
+} shm_type_t;
+
+/**
+ * Memory zone descriptor
+ *
+ * This struct is stored inside DPDK memzone to make it available for
+ * odp_shm_import().
+ */
+typedef struct {
+	/* Shared memory flags */
+	uint32_t flags;
+} shm_zone_t;
+
+/**
+ * Memory block descriptor
+ */
+typedef struct {
+	/* Memory block type */
+	shm_type_t type;
+	/* Memory block name */
+	char name[ODP_SHM_NAME_LEN];
+	/* DPDK memzone. If this pointer != NULL, the shm block is interpreted
+	 * as reserved. */
+	const struct rte_memzone *mz;
+} shm_block_t;
+
+/**
+ * Table of blocks describing allocated shared memory. This table is visible to
+ * every ODP thread (linux process or pthreads). It is allocated shared at odp
+ * init time and is therefore inherited by all.
+ */
+typedef struct {
+	odp_spinlock_t  lock;
+	shm_block_t block[ODP_CONFIG_SHM_BLOCKS];
+} shm_table_t;
+
+static shm_table_t *shm_tbl;
+
+/**
+ * Check if DPDK memzone name has been used already
+ */
+static odp_bool_t mz_name_used(const char *name)
+{
+	int idx;
+
+	for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+		if (shm_tbl->block[idx].mz &&
+		    strncmp(name, shm_tbl->block[idx].mz->name,
+			    RTE_MEMZONE_NAMESIZE) == 0)
+			return 1;
+	}
+	return 0;
+}
+
+/**
+ * Convert ODP shm name into unique DPDK memzone name
+ */
+static void name_to_mz_name(const char *name, char *mz_name)
+{
+	int i = 0;
+
+	/* Use pid and counter to make name unique */
+	do {
+		snprintf(mz_name, RTE_MEMZONE_NAMESIZE - 1, SHM_BLOCK_NAME,
+			 (odp_instance_t)odp_global_data.main_pid, i++, name);
+		mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
+	} while (mz_name_used(mz_name));
+}
+
+/**
+ * Convert DPDK memzone length into ODP shm block size
+ */
+static uint64_t shm_size(const struct rte_memzone *mz)
+{
+	return mz->len - sizeof(shm_zone_t);
+}
+
+/**
+ * Return a pointer to shm zone descriptor stored at the end of DPDK memzone
+ */
+static shm_zone_t *shm_zone(const struct rte_memzone *mz)
+{
+	return (shm_zone_t *)((uint8_t *)mz->addr + shm_size(mz));
+}
+
+static int find_free_block(void)
+{
+	int idx;
+
+	for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+		if (shm_tbl->block[idx].mz == NULL)
+			return idx;
+	}
+	return -1;
+}
+
+static inline uint32_t handle_to_idx(odp_shm_t shm)
+{
+	return _odp_typeval(shm) - 1;
+}
+
+static inline odp_shm_t idx_to_handle(uint32_t idx)
+{
+	return _odp_cast_scalar(odp_shm_t, idx + 1);
+}
+
+static inline odp_bool_t handle_is_valid(odp_shm_t shm)
+{
+	int idx = handle_to_idx(shm);
+
+	if (idx < 0 || idx >= ODP_CONFIG_SHM_BLOCKS ||
+	    shm_tbl->block[idx].mz == NULL) {
+		ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n",
+			odp_shm_to_u64(shm));
+		return 0;
+	}
+	return 1;
+}
+
+int _odp_shm_init_global(void)
+{
+	void *addr;
+
+	if ((getpid() != odp_global_data.main_pid) ||
+	    (syscall(SYS_gettid) != getpid()))
+		ODP_ERR("shm_init_global() must be performed by the main "
+			"ODP process!\n.");
+
+	/* Allocate space for the internal shared mem block table */
+	addr = mmap(NULL, sizeof(shm_table_t), PROT_READ | PROT_WRITE,
+		    MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+	if (addr == MAP_FAILED) {
+		ODP_ERR("Unable to mmap the shm block table\n");
+		return -1;
+	}
+
+	shm_tbl = addr;
+	memset(shm_tbl, 0, sizeof(shm_table_t));
+
+	odp_spinlock_init(&shm_tbl->lock);
+
+	return 0;
+}
+
+int _odp_shm_init_local(void)
+{
+	return 0;
+}
+
+int _odp_shm_term_global(void)
+{
+	shm_block_t *block;
+	int idx;
+
+	if ((getpid() != odp_global_data.main_pid) ||
+	    (syscall(SYS_gettid) != getpid()))
+		ODP_ERR("shm_term_global() must be performed by the main "
+			"ODP process!\n.");
+
+	/* Cleanup possibly non freed memory (and complain a bit) */
+	for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+		block = &shm_tbl->block[idx];
+		if (block->mz) {
+			ODP_ERR("block '%s' was never freed (cleaning up...)\n",
+				block->name);
+			rte_memzone_free(block->mz);
+		}
+	}
+	/* Free the shared memory block table */
+	if (munmap(shm_tbl, sizeof(shm_table_t)) < 0) {
+		ODP_ERR("Unable to munmap the shm block table\n");
+		return -1;
+	}
+	return 0;
+}
+
+int _odp_shm_term_local(void)
+{
+	return 0;
+}
+
+int odp_shm_capability(odp_shm_capability_t *capa)
+{
+	memset(capa, 0, sizeof(odp_shm_capability_t));
+
+	capa->max_blocks = ODP_CONFIG_SHM_BLOCKS;
+	capa->max_size = 0;
+	capa->max_align = SHM_MAX_ALIGN;
+
+	return 0;
+}
+
+odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align,
+			  uint32_t flags)
+{
+	shm_block_t *block;
+	const struct rte_memzone *mz;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t mz_flags = RTE_MEMZONE_1GB | RTE_MEMZONE_SIZE_HINT_ONLY;
+	int idx;
+
+	if (align > SHM_MAX_ALIGN) {
+		ODP_ERR("Align too large: %" PRIu64 "\n", align);
+		return ODP_SHM_INVALID;
+	}
+
+	/* DPDK requires alignment to be power of two */
+	if (!rte_is_power_of_2(align))
+		align = ROUNDUP_POWER2_U32(align);
+
+	odp_spinlock_lock(&shm_tbl->lock);
+
+	idx = find_free_block();
+	if (idx < 0) {
+		odp_spinlock_unlock(&shm_tbl->lock);
+		ODP_ERR("No free SHM blocks left\n");
+		return ODP_SHM_INVALID;
+	}
+	block = &shm_tbl->block[idx];
+
+	/* DPDK requires unique memzone names */
+	name_to_mz_name(name, mz_name);
+	/* Reserve extra space for storing shm zone descriptor */
+	mz = rte_memzone_reserve_aligned(mz_name, size + sizeof(shm_zone_t),
+					 rte_socket_id(), mz_flags, align);
+	if (mz == NULL) {
+		odp_spinlock_unlock(&shm_tbl->lock);
+		ODP_ERR("Reserving DPDK memzone failed\n");
+		return ODP_SHM_INVALID;
+	}
+
+	block->mz = mz;
+	snprintf(block->name, ODP_SHM_NAME_LEN - 1, "%s", name);
+	block->name[ODP_SHM_NAME_LEN - 1] = 0;
+	block->type = SHM_TYPE_LOCAL;
+	/* Note: ODP_SHM_SW_ONLY/ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are
+	 * currently ignored. */
+	shm_zone(mz)->flags = flags;
+
+	odp_spinlock_unlock(&shm_tbl->lock);
+
+	return idx_to_handle(idx);
+}
+
+odp_shm_t odp_shm_import(const char *remote_name, odp_instance_t odp_inst,
+			 const char *local_name)
+{
+	shm_block_t *block;
+	const struct rte_memzone *mz;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	int idx;
+
+	snprintf(mz_name, RTE_MEMZONE_NAMESIZE - 1, SHM_BLOCK_NAME, odp_inst, 0,
+		 remote_name);
+	mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
+
+	mz = rte_memzone_lookup(mz_name);
+	if (mz == NULL) {
+		ODP_ERR("Unable to find remote SHM block: %s\n", remote_name);
+		return ODP_SHM_INVALID;
+	}
+
+	if (!(shm_zone(mz)->flags & ODP_SHM_EXPORT)) {
+		ODP_ERR("Not exported SHM block!\n");
+		return ODP_SHM_INVALID;
+	}
+
+	odp_spinlock_lock(&shm_tbl->lock);
+
+	idx = find_free_block();
+	if (idx < 0) {
+		odp_spinlock_unlock(&shm_tbl->lock);
+		ODP_ERR("No free SHM blocks left\n");
+		return ODP_SHM_INVALID;
+	}
+	block = &shm_tbl->block[idx];
+
+	block->mz = mz;
+	snprintf(block->name, ODP_SHM_NAME_LEN - 1, "%s", local_name);
+	block->name[ODP_SHM_NAME_LEN - 1] = 0;
+	block->type = SHM_TYPE_REMOTE;
+
+	odp_spinlock_unlock(&shm_tbl->lock);
+
+	return idx_to_handle(idx);
+}
+
+int odp_shm_free(odp_shm_t shm)
+{
+	shm_block_t *block;
+	int ret = 0;
+
+	odp_spinlock_lock(&shm_tbl->lock);
+
+	if (!handle_is_valid(shm)) {
+		odp_spinlock_unlock(&shm_tbl->lock);
+		return -1;
+	}
+
+	block = &shm_tbl->block[handle_to_idx(shm)];
+
+	/* Only the creator of memzone can free it */
+	if (block->type == SHM_TYPE_LOCAL)
+		ret = rte_memzone_free(block->mz);
+
+	block->mz = NULL;
+
+	odp_spinlock_unlock(&shm_tbl->lock);
+
+	return ret;
+}
+
+odp_shm_t odp_shm_lookup(const char *name)
+{
+	int idx;
+
+	odp_spinlock_lock(&shm_tbl->lock);
+
+	for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+		if (shm_tbl->block[idx].mz &&
+		    strncmp(name, shm_tbl->block[idx].name,
+			    ODP_SHM_NAME_LEN) == 0) {
+			odp_spinlock_unlock(&shm_tbl->lock);
+			return idx_to_handle(idx);
+		}
+	}
+
+	odp_spinlock_unlock(&shm_tbl->lock);
+
+	return ODP_SHM_INVALID;
+}
+
+void *odp_shm_addr(odp_shm_t shm)
+{
+	void *addr;
+
+	odp_spinlock_lock(&shm_tbl->lock);
+
+	if (!handle_is_valid(shm)) {
+		odp_spinlock_unlock(&shm_tbl->lock);
+		return NULL;
+	}
+
+	addr = shm_tbl->block[handle_to_idx(shm)].mz->addr;
+
+	odp_spinlock_unlock(&shm_tbl->lock);
+
+	return addr;
+}
+
+int odp_shm_info(odp_shm_t shm, odp_shm_info_t *info)
+{
+	shm_block_t *block;
+	int idx = handle_to_idx(shm);
+
+	odp_spinlock_lock(&shm_tbl->lock);
+
+	if (!handle_is_valid(shm)) {
+		odp_spinlock_unlock(&shm_tbl->lock);
+		return -1;
+	}
+
+	block = &shm_tbl->block[idx];
+
+	memset(info, 0, sizeof(odp_shm_info_t));
+
+	info->name = block->name;
+	info->addr = block->mz->addr;
+	info->size = shm_size(block->mz);
+	info->page_size = block->mz->hugepage_sz;
+	info->flags = shm_zone(block->mz)->flags;
+
+	odp_spinlock_unlock(&shm_tbl->lock);
+
+	return 0;
+}
+
+void odp_shm_print_all(void)
+{
+	shm_block_t *block;
+	int idx;
+
+	odp_spinlock_lock(&shm_tbl->lock);
+
+	printf("\nShared memory blocks\n--------------------\n");
+
+	for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+		block = &shm_tbl->block[idx];
+		if (block->mz == NULL)
+			continue;
+		printf("  %s: addr: %p, len: %" PRIu64 " page size: "
+		       "%" PRIu64 "\n", block->name, block->mz->addr,
+		       shm_size(block->mz), block->mz->hugepage_sz);
+	}
+
+	odp_spinlock_unlock(&shm_tbl->lock);
+}
+
+uint64_t odp_shm_to_u64(odp_shm_t hdl)
+{
+	return _odp_pri(hdl);
+}