[v1,1/1] linux-gen: dpdk: make sure mbuf memory is allocated from huge pages

Message ID 1502265629-9493-2-git-send-email-odpbot@yandex.ru
State New
Headers show
Series
  • linux-gen: dpdk: make sure mbuf memory is allocated from huge pages
Related show

Commit Message

Github ODP bot Aug. 9, 2017, 8 a.m.
From: Matias Elo <matias.elo@nokia.com>


DPDK requires that mbuf memory is allocated from huge pages to ensure page
locking. Modify zero-copy dpdk pktio to fall back to packet copy if
transmitted ODP packet is not allocated from huge page memory.

Signed-off-by: Matias Elo <matias.elo@nokia.com>

---
/** Email created from pull request 119 (matiaselo:dev/dpdk_zero_copy)
 ** https://github.com/Linaro/odp/pull/119
 ** Patch: https://github.com/Linaro/odp/pull/119.patch
 ** Base sha: bb3400df72401d88aa8e79fb0244a2b90f3bd138
 ** Merge commit sha: efca05f2f045fc7c6844cef752f200bf9668ceec
 **/
 platform/linux-generic/include/odp_pool_internal.h |  1 +
 platform/linux-generic/odp_pool.c                  | 19 ++++++++
 platform/linux-generic/pktio/dpdk.c                | 53 +++++++++++++---------
 3 files changed, 51 insertions(+), 22 deletions(-)

Patch

diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h
index f2d2e2ca..edf75d6e 100644
--- a/platform/linux-generic/include/odp_pool_internal.h
+++ b/platform/linux-generic/include/odp_pool_internal.h
@@ -71,6 +71,7 @@  typedef struct pool_t {
 	uint8_t         *uarea_base_addr;
 
 	/* Used by DPDK zero-copy pktio */
+	uint8_t		mem_from_huge_pages;
 	pool_destroy_cb_fn ext_destroy;
 	void            *ext_desc;
 
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index f2410b6e..ef19ed6f 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -8,6 +8,7 @@ 
 #include <odp/api/shared_memory.h>
 #include <odp/api/align.h>
 #include <odp/api/ticketlock.h>
+#include <odp/api/system_info.h>
 
 #include <odp_pool_internal.h>
 #include <odp_internal.h>
@@ -281,6 +282,22 @@  static void init_buffers(pool_t *pool)
 	}
 }
 
+static bool shm_is_from_huge_pages(odp_shm_t shm)
+{
+	odp_shm_info_t info;
+	uint64_t huge_page_size = odp_sys_huge_page_size();
+
+	if (huge_page_size == 0)
+		return 0;
+
+	if (odp_shm_info(shm, &info)) {
+		ODP_ERR("Failed to fetch shm info\n");
+		return 0;
+	}
+
+	return (info.page_size >= huge_page_size);
+}
+
 static odp_pool_t pool_create(const char *name, odp_pool_param_t *params,
 			      uint32_t shmflags)
 {
@@ -406,6 +423,8 @@  static odp_pool_t pool_create(const char *name, odp_pool_param_t *params,
 		goto error;
 	}
 
+	pool->mem_from_huge_pages = shm_is_from_huge_pages(pool->shm);
+
 	pool->base_addr = odp_shm_addr(pool->shm);
 
 	pool->uarea_shm = ODP_SHM_INVALID;
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 9e3e583d..19310651 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -129,14 +129,13 @@  static void mbuf_init(struct rte_mempool *mp, struct rte_mbuf *mbuf,
 {
 	void *buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM;
 
-	rte_mem_lock_page(buf_addr);
-
 	memset(mbuf, 0, sizeof(struct rte_mbuf));
 
 	mbuf->priv_size = 0;
 	mbuf->buf_addr = buf_addr;
 	mbuf->buf_physaddr = rte_mem_virt2phy(buf_addr);
-	if (odp_unlikely(mbuf->buf_physaddr == RTE_BAD_PHYS_ADDR))
+	if (odp_unlikely(mbuf->buf_physaddr == RTE_BAD_PHYS_ADDR ||
+			 mbuf->buf_physaddr == 0))
 		ODP_ABORT("Failed to map virt addr to phy");
 
 	mbuf->buf_len = (uint16_t)rte_pktmbuf_data_room_size(mp);
@@ -163,6 +162,11 @@  static struct rte_mempool *mbuf_pool_create(const char *name,
 	unsigned num;
 	uint16_t data_room_size;
 
+	if (!(pool_entry->mem_from_huge_pages)) {
+		ODP_ERR("DPDK requires memory is allocated from huge pages\n");
+		return NULL;
+	}
+
 	num = pool_entry->num;
 	data_room_size = pool_entry->max_seg_len + CONFIG_PACKET_HEADROOM;
 	elt_size = sizeof(struct rte_mbuf) + (unsigned)data_room_size;
@@ -485,12 +489,11 @@  static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
 static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry,
 				   struct rte_mbuf *mbuf_table[],
 				   const odp_packet_t pkt_table[], uint16_t num,
-				   uint16_t *seg_count)
+				   uint16_t *copy_count)
 {
 	pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk;
 	int i;
-
-	*seg_count = 0;
+	*copy_count = 0;
 
 	for (i = 0; i < num; i++) {
 		odp_packet_t pkt = pkt_table[i];
@@ -502,20 +505,26 @@  static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry,
 		if (odp_unlikely(pkt_len > pkt_dpdk->mtu))
 			goto fail;
 
-		if (odp_likely(pkt_hdr->buf_hdr.segcount == 1)) {
-			if (odp_unlikely(pkt_hdr->extra_type !=
-					 PKT_EXTRA_TYPE_DPDK))
-				mbuf_init(pkt_dpdk->pkt_pool, mbuf, pkt_hdr);
-
+		if (odp_likely(pkt_hdr->buf_hdr.segcount == 1 &&
+			       pkt_hdr->extra_type == PKT_EXTRA_TYPE_DPDK)) {
 			mbuf_update(mbuf, pkt_hdr, pkt_len);
 		} else {
-			/* Fall back to packet copy */
-			if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf,
-						     &pkt, 1) != 1))
-				goto fail;
-			(*seg_count)++;
-		}
+			pool_t *pool_entry = pkt_hdr->buf_hdr.pool_ptr;
+
+			if (pkt_hdr->buf_hdr.segcount != 1 ||
+			    !pool_entry->mem_from_huge_pages) {
+				/* Fall back to packet copy */
+				if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf,
+							     &pkt, 1) != 1))
+					goto fail;
+				(*copy_count)++;
 
+			} else {
+				mbuf_init(pkt_dpdk->pkt_pool, mbuf,
+					  pkt_hdr);
+				mbuf_update(mbuf, pkt_hdr, pkt_len);
+			}
+		}
 		mbuf_table[i] = mbuf;
 	}
 	return i;
@@ -1148,7 +1157,7 @@  static int dpdk_send(pktio_entry_t *pktio_entry, int index,
 {
 	struct rte_mbuf *tx_mbufs[num];
 	pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk;
-	uint16_t seg_count = 0;
+	uint16_t copy_count = 0;
 	int tx_pkts;
 	int i;
 	int mbufs;
@@ -1158,7 +1167,7 @@  static int dpdk_send(pktio_entry_t *pktio_entry, int index,
 
 	if (ODP_DPDK_ZERO_COPY)
 		mbufs = pkt_to_mbuf_zero(pktio_entry, tx_mbufs, pkt_table, num,
-					 &seg_count);
+					 &copy_count);
 	else
 		mbufs = pkt_to_mbuf(pktio_entry, tx_mbufs, pkt_table, num);
 
@@ -1172,11 +1181,11 @@  static int dpdk_send(pktio_entry_t *pktio_entry, int index,
 		odp_ticketlock_unlock(&pkt_dpdk->tx_lock[index]);
 
 	if (ODP_DPDK_ZERO_COPY) {
-		/* Free copied segmented packets */
-		if (odp_unlikely(seg_count)) {
+		/* Free copied packets */
+		if (odp_unlikely(copy_count)) {
 			uint16_t freed = 0;
 
-			for (i = 0; i < mbufs && freed != seg_count; i++) {
+			for (i = 0; i < mbufs && freed != copy_count; i++) {
 				odp_packet_t pkt = pkt_table[i];
 				odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);