[CATERPILLAR,v6,4/4] linux-gen: packet: calculate the start of packet pointer

Message ID 1515614416-28481-5-git-send-email-odpbot@yandex.ru
State New
Headers show
Series
  • Reduce segmentation metadata
Related show

Commit Message

Github ODP bot Jan. 10, 2018, 8 p.m.
From: Brian Brooks <brian.brooks@arm.com>


The start of packet pointer was adjusted on every operation involving
headroom manipulation. This meant that this pointer had to be restored
to its pristine (aligned) offset when the buffer was recycled for reuse.

Instead of maintaining a pristine offset that could be used to restore
this pointer, use a (fixed) pointer to the start of the buffer i.e.
start of headroom. On operations involving headroom manipulation, only
modify the headroom offset value. This means that the start of packet
pointer must be calculated as start of buffer + headroom.

Signed-off-by: Brian Brooks <brian.brooks@arm.com>

Signed-off-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>

---
/** Email created from pull request 354 (brbrooks:caterpillar)
 ** https://github.com/Linaro/odp/pull/354
 ** Patch: https://github.com/Linaro/odp/pull/354.patch
 ** Base sha: 4d17f8ae64aba0e6f24877be30f86ae5880cef7e
 ** Merge commit sha: 4901013db66aec2119d9705f69569107629ee938
 **/
 platform/linux-generic/buffer/generic.c            |  4 +-
 .../include/odp/api/plat/packet_inlines.h          | 15 +++---
 .../include/odp/api/plat/packet_types.h            |  2 +-
 .../linux-generic/include/odp_buffer_internal.h    | 11 ++--
 .../linux-generic/include/odp_packet_internal.h    | 12 ++---
 platform/linux-generic/odp_packet.c                | 59 ++++++++++------------
 platform/linux-generic/pktio/dpdk.c                |  6 +--
 platform/linux-generic/pktio/ipc.c                 |  2 +-
 platform/linux-generic/pool/generic.c              | 13 ++---
 9 files changed, 58 insertions(+), 66 deletions(-)

Patch

diff --git a/platform/linux-generic/buffer/generic.c b/platform/linux-generic/buffer/generic.c
index 4acca3522..7f0060124 100644
--- a/platform/linux-generic/buffer/generic.c
+++ b/platform/linux-generic/buffer/generic.c
@@ -31,7 +31,7 @@  static void *generic_buffer_addr(odp_buffer_t buf)
 {
 	odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf);
 
-	return hdr->base_data;
+	return hdr->buf_start;
 }
 
 static uint32_t generic_buffer_size(odp_buffer_t buf)
@@ -61,7 +61,7 @@  int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf)
 			"  pool         %" PRIu64 "\n",
 			odp_pool_to_u64(pool->pool_hdl));
 	len += snprintf(&str[len], n - len,
-			"  addr         %p\n",          hdr->base_data);
+			"  addr         %p\n",          hdr->buf_start);
 	len += snprintf(&str[len], n - len,
 			"  size         %" PRIu32 "\n", hdr->size);
 	len += snprintf(&str[len], n - len,
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index 06bcf8557..af7c54ea7 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -35,12 +35,6 @@  static inline odp_packet_seg_t _odp_packet_seg_from_ndx(uint32_t ndx)
 }
 #endif
 
-/** @internal Inline function @param pkt @return */
-static inline void *_odp_packet_data(odp_packet_t pkt)
-{
-	return _odp_pkt_get(pkt, void *, data);
-}
-
 /** @internal Inline function @param pkt @return */
 static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt)
 {
@@ -65,6 +59,13 @@  static inline uint32_t _odp_packet_tailroom(odp_packet_t pkt)
 	return _odp_pkt_get(pkt, uint16_t, tailroom);
 }
 
+/** @internal Inline function @param pkt @return */
+static inline void *_odp_packet_data(odp_packet_t pkt)
+{
+	return _odp_pkt_get(pkt, uint8_t *, buf_start) +
+		_odp_packet_headroom(pkt);
+}
+
 /** @internal Inline function @param pkt @return */
 static inline odp_pool_t _odp_packet_pool(odp_packet_t pkt)
 {
@@ -110,7 +111,7 @@  static inline odp_time_t _odp_packet_ts(odp_packet_t pkt)
 /** @internal Inline function @param pkt @return */
 static inline void *_odp_packet_head(odp_packet_t pkt)
 {
-	return (uint8_t *)_odp_packet_data(pkt) - _odp_packet_headroom(pkt);
+	return _odp_pkt_get(pkt, void *, buf_start);
 }
 
 /** @internal Inline function @param pkt @return */
diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h
index 009a3aa7c..e3774cf0d 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_types.h
@@ -74,7 +74,7 @@  typedef enum {
 /** @internal Packet header field offsets for inline functions */
 typedef struct _odp_packet_inline_offset_t {
 	/** @internal field offset */
-	uint16_t data;
+	uint16_t buf_start;
 	/** @internal field offset */
 	uint16_t seg_len;
 	/** @internal field offset */
diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h
index 0046afb3a..dea7d6261 100644
--- a/platform/linux-generic/include/odp_buffer_internal.h
+++ b/platform/linux-generic/include/odp_buffer_internal.h
@@ -43,14 +43,13 @@  struct odp_buffer_hdr_t {
 	/* Pool type */
 	int8_t    type;
 
-	/* Offset used to restore base_data */
-	uint8_t pristine_offset;
+	/* uint8_t unused0; */
 
 	/* Next header which continues the segment list */
 	void *next_seg;
 
-	/* Pointer to start of segment */
-	uint8_t  *base_data;
+	/* Pointer to start of aligned buffer */
+	uint8_t *buf_start;
 
 	/* Pool pointer */
 	void *pool_ptr;
@@ -81,8 +80,8 @@  struct odp_buffer_hdr_t {
 	/* Event type. Maybe different than pool type (crypto compl event) */
 	int8_t    event_type;
 
-	/* Initial buffer tail pointer */
-	uint8_t  *buf_end;
+	/* Pointer to end of buffer */
+	uint8_t *buf_end;
 
 	/* User area pointer */
 	void    *uarea_addr;
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 8f381f097..91f7a1425 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -180,6 +180,11 @@  static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr)
 	return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr;
 }
 
+static inline uint8_t *packet_base_data(odp_packet_hdr_t *pkt_hdr)
+{
+	return pkt_hdr->buf_hdr.buf_start + pkt_hdr->headroom;
+}
+
 static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *pkt_hdr)
 {
 	int segcount = pkt_hdr->buf_hdr.segcount;
@@ -214,18 +219,13 @@  static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len)
 
 	pkt_hdr->buf_hdr.event_subtype = ODP_EVENT_PACKET_BASIC;
 
-	/* Restore each segment's base_data and size fields */
+	/* Restore each segment's size field */
 	while (len > pool->seg_len) {
-		pkt_hdr->buf_hdr.base_data =
-			&pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset];
 		pkt_hdr->buf_hdr.size = pool->seg_len;
 
 		len -= pool->seg_len;
-
 		pkt_hdr = pkt_hdr->buf_hdr.next_seg;
 	}
-	pkt_hdr->buf_hdr.base_data =
-		&pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset];
 	pkt_hdr->buf_hdr.size = len;
 
 	old_pkt_hdr->tailroom += pool->seg_len - len;
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index cae38fbda..6c570677d 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -28,7 +28,7 @@ 
 
 /* Fill in packet header field offsets for inline functions */
 const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
-	.data           = offsetof(odp_packet_hdr_t, buf_hdr.base_data),
+	.buf_start      = offsetof(odp_packet_hdr_t, buf_hdr.buf_start),
 	.seg_len        = offsetof(odp_packet_hdr_t, buf_hdr.size),
 	.frame_len      = offsetof(odp_packet_hdr_t, frame_len),
 	.headroom       = offsetof(odp_packet_hdr_t, headroom),
@@ -174,7 +174,7 @@  static void *packet_map(odp_packet_hdr_t *pkt_hdr, uint32_t offset,
 	if (seg_idx)
 		*seg_idx = skipped_segs;
 
-	return pkt_hdr->buf_hdr.base_data + offset;
+	return packet_base_data(pkt_hdr) + offset;
 }
 
 static void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr)
@@ -236,17 +236,11 @@  static int packet_alloc(pool_t *pool, uint32_t len, int num_pkt,
 	}
 
 	for (int i = 0; i < npkt; i++) {
-		packet_init_segs(&pkt_hdr[i * segs_per_pkt], segs_per_pkt);
-		packet_init(pkt_hdr[i * segs_per_pkt], len);
+		int seg = i * segs_per_pkt;
 
-		pkts[i] = (odp_packet_t)pkt_hdr[i * segs_per_pkt];
-	}
-
-	for (int i = 0; i < nbuf; i++) {
-		ODP_ASSERT(((intptr_t)pkt_hdr[i]->buf_hdr.base_data -
-			    (intptr_t)pkt_hdr[i]) <=
-			   (int)(sizeof(odp_packet_hdr_t) +
-				 pool->headroom + pool->align));
+		packet_init_segs(&pkt_hdr[seg], segs_per_pkt);
+		packet_init(pkt_hdr[seg], len);
+		pkts[i] = (odp_packet_t)pkt_hdr[seg];
 	}
 
 	return npkt;
@@ -407,8 +401,13 @@  odp_event_t odp_packet_to_event(odp_packet_t pkt)
 uint32_t odp_packet_buf_len(odp_packet_t pkt)
 {
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+	uint32_t len = 0;
 
-	return packet_buf_len(pkt_hdr) * pkt_hdr->buf_hdr.segcount;
+	while (pkt_hdr) {
+		len += packet_buf_len(pkt_hdr);
+		pkt_hdr = pkt_hdr->buf_hdr.next_seg;
+	}
+	return len;
 }
 
 void *odp_packet_tail(odp_packet_t pkt)
@@ -439,13 +438,11 @@  void *odp_packet_push_head(odp_packet_t pkt, uint32_t len)
 	if (len > pkt_hdr->headroom)
 		return NULL;
 
-	pkt_hdr->buf_hdr.base_data -= len;
 	pkt_hdr->buf_hdr.size += len;
-
 	pkt_hdr->headroom -= len;
 	pkt_hdr->frame_len += len;
 
-	return pkt_hdr->buf_hdr.base_data;
+	return packet_base_data(pkt_hdr);
 }
 
 void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len)
@@ -455,13 +452,11 @@  void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len)
 	if (len > pkt_hdr->frame_len)
 		return NULL;
 
-	pkt_hdr->buf_hdr.base_data += len;
 	pkt_hdr->buf_hdr.size -= len;
-
 	pkt_hdr->headroom += len;
 	pkt_hdr->frame_len -= len;
 
-	return pkt_hdr->buf_hdr.base_data;
+	return packet_base_data(pkt_hdr);
 }
 
 void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len)
@@ -476,7 +471,6 @@  void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len)
 	old_tail = seg->buf_hdr.buf_end - pkt_hdr->tailroom;
 
 	seg->buf_hdr.size += len;
-
 	pkt_hdr->tailroom -= len;
 	pkt_hdr->frame_len += len;
 
@@ -492,7 +486,6 @@  void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len)
 		return NULL;
 
 	seg->buf_hdr.size -= len;
-
 	pkt_hdr->tailroom += len;
 	pkt_hdr->frame_len -= len;
 
@@ -522,7 +515,7 @@  int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len,
 			return -1;
 
 		if (data_ptr)
-			*data_ptr = pkt_hdr->buf_hdr.base_data;
+			*data_ptr = packet_base_data(pkt_hdr);
 		if (seg_len)
 			*seg_len = pkt_hdr->buf_hdr.size;
 
@@ -530,15 +523,17 @@  int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len,
 	} else {
 		pool_t *pool = pkt_hdr->buf_hdr.pool_ptr;
 		odp_packet_t head;
+		uint16_t headroom;
 
 		if (odp_unlikely(pkt_hdr->frame_len + len > pool->max_len))
 			return -1;
 
-		pkt_hdr->frame_len += pkt_hdr->headroom;
-		pkt_hdr->buf_hdr.base_data -= pkt_hdr->headroom;
-		pkt_hdr->buf_hdr.size += pkt_hdr->headroom;
+		headroom = pkt_hdr->headroom;
+		pkt_hdr->buf_hdr.size += headroom;
+		pkt_hdr->headroom -= headroom;
+		pkt_hdr->frame_len += headroom;
 
-		if (packet_alloc(pool, len - pkt_hdr->headroom, 1, &head) != 1)
+		if (packet_alloc(pool, len - headroom, 1, &head) != 1)
 			return -1;
 
 		concat_seg((odp_packet_hdr_t *)(uintptr_t)head, pkt_hdr);
@@ -549,7 +544,7 @@  int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len,
 		packet_copy_md(packet_hdr(head), pkt_hdr);
 
 		if (data_ptr)
-			*data_ptr = packet_hdr(head)->buf_hdr.base_data;
+			*data_ptr = packet_base_data(packet_hdr(head));
 		if (seg_len)
 			*seg_len = packet_hdr(head)->buf_hdr.size;
 
@@ -574,7 +569,7 @@  int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len,
 			return -1;
 
 		if (data_ptr)
-			*data_ptr = pkt_hdr->buf_hdr.base_data;
+			*data_ptr = packet_base_data(pkt_hdr);
 		if (seg_len)
 			*seg_len = pkt_hdr->buf_hdr.size;
 
@@ -602,7 +597,7 @@  int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len,
 			return -1;
 
 		if (data_ptr)
-			*data_ptr = head->buf_hdr.base_data;
+			*data_ptr = packet_base_data(head);
 		if (seg_len)
 			*seg_len = head->buf_hdr.size;
 
@@ -817,7 +812,7 @@  void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg)
 
 	pkt_hdr = get_seg(pkt_hdr, _odp_packet_seg_to_ndx(seg));
 
-	return pkt_hdr->buf_hdr.base_data;
+	return packet_base_data(pkt_hdr);
 }
 
 uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg)
@@ -1224,7 +1219,7 @@  void odp_packet_print(odp_packet_t pkt)
 				hdr,
 				buffer_ref(&hdr->buf_hdr),
 				hdr->buf_hdr.size,
-				hdr->buf_hdr.base_data,
+				packet_base_data(hdr),
 				hdr->buf_hdr.buf_end,
 				hdr->buf_hdr.next_seg);
 
@@ -1682,7 +1677,7 @@  int packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
 		       odp_pktio_parser_layer_t layer)
 {
 	return packet_parse_common(&pkt_hdr->p,
-				   pkt_hdr->buf_hdr.base_data,
+				   packet_base_data(pkt_hdr),
 				   pkt_hdr->frame_len,
 				   pkt_hdr->buf_hdr.size,
 				   layer);
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 42df14ddd..9aeb4a3cc 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -106,7 +106,7 @@  static unsigned cache_size(uint32_t num)
 static inline uint16_t mbuf_data_off(struct rte_mbuf *mbuf,
 				     odp_packet_hdr_t *pkt_hdr)
 {
-	return (uint64_t)pkt_hdr->buf_hdr.base_data - (uint64_t)mbuf->buf_addr;
+	return (uint64_t)packet_base_data(pkt_hdr) - (uint64_t)mbuf->buf_addr;
 }
 
 /**
@@ -133,7 +133,7 @@  static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr,
 static void mbuf_init(struct rte_mempool *mp, struct rte_mbuf *mbuf,
 		      odp_packet_hdr_t *pkt_hdr)
 {
-	void *buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM;
+	void *buf_addr = packet_base_data(pkt_hdr) - RTE_PKTMBUF_HEADROOM;
 
 	memset(mbuf, 0, sizeof(struct rte_mbuf));
 
@@ -645,7 +645,7 @@  static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
 
 		/* Init buffer segments. Currently, only single segment packets
 		 * are supported. */
-		pkt_hdr->buf_hdr.base_data = data;
+		pkt_hdr->buf_hdr.buf_start = data;
 
 		packet_init(pkt_hdr, pkt_len);
 		pkt_hdr->input = pktio_entry->s.handle;
diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c
index 66c3dfad6..3c4699db8 100644
--- a/platform/linux-generic/pktio/ipc.c
+++ b/platform/linux-generic/pktio/ipc.c
@@ -658,7 +658,7 @@  static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry,
 
 		offsets[i] = (uint8_t *)pkt_hdr -
 			     (uint8_t *)odp_shm_addr(pool->shm);
-		data_pool_off = (uint8_t *)pkt_hdr->buf_hdr.base_data -
+		data_pool_off = packet_base_data(pkt_hdr) -
 				(uint8_t *)odp_shm_addr(pool->shm);
 
 		/* compile all function code even if ipc disabled with config */
diff --git a/platform/linux-generic/pool/generic.c b/platform/linux-generic/pool/generic.c
index 85a1ebe31..da907b9eb 100644
--- a/platform/linux-generic/pool/generic.c
+++ b/platform/linux-generic/pool/generic.c
@@ -247,11 +247,11 @@  static void init_buffers(pool_t *pool)
 		if (type == ODP_POOL_PACKET)
 			data = pkt_hdr->data;
 
+		/* Determine amount of offset needed for desired alignment. */
 		offset = pool->headroom;
-
-		/* move to correct align */
 		while (((uintptr_t)&data[offset]) % pool->align != 0)
 			offset++;
+		offset -= pool->headroom;
 
 		memset(buf_hdr, 0, (uintptr_t)data - (uintptr_t)buf_hdr);
 
@@ -268,12 +268,9 @@  static void init_buffers(pool_t *pool)
 
 		odp_atomic_init_u32(&buf_hdr->ref_cnt, 0);
 
-		ODP_ASSERT(offset <= 255);
-		buf_hdr->pristine_offset = offset;
-
-		buf_hdr->base_data = &data[offset];
-		buf_hdr->buf_end =
-			&data[offset + pool->seg_len + pool->tailroom];
+		buf_hdr->buf_start = &data[offset];
+		buf_hdr->buf_end = buf_hdr->buf_start +
+			pool->headroom + pool->seg_len + pool->tailroom;
 
 		/* Store buffer index into the global pool */
 		ring_enq(ring, mask, i);