1 files changed, 153 insertions, 36 deletions

diff --git a/sys/net/netmap_user.h b/sys/net/netmap_user.h
index 3f2858304caf..bd6fe0db22ae 100644
--- a/sys/net/netmap_user.h
+++ b/sys/net/netmap_user.h
@@ -1,6 +1,5 @@
 /*
- * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
- * Copyright (C) 2013 Universita` di Pisa
+ * Copyright (C) 2011-2014 Universita` di Pisa. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -28,8 +27,8 @@
 /*
  * $FreeBSD$
  *
- * This header contains the macros used to manipulate netmap structures
- * and packets in userspace. See netmap(4) for more information.
+ * Functions and macros to manipulate netmap structures and packets
+ * in userspace. See netmap(4) for more information.
  *
  * The address of the struct netmap_if, say nifp, is computed from the
  * value returned from ioctl(.., NIOCREG, ...) and the mmap region:
@@ -44,17 +43,20 @@
  *		we can access ring->nr_cur, ring->nr_avail, ring->nr_flags
  *
  *	ring->slot[i] gives us the i-th slot (we can access
- *		directly plen, flags, bufindex)
+ *		directly len, flags, buf_idx)
  *
  *	char *buf = NETMAP_BUF(ring, x) returns a pointer to
  *		the buffer numbered x
  *
- * Since rings are circular, we have macros to compute the next index
- *	i = NETMAP_RING_NEXT(ring, i);
+ * All ring indexes (head, cur, tail) should always move forward.
+ * To compute the next index in a circular ring you can use
+ *	i = nm_ring_next(ring, i);
  *
  * To ease porting apps from pcap to netmap we supply a few fuctions
- * that can be called to open, close and read from netmap in a way
- * similar to libpcap.
+ * that can be called to open, close, read and write on netmap in a way
+ * similar to libpcap. Note that the read/write function depend on
+ * an ioctl()/select()/poll() being issued to refill rings or push
+ * packets out.
  *
  * In order to use these, include #define NETMAP_WITH_LIBS
  * in the source file that invokes these functions.
@@ -65,12 +67,19 @@
 
 #include <stdint.h>
 #include <net/if.h>		/* IFNAMSIZ */
+
+#ifndef likely
+#define likely(x)	__builtin_expect(!!(x), 1)
+#define unlikely(x)	__builtin_expect(!!(x), 0)
+#endif /* likely and unlikely */
+
 #include <net/netmap.h>
 
+/* helper macro */
 #define _NETMAP_OFFSET(type, ptr, offset) \
 	((type)(void *)((char *)(ptr) + (offset)))
 
-#define NETMAP_IF(b, o)	_NETMAP_OFFSET(struct netmap_if *, b, o)
+#define NETMAP_IF(_base, _ofs)	_NETMAP_OFFSET(struct netmap_if *, _base, _ofs)
 
 #define NETMAP_TXRING(nifp, index) _NETMAP_OFFSET(struct netmap_ring *, \
 	nifp, (nifp)->ring_ofs[index] )
@@ -85,18 +94,34 @@
 	( ((char *)(buf) - ((char *)(ring) + (ring)->buf_ofs) ) / \
 		(ring)->nr_buf_size )
 
-#define	NETMAP_RING_NEXT(r, i)				\
-	((i)+1 == (r)->num_slots ? 0 : (i) + 1 )
 
-#define	NETMAP_RING_FIRST_RESERVED(r)			\
-	( (r)->cur < (r)->reserved ?			\
-	  (r)->cur + (r)->num_slots - (r)->reserved :	\
-	  (r)->cur - (r)->reserved )
+static inline uint32_t
+nm_ring_next(struct netmap_ring *r, uint32_t i)
+{
+	return ( unlikely(i + 1 == r->num_slots) ? 0 : i + 1);
+}
+
 
 /*
- * Return 1 if the given tx ring is empty.
+ * Return 1 if we have pending transmissions in the tx ring.
+ * When everything is complete ring->cur = ring->tail + 1 (modulo ring size)
  */
-#define NETMAP_TX_RING_EMPTY(r)	((r)->avail >= (r)->num_slots - 1)
+static inline int
+nm_tx_pending(struct netmap_ring *r)
+{
+	return nm_ring_next(r, r->tail) != r->cur;
+}
+
+
+static inline uint32_t
+nm_ring_space(struct netmap_ring *ring)
+{
+        int ret = ring->tail - ring->cur;
+        if (ret < 0)
+                ret += ring->num_slots;
+        return ret;
+}
+
 
 #ifdef NETMAP_WITH_LIBS
 /*
@@ -113,7 +138,12 @@
 #include <sys/ioctl.h>
 #include <sys/errno.h>	/* EINVAL */
 #include <fcntl.h>	/* O_RDWR */
-#include <malloc.h>
+#include <unistd.h>	/* close() */
+#ifdef __FreeBSD__
+#include <stdlib.h>
+#else
+#include <malloc.h>	/* on FreeBSD it is stdlib.h */
+#endif
 
 struct nm_hdr_t {	/* same as pcap_pkthdr */
 	struct timeval	ts;
@@ -139,30 +169,73 @@ struct nm_desc_t {
 #define IS_NETMAP_DESC(d)	(P2NMD(d)->self == P2NMD(d))
 #define NETMAP_FD(d)		(P2NMD(d)->fd)
 
+
+/*
+ * this is a slightly optimized copy routine which rounds
+ * to multiple of 64 bytes and is often faster than dealing
+ * with other odd sizes. We assume there is enough room
+ * in the source and destination buffers.
+ *
+ * XXX only for multiples of 64 bytes, non overlapped.
+ */
+static inline void
+pkt_copy(const void *_src, void *_dst, int l)
+{
+	const uint64_t *src = _src;
+	uint64_t *dst = _dst;
+	if (unlikely(l >= 1024)) {
+		memcpy(dst, src, l);
+		return;
+	}
+	for (; likely(l > 0); l-=64) {
+		*dst++ = *src++;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		*dst++ = *src++;
+	}
+}
+
+
 /*
  * The callback, invoked on each received packet. Same as libpcap
  */
 typedef void (*nm_cb_t)(u_char *, const struct nm_hdr_t *, const u_char *d);
 
 /*
- * The open routine accepts an ifname (netmap:foo or vale:foo) and
- * optionally a second (string) argument indicating the ring number
+ *--- the pcap-like API ---
+ *
+ * nm_open() opens a file descriptor, binds to a port and maps memory.
+ *
+ * ifname	(netmap:foo or vale:foo) is the port name
+ * flags	can be NETMAP_SW_RING or NETMAP_HW_RING etc.
+ * ring_no 	only used if NETMAP_HW_RING is specified, is interpreted
+ *		as a string or integer indicating the ring number
+ * ring_flags	is stored in all ring flags (e.g. for transparent mode)
  * to open. If successful, t opens the fd and maps the memory.
  */
+ 
 static struct nm_desc_t *nm_open(const char *ifname,
 	 const char *ring_no, int flags, int ring_flags);
 
 /*
- * nm_dispatch() is the same as pcap_dispatch()
- * nm_next() is the same as pcap_next()
+ * nm_close()	closes and restores the port to its previous state
  */
-static int nm_dispatch(struct nm_desc_t *, int, nm_cb_t, u_char *);
-static u_char *nm_next(struct nm_desc_t *, struct nm_hdr_t *);
+
+static int nm_close(struct nm_desc_t *);
 
 /*
- * unmap memory, close file descriptor and free the descriptor.
+ * nm_inject() is the same as pcap_inject()
+ * nm_dispatch() is the same as pcap_dispatch()
+ * nm_nextpkt() is the same as pcap_next()
  */
-static int nm_close(struct nm_desc_t *);
+
+static int nm_inject(struct nm_desc_t *, const void *, size_t);
+static int nm_dispatch(struct nm_desc_t *, int, nm_cb_t, u_char *);
+static u_char *nm_nextpkt(struct nm_desc_t *, struct nm_hdr_t *);
 
 
 /*
@@ -240,6 +313,12 @@ fail:
 static int
 nm_close(struct nm_desc_t *d)
 {
+	/*
+	 * ugly trick to avoid unused warnings
+	 */
+	static void *__xxzt[] __attribute__ ((unused))  =
+		{ nm_open, nm_inject, nm_dispatch, nm_nextpkt } ;
+
 	if (d == NULL || d->self != d)
 		return EINVAL;
 	if (d->mem)
@@ -253,9 +332,45 @@ nm_close(struct nm_desc_t *d)
 
 
 /*
+ * Same prototype as pcap_inject(), only need to cast.
+ */
+static int
+nm_inject(struct nm_desc_t *d, const void *buf, size_t size)
+{
+	u_int c, n = d->last_ring - d->first_ring + 1;
+
+	if (0) fprintf(stderr, "%s rings %d %d %d\n", __FUNCTION__,
+		d->first_ring, d->cur_ring, d->last_ring);
+	for (c = 0; c < n ; c++) {
+		/* compute current ring to use */
+		struct netmap_ring *ring;
+		uint32_t i, idx;
+		uint32_t ri = d->cur_ring + c;
+
+		if (ri > d->last_ring)
+			ri = d->first_ring;
+		ring = NETMAP_TXRING(d->nifp, ri);
+		if (nm_ring_empty(ring)) {
+			if (0) fprintf(stderr, "%s ring %d cur %d tail %d\n",
+				__FUNCTION__,
+				ri, ring->cur, ring->tail);
+			continue;
+		}
+		i = ring->cur;
+		idx = ring->slot[i].buf_idx;
+		ring->slot[i].len = size;
+		pkt_copy(buf, NETMAP_BUF(ring, idx), size);
+		d->cur_ring = ri;
+		ring->head = ring->cur = nm_ring_next(ring, i);
+		return size;
+	}
+	return 0; /* fail */
+}
+
+
+/*
  * Same prototype as pcap_dispatch(), only need to cast.
  */
-inline /* not really, but disable unused warnings */
 static int
 nm_dispatch(struct nm_desc_t *d, int cnt, nm_cb_t cb, u_char *arg)
 {
@@ -276,7 +391,7 @@ nm_dispatch(struct nm_desc_t *d, int cnt, nm_cb_t cb, u_char *arg)
 		if (ri > d->last_ring)
 			ri = d->first_ring;
 		ring = NETMAP_RXRING(d->nifp, ri);
-		for ( ; ring->avail > 0 && cnt != got; got++) {
+		for ( ; !nm_ring_empty(ring) && cnt != got; got++) {
 			u_int i = ring->cur;
 			u_int idx = ring->slot[i].buf_idx;
 			u_char *buf = (u_char *)NETMAP_BUF(ring, idx);
@@ -285,24 +400,22 @@ nm_dispatch(struct nm_desc_t *d, int cnt, nm_cb_t cb, u_char *arg)
 			d->hdr.len = d->hdr.caplen = ring->slot[i].len;
 			d->hdr.ts = ring->ts;
 			cb(arg, &d->hdr, buf);
-			ring->cur = NETMAP_RING_NEXT(ring, i);
-			ring->avail--;
+			ring->head = ring->cur = nm_ring_next(ring, i);
 		}
 	}
 	d->cur_ring = ri;
 	return got;
 }
 
-inline /* not really, but disable unused warnings */
 static u_char *
-nm_next(struct nm_desc_t *d, struct nm_hdr_t *hdr)
+nm_nextpkt(struct nm_desc_t *d, struct nm_hdr_t *hdr)
 {
 	int ri = d->cur_ring;
 
 	do {
 		/* compute current ring to use */
 		struct netmap_ring *ring = NETMAP_RXRING(d->nifp, ri);
-		if (ring->avail > 0) {
+		if (!nm_ring_empty(ring)) {
 			u_int i = ring->cur;
 			u_int idx = ring->slot[i].buf_idx;
 			u_char *buf = (u_char *)NETMAP_BUF(ring, idx);
@@ -310,8 +423,12 @@ nm_next(struct nm_desc_t *d, struct nm_hdr_t *hdr)
 			// prefetch(buf);
 			hdr->ts = ring->ts;
 			hdr->len = hdr->caplen = ring->slot[i].len;
-			ring->cur = NETMAP_RING_NEXT(ring, i);
-			ring->avail--;
+			ring->cur = nm_ring_next(ring, i);
+			/* we could postpone advancing head if we want
+			 * to hold the buffer. This can be supported in
+			 * the future.
+			 */
+			ring->head = ring->cur;
 			d->cur_ring = ri;
 			return buf;
 		}