[if_dwc] add support for multi-descriptor packets in TX path

Original if_dwc driver used m_defrag as an implementation shortcut but on
1000Mb networks it affects performance. Implement multi-descriptor support for
TX path.

Tested on RK3399-Firefly, patch adds ~15% of network throughput.

Reviewed By:	manu
Differential Revision:	https://reviews.freebsd.org/D27520

2 files changed, 116 insertions, 43 deletions

diff --git a/sys/dev/dwc/if_dwc.c b/sys/dev/dwc/if_dwc.c
index ee871c268ea66..776d0d0dc3925 100644
--- a/sys/dev/dwc/if_dwc.c
+++ b/sys/dev/dwc/if_dwc.c
@@ -627,7 +627,7 @@ dwc_get1paddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 
 inline static void
 dwc_setup_txdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr,
-  uint32_t len, uint32_t flags)
+  uint32_t len, uint32_t flags, bool first, bool last)
 {
 	uint32_t desc0, desc1;
 
@@ -635,55 +635,72 @@ dwc_setup_txdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr,
 	if (paddr == 0 || len == 0) {
 		desc0 = 0;
 		desc1 = 0;
-		--sc->txcount;
+		--sc->tx_desccount;
 	} else {
 		if (sc->mactype != DWC_GMAC_EXT_DESC) {
 			desc0 = 0;
-			desc1 = NTDESC1_TCH | NTDESC1_FS | NTDESC1_LS |
-			    NTDESC1_IC | len | flags;
+			desc1 = NTDESC1_TCH | len | flags;
+			if (first)
+				desc1 |=  NTDESC1_FS;
+			if (last)
+				desc1 |= NTDESC1_LS | NTDESC1_IC;
 		} else {
-			desc0 = ETDESC0_TCH | ETDESC0_FS | ETDESC0_LS |
-			    ETDESC0_IC | flags;
+			desc0 = ETDESC0_TCH | flags;
+			if (first)
+				desc0 |= ETDESC0_FS;
+			if (last)
+				desc0 |= ETDESC0_LS | ETDESC0_IC;
 			desc1 = len;
 		}
-		++sc->txcount;
+		++sc->tx_desccount;
 	}
 
 	sc->txdesc_ring[idx].addr1 = (uint32_t)(paddr);
 	sc->txdesc_ring[idx].desc0 = desc0;
 	sc->txdesc_ring[idx].desc1 = desc1;
+}
 
-	if (paddr && len) {
-		wmb();
-		sc->txdesc_ring[idx].desc0 |= TDESC0_OWN;
-		wmb();
-	}
+inline static void
+dwc_set_owner(struct dwc_softc *sc, int idx)
+{
+	wmb();
+	sc->txdesc_ring[idx].desc0 |= TDESC0_OWN;
+	wmb();
 }
 
 static int
 dwc_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp)
 {
-	struct bus_dma_segment seg;
+	struct bus_dma_segment segs[TX_MAP_MAX_SEGS];
 	int error, nsegs;
 	struct mbuf * m;
 	uint32_t flags = 0;
+	int i;
+	int first, last;
 
-	if ((m = m_defrag(*mp, M_NOWAIT)) == NULL)
+	error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map,
+	    *mp, segs, &nsegs, 0);
+	if (error == EFBIG) {
+		/*
+		 * The map may be partially mapped from the first call.
+		 * Make sure to reset it.
+		 */
+		bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map);
+		if ((m = m_defrag(*mp, M_NOWAIT)) == NULL)
+			return (ENOMEM);
+		*mp = m;
+		error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map,
+		    *mp, segs, &nsegs, 0);
+	}
+	if (error != 0)
 		return (ENOMEM);
-	*mp = m;
 
-	error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map,
-	    m, &seg, &nsegs, 0);
-	if (error != 0) {
+	if (sc->tx_desccount + nsegs > TX_DESC_COUNT) {
+		bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map);
 		return (ENOMEM);
 	}
 
-	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
-
-	bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map,
-	    BUS_DMASYNC_PREWRITE);
-
-	sc->txbuf_map[idx].mbuf = m;
+	m = *mp;
 
 	if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0) {
 		if ((m->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_UDP)) != 0) {
@@ -699,7 +716,27 @@ dwc_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp)
 		}
 	}
 
-	dwc_setup_txdesc(sc, idx, seg.ds_addr, seg.ds_len, flags);
+	bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map,
+	    BUS_DMASYNC_PREWRITE);
+
+	sc->txbuf_map[idx].mbuf = m;
+
+	first = sc->tx_desc_head;
+	for (i = 0; i < nsegs; i++) {
+		dwc_setup_txdesc(sc, sc->tx_desc_head,
+		    segs[i].ds_addr, segs[i].ds_len,
+		    (i == 0) ? flags : 0, /* only first desc needs flags */
+		    (i == 0),
+		    (i == nsegs - 1));
+		if (i > 0)
+			dwc_set_owner(sc, sc->tx_desc_head);
+		last = sc->tx_desc_head;
+		sc->tx_desc_head = next_txidx(sc, sc->tx_desc_head);
+	}
+
+	sc->txbuf_map[idx].last_desc_idx = last;
+
+	dwc_set_owner(sc, first);
 
 	return (0);
 }
@@ -900,7 +937,8 @@ setup_dma(struct dwc_softc *sc)
 	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 	    BUS_SPACE_MAXADDR,		/* highaddr */
 	    NULL, NULL,			/* filter, filterarg */
-	    MCLBYTES, 1, 		/* maxsize, nsegments */
+	    MCLBYTES*TX_MAP_MAX_SEGS,	/* maxsize */
+	    TX_MAP_MAX_SEGS,		/* nsegments */
 	    MCLBYTES,			/* maxsegsize */
 	    0,				/* flags */
 	    NULL, NULL,			/* lockfunc, lockarg */
@@ -911,7 +949,7 @@ setup_dma(struct dwc_softc *sc)
 		goto out;
 	}
 
-	for (idx = 0; idx < TX_DESC_COUNT; idx++) {
+	for (idx = 0; idx < TX_MAP_COUNT; idx++) {
 		error = bus_dmamap_create(sc->txbuf_tag, BUS_DMA_COHERENT,
 		    &sc->txbuf_map[idx].map);
 		if (error != 0) {
@@ -919,9 +957,11 @@ setup_dma(struct dwc_softc *sc)
 			    "could not create TX buffer DMA map.\n");
 			goto out;
 		}
-		dwc_setup_txdesc(sc, idx, 0, 0, 0);
 	}
 
+	for (idx = 0; idx < TX_DESC_COUNT; idx++)
+		dwc_setup_txdesc(sc, idx, 0, 0, 0, false, false);
+
 	/*
 	 * Set up RX descriptor ring, descriptors, dma maps, and mbufs.
 	 */
@@ -1029,7 +1069,12 @@ dwc_txstart_locked(struct dwc_softc *sc)
 	enqueued = 0;
 
 	for (;;) {
-		if (sc->txcount == (TX_DESC_COUNT - 1)) {
+		if (sc->tx_desccount > (TX_DESC_COUNT - TX_MAP_MAX_SEGS  + 1)) {
+			if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
+			break;
+		}
+
+		if (sc->tx_mapcount == (TX_MAP_COUNT - 1)) {
 			if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
 			break;
 		}
@@ -1037,12 +1082,14 @@ dwc_txstart_locked(struct dwc_softc *sc)
 		m = if_dequeue(ifp);
 		if (m == NULL)
 			break;
-		if (dwc_setup_txbuf(sc, sc->tx_idx_head, &m) != 0) {
+		if (dwc_setup_txbuf(sc, sc->tx_map_head, &m) != 0) {
 			if_sendq_prepend(ifp, m);
+			if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
 			break;
 		}
 		if_bpfmtap(ifp, m);
-		sc->tx_idx_head = next_txidx(sc, sc->tx_idx_head);
+		sc->tx_map_head = next_txidx(sc, sc->tx_map_head);
+		sc->tx_mapcount++;
 		++enqueued;
 	}
 
@@ -1193,28 +1240,46 @@ dwc_txfinish_locked(struct dwc_softc *sc)
 	struct dwc_bufmap *bmap;
 	struct dwc_hwdesc *desc;
 	struct ifnet *ifp;
+	int idx, last_idx;
+	bool map_finished;
 
 	DWC_ASSERT_LOCKED(sc);
 
 	ifp = sc->ifp;
-	while (sc->tx_idx_tail != sc->tx_idx_head) {
-		desc = &sc->txdesc_ring[sc->tx_idx_tail];
-		if ((desc->desc0 & TDESC0_OWN) != 0)
+	/* check if all descriptors of the map are done */
+	while (sc->tx_map_tail != sc->tx_map_head) {
+		map_finished = true;
+		bmap = &sc->txbuf_map[sc->tx_map_tail];
+		idx = sc->tx_desc_tail;
+		last_idx = next_txidx(sc, bmap->last_desc_idx);
+		while (idx != last_idx) {
+			desc = &sc->txdesc_ring[idx];
+			if ((desc->desc0 & TDESC0_OWN) != 0) {
+				map_finished = false;
+				break;
+			}
+			idx = next_txidx(sc, idx);
+		}
+
+		if (!map_finished)
 			break;
-		bmap = &sc->txbuf_map[sc->tx_idx_tail];
 		bus_dmamap_sync(sc->txbuf_tag, bmap->map,
 		    BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(sc->txbuf_tag, bmap->map);
 		m_freem(bmap->mbuf);
 		bmap->mbuf = NULL;
-		dwc_setup_txdesc(sc, sc->tx_idx_tail, 0, 0, 0);
-		sc->tx_idx_tail = next_txidx(sc, sc->tx_idx_tail);
+		sc->tx_mapcount--;
+		while (sc->tx_desc_tail != last_idx) {
+			dwc_setup_txdesc(sc, sc->tx_desc_tail, 0, 0, 0, false, false);
+			sc->tx_desc_tail = next_txidx(sc, sc->tx_desc_tail);
+		}
+		sc->tx_map_tail = next_txidx(sc, sc->tx_map_tail);
 		if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
 		if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
 	}
 
 	/* If there are no buffers outstanding, muzzle the watchdog. */
-	if (sc->tx_idx_tail == sc->tx_idx_head) {
+	if (sc->tx_desc_tail == sc->tx_desc_head) {
 		sc->tx_watchdog_count = 0;
 	}
 }
@@ -1503,7 +1568,8 @@ dwc_attach(device_t dev)
 	sc = device_get_softc(dev);
 	sc->dev = dev;
 	sc->rx_idx = 0;
-	sc->txcount = TX_DESC_COUNT;
+	sc->tx_desccount = TX_DESC_COUNT;
+	sc->tx_mapcount = 0;
 	sc->mii_clk = IF_DWC_MII_CLK(dev);
 	sc->mactype = IF_DWC_MAC_TYPE(dev);
 
@@ -1610,7 +1676,7 @@ dwc_attach(device_t dev)
 	if_setstartfn(ifp, dwc_txstart);
 	if_setioctlfn(ifp, dwc_ioctl);
 	if_setinitfn(ifp, dwc_init);
-	if_setsendqlen(ifp, TX_DESC_COUNT - 1);
+	if_setsendqlen(ifp, TX_MAP_COUNT - 1);
 	if_setsendqready(sc->ifp);
 	if_sethwassist(sc->ifp, CSUM_IP | CSUM_UDP | CSUM_TCP);
 	if_setcapabilities(sc->ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM);
diff --git a/sys/dev/dwc/if_dwcvar.h b/sys/dev/dwc/if_dwcvar.h
index 0470b29cb0e1c..97ae0ea681c89 100644
--- a/sys/dev/dwc/if_dwcvar.h
+++ b/sys/dev/dwc/if_dwcvar.h
@@ -47,11 +47,15 @@
 #define	RX_DESC_COUNT	1024
 #define	RX_DESC_SIZE	(sizeof(struct dwc_hwdesc) * RX_DESC_COUNT)
 #define	TX_DESC_COUNT	1024
+#define	TX_MAP_COUNT	TX_DESC_COUNT
 #define	TX_DESC_SIZE	(sizeof(struct dwc_hwdesc) * TX_DESC_COUNT)
+#define	TX_MAP_MAX_SEGS	32
 
 struct dwc_bufmap {
 	bus_dmamap_t		map;
 	struct mbuf		*mbuf;
+	/* Only used for TX descirptors */
+	int			last_desc_idx;
 };
 
 struct dwc_softc {
@@ -89,9 +93,12 @@ struct dwc_softc {
 	bus_addr_t		txdesc_ring_paddr;
 	bus_dma_tag_t		txbuf_tag;
 	struct dwc_bufmap	txbuf_map[TX_DESC_COUNT];
-	uint32_t		tx_idx_head;
-	uint32_t		tx_idx_tail;
-	int			txcount;
+	uint32_t		tx_desc_head;
+	uint32_t		tx_desc_tail;
+	uint32_t		tx_map_head;
+	uint32_t		tx_map_tail;
+	int			tx_desccount;
+	int			tx_mapcount;
 };
 
 #endif	/* __IF_DWCVAR_H__ */