2 files changed, 203 insertions, 29 deletions

diff --git a/sys/dev/em/if_em.c b/sys/dev/em/if_em.c
index 8fbdf22f2ab45..47aecf7c43825 100644
--- a/sys/dev/em/if_em.c
+++ b/sys/dev/em/if_em.c
@@ -86,7 +86,7 @@ int	em_display_debug_stats = 0;
  *  Driver version
  *********************************************************************/
 
-char em_driver_version[] = "Version - 6.1.4";
+char em_driver_version[] = "Version - 6.1.4 - TSO";
 
 
 /*********************************************************************
@@ -231,6 +231,8 @@ static void	em_receive_checksum(struct adapter *, struct em_rx_desc *,
 		    struct mbuf *);
 static void	em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 		    uint32_t *, uint32_t *);
+static boolean_t em_tso_setup(struct adapter *, struct mbuf *,
+		    uint32_t *, uint32_t *);
 static void	em_set_promisc(struct adapter *);
 static void	em_disable_promisc(struct adapter *);
 static void	em_set_multi(struct adapter *);
@@ -304,6 +306,7 @@ MODULE_DEPEND(em, ether, 1, 1, 1);
 
 #define E1000_TICKS_TO_USECS(ticks)	((1024 * (ticks) + 500) / 1000)
 #define E1000_USECS_TO_TICKS(usecs)	((1000 * (usecs) + 512) / 1024)
+#define M_TSO_LEN			66
 
 static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
 static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
@@ -907,6 +910,10 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 			ifp->if_capenable ^= IFCAP_HWCSUM;
 			reinit = 1;
 		}
+		if (mask & IFCAP_TSO) {
+			ifp->if_capenable ^= IFCAP_TSO;
+			reinit = 1;
+		}
 		if (mask & IFCAP_VLAN_HWTAGGING) {
 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 			reinit = 1;
@@ -1075,11 +1082,12 @@ em_init_locked(struct adapter *adapter)
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
+	ifp->if_hwassist = 0;
 	if (adapter->hw.mac_type >= em_82543) {
 		if (ifp->if_capenable & IFCAP_TXCSUM)
 			ifp->if_hwassist = EM_CHECKSUM_FEATURES;
-		else
-			ifp->if_hwassist = 0;
+		if (ifp->if_capenable & IFCAP_TSO)
+			ifp->if_hwassist |= EM_TCPSEG_FEATURES;
 	}
 
 	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
@@ -1441,11 +1449,13 @@ em_encap(struct adapter *adapter, struct mbuf **m_headp)
 	struct m_tag		*mtag;
 	uint32_t		txd_upper, txd_lower, txd_used, txd_saved;
 	int			nsegs, i, j;
-	int			error;
+	int			error, do_tso, tso_desc = 0;
 
 	m_head = *m_headp;
 	current_tx_desc = NULL;
-	txd_used = txd_saved = 0;
+	txd_upper = txd_lower = txd_used = txd_saved = 0;
+
+        do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
 
 	/*
 	 * Force a cleanup if number of TX descriptors
@@ -1498,6 +1508,17 @@ em_encap(struct adapter *adapter, struct mbuf **m_headp)
 		*m_headp = m_head;
 	}
 
+        /*
+         * TSO workaround:
+         *  If an mbuf is only header we need
+         *     to pull 4 bytes of data into it.
+         */
+        if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
+                m_head = m_pullup(m_head, M_TSO_LEN + 4);
+                if (m_head == NULL)
+                        return (ENOBUFS);
+        }
+
 	/*
 	 * Map the packet for DMA.
 	 */
@@ -1536,24 +1557,42 @@ em_encap(struct adapter *adapter, struct mbuf **m_headp)
 		return (EIO);
 	}
 
-	if (nsegs > adapter->num_tx_desc_avail) {
+        /*
+         * TSO Hardware workaround, if this packet is not
+         * TSO, and is only a single descriptor long, and
+         * it follows a TSO burst, then we need to add a
+         * sentinel descriptor to prevent premature writeback.
+         */
+        if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
+                if (nsegs == 1)
+                        tso_desc = TRUE;
+                adapter->tx_tso = FALSE;
+        }
+
+	if (nsegs > adapter->num_tx_desc_avail - 2) {
 		adapter->no_tx_desc_avail2++;
 		bus_dmamap_unload(adapter->txtag, map);
 		return (ENOBUFS);
 	}
 
+        /* Do hardware assists */
 	m_head = *m_headp;
-	if (ifp->if_hwassist > 0)
-		em_transmit_checksum_setup(adapter,  m_head, &txd_upper, &txd_lower);
-	else
-		txd_upper = txd_lower = 0;
+        if ( ifp->if_hwassist > 0) {
+                if (em_tso_setup(adapter, m_head, &txd_upper, &txd_lower)) {
+                        /* we need to make a final sentinel transmit desc */
+                        tso_desc = TRUE;
+                } else
+                        em_transmit_checksum_setup(adapter,  m_head,
+                            &txd_upper, &txd_lower);
+        }
 
 	i = adapter->next_avail_tx_desc;
-	if (adapter->pcix_82544) {
+	if (adapter->pcix_82544)
 		txd_saved = i;
-		txd_used = 0;
-	}
+
 	for (j = 0; j < nsegs; j++) {
+                bus_size_t seg_len;
+                bus_addr_t seg_addr;
 		/* If adapter is 82544 and on PCIX bus. */
 		if(adapter->pcix_82544) {
 			DESC_ARRAY	desc_array;
@@ -1587,26 +1626,57 @@ em_encap(struct adapter *adapter, struct mbuf **m_headp)
 				txd_used++;
 			}
 		} else {
-			tx_buffer = &adapter->tx_buffer_area[i];
-			current_tx_desc = &adapter->tx_desc_base[i];
-
-			current_tx_desc->buffer_addr = htole64(segs[j].ds_addr);
-			current_tx_desc->lower.data = htole32(
-				adapter->txd_cmd | txd_lower | segs[j].ds_len);
-			current_tx_desc->upper.data = htole32(txd_upper);
-
-			if (++i == adapter->num_tx_desc)
-				i = 0;
-
-			tx_buffer->m_head = NULL;
+                       tx_buffer = &adapter->tx_buffer_area[i];
+                        current_tx_desc = &adapter->tx_desc_base[i];
+                        seg_addr = htole64(segs[j].ds_addr);
+                        seg_len  = segs[j].ds_len;
+                        /*
+                        ** TSO Workaround:
+                        ** If this is the last descriptor, we want to
+                        ** split it so we have a small final sentinel
+                        */
+                        if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
+                                seg_len -= 4;
+                                current_tx_desc->buffer_addr = seg_addr;
+                                current_tx_desc->lower.data = htole32(
+                                adapter->txd_cmd | txd_lower | seg_len);
+                                current_tx_desc->upper.data =
+                                    htole32(txd_upper);
+                                if (++i == adapter->num_tx_desc)
+                                        i = 0;
+                                /* Now make the sentinel */
+                                ++txd_used; /* using an extra txd */
+                                current_tx_desc = &adapter->tx_desc_base[i];
+                                tx_buffer = &adapter->tx_buffer_area[i];
+                                current_tx_desc->buffer_addr =
+                                    seg_addr + seg_len;
+                                current_tx_desc->lower.data = htole32(
+                                adapter->txd_cmd | txd_lower | 4);
+                                current_tx_desc->upper.data =
+                                    htole32(txd_upper);
+                                if (++i == adapter->num_tx_desc)
+                                        i = 0;
+                        } else {
+                                current_tx_desc->buffer_addr = seg_addr;
+                                current_tx_desc->lower.data = htole32(
+                                adapter->txd_cmd | txd_lower | seg_len);
+                                current_tx_desc->upper.data =
+                                    htole32(txd_upper);
+                                if (++i == adapter->num_tx_desc)
+                                        i = 0;
+                        }
+                        tx_buffer->m_head = NULL;
 		}
 	}
 
 	adapter->next_avail_tx_desc = i;
 	if (adapter->pcix_82544)
 		adapter->num_tx_desc_avail -= txd_used;
-	else
+	else {
 		adapter->num_tx_desc_avail -= nsegs;
+                if (tso_desc) /* TSO used an extra for sentinel */
+                        adapter->num_tx_desc_avail -= txd_used;
+        }
 
 	if (mtag != NULL) {
 		/* Set the vlan id. */
@@ -2266,6 +2336,13 @@ em_setup_interface(device_t dev, struct adapter *adapter)
 		ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
 	}
 
+        /* Enable TSO if available */
+        if ((adapter->hw.mac_type > em_82544) &&
+            (adapter->hw.mac_type != em_82547)) {
+                ifp->if_capabilities |= IFCAP_TSO;
+                ifp->if_capenable |= IFCAP_TSO;
+        }
+
 	/*
 	 * Tell the upper layer(s) we support long frames.
 	 */
@@ -2484,15 +2561,23 @@ em_allocate_transmit_structures(struct adapter *adapter)
 static int
 em_setup_transmit_structures(struct adapter *adapter)
 {
+        struct ifnet   *ifp = adapter->ifp;
 	device_t dev = adapter->dev;
 	struct em_buffer *tx_buffer;
-	bus_size_t size;
+	bus_size_t size, segsize;
 	int error, i;
 
 	/*
 	 * Setup DMA descriptor areas.
 	 */
-	size = roundup2(adapter->hw.max_frame_size, MCLBYTES);
+	segsize = size = roundup2(adapter->hw.max_frame_size, MCLBYTES);
+
+        /* Overrides for TSO - want large sizes */
+        if (ifp->if_hwassist & EM_TCPSEG_FEATURES) {
+                size = EM_TSO_SIZE;
+                segsize = PAGE_SIZE;
+        }
+
 	if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 				1, 0,			/* alignment, bounds */
 				BUS_SPACE_MAXADDR,	/* lowaddr */
@@ -2500,7 +2585,7 @@ em_setup_transmit_structures(struct adapter *adapter)
 				NULL, NULL,		/* filter, filterarg */
 				size,			/* maxsize */
 				EM_MAX_SCATTER,		/* nsegments */
-				size,			/* maxsegsize */
+				segsize,		/* maxsegsize */
 				0,			/* flags */
 				NULL,		/* lockfunc */
 				NULL,		/* lockarg */
@@ -2762,6 +2847,84 @@ em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
 
 /**********************************************************************
  *
+ *  Setup work for hardware segmentation offload (TSO)
+ *
+ **********************************************************************/
+static boolean_t
+em_tso_setup(struct adapter *adapter,
+             struct mbuf *mp,
+             uint32_t *txd_upper,
+             uint32_t *txd_lower)
+{
+        struct em_context_desc *TXD;
+        struct em_buffer *tx_buffer;
+        struct ip *ip;
+        struct tcphdr *th;
+        int curr_txd, hdr_len, ip_hlen, tcp_hlen;
+
+        if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) ||
+            (mp->m_pkthdr.len <= E1000_TX_BUFFER_SIZE)) {
+                return FALSE;
+        }
+
+        *txd_lower = (E1000_TXD_CMD_DEXT |
+                      E1000_TXD_DTYP_D |
+                      E1000_TXD_CMD_TSE);
+
+        *txd_upper = (E1000_TXD_POPTS_IXSM |
+                      E1000_TXD_POPTS_TXSM) << 8;
+
+        curr_txd = adapter->next_avail_tx_desc;
+        tx_buffer = &adapter->tx_buffer_area[curr_txd];
+        TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
+
+        mp->m_data += sizeof(struct ether_header);
+        ip = mtod(mp, struct ip *);
+        ip->ip_len = 0;
+        ip->ip_sum = 0;
+        ip_hlen = ip->ip_hl << 2 ;
+        th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
+        tcp_hlen = th->th_off << 2;
+
+        hdr_len = ETHER_HDR_LEN + ip_hlen + tcp_hlen;
+	th->th_sum = in_pseudo(ip->ip_src.s_addr,
+	    ip->ip_dst.s_addr, htons(IPPROTO_TCP));
+
+        mp->m_data -= sizeof(struct ether_header);
+        TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
+        TXD->lower_setup.ip_fields.ipcso =
+                ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
+        TXD->lower_setup.ip_fields.ipcse =
+                htole16(ETHER_HDR_LEN + ip_hlen - 1);
+
+        TXD->upper_setup.tcp_fields.tucss =
+                ETHER_HDR_LEN + ip_hlen;
+        TXD->upper_setup.tcp_fields.tucse = 0;
+        TXD->upper_setup.tcp_fields.tucso =
+                ETHER_HDR_LEN + ip_hlen +
+                offsetof(struct tcphdr, th_sum);
+        TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
+        TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
+        TXD->cmd_and_length = htole32(adapter->txd_cmd |
+                                E1000_TXD_CMD_DEXT |
+                                E1000_TXD_CMD_TSE |
+                                E1000_TXD_CMD_IP | E1000_TXD_CMD_TCP |
+                                (mp->m_pkthdr.len - (hdr_len)));
+
+        tx_buffer->m_head = NULL;
+
+        if (++curr_txd == adapter->num_tx_desc)
+                curr_txd = 0;
+
+        adapter->num_tx_desc_avail--;
+        adapter->next_avail_tx_desc = curr_txd;
+        adapter->tx_tso = TRUE;
+
+        return TRUE;
+}
+
+/**********************************************************************
+ *
  *  Examine each tx_buffer in the used queue. If the hardware is done
  *  processing the packet then free associated resources. The
  *  tx_buffer is put back on the free queue.
@@ -3706,6 +3869,10 @@ em_print_hw_stats(struct adapter *adapter)
 	    (long long)adapter->stats.gprc);
 	device_printf(dev, "Good Packets Xmtd = %lld\n",
 	    (long long)adapter->stats.gptc);
+        device_printf(dev, "TSO Contexts Xmtd = %lld\n",
+            (long long)adapter->stats.tsctc);
+        device_printf(dev, "TSO Contexts Failed = %lld\n",
+            (long long)adapter->stats.tsctfc);
 }
 
 static int
diff --git a/sys/dev/em/if_em.h b/sys/dev/em/if_em.h
index 2765dff4ed6a4..99a7ba895b4f3 100644
--- a/sys/dev/em/if_em.h
+++ b/sys/dev/em/if_em.h
@@ -138,6 +138,11 @@ POSSIBILITY OF SUCH DAMAGE.
 #define EM_CHECKSUM_FEATURES            (CSUM_TCP | CSUM_UDP)
 
 /*
+ * Inform the stack about transmit segmentation offload capabilities.
+ */
+#define EM_TCPSEG_FEATURES		CSUM_TSO
+
+/*
  * This parameter controls the duration of transmit watchdog timer.
  */
 #define EM_TX_TIMEOUT                   5    /* set to 5 seconds */
@@ -225,6 +230,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #define EM_RXBUFFER_16384      16384
 
 #define EM_MAX_SCATTER            64
+#define EM_TSO_SIZE		65535
 
 typedef enum _XSUM_CONTEXT_T {
 	OFFLOAD_NONE,
@@ -308,6 +314,7 @@ struct adapter {
         uint32_t		txd_cmd;
 	struct em_buffer	*tx_buffer_area;
 	bus_dma_tag_t		txtag;		/* dma tag for tx */
+	uint32_t		tx_tso;		/* last tx was tso */
 
 	/* 
 	 * Receive definitions