diff options
Diffstat (limited to 'sys/dev/netmap/if_lem_netmap.h')
| -rw-r--r-- | sys/dev/netmap/if_lem_netmap.h | 208 |
1 files changed, 197 insertions, 11 deletions
diff --git a/sys/dev/netmap/if_lem_netmap.h b/sys/dev/netmap/if_lem_netmap.h index 4fce5c988d09..272f02c7b9d0 100644 --- a/sys/dev/netmap/if_lem_netmap.h +++ b/sys/dev/netmap/if_lem_netmap.h @@ -39,6 +39,7 @@ #include <vm/pmap.h> /* vtophys ? */ #include <dev/netmap/netmap_kern.h> +extern int netmap_adaptive_io; /* * Register/unregister. We are already under netmap lock. @@ -84,10 +85,10 @@ lem_netmap_reg(struct netmap_adapter *na, int onoff) * Reconcile kernel and user view of the transmit ring. */ static int -lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) +lem_netmap_txsync(struct netmap_kring *kring, int flags) { + struct netmap_adapter *na = kring->na; struct ifnet *ifp = na->ifp; - struct netmap_kring *kring = &na->tx_rings[ring_nr]; struct netmap_ring *ring = kring->ring; u_int nm_i; /* index into the netmap ring */ u_int nic_i; /* index into the NIC ring */ @@ -98,6 +99,10 @@ lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) /* device-specific */ struct adapter *adapter = ifp->if_softc; +#ifdef NIC_PARAVIRT + struct paravirt_csb *csb = adapter->csb; + uint64_t *csbd = (uint64_t *)(csb + 1); +#endif /* NIC_PARAVIRT */ bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_POSTREAD); @@ -108,12 +113,25 @@ lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) nm_i = kring->nr_hwcur; if (nm_i != head) { /* we have new packets to send */ +#ifdef NIC_PARAVIRT + int do_kick = 0; + uint64_t t = 0; // timestamp + int n = head - nm_i; + if (n < 0) + n += lim + 1; + if (csb) { + t = rdtsc(); /* last timestamp */ + csbd[16] += t - csbd[0]; /* total Wg */ + csbd[17] += n; /* Wg count */ + csbd[0] = t; + } +#endif /* NIC_PARAVIRT */ nic_i = netmap_idx_k2n(kring, nm_i); while (nm_i != head) { struct netmap_slot *slot = &ring->slot[nm_i]; u_int len = slot->len; uint64_t paddr; - void *addr = PNMB(slot, &paddr); + void *addr = PNMB(na, slot, &paddr); /* device-specific */ struct e1000_tx_desc *curr = &adapter->tx_desc_base[nic_i]; @@ -122,12 +140,12 @@ lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) nic_i == 0 || nic_i == report_frequency) ? E1000_TXD_CMD_RS : 0; - NM_CHECK_ADDR_LEN(addr, len); + NM_CHECK_ADDR_LEN(na, addr, len); if (slot->flags & NS_BUF_CHANGED) { /* buffer has changed, reload map */ curr->buffer_addr = htole64(paddr); - netmap_reload_map(adapter->txtag, txbuf->map, addr); + netmap_reload_map(na, adapter->txtag, txbuf->map, addr); } slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); @@ -140,6 +158,7 @@ lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); + // XXX might try an early kick } kring->nr_hwcur = head; @@ -147,8 +166,38 @@ lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); +#ifdef NIC_PARAVIRT + /* set unconditionally, then also kick if needed */ + if (csb) { + t = rdtsc(); + if (csb->host_need_txkick == 2) { + /* can compute an update of delta */ + int64_t delta = t - csbd[3]; + if (delta < 0) + delta = -delta; + if (csbd[8] == 0 || delta < csbd[8]) { + csbd[8] = delta; + csbd[9]++; + } + csbd[10]++; + } + csb->guest_tdt = nic_i; + csbd[18] += t - csbd[0]; // total wp + csbd[19] += n; + } + if (!csb || !csb->guest_csb_on || (csb->host_need_txkick & 1)) + do_kick = 1; + if (do_kick) +#endif /* NIC_PARAVIRT */ /* (re)start the tx unit up to slot nic_i (excluded) */ E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), nic_i); +#ifdef NIC_PARAVIRT + if (do_kick) { + uint64_t t1 = rdtsc(); + csbd[20] += t1 - t; // total Np + csbd[21]++; + } +#endif /* NIC_PARAVIRT */ } /* @@ -157,6 +206,93 @@ lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) if (ticks != kring->last_reclaim || flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) { kring->last_reclaim = ticks; /* record completed transmissions using TDH */ +#ifdef NIC_PARAVIRT + /* host updates tdh unconditionally, and we have + * no side effects on reads, so we can read from there + * instead of exiting. + */ + if (csb) { + static int drain = 0, nodrain=0, good = 0, bad = 0, fail = 0; + u_int x = adapter->next_tx_to_clean; + csbd[19]++; // XXX count reclaims + nic_i = csb->host_tdh; + if (csb->guest_csb_on) { + if (nic_i == x) { + bad++; + csbd[24]++; // failed reclaims + /* no progress, request kick and retry */ + csb->guest_need_txkick = 1; + mb(); // XXX barrier + nic_i = csb->host_tdh; + } else { + good++; + } + if (nic_i != x) { + csb->guest_need_txkick = 2; + if (nic_i == csb->guest_tdt) + drain++; + else + nodrain++; +#if 1 + if (netmap_adaptive_io) { + /* new mechanism: last half ring (or so) + * released one slot at a time. + * This effectively makes the system spin. + * + * Take next_to_clean + 1 as a reference. + * tdh must be ahead or equal + * On entry, the logical order is + * x < tdh = nic_i + * We first push tdh up to avoid wraps. + * The limit is tdh-ll (half ring). + * if tdh-256 < x we report x; + * else we report tdh-256 + */ + u_int tdh = nic_i; + u_int ll = csbd[15]; + u_int delta = lim/8; + if (netmap_adaptive_io == 2 || ll > delta) + csbd[15] = ll = delta; + else if (netmap_adaptive_io == 1 && ll > 1) { + csbd[15]--; + } + + if (nic_i >= kring->nkr_num_slots) { + RD(5, "bad nic_i %d on input", nic_i); + } + x = nm_next(x, lim); + if (tdh < x) + tdh += lim + 1; + if (tdh <= x + ll) { + nic_i = x; + csbd[25]++; //report n + 1; + } else { + tdh = nic_i; + if (tdh < ll) + tdh += lim + 1; + nic_i = tdh - ll; + csbd[26]++; // report tdh - ll + } + } +#endif + } else { + /* we stop, count whether we are idle or not */ + int bh_active = csb->host_need_txkick & 2 ? 4 : 0; + csbd[27+ csb->host_need_txkick]++; + if (netmap_adaptive_io == 1) { + if (bh_active && csbd[15] > 1) + csbd[15]--; + else if (!bh_active && csbd[15] < lim/2) + csbd[15]++; + } + bad--; + fail++; + } + } + RD(1, "drain %d nodrain %d good %d retry %d fail %d", + drain, nodrain, good, bad, fail); + } else +#endif /* !NIC_PARAVIRT */ nic_i = E1000_READ_REG(&adapter->hw, E1000_TDH(0)); if (nic_i >= kring->nkr_num_slots) { /* XXX can it happen ? */ D("TDH wrap %d", nic_i); @@ -176,10 +312,10 @@ lem_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) * Reconcile kernel and user view of the receive ring. */ static int -lem_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) +lem_netmap_rxsync(struct netmap_kring *kring, int flags) { + struct netmap_adapter *na = kring->na; struct ifnet *ifp = na->ifp; - struct netmap_kring *kring = &na->rx_rings[ring_nr]; struct netmap_ring *ring = kring->ring; u_int nm_i; /* index into the netmap ring */ u_int nic_i; /* index into the NIC ring */ @@ -190,10 +326,21 @@ lem_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) /* device-specific */ struct adapter *adapter = ifp->if_softc; +#ifdef NIC_PARAVIRT + struct paravirt_csb *csb = adapter->csb; + uint32_t csb_mode = csb && csb->guest_csb_on; + uint32_t do_host_rxkick = 0; +#endif /* NIC_PARAVIRT */ if (head > lim) return netmap_ring_reinit(kring); +#ifdef NIC_PARAVIRT + if (csb_mode) { + force_update = 1; + csb->guest_need_rxkick = 0; + } +#endif /* NIC_PARAVIRT */ /* XXX check sync modes */ bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); @@ -212,11 +359,28 @@ lem_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) uint32_t staterr = le32toh(curr->status); int len; +#ifdef NIC_PARAVIRT + if (csb_mode) { + if ((staterr & E1000_RXD_STAT_DD) == 0) { + /* don't bother to retry if more than 1 pkt */ + if (n > 1) + break; + csb->guest_need_rxkick = 1; + wmb(); + staterr = le32toh(curr->status); + if ((staterr & E1000_RXD_STAT_DD) == 0) { + break; + } else { /* we are good */ + csb->guest_need_rxkick = 0; + } + } + } else +#endif /* NIC_PARAVIRT */ if ((staterr & E1000_RXD_STAT_DD) == 0) break; len = le16toh(curr->length) - 4; // CRC if (len < 0) { - D("bogus pkt size %d nic idx %d", len, nic_i); + RD(5, "bogus pkt (%d) size %d nic idx %d", n, len, nic_i); len = 0; } ring->slot[nm_i].len = len; @@ -228,6 +392,18 @@ lem_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) nic_i = nm_next(nic_i, lim); } if (n) { /* update the state variables */ +#ifdef NIC_PARAVIRT + if (csb_mode) { + if (n > 1) { + /* leave one spare buffer so we avoid rxkicks */ + nm_i = nm_prev(nm_i, lim); + nic_i = nm_prev(nic_i, lim); + n--; + } else { + csb->guest_need_rxkick = 1; + } + } +#endif /* NIC_PARAVIRT */ ND("%d new packets at nic %d nm %d tail %d", n, adapter->next_rx_desc_to_check, @@ -249,23 +425,27 @@ lem_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) for (n = 0; nm_i != head; n++) { struct netmap_slot *slot = &ring->slot[nm_i]; uint64_t paddr; - void *addr = PNMB(slot, &paddr); + void *addr = PNMB(na, slot, &paddr); struct e1000_rx_desc *curr = &adapter->rx_desc_base[nic_i]; struct em_buffer *rxbuf = &adapter->rx_buffer_area[nic_i]; - if (addr == netmap_buffer_base) /* bad buf */ + if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ goto ring_reset; if (slot->flags & NS_BUF_CHANGED) { /* buffer has changed, reload map */ curr->buffer_addr = htole64(paddr); - netmap_reload_map(adapter->rxtag, rxbuf->map, addr); + netmap_reload_map(na, adapter->rxtag, rxbuf->map, addr); slot->flags &= ~NS_BUF_CHANGED; } curr->status = 0; bus_dmamap_sync(adapter->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD); +#ifdef NIC_PARAVIRT + if (csb_mode && csb->host_rxkick_at == nic_i) + do_host_rxkick = 1; +#endif /* NIC_PARAVIRT */ nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } @@ -277,6 +457,12 @@ lem_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) * so move nic_i back by one unit */ nic_i = nm_prev(nic_i, lim); +#ifdef NIC_PARAVIRT + /* set unconditionally, then also kick if needed */ + if (csb) + csb->guest_rdt = nic_i; + if (!csb_mode || do_host_rxkick) +#endif /* NIC_PARAVIRT */ E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), nic_i); } |