From b8a6e03fac922677455d8e0977831506cf8212e8 Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Mon, 7 Oct 2019 22:40:05 +0000 Subject: Widen NET_EPOCH coverage. When epoch(9) was introduced to network stack, it was basically dropped in place of existing locking, which was mutexes and rwlocks. For the sake of performance mutex covered areas were as small as possible, so became epoch covered areas. However, epoch doesn't introduce any contention, it just delays memory reclaim. So, there is no point to minimise epoch covered areas in sense of performance. Meanwhile entering/exiting epoch also has non-zero CPU usage, so doing this less often is a win. Not the least is also code maintainability. In the new paradigm we can assume that at any stage of processing a packet, we are inside network epoch. This makes coding both input and output path way easier. On output path we already enter epoch quite early - in the ip_output(), in the ip6_output(). This patch does the same for the input path. All ISR processing, network related callouts, other ways of packet injection to the network stack shall be performed in net_epoch. Any leaf function that walks network configuration now asserts epoch. Tricky part is configuration code paths - ioctls, sysctls. They also call into leaf functions, so some need to be changed. This patch would introduce more epoch recursions (see EPOCH_TRACE) than we had before. They will be cleaned up separately, as several of them aren't trivial. Note, that unlike a lock recursion the epoch recursion is safe and just wastes a bit of resources. Reviewed by: gallatin, hselasky, cy, adrian, kristof Differential Revision: https://reviews.freebsd.org/D19111 --- sys/contrib/ipfilter/netinet/ip_fil_freebsd.c | 3 + sys/dev/firewire/if_fwip.c | 3 + sys/dev/iicbus/if_ic.c | 4 + sys/dev/usb/net/if_usie.c | 3 + sys/dev/usb/net/uhso.c | 3 + sys/dev/usb/net/usb_ethernet.c | 21 ++- sys/kern/uipc_socket.c | 2 + sys/net/if.c | 83 ++++++----- sys/net/if_ethersubr.c | 12 +- sys/net/if_gif.c | 2 + sys/net/if_me.c | 2 + sys/net/if_stf.c | 2 + sys/net/if_tuntap.c | 3 + sys/net/if_vlan.c | 73 +++------- sys/net/netisr.c | 4 + sys/net/route.c | 5 +- sys/net/rtsock.c | 21 ++- sys/netgraph/ng_ether.c | 3 + sys/netgraph/ng_iface.c | 3 + sys/netgraph/ng_ip_input.c | 7 +- sys/netinet/if_ether.c | 37 +++-- sys/netinet/igmp.c | 36 ++--- sys/netinet/in.c | 14 +- sys/netinet/in_mcast.c | 30 ++-- sys/netinet/in_rmx.c | 13 +- sys/netinet/in_var.h | 1 + sys/netinet/ip_carp.c | 22 ++- sys/netinet/ip_encap.c | 8 +- sys/netinet/ip_icmp.c | 21 +-- sys/netinet/ip_input.c | 21 +-- sys/netinet/ip_mroute.c | 5 - sys/netinet/ip_options.c | 10 +- sys/netinet/ip_output.c | 5 +- sys/netinet/sctp_bsd_addr.c | 7 +- sys/netinet6/icmp6.c | 16 +-- sys/netinet6/in6.c | 35 ++--- sys/netinet6/in6_ifattach.c | 41 ++---- sys/netinet6/in6_mcast.c | 10 +- sys/netinet6/in6_var.h | 9 +- sys/netinet6/ip6_output.c | 3 + sys/netinet6/mld6.c | 27 ++-- sys/netinet6/nd6.c | 41 +++--- sys/netinet6/nd6_nbr.c | 12 +- sys/netinet6/nd6_rtr.c | 13 +- sys/netinet6/raw_ip6.c | 12 +- sys/netipsec/xform_ipcomp.c | 4 + sys/netpfil/ipfw/ip_dn_io.c | 3 + sys/netpfil/pf/pf_if.c | 151 +++++++++++++-------- sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c | 3 + 49 files changed, 411 insertions(+), 458 deletions(-) diff --git a/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c b/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c index ef03cf678ed73..5f16d8b84c93a 100644 --- a/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c +++ b/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c @@ -1312,8 +1312,10 @@ ipf_inject(fin, m) fr_info_t *fin; mb_t *m; { + struct epoch_tracker et; int error = 0; + NET_EPOCH_ENTER(et); if (fin->fin_out == 0) { netisr_dispatch(NETISR_IP, m); } else { @@ -1321,6 +1323,7 @@ ipf_inject(fin, m) fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off); error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL); } + NET_EPOCH_EXIT(et); return error; } diff --git a/sys/dev/firewire/if_fwip.c b/sys/dev/firewire/if_fwip.c index 35a58c013707c..6d2b3fb59221c 100644 --- a/sys/dev/firewire/if_fwip.c +++ b/sys/dev/firewire/if_fwip.c @@ -708,6 +708,7 @@ fwip_start_send (void *arg, int count) static void fwip_stream_input(struct fw_xferq *xferq) { + struct epoch_tracker et; struct mbuf *m, *m0; struct m_tag *mtag; struct ifnet *ifp; @@ -720,6 +721,7 @@ fwip_stream_input(struct fw_xferq *xferq) fwip = (struct fwip_softc *)xferq->sc; ifp = fwip->fw_softc.fwip_ifp; + NET_EPOCH_ENTER(et); while ((sxfer = STAILQ_FIRST(&xferq->stvalid)) != NULL) { STAILQ_REMOVE_HEAD(&xferq->stvalid, link); fp = mtod(sxfer->mbuf, struct fw_pkt *); @@ -808,6 +810,7 @@ fwip_stream_input(struct fw_xferq *xferq) firewire_input(ifp, m, src); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); } + NET_EPOCH_EXIT(et); if (STAILQ_FIRST(&xferq->stfree) != NULL) fwip->fd.fc->irx_enable(fwip->fd.fc, fwip->dma_ch); } diff --git a/sys/dev/iicbus/if_ic.c b/sys/dev/iicbus/if_ic.c index ca50fbb11c339..4dac861412305 100644 --- a/sys/dev/iicbus/if_ic.c +++ b/sys/dev/iicbus/if_ic.c @@ -309,9 +309,13 @@ icintr(device_t dev, int event, char *ptr) BPF_TAP(sc->ic_ifp, sc->ic_ifbuf, len + ICHDRLEN); top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, sc->ic_ifp, 0); if (top) { + struct epoch_tracker et; + mtx_unlock(&sc->ic_lock); M_SETFIB(top, sc->ic_ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(NETISR_IP, top); + NET_EPOCH_EXIT(et); mtx_lock(&sc->ic_lock); } break; diff --git a/sys/dev/usb/net/if_usie.c b/sys/dev/usb/net/if_usie.c index a6e9c62f25378..eb92f4aabe4b4 100644 --- a/sys/dev/usb/net/if_usie.c +++ b/sys/dev/usb/net/if_usie.c @@ -773,6 +773,7 @@ tr_setup: static void usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error) { + struct epoch_tracker et; struct usie_softc *sc = usbd_xfer_softc(xfer); struct ifnet *ifp = sc->sc_ifp; struct mbuf *m0; @@ -852,6 +853,7 @@ tr_setup: err = pkt = 0; /* HW can aggregate multiple frames in a single USB xfer */ + NET_EPOCH_ENTER(et); for (;;) { rxd = mtod(m, struct usie_desc *); @@ -918,6 +920,7 @@ tr_setup: m->m_data += diff; m->m_pkthdr.len = (m->m_len -= diff); } + NET_EPOCH_EXIT(et); mtx_lock(&sc->sc_mtx); diff --git a/sys/dev/usb/net/uhso.c b/sys/dev/usb/net/uhso.c index 8e826fd9e6a75..127379fe22b04 100644 --- a/sys/dev/usb/net/uhso.c +++ b/sys/dev/usb/net/uhso.c @@ -1664,6 +1664,7 @@ tr_setup: static void uhso_if_rxflush(void *arg) { + struct epoch_tracker et; struct uhso_softc *sc = arg; struct ifnet *ifp = sc->sc_ifp; uint8_t *cp; @@ -1677,6 +1678,7 @@ uhso_if_rxflush(void *arg) m = NULL; mwait = sc->sc_mwait; + NET_EPOCH_ENTER(et); for (;;) { if (m == NULL) { if ((m = mbufq_dequeue(&sc->sc_rxq)) == NULL) @@ -1787,6 +1789,7 @@ uhso_if_rxflush(void *arg) m = m0 != NULL ? m0 : NULL; mtx_lock(&sc->sc_mtx); } + NET_EPOCH_EXIT(et); sc->sc_mwait = mwait; } diff --git a/sys/dev/usb/net/usb_ethernet.c b/sys/dev/usb/net/usb_ethernet.c index b637f1ace0e17..b1d1ff202fc47 100644 --- a/sys/dev/usb/net/usb_ethernet.c +++ b/sys/dev/usb/net/usb_ethernet.c @@ -645,22 +645,21 @@ void uether_rxflush(struct usb_ether *ue) { struct ifnet *ifp = ue->ue_ifp; - struct mbuf *m; + struct epoch_tracker et; + struct mbuf *m, *n; UE_LOCK_ASSERT(ue, MA_OWNED); - for (;;) { - m = mbufq_dequeue(&ue->ue_rxq); - if (m == NULL) - break; - - /* - * The USB xfer has been resubmitted so its safe to unlock now. - */ - UE_UNLOCK(ue); + n = mbufq_flush(&ue->ue_rxq); + UE_UNLOCK(ue); + NET_EPOCH_ENTER(et); + while ((m = n) != NULL) { + n = STAILQ_NEXT(m, m_stailqpkt); + m->m_nextpkt = NULL; ifp->if_input(ifp, m); - UE_LOCK(ue); } + NET_EPOCH_EXIT(et); + UE_LOCK(ue); } /* diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index 1f480462db2a7..dbb6ab4d261a0 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -146,6 +146,8 @@ __FBSDID("$FreeBSD$"); #include #include +#include /* XXXGL: net_epoch should move out there */ +#include /* XXXGL: net_epoch should move out there */ #include diff --git a/sys/net/if.c b/sys/net/if.c index 406c57c8efb4e..089a481c3b1dc 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -351,17 +351,14 @@ ifnet_byindex(u_short idx) struct ifnet * ifnet_byindex_ref(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); - if (ifp == NULL || (ifp->if_flags & IFF_DYING)) { - NET_EPOCH_EXIT(et); + if (ifp == NULL || (ifp->if_flags & IFF_DYING)) return (NULL); - } if_ref(ifp); - NET_EPOCH_EXIT(et); return (ifp); } @@ -425,15 +422,14 @@ ifnet_setbyindex(u_short idx, struct ifnet *ifp) struct ifaddr * ifaddr_byindex(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; struct ifaddr *ifa = NULL; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); if (ifp != NULL && (ifa = ifp->if_addr) != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); return (ifa); } @@ -1640,39 +1636,32 @@ ifgr_groups_get(void *ifgrp) static int if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp) { - struct epoch_tracker et; int len, error; struct ifg_list *ifgl; struct ifg_req ifgrq, *ifgp; + NET_EPOCH_ASSERT(); + if (ifgr->ifgr_len == 0) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) ifgr->ifgr_len += sizeof(struct ifg_req); - NET_EPOCH_EXIT(et); return (0); } len = ifgr->ifgr_len; ifgp = ifgr_groups_get(ifgr); /* XXX: wire */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { - if (len < sizeof(ifgrq)) { - NET_EPOCH_EXIT(et); + if (len < sizeof(ifgrq)) return (EINVAL); - } bzero(&ifgrq, sizeof ifgrq); strlcpy(ifgrq.ifgrq_group, ifgl->ifgl_group->ifg_group, sizeof(ifgrq.ifgrq_group)); - if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) { - NET_EPOCH_EXIT(et); + if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) return (error); - } len -= sizeof(ifgrq); ifgp++; } - NET_EPOCH_EXIT(et); return (0); } @@ -1972,7 +1961,8 @@ ifa_ifwithaddr(const struct sockaddr *addr) struct ifnet *ifp; struct ifaddr *ifa; - MPASS(in_epoch(net_epoch_preempt)); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) @@ -2324,17 +2314,22 @@ if_link_state_change(struct ifnet *ifp, int link_state) ifp->if_link_state = link_state; + /* XXXGL: reference ifp? */ taskqueue_enqueue(taskqueue_swi, &ifp->if_linktask); } static void do_link_state_change(void *arg, int pending) { - struct ifnet *ifp = (struct ifnet *)arg; - int link_state = ifp->if_link_state; - CURVNET_SET(ifp->if_vnet); + struct epoch_tracker et; + struct ifnet *ifp; + int link_state; + + NET_EPOCH_ENTER(et); + ifp = arg; + link_state = ifp->if_link_state; - /* Notify that the link state has changed. */ + CURVNET_SET(ifp->if_vnet); rt_ifmsg(ifp); if (ifp->if_vlantrunk != NULL) (*vlan_link_state_p)(ifp); @@ -2360,6 +2355,7 @@ do_link_state_change(void *arg, int pending) (link_state == LINK_STATE_UP) ? "UP" : "DOWN" ); EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state); CURVNET_RESTORE(); + NET_EPOCH_EXIT(et); } /* @@ -2912,9 +2908,14 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) break; case CASE_IOC_IFGROUPREQ(SIOCGIFGROUP): - if ((error = if_getgroup((struct ifgroupreq *)data, ifp))) - return (error); + { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); + error = if_getgroup((struct ifgroupreq *)data, ifp); + NET_EPOCH_EXIT(et); break; + } case CASE_IOC_IFGROUPREQ(SIOCDIFGROUP): error = priv_check(td, PRIV_NET_DELIFGROUP); @@ -2983,7 +2984,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) #ifdef COMPAT_FREEBSD32 caddr_t saved_data = NULL; struct ifmediareq ifmr; - struct ifmediareq *ifmrp; + struct ifmediareq *ifmrp = NULL; #endif struct ifnet *ifp; struct ifreq *ifr; @@ -2999,12 +3000,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) } #endif - switch (cmd) { case SIOCGIFCONF: error = ifconf(cmd, data); - CURVNET_RESTORE(); - return (error); + goto out_noref; #ifdef COMPAT_FREEBSD32 case SIOCGIFCONF32: @@ -3017,16 +3016,14 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) ifc.ifc_buf = PTRIN(ifc32->ifc_buf); error = ifconf(SIOCGIFCONF, (void *)&ifc); - CURVNET_RESTORE(); if (error == 0) ifc32->ifc_len = ifc.ifc_len; - return (error); + goto out_noref; } #endif } #ifdef COMPAT_FREEBSD32 - ifmrp = NULL; switch (cmd) { case SIOCGIFMEDIA32: case SIOCGIFXMEDIA32: @@ -3618,16 +3615,15 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa) struct ifmultiaddr *ifma; int lastref; #ifdef INVARIANTS - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); KASSERT(ifp != NULL, ("%s: ifnet went away", __func__)); #endif @@ -3693,16 +3689,15 @@ if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int flags) if (ifp == NULL) { printf("%s: ifma_ifp seems to be detached\n", __func__); } else { - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); } #endif /* @@ -3826,11 +3821,11 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) struct sockaddr_dl *sdl; struct ifaddr *ifa; struct ifreq ifr; - struct epoch_tracker et; int rc; + NET_EPOCH_ASSERT(); + rc = 0; - NET_EPOCH_ENTER(et); ifa = ifp->if_addr; if (ifa == NULL) { rc = EINVAL; @@ -3864,7 +3859,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) * to re-init it in order to reprogram its * address filter. */ - NET_EPOCH_EXIT(et); if ((ifp->if_flags & IFF_UP) != 0) { if (ifp->if_ioctl) { ifp->if_flags &= ~IFF_UP; @@ -3879,8 +3873,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) } EVENTHANDLER_INVOKE(iflladdr_event, ifp); return (0); - out: - NET_EPOCH_EXIT(et); +out: return (rc); } diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c index 991ee08826254..7a750558890ab 100644 --- a/sys/net/if_ethersubr.c +++ b/sys/net/if_ethersubr.c @@ -800,6 +800,7 @@ VNET_SYSUNINIT(vnet_ether_uninit, SI_SUB_PROTO_IF, SI_ORDER_ANY, static void ether_input(struct ifnet *ifp, struct mbuf *m) { + struct epoch_tracker et; struct mbuf *mn; /* @@ -807,22 +808,24 @@ ether_input(struct ifnet *ifp, struct mbuf *m) * m_nextpkt. We split them up into separate packets here and pass * them up. This allows the drivers to amortize the receive lock. */ + CURVNET_SET_QUIET(ifp->if_vnet); + NET_EPOCH_ENTER(et); while (m) { mn = m->m_nextpkt; m->m_nextpkt = NULL; /* - * We will rely on rcvif being set properly in the deferred context, - * so assert it is correct here. + * We will rely on rcvif being set properly in the deferred + * context, so assert it is correct here. */ MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); KASSERT(m->m_pkthdr.rcvif == ifp, ("%s: ifnet mismatch m %p " "rcvif %p ifp %p", __func__, m, m->m_pkthdr.rcvif, ifp)); - CURVNET_SET_QUIET(ifp->if_vnet); netisr_dispatch(NETISR_ETHER, m); - CURVNET_RESTORE(); m = mn; } + NET_EPOCH_EXIT(et); + CURVNET_RESTORE(); } /* @@ -835,6 +838,7 @@ ether_demux(struct ifnet *ifp, struct mbuf *m) int i, isr; u_short ether_type; + NET_EPOCH_ASSERT(); KASSERT(ifp != NULL, ("%s: NULL interface pointer", __func__)); /* Do not grab PROMISC frames in case we are re-entered. */ diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c index 6a5571a996f58..af911d09c945a 100644 --- a/sys/net/if_gif.c +++ b/sys/net/if_gif.c @@ -415,6 +415,8 @@ gif_input(struct mbuf *m, struct ifnet *ifp, int proto, uint8_t ecn) struct ifnet *oldifp; int isr, n, af; + NET_EPOCH_ASSERT(); + if (ifp == NULL) { /* just in case */ m_freem(m); diff --git a/sys/net/if_me.c b/sys/net/if_me.c index c9679d23e67bd..05bc50cfb9175 100644 --- a/sys/net/if_me.c +++ b/sys/net/if_me.c @@ -451,6 +451,8 @@ me_input(struct mbuf *m, int off, int proto, void *arg) struct ip *ip; int hlen; + NET_EPOCH_ASSERT(); + ifp = ME2IFP(sc); /* checks for short packets */ hlen = sizeof(struct mobhdr); diff --git a/sys/net/if_stf.c b/sys/net/if_stf.c index 225bd30b6bd3e..aadc58ec5a48f 100644 --- a/sys/net/if_stf.c +++ b/sys/net/if_stf.c @@ -613,6 +613,8 @@ in_stf_input(struct mbuf *m, int off, int proto, void *arg) u_int8_t otos, itos; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + if (proto != IPPROTO_IPV6) { m_freem(m); return (IPPROTO_DONE); diff --git a/sys/net/if_tuntap.c b/sys/net/if_tuntap.c index ff6c3291d7f41..eb197207ca467 100644 --- a/sys/net/if_tuntap.c +++ b/sys/net/if_tuntap.c @@ -1662,6 +1662,7 @@ tunwrite_l2(struct tuntap_softc *tp, struct mbuf *m) static int tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m) { + struct epoch_tracker et; struct ifnet *ifp; int family, isr; @@ -1702,7 +1703,9 @@ tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m) if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); CURVNET_SET(ifp->if_vnet); M_SETFIB(m, ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return (0); } diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c index 76a0f29d63c35..2f6aa69ca0000 100644 --- a/sys/net/if_vlan.c +++ b/sys/net/if_vlan.c @@ -255,7 +255,6 @@ static struct sx _VLAN_SX_ID; #define TRUNK_LOCK_DESTROY(trunk) mtx_destroy(&(trunk)->lock) #define TRUNK_WLOCK(trunk) mtx_lock(&(trunk)->lock) #define TRUNK_WUNLOCK(trunk) mtx_unlock(&(trunk)->lock) -#define TRUNK_LOCK_ASSERT(trunk) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(trunk)->lock)) #define TRUNK_WLOCK_ASSERT(trunk) mtx_assert(&(trunk)->lock, MA_OWNED); /* @@ -704,18 +703,17 @@ vlan_ifdetach(void *arg __unused, struct ifnet *ifp) static struct ifnet * vlan_trunkdev(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlan *ifv; + NET_EPOCH_ASSERT(); + if (ifp->if_type != IFT_L2VLAN) return (NULL); - NET_EPOCH_ENTER(et); ifv = ifp->if_softc; ifp = NULL; if (ifv->ifv_trunk) ifp = PARENT(ifv); - NET_EPOCH_EXIT(et); return (ifp); } @@ -787,21 +785,18 @@ vlan_setcookie(struct ifnet *ifp, void *cookie) static struct ifnet * vlan_devat(struct ifnet *ifp, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return (NULL); - } ifp = NULL; ifv = vlan_gethash(trunk, vid); if (ifv) ifp = ifv->ifv_ifp; - NET_EPOCH_EXIT(et); return (ifp); } @@ -1140,16 +1135,15 @@ vlan_init(void *foo __unused) static int vlan_transmit(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlan *ifv; struct ifnet *p; int error, len, mcast; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifv = ifp->if_softc; if (TRUNK(ifv) == NULL) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } @@ -1169,7 +1163,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) vst = mst_to_vst(mst); if (vst->tag->ifp != p) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (EAGAIN); } @@ -1185,14 +1178,12 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) */ if (!UP_AND_RUNNING(p)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } if (!ether_8021q_frame(&m, ifp, p, ifv->ifv_vid, ifv->ifv_pcp)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (0); } @@ -1206,7 +1197,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast); } else if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (error); } @@ -1214,19 +1204,17 @@ static int vlan_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct route *ro) { - struct epoch_tracker et; struct ifvlan *ifv; struct ifnet *p; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifv = ifp->if_softc; if (TRUNK(ifv) == NULL) { - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } p = PARENT(ifv); - NET_EPOCH_EXIT(et); return p->if_output(ifp, m, dst, ro); } @@ -1242,16 +1230,15 @@ vlan_qflush(struct ifnet *ifp __unused) static void vlan_input(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; struct m_tag *mtag; uint16_t vid, tag; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; if (trunk == NULL) { - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1274,7 +1261,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) if (m->m_len < sizeof(*evl) && (m = m_pullup(m, sizeof(*evl))) == NULL) { if_printf(ifp, "cannot pullup VLAN header\n"); - NET_EPOCH_EXIT(et); return; } evl = mtod(m, struct ether_vlan_header *); @@ -1297,7 +1283,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) __func__, ifp->if_xname, ifp->if_type); #endif if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1307,7 +1292,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) ifv = vlan_gethash(trunk, vid); if (ifv == NULL || !UP_AND_RUNNING(ifv->ifv_ifp)) { - NET_EPOCH_EXIT(et); if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); m_freem(m); return; @@ -1327,7 +1311,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) sizeof(uint8_t), M_NOWAIT); if (mtag == NULL) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1338,7 +1321,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) m->m_pkthdr.rcvif = ifv->ifv_ifp; if_inc_counter(ifv->ifv_ifp, IFCOUNTER_IPACKETS, 1); - NET_EPOCH_EXIT(et); /* Pass it back through the parent's input routine. */ (*ifv->ifv_ifp->if_input)(ifv->ifv_ifp, m); @@ -1364,11 +1346,12 @@ vlan_lladdr_fn(void *arg, int pending __unused) static int vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifnet *ifp; int error = 0; + NET_EPOCH_ASSERT(); + /* * We can handle non-ethernet hardware types as long as * they handle the tagging and headers themselves. @@ -1469,9 +1452,7 @@ vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid) ifp->if_link_state = p->if_link_state; - NET_EPOCH_ENTER(et); vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); /* * Set up our interface address to reflect the underlying @@ -1643,17 +1624,14 @@ vlan_setflags(struct ifnet *ifp, int status) static void vlan_link_state(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - /* Called from a taskqueue_swi task, so we cannot sleep. */ - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return; - } TRUNK_WLOCK(trunk); VLAN_FOREACH(ifv, trunk) { @@ -1662,7 +1640,6 @@ vlan_link_state(struct ifnet *ifp) trunk->parent->if_link_state); } TRUNK_WUNLOCK(trunk); - NET_EPOCH_EXIT(et); } static void @@ -1674,8 +1651,9 @@ vlan_capabilities(struct ifvlan *ifv) int cap = 0, ena = 0, mena; u_long hwa = 0; - VLAN_SXLOCK_ASSERT(); NET_EPOCH_ASSERT(); + VLAN_SXLOCK_ASSERT(); + p = PARENT(ifv); ifp = ifv->ifv_ifp; @@ -1791,7 +1769,6 @@ vlan_capabilities(struct ifvlan *ifv) static void vlan_trunk_capabilities(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; @@ -1801,11 +1778,8 @@ vlan_trunk_capabilities(struct ifnet *ifp) VLAN_SUNLOCK(); return; } - NET_EPOCH_ENTER(et); - VLAN_FOREACH(ifv, trunk) { + VLAN_FOREACH(ifv, trunk) vlan_capabilities(ifv); - } - NET_EPOCH_EXIT(et); VLAN_SUNLOCK(); } @@ -1820,6 +1794,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) struct vlanreq vlr; int error = 0; + NET_EPOCH_ASSERT(); + ifr = (struct ifreq *)data; ifa = (struct ifaddr *) data; ifv = ifp->if_softc; @@ -1996,13 +1972,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) VLAN_SLOCK(); ifv->ifv_capenable = ifr->ifr_reqcap; trunk = TRUNK(ifv); - if (trunk != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); + if (trunk != NULL) vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); - } VLAN_SUNLOCK(); break; diff --git a/sys/net/netisr.c b/sys/net/netisr.c index 049fe0fc32d86..ca269815d2fec 100644 --- a/sys/net/netisr.c +++ b/sys/net/netisr.c @@ -861,6 +861,7 @@ static u_int netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) { struct netisr_work local_npw, *npwp; + struct epoch_tracker et; u_int handled; struct mbuf *m; @@ -890,6 +891,7 @@ netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) npwp->nw_len = 0; nwsp->nws_pendingbits &= ~(1 << proto); NWS_UNLOCK(nwsp); + NET_EPOCH_ENTER(et); while ((m = local_npw.nw_head) != NULL) { local_npw.nw_head = m->m_nextpkt; m->m_nextpkt = NULL; @@ -902,6 +904,7 @@ netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) netisr_proto[proto].np_handler(m); CURVNET_RESTORE(); } + NET_EPOCH_EXIT(et); KASSERT(local_npw.nw_len == 0, ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); if (netisr_proto[proto].np_drainedcpu) @@ -1088,6 +1091,7 @@ netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) int dosignal, error; u_int cpuid, dispatch_policy; + NET_EPOCH_ASSERT(); KASSERT(proto < NETISR_MAXPROT, ("%s: invalid proto %u", __func__, proto)); #ifdef NETISR_LOCKING diff --git a/sys/net/route.c b/sys/net/route.c index 1dd12da6ded12..ee3550441cf90 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -593,12 +593,12 @@ rtredirect_fib(struct sockaddr *dst, int error = 0; short *stat = NULL; struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct rib_head *rnh; + NET_EPOCH_ASSERT(); + ifa = NULL; - NET_EPOCH_ENTER(et); rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) { error = EAFNOSUPPORT; @@ -693,7 +693,6 @@ done: if (rt) RTFREE_LOCKED(rt); out: - NET_EPOCH_EXIT(et); if (error) V_rtstat.rts_badredirect++; else if (stat != NULL) diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c index 63dd1dedb4817..cec807eb61558 100644 --- a/sys/net/rtsock.c +++ b/sys/net/rtsock.c @@ -560,6 +560,7 @@ route_output(struct mbuf *m, struct socket *so, ...) struct rib_head *rnh; struct rt_addrinfo info; struct sockaddr_storage ss; + struct epoch_tracker et; #ifdef INET6 struct sockaddr_in6 *sin6; int i, rti_need_deembed = 0; @@ -579,6 +580,7 @@ route_output(struct mbuf *m, struct socket *so, ...) return (ENOBUFS); if ((m->m_flags & M_PKTHDR) == 0) panic("route_output"); + NET_EPOCH_ENTER(et); len = m->m_pkthdr.len; if (len < sizeof(*rtm) || len != mtod(m, struct rt_msghdr *)->rtm_msglen) @@ -803,11 +805,11 @@ route_output(struct mbuf *m, struct socket *so, ...) NET_EPOCH_ENTER(et); ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1, RT_ALL_FIBS); + NET_EPOCH_EXIT(et); if (ifa != NULL) rt_maskedcopy(ifa->ifa_addr, &laddr, ifa->ifa_netmask); - NET_EPOCH_EXIT(et); } else rt_maskedcopy(rt->rt_ifa->ifa_addr, &laddr, @@ -898,6 +900,7 @@ report: } flush: + NET_EPOCH_EXIT(et); if (rt != NULL) RTFREE(rt); /* @@ -1761,11 +1764,9 @@ sysctl_iflist(int af, struct walkarg *w) struct rt_addrinfo info; int len, error = 0; struct sockaddr_storage ss; - struct epoch_tracker et; bzero((caddr_t)&info, sizeof(info)); bzero(&ifd, sizeof(ifd)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1815,7 +1816,6 @@ sysctl_iflist(int af, struct walkarg *w) info.rti_info[RTAX_BRD] = NULL; } done: - NET_EPOCH_EXIT(et); return (error); } @@ -1823,16 +1823,16 @@ static int sysctl_ifmalist(int af, struct walkarg *w) { struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct ifmultiaddr *ifma; struct ifnet *ifp; int error, len; + NET_EPOCH_ASSERT(); + error = 0; bzero((caddr_t)&info, sizeof(info)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1867,7 +1867,6 @@ sysctl_ifmalist(int af, struct walkarg *w) if (error != 0) break; } - NET_EPOCH_EXIT(et); return (error); } @@ -1875,6 +1874,7 @@ static int sysctl_rtsock(SYSCTL_HANDLER_ARGS) { RIB_RLOCK_TRACKER; + struct epoch_tracker et; int *name = (int *)arg1; u_int namelen = arg2; struct rib_head *rnh = NULL; /* silence compiler. */ @@ -1918,8 +1918,8 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) w.w_tmemsize = 65536; w.w_tmem = malloc(w.w_tmemsize, M_TEMP, M_WAITOK); + NET_EPOCH_ENTER(et); switch (w.w_op) { - case NET_RT_DUMP: case NET_RT_FLAGS: if (af == 0) { /* dump all tables */ @@ -1946,13 +1946,9 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) for (error = 0; error == 0 && i <= lim; i++) { rnh = rt_tables_get_rnh(fib, i); if (rnh != NULL) { - struct epoch_tracker et; - RIB_RLOCK(rnh); - NET_EPOCH_ENTER(et); error = rnh->rnh_walktree(&rnh->head, sysctl_dumpentry, &w); - NET_EPOCH_EXIT(et); RIB_RUNLOCK(rnh); } else if (af != 0) error = EAFNOSUPPORT; @@ -1968,6 +1964,7 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) error = sysctl_ifmalist(af, &w); break; } + NET_EPOCH_EXIT(et); free(w.w_tmem, M_TEMP); return (error); diff --git a/sys/netgraph/ng_ether.c b/sys/netgraph/ng_ether.c index 575f3740a6f1e..3b1a9efc849e1 100644 --- a/sys/netgraph/ng_ether.c +++ b/sys/netgraph/ng_ether.c @@ -711,6 +711,7 @@ ng_ether_rcv_lower(hook_p hook, item_p item) static int ng_ether_rcv_upper(hook_p hook, item_p item) { + struct epoch_tracker et; struct mbuf *m; const node_p node = NG_HOOK_NODE(hook); const priv_p priv = NG_NODE_PRIVATE(node); @@ -738,7 +739,9 @@ ng_ether_rcv_upper(hook_p hook, item_p item) } /* Route packet back in */ + NET_EPOCH_ENTER(et); ether_demux(ifp, m); + NET_EPOCH_EXIT(et); return (0); } diff --git a/sys/netgraph/ng_iface.c b/sys/netgraph/ng_iface.c index f1ddce70c86d5..605a1f7d9e24b 100644 --- a/sys/netgraph/ng_iface.c +++ b/sys/netgraph/ng_iface.c @@ -690,6 +690,7 @@ ng_iface_rcvdata(hook_p hook, item_p item) const priv_p priv = NG_NODE_PRIVATE(NG_HOOK_NODE(hook)); const iffam_p iffam = get_iffam_from_hook(priv, hook); struct ifnet *const ifp = priv->ifp; + struct epoch_tracker et; struct mbuf *m; int isr; @@ -731,7 +732,9 @@ ng_iface_rcvdata(hook_p hook, item_p item) } random_harvest_queue(m, sizeof(*m), RANDOM_NET_NG); M_SETFIB(m, ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); return (0); } diff --git a/sys/netgraph/ng_ip_input.c b/sys/netgraph/ng_ip_input.c index 5e6dce38f6d18..88e0e11c08d32 100644 --- a/sys/netgraph/ng_ip_input.c +++ b/sys/netgraph/ng_ip_input.c @@ -125,8 +125,13 @@ ngipi_rcvdata(hook_p hook, item_p item) NG_FREE_ITEM(item); if (curthread->td_ng_outbound) netisr_queue(NETISR_IP, m); - else + else { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); netisr_dispatch(NETISR_IP, m); + NET_EPOCH_EXIT(et); + } return 0; } diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c index 0c85349f88a8f..cb3c335d0897b 100644 --- a/sys/netinet/if_ether.c +++ b/sys/netinet/if_ether.c @@ -258,12 +258,16 @@ arptimer(void *arg) if (r_skip_req == 0 && lle->la_preempt > 0) { /* Entry was used, issue refresh request */ + struct epoch_tracker et; struct in_addr dst; + dst = lle->r_l3addr.addr4; lle->la_preempt--; callout_schedule(&lle->lle_timer, hz * V_arpt_rexmit); LLE_WUNLOCK(lle); + NET_EPOCH_ENTER(et); arprequest(ifp, NULL, &dst, NULL); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return; } @@ -362,15 +366,15 @@ arprequest_internal(struct ifnet *ifp, const struct in_addr *sip, struct route ro; int error; + NET_EPOCH_ASSERT(); + if (sip == NULL) { /* * The caller did not supply a source address, try to find * a compatible one among those assigned to this interface. */ - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -388,7 +392,6 @@ arprequest_internal(struct ifnet *ifp, const struct in_addr *sip, IA_MASKSIN(ifa)->sin_addr.s_addr)) break; /* found it. */ } - NET_EPOCH_EXIT(et); if (sip == NULL) { printf("%s: cannot find matching address\n", __func__); return (EADDRNOTAVAIL); @@ -475,18 +478,15 @@ arpresolve_full(struct ifnet *ifp, int is_gw, int flags, struct mbuf *m, char *lladdr; int ll_len; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; if (plle != NULL) *plle = NULL; - if ((flags & LLE_CREATE) == 0) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); + if ((flags & LLE_CREATE) == 0) la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); - NET_EPOCH_EXIT(et); - } if (la == NULL && (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { la = lltable_alloc_entry(LLTABLE(ifp), 0, dst); if (la == NULL) { @@ -623,9 +623,10 @@ arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, const struct sockaddr *dst, u_char *desten, uint32_t *pflags, struct llentry **plle) { - struct epoch_tracker et; struct llentry *la = NULL; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; if (plle != NULL) @@ -645,7 +646,6 @@ arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, } } - NET_EPOCH_ENTER(et); la = lla_lookup(LLTABLE(ifp), plle ? LLE_EXCLUSIVE : LLE_UNLOCKED, dst); if (la != NULL && (la->r_flags & RLLE_VALID) != 0) { /* Entry found, let's copy lle info */ @@ -659,12 +659,10 @@ arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, *plle = la; LLE_WUNLOCK(la); } - NET_EPOCH_EXIT(et); return (0); } if (plle && la) LLE_WUNLOCK(la); - NET_EPOCH_EXIT(et); return (arpresolve_full(ifp, is_gw, la == NULL ? LLE_CREATE : 0, m, dst, desten, pflags, plle)); @@ -809,7 +807,8 @@ in_arpinput(struct mbuf *m) int lladdr_off; int error; char addrbuf[INET_ADDRSTRLEN]; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); sin.sin_len = sizeof(struct sockaddr_in); sin.sin_family = AF_INET; @@ -902,17 +901,14 @@ in_arpinput(struct mbuf *m) * No match, use the first inet address on the receive interface * as a dummy address for the rest of the function. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && (ifa->ifa_carp == NULL || (*carp_iamatch_p)(ifa, &enaddr))) { ia = ifatoia(ifa); ifa_ref(ifa); - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); /* * If bridging, fall back to using any inet address. @@ -969,9 +965,7 @@ match: sin.sin_family = AF_INET; sin.sin_addr = isaddr; dst = (struct sockaddr *)&sin; - NET_EPOCH_ENTER(et); la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); - NET_EPOCH_EXIT(et); if (la != NULL) arp_check_update_lle(ah, isaddr, ifp, bridged, la); else if (itaddr.s_addr == myaddr.s_addr) { @@ -1049,9 +1043,7 @@ reply: struct llentry *lle = NULL; sin.sin_addr = itaddr; - NET_EPOCH_ENTER(et); lle = lla_lookup(LLTABLE(ifp), 0, (struct sockaddr *)&sin); - NET_EPOCH_EXIT(et); if ((lle != NULL) && (lle->la_flags & LLE_PUB)) { (void)memcpy(ar_tha(ah), ar_sha(ah), ah->ar_hln); @@ -1430,6 +1422,7 @@ garp_timer_start(struct ifaddr *ifa) void arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa) { + struct epoch_tracker et; const struct sockaddr_in *dst_in; const struct sockaddr *dst; @@ -1441,7 +1434,9 @@ arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa) if (ntohl(dst_in->sin_addr.s_addr) == INADDR_ANY) return; + NET_EPOCH_ENTER(et); arp_announce_ifaddr(ifp, dst_in->sin_addr, IF_LLADDR(ifp)); + NET_EPOCH_EXIT(et); if (garp_rexmit_count > 0) { garp_timer_start(ifa); } diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c index beb0f98209bea..81d09c926998b 100644 --- a/sys/netinet/igmp.c +++ b/sys/netinet/igmp.c @@ -487,8 +487,10 @@ out_locked: static void igmp_dispatch_queue(struct mbufq *mq, int limit, const int loop) { + struct epoch_tracker et; struct mbuf *m; + NET_EPOCH_ENTER(et); while ((m = mbufq_dequeue(mq)) != NULL) { CTR3(KTR_IGMPV3, "%s: dispatch %p from %p", __func__, mq, m); if (loop) @@ -497,6 +499,7 @@ igmp_dispatch_queue(struct mbufq *mq, int limit, const int loop) if (--limit == 0) break; } + NET_EPOCH_EXIT(et); } /* @@ -692,11 +695,12 @@ static int igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, const struct igmp *igmp) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct igmp_ifsoftc *igi; struct in_multi *inm; + NET_EPOCH_ASSERT(); + /* * IGMPv1 Host Mmembership Queries SHOULD always be addressed to * 224.0.0.1. They are always treated as General Queries. @@ -734,7 +738,6 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, * for the interface on which the query arrived, * except those which are already running. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -762,7 +765,6 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, break; } } - NET_EPOCH_EXIT(et); out_locked: IGMP_UNLOCK(); @@ -778,13 +780,14 @@ static int igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, const struct igmp *igmp) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct igmp_ifsoftc *igi; struct in_multi *inm; int is_general_query; uint16_t timer; + NET_EPOCH_ASSERT(); + is_general_query = 0; /* @@ -836,7 +839,6 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, */ CTR2(KTR_IGMPV3, "process v2 general query on ifp %p(%s)", ifp, ifp->if_xname); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -844,7 +846,6 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, inm = (struct in_multi *)ifma->ifma_protospec; igmp_v2_update_group(inm, timer); } - NET_EPOCH_EXIT(et); } else { /* * Group-specific IGMPv2 query, we need only @@ -1220,13 +1221,9 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip, * Replace 0.0.0.0 with the subnet address if told to do so. */ if (V_igmp_recvifkludge && in_nullhost(ip->ip_src)) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); - NET_EPOCH_EXIT(et); } CTR3(KTR_IGMPV3, "process v1 report 0x%08x on ifp %p(%s)", @@ -1311,7 +1308,6 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, /*const*/ struct igmp *igmp) { struct rm_priotracker in_ifa_tracker; - struct epoch_tracker et; struct in_ifaddr *ia; struct in_multi *inm; @@ -1320,23 +1316,19 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, * leave requires knowing that we are the only member of a * group. */ - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL && in_hosteq(ip->ip_src, IA_SIN(ia)->sin_addr)) { - NET_EPOCH_EXIT(et); return (0); } IGMPSTAT_INC(igps_rcv_reports); if (ifp->if_flags & IFF_LOOPBACK) { - NET_EPOCH_EXIT(et); return (0); } if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr)) || !in_hosteq(igmp->igmp_group, ip->ip_dst)) { - NET_EPOCH_EXIT(et); IGMPSTAT_INC(igps_rcv_badreports); return (EINVAL); } @@ -1352,7 +1344,6 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); } - NET_EPOCH_EXIT(et); CTR3(KTR_IGMPV3, "process v2 report 0x%08x on ifp %p(%s)", ntohl(igmp->igmp_group.s_addr), ifp, ifp->if_xname); @@ -1994,13 +1985,14 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) struct ifnet *ifp; struct in_multi *inm; struct in_multi_head inm_free_tmp; - struct epoch_tracker et; CTR3(KTR_IGMPV3, "%s: cancel v3 timers on ifp %p(%s)", __func__, igi->igi_ifp, igi->igi_ifp->if_xname); IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); + NET_EPOCH_ASSERT(); + SLIST_INIT(&inm_free_tmp); /* @@ -2015,7 +2007,6 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) * for all memberships scoped to this link. */ ifp = igi->igi_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -2060,7 +2051,6 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) inm->inm_timer = 0; mbufq_drain(&inm->inm_scq); } - NET_EPOCH_EXIT(et); inm_release_list_deferred(&inm_free_tmp); } @@ -2189,6 +2179,7 @@ igmp_v1v2_queue_report(struct in_multi *inm, const int type) struct ip *ip; struct mbuf *m; + NET_EPOCH_ASSERT(); IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); @@ -3303,7 +3294,6 @@ igmp_v3_merge_state_changes(struct in_multi *inm, struct mbufq *scq) static void igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct ifnet *ifp; struct in_multi *inm; @@ -3311,6 +3301,7 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); + NET_EPOCH_ASSERT(); KASSERT(igi->igi_version == IGMP_VERSION_3, ("%s: called when version %d", __func__, igi->igi_version)); @@ -3326,7 +3317,6 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) ifp = igi->igi_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -3357,7 +3347,6 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) break; } } - NET_EPOCH_EXIT(et); send: loop = (igi->igi_flags & IGIF_LOOPBACK) ? 1 : 0; @@ -3529,14 +3518,11 @@ igmp_v3_encap_report(struct ifnet *ifp, struct mbuf *m) ip->ip_src.s_addr = INADDR_ANY; if (m->m_flags & M_IGMP_LOOP) { - struct epoch_tracker et; struct in_ifaddr *ia; - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) ip->ip_src = ia->ia_addr.sin_addr; - NET_EPOCH_EXIT(et); } ip->ip_dst.s_addr = htonl(INADDR_ALLRPTS_GROUP); diff --git a/sys/netinet/in.c b/sys/netinet/in.c index cbb171395662f..c83f0ae83fc44 100644 --- a/sys/netinet/in.c +++ b/sys/netinet/in.c @@ -139,21 +139,18 @@ in_localip(struct in_addr in) int in_ifhasaddr(struct ifnet *ifp, struct in_addr in) { - struct epoch_tracker et; struct ifaddr *ifa; struct in_ifaddr *ia; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = (struct in_ifaddr *)ifa; - if (ia->ia_addr.sin_addr.s_addr == in.s_addr) { - NET_EPOCH_EXIT(et); + if (ia->ia_addr.sin_addr.s_addr == in.s_addr) return (1); - } } - NET_EPOCH_EXIT(et); return (0); } @@ -965,10 +962,11 @@ in_ifaddr_broadcast(struct in_addr in, struct in_ifaddr *ia) int in_broadcast(struct in_addr in, struct ifnet *ifp) { - struct epoch_tracker et; struct ifaddr *ifa; int found; + NET_EPOCH_ASSERT(); + if (in.s_addr == INADDR_BROADCAST || in.s_addr == INADDR_ANY) return (1); @@ -979,14 +977,12 @@ in_broadcast(struct in_addr in, struct ifnet *ifp) * Look through the list of addresses for a match * with a broadcast address. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && in_ifaddr_broadcast(in, (struct in_ifaddr *)ifa)) { found = 1; break; } - NET_EPOCH_EXIT(et); return (found); } diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c index ac5c6a03ab549..a91d88277fceb 100644 --- a/sys/netinet/in_mcast.c +++ b/sys/netinet/in_mcast.c @@ -388,13 +388,12 @@ inm_lookup_locked(struct ifnet *ifp, const struct in_addr ina) struct in_multi * inm_lookup(struct ifnet *ifp, const struct in_addr ina) { - struct epoch_tracker et; struct in_multi *inm; IN_MULTI_LIST_LOCK_ASSERT(); - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + inm = inm_lookup_locked(ifp, ina); - NET_EPOCH_EXIT(et); return (inm); } @@ -1199,10 +1198,13 @@ int in_joingroup(struct ifnet *ifp, const struct in_addr *gina, /*const*/ struct in_mfilter *imf, struct in_multi **pinm) { + struct epoch_tracker et; int error; IN_MULTI_LOCK(); + NET_EPOCH_ENTER(et); error = in_joingroup_locked(ifp, gina, imf, pinm); + NET_EPOCH_EXIT(et); IN_MULTI_UNLOCK(); return (error); @@ -1225,6 +1227,7 @@ in_joingroup_locked(struct ifnet *ifp, const struct in_addr *gina, struct in_multi *inm; int error; + NET_EPOCH_ASSERT(); IN_MULTI_LOCK_ASSERT(); IN_MULTI_LIST_UNLOCK_ASSERT(); @@ -1282,11 +1285,14 @@ in_joingroup_locked(struct ifnet *ifp, const struct in_addr *gina, int in_leavegroup(struct in_multi *inm, /*const*/ struct in_mfilter *imf) { + struct epoch_tracker et; int error; + NET_EPOCH_ENTER(et); IN_MULTI_LOCK(); error = in_leavegroup_locked(inm, imf); IN_MULTI_UNLOCK(); + NET_EPOCH_EXIT(et); return (error); } @@ -1310,11 +1316,12 @@ in_leavegroup_locked(struct in_multi *inm, /*const*/ struct in_mfilter *imf) struct in_mfilter timf; int error; - error = 0; - + NET_EPOCH_ASSERT(); IN_MULTI_LOCK_ASSERT(); IN_MULTI_LIST_UNLOCK_ASSERT(); + error = 0; + CTR5(KTR_IGMPV3, "%s: leave inm %p, 0x%08x/%s, imf %p", __func__, inm, ntohl(inm->inm_addr.s_addr), (inm_is_ifp_detached(inm) ? "null" : inm->inm_ifp->if_xname), @@ -1811,15 +1818,11 @@ inp_getmoptions(struct inpcb *inp, struct sockopt *sopt) if (!in_nullhost(imo->imo_multicast_addr)) { mreqn.imr_address = imo->imo_multicast_addr; } else if (ifp != NULL) { - struct epoch_tracker et; - mreqn.imr_ifindex = ifp->if_index; - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) mreqn.imr_address = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); } } INP_WUNLOCK(inp); @@ -2908,8 +2911,10 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) return (EINVAL); } + NET_EPOCH_ENTER(et); ifp = ifnet_byindex(ifindex); if (ifp == NULL) { + NET_EPOCH_EXIT(et); CTR2(KTR_IGMPV3, "%s: no ifp for ifindex %u", __func__, ifindex); return (ENOENT); @@ -2917,12 +2922,13 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) retval = sysctl_wire_old_buffer(req, sizeof(uint32_t) + (in_mcast_maxgrpsrc * sizeof(struct in_addr))); - if (retval) + if (retval) { + NET_EPOCH_EXIT(et); return (retval); + } IN_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -2951,9 +2957,9 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) break; } } - NET_EPOCH_EXIT(et); IN_MULTI_LIST_UNLOCK(); + NET_EPOCH_EXIT(et); return (retval); } diff --git a/sys/netinet/in_rmx.c b/sys/netinet/in_rmx.c index 2cface409d7fc..cc866e33cfaa5 100644 --- a/sys/netinet/in_rmx.c +++ b/sys/netinet/in_rmx.c @@ -80,12 +80,17 @@ in_addroute(void *v_arg, void *n_arg, struct radix_head *head, * dubious since it's so easy to inspect the address). */ if (rt->rt_flags & RTF_HOST) { - if (in_broadcast(sin->sin_addr, rt->rt_ifp)) { + struct epoch_tracker et; + bool bcast; + + NET_EPOCH_ENTER(et); + bcast = in_broadcast(sin->sin_addr, rt->rt_ifp); + NET_EPOCH_EXIT(et); + if (bcast) rt->rt_flags |= RTF_BROADCAST; - } else if (satosin(rt->rt_ifa->ifa_addr)->sin_addr.s_addr == - sin->sin_addr.s_addr) { + else if (satosin(rt->rt_ifa->ifa_addr)->sin_addr.s_addr == + sin->sin_addr.s_addr) rt->rt_flags |= RTF_LOCAL; - } } if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) rt->rt_flags |= RTF_MULTICAST; diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h index 50112481f2364..96879db0abb78 100644 --- a/sys/netinet/in_var.h +++ b/sys/netinet/in_var.h @@ -171,6 +171,7 @@ do { \ /* struct in_ifaddr *ia; */ \ /* struct rm_priotracker *t; */ \ do { \ + NET_EPOCH_ASSERT(); \ IN_IFADDR_RLOCK((t)); \ for ((ia) = CK_STAILQ_FIRST(&V_in_ifaddrhead); \ (ia) != NULL && (ia)->ia_ifp != (ifp); \ diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c index 819a0f9c75e6b..ceccd5a02f3a4 100644 --- a/sys/netinet/ip_carp.c +++ b/sys/netinet/ip_carp.c @@ -644,9 +644,10 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af) struct carp_softc *sc; uint64_t tmp_counter; struct timeval sc_tv, ch_tv; - struct epoch_tracker et; int error; + NET_EPOCH_ASSERT(); + /* * Verify that the VHID is valid on the receiving interface. * @@ -658,7 +659,6 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af) * (these should never happen, and as noted above, we may * miss real loops; this is just a double-check). */ - NET_EPOCH_ENTER(et); error = 0; match = NULL; IFNET_FOREACH_IFA(ifp, ifa) { @@ -672,7 +672,6 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af) ifa = error ? NULL : match; if (ifa != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); if (ifa == NULL) { if (error == ELOOP) { @@ -880,19 +879,18 @@ carp_send_ad_error(struct carp_softc *sc, int error) static struct ifaddr * carp_best_ifa(int af, struct ifnet *ifp) { - struct epoch_tracker et; struct ifaddr *ifa, *best; + NET_EPOCH_ASSERT(); + if (af >= AF_MAX) return (NULL); best = NULL; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == af && (best == NULL || ifa_preferred(best, ifa))) best = ifa; } - NET_EPOCH_EXIT(et); if (best != NULL) ifa_ref(best); return (best); @@ -1169,11 +1167,11 @@ carp_send_na(struct carp_softc *sc) struct ifaddr * carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) { - struct epoch_tracker et; struct ifaddr *ifa; + NET_EPOCH_ASSERT(); + ifa = NULL; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1185,7 +1183,6 @@ carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) ifa_ref(ifa); break; } - NET_EPOCH_EXIT(et); return (ifa); } @@ -1193,18 +1190,16 @@ carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) caddr_t carp_macmatch6(struct ifnet *ifp, struct mbuf *m, const struct in6_addr *taddr) { - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + IFNET_FOREACH_IFA(ifp, ifa) if (ifa->ifa_addr->sa_family == AF_INET6 && IN6_ARE_ADDR_EQUAL(taddr, IFA_IN6(ifa))) { struct carp_softc *sc = ifa->ifa_carp; struct m_tag *mtag; - NET_EPOCH_EXIT(et); - mtag = m_tag_get(PACKET_TAG_CARP, sizeof(struct carp_softc *), M_NOWAIT); if (mtag == NULL) @@ -1216,7 +1211,6 @@ carp_macmatch6(struct ifnet *ifp, struct mbuf *m, const struct in6_addr *taddr) return (LLADDR(&sc->sc_addr)); } - NET_EPOCH_EXIT(et); return (NULL); } diff --git a/sys/netinet/ip_encap.c b/sys/netinet/ip_encap.c index ebcce2958a323..15ba528e7e0c5 100644 --- a/sys/netinet/ip_encap.c +++ b/sys/netinet/ip_encap.c @@ -125,11 +125,9 @@ MTX_SYSINIT(srcaddrmtx, &srcaddrmtx, "srcaddrmtx", MTX_DEF); #define ENCAP_WLOCK() mtx_lock(&encapmtx) #define ENCAP_WUNLOCK() mtx_unlock(&encapmtx) #define ENCAP_RLOCK_TRACKER struct epoch_tracker encap_et -#define ENCAP_RLOCK() \ - epoch_enter_preempt(net_epoch_preempt, &encap_et) -#define ENCAP_RUNLOCK() \ - epoch_exit_preempt(net_epoch_preempt, &encap_et) -#define ENCAP_WAIT() epoch_wait_preempt(net_epoch_preempt) +#define ENCAP_RLOCK() NET_EPOCH_ENTER(encap_et) +#define ENCAP_RUNLOCK() NET_EPOCH_EXIT(encap_et) +#define ENCAP_WAIT() NET_EPOCH_WAIT() #define SRCADDR_WLOCK() mtx_lock(&srcaddrmtx) #define SRCADDR_WUNLOCK() mtx_unlock(&srcaddrmtx) diff --git a/sys/netinet/ip_icmp.c b/sys/netinet/ip_icmp.c index 3e5ce6283f0cc..0bf9b86756f8b 100644 --- a/sys/netinet/ip_icmp.c +++ b/sys/netinet/ip_icmp.c @@ -393,7 +393,6 @@ freeit: int icmp_input(struct mbuf **mp, int *offp, int proto) { - struct epoch_tracker et; struct icmp *icp; struct in_ifaddr *ia; struct mbuf *m = *mp; @@ -405,6 +404,8 @@ icmp_input(struct mbuf **mp, int *offp, int proto) void (*ctlfunc)(int, struct sockaddr *, void *); int fibnum; + NET_EPOCH_ASSERT(); + *mp = NULL; /* @@ -421,7 +422,6 @@ icmp_input(struct mbuf **mp, int *offp, int proto) inet_ntoa_r(ip->ip_dst, dstbuf), icmplen); } #endif - NET_EPOCH_ENTER(et); if (icmplen < ICMP_MINLEN) { ICMPSTAT_INC(icps_tooshort); goto freeit; @@ -429,7 +429,6 @@ icmp_input(struct mbuf **mp, int *offp, int proto) i = hlen + min(icmplen, ICMP_ADVLENMIN); if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { ICMPSTAT_INC(icps_tooshort); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -547,7 +546,6 @@ icmp_input(struct mbuf **mp, int *offp, int proto) if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { /* This should actually not happen */ ICMPSTAT_INC(icps_tooshort); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -640,7 +638,6 @@ reflect: ICMPSTAT_INC(icps_reflect); ICMPSTAT_INC(icps_outhist[icp->icmp_type]); icmp_reflect(m); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); case ICMP_REDIRECT: @@ -717,13 +714,11 @@ reflect: } raw: - NET_EPOCH_EXIT(et); *mp = m; rip_input(mp, offp, proto); return (IPPROTO_DONE); freeit: - NET_EPOCH_EXIT(et); m_freem(m); return (IPPROTO_DONE); } @@ -735,7 +730,6 @@ static void icmp_reflect(struct mbuf *m) { struct rm_priotracker in_ifa_tracker; - struct epoch_tracker et; struct ip *ip = mtod(m, struct ip *); struct ifaddr *ifa; struct ifnet *ifp; @@ -745,6 +739,8 @@ icmp_reflect(struct mbuf *m) struct mbuf *opts = NULL; int optlen = (ip->ip_hl << 2) - sizeof(struct ip); + NET_EPOCH_ASSERT(); + if (IN_MULTICAST(ntohl(ip->ip_src.s_addr)) || IN_EXPERIMENTAL(ntohl(ip->ip_src.s_addr)) || IN_ZERONET(ntohl(ip->ip_src.s_addr)) ) { @@ -779,7 +775,6 @@ icmp_reflect(struct mbuf *m) */ ifp = m->m_pkthdr.rcvif; if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -787,11 +782,9 @@ icmp_reflect(struct mbuf *m) if (satosin(&ia->ia_broadaddr)->sin_addr.s_addr == t.s_addr) { t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } } - NET_EPOCH_EXIT(et); } /* * If the packet was transiting through us, use the address of @@ -800,16 +793,13 @@ icmp_reflect(struct mbuf *m) * criteria apply. */ if (V_icmp_rfi && ifp != NULL) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); } /* * If the incoming packet was not addressed directly to us, use @@ -818,16 +808,13 @@ icmp_reflect(struct mbuf *m) * with normal source selection. */ if (V_reply_src[0] != '\0' && (ifp = ifunit(V_reply_src))) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); } /* * If the packet was transiting through us, use the address of diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index 03f5fed3e82b4..8991262cc1200 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -457,6 +457,7 @@ ip_input(struct mbuf *m) struct in_addr odst; /* original dst address */ M_ASSERTPKTHDR(m); + NET_EPOCH_ASSERT(); if (m->m_flags & M_FASTFWD_OURS) { m->m_flags &= ~M_FASTFWD_OURS; @@ -708,9 +709,6 @@ passin: * into the stack for SIMPLEX interfaces handled by ether_output(). */ if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -720,7 +718,6 @@ passin: counter_u64_add(ia->ia_ifa.ifa_ipackets, 1); counter_u64_add(ia->ia_ifa.ifa_ibytes, m->m_pkthdr.len); - NET_EPOCH_EXIT(et); goto ours; } #ifdef BOOTP_COMPAT @@ -728,12 +725,10 @@ passin: counter_u64_add(ia->ia_ifa.ifa_ipackets, 1); counter_u64_add(ia->ia_ifa.ifa_ibytes, m->m_pkthdr.len); - NET_EPOCH_EXIT(et); goto ours; } #endif } - NET_EPOCH_EXIT(et); ia = NULL; } /* RFC 3927 2.7: Do not forward datagrams for 169.254.0.0/16. */ @@ -953,9 +948,10 @@ ip_forward(struct mbuf *m, int srcrt) struct sockaddr_in *sin; struct in_addr dest; struct route ro; - struct epoch_tracker et; int error, type = 0, code = 0, mtu = 0; + NET_EPOCH_ASSERT(); + if (m->m_flags & (M_BCAST|M_MCAST) || in_canforward(ip->ip_dst) == 0) { IPSTAT_INC(ips_cantforward); m_freem(m); @@ -982,7 +978,6 @@ ip_forward(struct mbuf *m, int srcrt) #else in_rtalloc_ign(&ro, 0, M_GETFIB(m)); #endif - NET_EPOCH_ENTER(et); if (ro.ro_rt != NULL) { ia = ifatoia(ro.ro_rt->rt_ifa); } else @@ -1030,7 +1025,7 @@ ip_forward(struct mbuf *m, int srcrt) m_freem(mcopy); if (error != EINPROGRESS) IPSTAT_INC(ips_cantforward); - goto out; + return; } /* No IPsec processing required */ } @@ -1083,11 +1078,11 @@ ip_forward(struct mbuf *m, int srcrt) else { if (mcopy) m_freem(mcopy); - goto out; + return; } } if (mcopy == NULL) - goto out; + return; switch (error) { @@ -1130,11 +1125,9 @@ ip_forward(struct mbuf *m, int srcrt) case ENOBUFS: case EACCES: /* ipfw denied packet */ m_freem(mcopy); - goto out; + return; } icmp_error(mcopy, type, code, dest.s_addr, mtu); - out: - NET_EPOCH_EXIT(et); } #define CHECK_SO_CT(sp, ct) \ diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c index 14ebb6ef10079..dd43b467f1f38 100644 --- a/sys/netinet/ip_mroute.c +++ b/sys/netinet/ip_mroute.c @@ -874,18 +874,13 @@ add_vif(struct vifctl *vifcp) */ ifp = NULL; } else { - struct epoch_tracker et; - sin.sin_addr = vifcp->vifc_lcl_addr; - NET_EPOCH_ENTER(et); ifa = ifa_ifwithaddr((struct sockaddr *)&sin); if (ifa == NULL) { - NET_EPOCH_EXIT(et); VIF_UNLOCK(); return EADDRNOTAVAIL; } ifp = ifa->ifa_ifp; - NET_EPOCH_EXIT(et); } if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) { diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c index 87202eb806c41..cb9920d86d9eb 100644 --- a/sys/netinet/ip_options.c +++ b/sys/netinet/ip_options.c @@ -109,7 +109,8 @@ ip_dooptions(struct mbuf *m, int pass) uint32_t ntime; struct nhop4_extended nh_ext; struct sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET }; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); /* Ignore or reject packets with IP options. */ if (V_ip_doopts == 0) @@ -117,10 +118,9 @@ ip_dooptions(struct mbuf *m, int pass) else if (V_ip_doopts == 2) { type = ICMP_UNREACH; code = ICMP_UNREACH_FILTER_PROHIB; - goto bad_unlocked; + goto bad; } - NET_EPOCH_ENTER(et); dst = ip->ip_dst; cp = (u_char *)(ip + 1); cnt = (ip->ip_hl << 2) - sizeof (struct ip); @@ -226,7 +226,6 @@ dropit: #endif IPSTAT_INC(ips_cantforward); m_freem(m); - NET_EPOCH_EXIT(et); return (1); } } @@ -381,15 +380,12 @@ dropit: cp[IPOPT_OFFSET] += sizeof(uint32_t); } } - NET_EPOCH_EXIT(et); if (forward && V_ipforwarding) { ip_forward(m, 1); return (1); } return (0); bad: - NET_EPOCH_EXIT(et); -bad_unlocked: icmp_error(m, type, code, 0, 0); IPSTAT_INC(ips_badoptions); return (1); diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index cbd2d72188fa9..9266dce3f7c26 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -1078,7 +1078,8 @@ in_delayed_cksum(struct mbuf *m) int ip_ctloutput(struct socket *so, struct sockopt *sopt) { - struct inpcb *inp = sotoinpcb(so); + struct inpcb *inp = sotoinpcb(so); + struct epoch_tracker et; int error, optval; #ifdef RSS uint32_t rss_bucket; @@ -1517,7 +1518,9 @@ ip_ctloutput(struct socket *so, struct sockopt *sopt) case IP_MULTICAST_TTL: case IP_MULTICAST_LOOP: case IP_MSFILTER: + NET_EPOCH_ENTER(et); error = inp_getmoptions(inp, sopt); + NET_EPOCH_EXIT(et); break; #if defined(IPSEC) || defined(IPSEC_SUPPORT) diff --git a/sys/netinet/sctp_bsd_addr.c b/sys/netinet/sctp_bsd_addr.c index e0432fb6b689b..df8841c767c39 100644 --- a/sys/netinet/sctp_bsd_addr.c +++ b/sys/netinet/sctp_bsd_addr.c @@ -198,6 +198,7 @@ sctp_init_ifns_for_vrf(int vrfid) * make sure we lock any IFA that exists as we float through the * list of IFA's */ + struct epoch_tracker et; struct ifnet *ifn; struct ifaddr *ifa; struct sctp_ifa *sctp_ifa; @@ -207,14 +208,12 @@ sctp_init_ifns_for_vrf(int vrfid) #endif IFNET_RLOCK(); + NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { - struct epoch_tracker et; - if (sctp_is_desired_interface_type(ifn) == 0) { /* non desired type */ continue; } - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifn->if_addrhead, ifa_link) { if (ifa->ifa_addr == NULL) { continue; @@ -267,8 +266,8 @@ sctp_init_ifns_for_vrf(int vrfid) sctp_ifa->localifa_flags &= ~SCTP_ADDR_DEFER_USE; } } - NET_EPOCH_EXIT(et); } + NET_EPOCH_EXIT(et); IFNET_RUNLOCK(); } diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c index b18de150fd332..f2d0321ffdab7 100644 --- a/sys/netinet6/icmp6.c +++ b/sys/netinet6/icmp6.c @@ -1660,13 +1660,14 @@ static int ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, struct in6_addr *subj) { - struct epoch_tracker et; struct ifnet *ifp; struct in6_ifaddr *ifa6; struct ifaddr *ifa; int addrs = 0, addrsofif, iffound = 0; int niflags = ni6->ni_flags; + NET_EPOCH_ASSERT(); + if ((niflags & NI_NODEADDR_FLAG_ALL) == 0) { switch (ni6->ni_code) { case ICMP6_NI_SUBJ_IPV6: @@ -1682,7 +1683,6 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, } } - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { addrsofif = 0; CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -1737,13 +1737,11 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, } if (iffound) { *ifpp = ifp; - NET_EPOCH_EXIT(et); return (addrsofif); } addrs += addrsofif; } - NET_EPOCH_EXIT(et); return (addrs); } @@ -1752,7 +1750,6 @@ static int ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, struct ifnet *ifp0, int resid) { - struct epoch_tracker et; struct ifnet *ifp; struct in6_ifaddr *ifa6; struct ifaddr *ifa; @@ -1762,10 +1759,11 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, int niflags = ni6->ni_flags; u_int32_t ltime; + NET_EPOCH_ASSERT(); + if (ifp0 == NULL && !(niflags & NI_NODEADDR_FLAG_ALL)) return (0); /* needless to copy */ - NET_EPOCH_ENTER(et); ifp = ifp0 ? ifp0 : CK_STAILQ_FIRST(&V_ifnet); again: @@ -1829,7 +1827,6 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, * Set the truncate flag and return. */ nni6->ni_flags |= NI_NODEADDR_FLAG_TRUNCATE; - NET_EPOCH_EXIT(et); return (copied); } @@ -1881,8 +1878,6 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, goto again; } - NET_EPOCH_EXIT(et); - return (copied); } @@ -2563,14 +2558,11 @@ icmp6_redirect_output(struct mbuf *m0, struct rtentry *rt) { /* target lladdr option */ - struct epoch_tracker et; int len; struct nd_opt_hdr *nd_opt; char *lladdr; - NET_EPOCH_ENTER(et); ln = nd6_lookup(router_ll6, 0, ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) goto nolladdropt; diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c index 4de6e8d2dfe08..c92ee171b7848 100644 --- a/sys/netinet6/in6.c +++ b/sys/netinet6/in6.c @@ -1474,10 +1474,10 @@ done: struct in6_ifaddr * in6ifa_ifpforlinklocal(struct ifnet *ifp, int ignoreflags) { - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1489,7 +1489,6 @@ in6ifa_ifpforlinklocal(struct ifnet *ifp, int ignoreflags) break; } } - NET_EPOCH_EXIT(et); return ((struct in6_ifaddr *)ifa); } @@ -1549,13 +1548,13 @@ in6ifa_ifpwithaddr(struct ifnet *ifp, const struct in6_addr *addr) struct in6_ifaddr * in6ifa_llaonifp(struct ifnet *ifp) { - struct epoch_tracker et; struct sockaddr_in6 *sin6; struct ifaddr *ifa; + NET_EPOCH_ASSERT(); + if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) return (NULL); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1565,7 +1564,6 @@ in6ifa_llaonifp(struct ifnet *ifp) IN6_IS_ADDR_MC_NODELOCAL(&sin6->sin6_addr)) break; } - NET_EPOCH_EXIT(et); return ((struct in6_ifaddr *)ifa); } @@ -1702,26 +1700,23 @@ int in6_ifhasaddr(struct ifnet *ifp, struct in6_addr *addr) { struct in6_addr in6; - struct epoch_tracker et; struct ifaddr *ifa; struct in6_ifaddr *ia6; + NET_EPOCH_ASSERT(); + in6 = *addr; if (in6_clearscope(&in6)) return (0); in6_setscope(&in6, ifp, NULL); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ia6 = (struct in6_ifaddr *)ifa; - if (IN6_ARE_ADDR_EQUAL(&ia6->ia_addr.sin6_addr, &in6)) { - NET_EPOCH_EXIT(et); + if (IN6_ARE_ADDR_EQUAL(&ia6->ia_addr.sin6_addr, &in6)) return (1); - } } - NET_EPOCH_EXIT(et); return (0); } @@ -1825,12 +1820,13 @@ in6_prefixlen2mask(struct in6_addr *maskp, int len) struct in6_ifaddr * in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) { - struct epoch_tracker et; int dst_scope = in6_addrscope(dst), blen = -1, tlen; struct ifaddr *ifa; struct in6_ifaddr *besta = NULL; struct in6_ifaddr *dep[2]; /* last-resort: deprecated */ + NET_EPOCH_ASSERT(); + dep[0] = dep[1] = NULL; /* @@ -1839,7 +1835,6 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) * If two or more, return one which matches the dst longest. * If none, return one of global addresses assigned other ifs. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1873,7 +1868,6 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) } if (besta) { ifa_ref(&besta->ia_ifa); - NET_EPOCH_EXIT(et); return (besta); } @@ -1894,23 +1888,19 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) if (ifa != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); return (struct in6_ifaddr *)ifa; } /* use the last-resort values, that are, deprecated addresses */ if (dep[0]) { ifa_ref((struct ifaddr *)dep[0]); - NET_EPOCH_EXIT(et); return dep[0]; } if (dep[1]) { ifa_ref((struct ifaddr *)dep[1]); - NET_EPOCH_EXIT(et); return dep[1]; } - NET_EPOCH_EXIT(et); return NULL; } @@ -1940,12 +1930,12 @@ in6_if_up(struct ifnet *ifp) arc4random() % (MAX_RTR_SOLICITATION_DELAY * hz)); } } - NET_EPOCH_EXIT(et); /* * special cases, like 6to4, are handled in in6_ifattach */ in6_ifattach(ifp, NULL); + NET_EPOCH_EXIT(et); } int @@ -2152,6 +2142,7 @@ in6_lltable_rtcheck(struct ifnet *ifp, char ip6buf[INET6_ADDRSTRLEN]; int fibnum; + NET_EPOCH_ASSERT(); KASSERT(l3addr->sa_family == AF_INET6, ("sin_family %d", l3addr->sa_family)); @@ -2160,19 +2151,15 @@ in6_lltable_rtcheck(struct ifnet *ifp, fibnum = V_rt_add_addr_allfibs ? RT_DEFAULT_FIB : ifp->if_fib; error = fib6_lookup_nh_basic(fibnum, &dst, scopeid, 0, 0, &nh6); if (error != 0 || (nh6.nh_flags & NHF_GATEWAY) || nh6.nh_ifp != ifp) { - struct epoch_tracker et; struct ifaddr *ifa; /* * Create an ND6 cache for an IPv6 neighbor * that is not covered by our own prefix. */ - NET_EPOCH_ENTER(et); ifa = ifaof_ifpforaddr(l3addr, ifp); if (ifa != NULL) { - NET_EPOCH_EXIT(et); return 0; } - NET_EPOCH_EXIT(et); log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n", ip6_sprintf(ip6buf, &sin6->sin6_addr)); return EINVAL; diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c index 7ce680f2f91ad..fd46f4494a432 100644 --- a/sys/netinet6/in6_ifattach.c +++ b/sys/netinet6/in6_ifattach.c @@ -240,7 +240,6 @@ generate_tmp_ifid(u_int8_t *seed0, const u_int8_t *seed1, u_int8_t *ret) int in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) { - struct epoch_tracker et; struct ifaddr *ifa; struct sockaddr_dl *sdl; u_int8_t *addr; @@ -249,7 +248,8 @@ in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) static u_int8_t allone[8] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_LINK) continue; @@ -261,12 +261,10 @@ in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) goto found; } - NET_EPOCH_EXIT(et); return -1; found: - IF_ADDR_LOCK_ASSERT(ifp); addr = LLADDR(sdl); addrlen = sdl->sdl_alen; @@ -283,24 +281,18 @@ found: addrlen = 8; /* look at IEEE802/EUI64 only */ - if (addrlen != 8 && addrlen != 6) { - NET_EPOCH_EXIT(et); + if (addrlen != 8 && addrlen != 6) return -1; - } /* * check for invalid MAC address - on bsdi, we see it a lot * since wildboar configures all-zero MAC on pccard before * card insertion. */ - if (bcmp(addr, allzero, addrlen) == 0) { - NET_EPOCH_EXIT(et); + if (bcmp(addr, allzero, addrlen) == 0) return -1; - } - if (bcmp(addr, allone, addrlen) == 0) { - NET_EPOCH_EXIT(et); + if (bcmp(addr, allone, addrlen) == 0) return -1; - } /* make EUI64 address */ if (addrlen == 8) @@ -325,27 +317,21 @@ found: * identifier source (can be renumbered). * we don't do this. */ - NET_EPOCH_EXIT(et); return -1; case IFT_INFINIBAND: - if (addrlen != 20) { - NET_EPOCH_EXIT(et); + if (addrlen != 20) return -1; - } bcopy(addr + 12, &in6->s6_addr[8], 8); break; default: - NET_EPOCH_EXIT(et); return -1; } /* sanity check: g bit must not indicate "group" */ - if (EUI64_GROUP(in6)) { - NET_EPOCH_EXIT(et); + if (EUI64_GROUP(in6)) return -1; - } /* convert EUI64 into IPv6 interface identifier */ EUI64_TO_IFID(in6); @@ -355,12 +341,9 @@ found: * subnet router anycast */ if ((in6->s6_addr[8] & ~(EUI64_GBIT | EUI64_UBIT)) == 0x00 && - bcmp(&in6->s6_addr[9], allzero, 7) == 0) { - NET_EPOCH_EXIT(et); + bcmp(&in6->s6_addr[9], allzero, 7) == 0) return -1; - } - NET_EPOCH_EXIT(et); return 0; } @@ -375,9 +358,10 @@ static int get_ifid(struct ifnet *ifp0, struct ifnet *altifp, struct in6_addr *in6) { - struct epoch_tracker et; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + /* first, try to get it from the interface itself */ if (in6_get_hw_ifid(ifp0, in6) == 0) { nd6log((LOG_DEBUG, "%s: got interface identifier from itself\n", @@ -393,7 +377,6 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp, } /* next, try to get it from some other hardware interface */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp == ifp0) continue; @@ -408,11 +391,9 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp, nd6log((LOG_DEBUG, "%s: borrow interface identifier from %s\n", if_name(ifp0), if_name(ifp))); - NET_EPOCH_EXIT(et); goto success; } } - NET_EPOCH_EXIT(et); /* last resort: get from random number source */ if (get_rand_ifid(ifp, in6) == 0) { @@ -686,6 +667,8 @@ in6_ifattach(struct ifnet *ifp, struct ifnet *altifp) { struct in6_ifaddr *ia; + NET_EPOCH_ASSERT(); + if (ifp->if_afdata[AF_INET6] == NULL) return; /* diff --git a/sys/netinet6/in6_mcast.c b/sys/netinet6/in6_mcast.c index 904154f56da58..5e7969d2a6dab 100644 --- a/sys/netinet6/in6_mcast.c +++ b/sys/netinet6/in6_mcast.c @@ -2770,8 +2770,10 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) return (EINVAL); } + NET_EPOCH_ENTER(et); ifp = ifnet_byindex(ifindex); if (ifp == NULL) { + NET_EPOCH_EXIT(et); CTR2(KTR_MLD, "%s: no ifp for ifindex %u", __func__, ifindex); return (ENOENT); @@ -2783,12 +2785,13 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) retval = sysctl_wire_old_buffer(req, sizeof(uint32_t) + (in6_mcast_maxgrpsrc * sizeof(struct in6_addr))); - if (retval) + if (retval) { + NET_EPOCH_EXIT(et); return (retval); + } IN6_MULTI_LOCK(); IN6_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -2816,10 +2819,9 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) break; } } - NET_EPOCH_EXIT(et); - IN6_MULTI_LIST_UNLOCK(); IN6_MULTI_UNLOCK(); + NET_EPOCH_EXIT(et); return (retval); } diff --git a/sys/netinet6/in6_var.h b/sys/netinet6/in6_var.h index 3e535310b29e1..9856acd87429d 100644 --- a/sys/netinet6/in6_var.h +++ b/sys/netinet6/in6_var.h @@ -785,6 +785,8 @@ in6m_ifmultiaddr_get_inm(struct ifmultiaddr *ifma) * Look up an in6_multi record for an IPv6 multicast address * on the interface ifp. * If no record found, return NULL. + * + * SMPng: The IN6_MULTI_LOCK and must be held and must be in network epoch. */ static __inline struct in6_multi * in6m_lookup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr) @@ -805,18 +807,17 @@ in6m_lookup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr) /* * Wrapper for in6m_lookup_locked(). * - * SMPng: Assumes that neithr the IN6_MULTI_LOCK() or IF_ADDR_LOCK() are held. + * SMPng: Assumes network epoch entered and that IN6_MULTI_LOCK() isn't held. */ static __inline struct in6_multi * in6m_lookup(struct ifnet *ifp, const struct in6_addr *mcaddr) { - struct epoch_tracker et; struct in6_multi *inm; + NET_EPOCH_ASSERT(); + IN6_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); inm = in6m_lookup_locked(ifp, mcaddr); - NET_EPOCH_EXIT(et); IN6_MULTI_LIST_UNLOCK(); return (inm); diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c index 08dffde61721d..4cf4288b1b5bd 100644 --- a/sys/netinet6/ip6_output.c +++ b/sys/netinet6/ip6_output.c @@ -384,6 +384,7 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, struct mbuf *m = m0; struct mbuf *mprev = NULL; int hlen, tlen, len; + struct epoch_tracker et; struct route_in6 ip6route; struct rtentry *rt = NULL; struct sockaddr_in6 *dst, src_sa, dst_sa; @@ -586,6 +587,7 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, ro = &opt->ip6po_route; dst = (struct sockaddr_in6 *)&ro->ro_dst; fibnum = (inp != NULL) ? inp->inp_inc.inc_fibnum : M_GETFIB(m); + NET_EPOCH_ENTER(et); again: /* * if specified, try to fill in the traffic class field. @@ -1186,6 +1188,7 @@ sendorfree: IP6STAT_INC(ip6s_fragmented); done: + NET_EPOCH_EXIT(et); if (ro == &ip6route) RO_RTFREE(ro); return (error); diff --git a/sys/netinet6/mld6.c b/sys/netinet6/mld6.c index 6ea63b47b47af..868edeaff8c57 100644 --- a/sys/netinet6/mld6.c +++ b/sys/netinet6/mld6.c @@ -636,7 +636,6 @@ static int mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, /*const*/ struct mld_hdr *mld) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct mld_ifsoftc *mli; struct in6_multi *inm; @@ -646,6 +645,8 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + is_general_query = 0; if (!mld_v1enable) { @@ -704,7 +705,6 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, if (timer == 0) timer = 1; - NET_EPOCH_ENTER(et); if (is_general_query) { /* * For each reporting group joined on this @@ -735,7 +735,6 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, in6_clearscope(&mld->mld_addr); } - NET_EPOCH_EXIT(et); MLD_UNLOCK(); IN6_MULTI_LIST_UNLOCK(); @@ -824,6 +823,8 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + if (!mld_v2enable) { CTR3(KTR_MLD, "ignore v2 query src %s on ifp %p(%s)", ip6_sprintf(ip6tbuf, &ip6->ip6_src), @@ -937,8 +938,6 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, V_interface_timers_running6 = 1; } } else { - struct epoch_tracker et; - /* * MLDv2 Group-specific or Group-and-source-specific Query. * @@ -947,18 +946,14 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, * Queries for groups we are not a member of on this * link are simply ignored. */ - NET_EPOCH_ENTER(et); inm = in6m_lookup_locked(ifp, &mld->mld_addr); - if (inm == NULL) { - NET_EPOCH_EXIT(et); + if (inm == NULL) goto out_locked; - } if (nsrc > 0) { if (!ratecheck(&inm->in6m_lastgsrtv, &V_mld_gsrdelay)) { CTR1(KTR_MLD, "%s: GS query throttled.", __func__); - NET_EPOCH_EXIT(et); goto out_locked; } } @@ -976,7 +971,6 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, /* XXX Clear embedded scope ID as userland won't expect it. */ in6_clearscope(&mld->mld_addr); - NET_EPOCH_EXIT(et); } out_locked: @@ -1110,13 +1104,14 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, /*const*/ struct mld_hdr *mld) { struct in6_addr src, dst; - struct epoch_tracker et; struct in6_ifaddr *ia; struct in6_multi *inm; #ifdef KTR char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + if (!mld_v1enable) { CTR3(KTR_MLD, "ignore v1 report %s on ifp %p(%s)", ip6_sprintf(ip6tbuf, &mld->mld_addr), @@ -1186,7 +1181,6 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, IN6_MULTI_LIST_LOCK(); MLD_LOCK(); - NET_EPOCH_ENTER(et); /* * MLDv1 report suppression. @@ -1234,7 +1228,6 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, } out_locked: - NET_EPOCH_EXIT(et); MLD_UNLOCK(); IN6_MULTI_LIST_UNLOCK(); @@ -1412,8 +1405,8 @@ mld_fasttimo_vnet(struct in6_multi_head *inmh) mbufq_init(&scq, MLD_MAX_STATE_CHANGE_PACKETS); } - IF_ADDR_WLOCK(ifp); NET_EPOCH_ENTER(et); + IF_ADDR_WLOCK(ifp); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -2995,12 +2988,12 @@ mld_v2_merge_state_changes(struct in6_multi *inm, struct mbufq *scq) static void mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct ifnet *ifp; struct in6_multi *inm; int retval; + NET_EPOCH_ASSERT(); IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); @@ -3018,7 +3011,6 @@ mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) ifp = mli->mli_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -3047,7 +3039,6 @@ mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) break; } } - NET_EPOCH_EXIT(et); send: mld_dispatch_queue(&mli->mli_gq, MLD_MAX_RESPONSE_BURST); diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index e359b72cb4555..fae88877a2f55 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -732,6 +732,7 @@ nd6_llinfo_setstate(struct llentry *lle, int newstate) static __noinline void nd6_llinfo_timer(void *arg) { + struct epoch_tracker et; struct llentry *ln; struct in6_addr *dst, *pdst, *psrc, src; struct ifnet *ifp; @@ -768,6 +769,7 @@ nd6_llinfo_timer(void *arg) CURVNET_RESTORE(); return; } + NET_EPOCH_ENTER(et); ndi = ND_IFINFO(ifp); send_ns = 0; dst = &ln->r_l3addr.addr6; @@ -886,6 +888,7 @@ done: if (ln != NULL) LLE_FREE_LOCKED(ln); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); } @@ -897,6 +900,7 @@ void nd6_timer(void *arg) { CURVNET_SET((struct vnet *) arg); + struct epoch_tracker et; struct nd_drhead drq; struct nd_prhead prl; struct nd_defrouter *dr, *ndr; @@ -927,6 +931,7 @@ nd6_timer(void *arg) * * XXXRW: in6_ifaddrhead locking. */ + NET_EPOCH_ENTER(et); addrloop: CK_STAILQ_FOREACH_SAFE(ia6, &V_in6_ifaddrhead, ia_link, nia6) { /* check address lifetime */ @@ -1014,6 +1019,7 @@ nd6_timer(void *arg) ia6->ia6_flags &= ~IN6_IFF_DEPRECATED; } } + NET_EPOCH_EXIT(et); ND6_WLOCK(); restart: @@ -1069,13 +1075,13 @@ restart: static int regen_tmpaddr(struct in6_ifaddr *ia6) { - struct epoch_tracker et; struct ifaddr *ifa; struct ifnet *ifp; struct in6_ifaddr *public_ifa6 = NULL; + NET_EPOCH_ASSERT(); + ifp = ia6->ia_ifa.ifa_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *it6; @@ -1116,7 +1122,6 @@ regen_tmpaddr(struct in6_ifaddr *ia6) } if (public_ifa6 != NULL) ifa_ref(&public_ifa6->ia_ifa); - NET_EPOCH_EXIT(et); if (public_ifa6 != NULL) { int e; @@ -1387,10 +1392,10 @@ restart: int nd6_is_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp) { - struct epoch_tracker et; struct llentry *lle; int rc = 0; + NET_EPOCH_ASSERT(); IF_AFDATA_UNLOCK_ASSERT(ifp); if (nd6_is_new_addr_neighbor(addr, ifp)) return (1); @@ -1399,12 +1404,10 @@ nd6_is_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp) * Even if the address matches none of our addresses, it might be * in the neighbor cache. */ - NET_EPOCH_ENTER(et); if ((lle = nd6_lookup(&addr->sin6_addr, 0, ifp)) != NULL) { LLE_RUNLOCK(lle); rc = 1; } - NET_EPOCH_EXIT(et); return (rc); } @@ -1688,6 +1691,7 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) struct ifaddr *ifa; struct in6_ifaddr *ia; + NET_EPOCH_ENTER(et); if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) && !(ND.flags & ND6_IFF_IFDISABLED)) { /* ifdisabled 1->0 transision */ @@ -1698,7 +1702,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) * do not clear ND6_IFF_IFDISABLED. * See RFC 4862, Section 5.4.5. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1707,7 +1710,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) break; } - NET_EPOCH_EXIT(et); if (ifa != NULL) { /* LLA is duplicated. */ @@ -1728,7 +1730,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) ND_IFINFO(ifp)->flags |= ND6_IFF_IFDISABLED; if (V_ip6_dad_count > 0 && (ND_IFINFO(ifp)->flags & ND6_IFF_NO_DAD) == 0) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != @@ -1737,7 +1738,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) ia = (struct in6_ifaddr *)ifa; ia->ia6_flags |= IN6_IFF_TENTATIVE; } - NET_EPOCH_EXIT(et); } } @@ -1756,7 +1756,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) * address is assigned, and IFF_UP, try to * assign one. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != @@ -1766,15 +1765,15 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) if (IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) break; } - NET_EPOCH_EXIT(et); if (ifa != NULL) /* No LLA is configured. */ in6_ifattach(ifp, NULL); } } - } ND_IFINFO(ifp)->flags = ND.flags; + NET_EPOCH_EXIT(et); break; + } #undef ND case SIOCSNDFLUSH_IN6: /* XXX: the ioctl name is confusing... */ /* sync kernel routing table with the default router list */ @@ -1970,12 +1969,12 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr, int flags; uint16_t router = 0; struct sockaddr_in6 sin6; - struct epoch_tracker et; struct mbuf *chain = NULL; u_char linkhdr[LLE_MAX_LINKHDR]; size_t linkhdrsize; int lladdr_off; + NET_EPOCH_ASSERT(); IF_AFDATA_UNLOCK_ASSERT(ifp); KASSERT(ifp != NULL, ("%s: ifp == NULL", __func__)); @@ -1995,9 +1994,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr, * description on it in NS section (RFC 2461 7.2.3). */ flags = lladdr ? LLE_EXCLUSIVE : 0; - NET_EPOCH_ENTER(et); ln = nd6_lookup(from, flags, ifp); - NET_EPOCH_EXIT(et); is_newentry = 0; if (ln == NULL) { flags |= LLE_EXCLUSIVE; @@ -2256,10 +2253,11 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, const struct sockaddr *sa_dst, u_char *desten, uint32_t *pflags, struct llentry **plle) { - struct epoch_tracker et; struct llentry *ln = NULL; const struct sockaddr_in6 *dst6; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; @@ -2285,7 +2283,6 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, } } - NET_EPOCH_ENTER(et); ln = nd6_lookup(&dst6->sin6_addr, plle ? LLE_EXCLUSIVE : LLE_UNLOCKED, ifp); if (ln != NULL && (ln->r_flags & RLLE_VALID) != 0) { @@ -2305,11 +2302,9 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, *plle = ln; LLE_WUNLOCK(ln); } - NET_EPOCH_EXIT(et); return (0); } else if (plle && ln) LLE_WUNLOCK(ln); - NET_EPOCH_EXIT(et); return (nd6_resolve_slow(ifp, 0, m, dst6, desten, pflags, plle)); } @@ -2336,6 +2331,8 @@ nd6_resolve_slow(struct ifnet *ifp, int flags, struct mbuf *m, int send_ns, ll_len; char *lladdr; + NET_EPOCH_ASSERT(); + /* * Address resolution or Neighbor Unreachability Detection * for the next hop. @@ -2343,11 +2340,7 @@ nd6_resolve_slow(struct ifnet *ifp, int flags, struct mbuf *m, * or an anycast address(i.e. not a multicast). */ if (lle == NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); lle = nd6_lookup(&dst->sin6_addr, LLE_EXCLUSIVE, ifp); - NET_EPOCH_EXIT(et); if ((lle == NULL) && nd6_is_addr_neighbor(dst, ifp)) { /* * Since nd6_is_addr_neighbor() internally calls nd6_lookup(), diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c index cbac0dcb724ea..3624909cc164c 100644 --- a/sys/netinet6/nd6_nbr.c +++ b/sys/netinet6/nd6_nbr.c @@ -614,7 +614,6 @@ nd6_ns_output(struct ifnet *ifp, const struct in6_addr *saddr6, void nd6_na_input(struct mbuf *m, int off, int icmp6len) { - struct epoch_tracker et; struct ifnet *ifp = m->m_pkthdr.rcvif; struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); struct nd_neighbor_advert *nd_na; @@ -637,6 +636,8 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) int lladdr_off; char ip6bufs[INET6_ADDRSTRLEN], ip6bufd[INET6_ADDRSTRLEN]; + NET_EPOCH_ASSERT(); + /* RFC 6980: Nodes MUST silently ignore fragments */ if(m->m_flags & M_FRAGMENTED) goto freeit; @@ -742,9 +743,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) * If no neighbor cache entry is found, NA SHOULD silently be * discarded. */ - NET_EPOCH_ENTER(et); ln = nd6_lookup(&taddr6, LLE_EXCLUSIVE, ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) { goto freeit; } @@ -1193,8 +1192,13 @@ static void nd6_dad_starttimer(struct dadq *dp, int ticks, int send_ns) { - if (send_ns != 0) + if (send_ns != 0) { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); nd6_dad_ns_output(dp); + NET_EPOCH_EXIT(et); + } callout_reset(&dp->dad_timer_ch, ticks, (void (*)(void *))nd6_dad_timer, (void *)dp); } diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c index d0b8dc51eba7e..03442745f96ad 100644 --- a/sys/netinet6/nd6_rtr.c +++ b/sys/netinet6/nd6_rtr.c @@ -1260,7 +1260,10 @@ nd6_prelist_add(struct nd_prefixctl *pr, struct nd_defrouter *dr, /* ND_OPT_PI_FLAG_ONLINK processing */ if (new->ndpr_raf_onlink) { + struct epoch_tracker et; + ND6_ONLINK_LOCK(); + NET_EPOCH_ENTER(et); if ((error = nd6_prefix_onlink(new)) != 0) { nd6log((LOG_ERR, "nd6_prelist_add: failed to make " "the prefix %s/%d on-link on %s (errno=%d)\n", @@ -1268,6 +1271,7 @@ nd6_prelist_add(struct nd_prefixctl *pr, struct nd_defrouter *dr, pr->ndpr_plen, if_name(pr->ndpr_ifp), error)); /* proceed anyway. XXX: is it correct? */ } + NET_EPOCH_EXIT(et); ND6_ONLINK_UNLOCK(); } @@ -1351,7 +1355,8 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, int auth; struct in6_addrlifetime lt6_tmp; char ip6buf[INET6_ADDRSTRLEN]; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); auth = 0; if (m) { @@ -1465,7 +1470,6 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, * consider autoconfigured addresses while RFC2462 simply said * "address". */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *ifa6; u_int32_t remaininglifetime; @@ -1588,7 +1592,6 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, ifa6->ia6_lifetime = lt6_tmp; ifa6->ia6_updatetime = time_uptime; } - NET_EPOCH_EXIT(et); if (ia6_match == NULL && new->ndpr_vltime) { int ifidlen; @@ -1684,10 +1687,9 @@ find_pfxlist_reachable_router(struct nd_prefix *pr) ND6_LOCK_ASSERT(); + NET_EPOCH_ENTER(et); LIST_FOREACH(pfxrtr, &pr->ndpr_advrtrs, pfr_entry) { - NET_EPOCH_ENTER(et); ln = nd6_lookup(&pfxrtr->router->rtaddr, 0, pfxrtr->router->ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) continue; canreach = ND6_IS_LLINFO_PROBREACH(ln); @@ -1695,6 +1697,7 @@ find_pfxlist_reachable_router(struct nd_prefix *pr) if (canreach) break; } + NET_EPOCH_EXIT(et); return (pfxrtr); } diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c index a0d06230e8523..ef903bc6bd668 100644 --- a/sys/netinet6/raw_ip6.c +++ b/sys/netinet6/raw_ip6.c @@ -734,12 +734,12 @@ rip6_disconnect(struct socket *so) static int rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td) { - struct epoch_tracker et; struct inpcb *inp; struct sockaddr_in6 *addr = (struct sockaddr_in6 *)nam; struct ifaddr *ifa = NULL; int error = 0; + NET_EPOCH_ASSERT(); inp = sotoinpcb(so); KASSERT(inp != NULL, ("rip6_bind: inp == NULL")); @@ -752,20 +752,14 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td) if ((error = sa6_embedscope(addr, V_ip6_use_defzone)) != 0) return (error); - NET_EPOCH_ENTER(et); if (!IN6_IS_ADDR_UNSPECIFIED(&addr->sin6_addr) && - (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) { - NET_EPOCH_EXIT(et); + (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) return (EADDRNOTAVAIL); - } if (ifa != NULL && ((struct in6_ifaddr *)ifa)->ia6_flags & (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY| - IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) { - NET_EPOCH_EXIT(et); + IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) return (EADDRNOTAVAIL); - } - NET_EPOCH_EXIT(et); INP_INFO_WLOCK(&V_ripcbinfo); INP_WLOCK(inp); inp->in6p_laddr = addr->sin6_addr; diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c index df09eb36cfa3c..96cffd6305a47 100644 --- a/sys/netipsec/xform_ipcomp.c +++ b/sys/netipsec/xform_ipcomp.c @@ -52,6 +52,8 @@ #include #include +#include /* XXXGL: net_epoch should move out there */ +#include /* XXXGL: net_epoch should move out there */ #include #include @@ -122,6 +124,8 @@ ipcomp_nonexp_input(struct mbuf *m, int off, int proto, void *arg __unused) { int isr; + NET_EPOCH_ASSERT(); + switch (proto) { #ifdef INET case IPPROTO_IPV4: diff --git a/sys/netpfil/ipfw/ip_dn_io.c b/sys/netpfil/ipfw/ip_dn_io.c index 0bd8690767795..c5f0f15187518 100644 --- a/sys/netpfil/ipfw/ip_dn_io.c +++ b/sys/netpfil/ipfw/ip_dn_io.c @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include #include /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ +#include /* NET_EPOCH_... */ #include #include @@ -741,6 +742,8 @@ dummynet_send(struct mbuf *m) { struct mbuf *n; + NET_EPOCH_ASSERT(); + for (; m != NULL; m = n) { struct ifnet *ifp = NULL; /* gcc 3.4.6 complains */ struct m_tag *tag; diff --git a/sys/netpfil/pf/pf_if.c b/sys/netpfil/pf/pf_if.c index f187db2a224c1..62653c877122f 100644 --- a/sys/netpfil/pf/pf_if.c +++ b/sys/netpfil/pf/pf_if.c @@ -76,8 +76,8 @@ eventhandler_tag pfi_change_group_cookie; eventhandler_tag pfi_detach_group_cookie; eventhandler_tag pfi_ifaddr_event_cookie; -static void pfi_attach_ifnet(struct ifnet *); -static void pfi_attach_ifgroup(struct ifg_group *); +static void pfi_attach_ifnet(struct ifnet *, struct pfi_kif *); +static void pfi_attach_ifgroup(struct ifg_group *, struct pfi_kif *); static void pfi_kif_update(struct pfi_kif *); static void pfi_dynaddr_update(struct pfi_dynaddr *dyn); @@ -114,25 +114,49 @@ MTX_SYSINIT(pfi_unlnkdkifs_mtx, &pfi_unlnkdkifs_mtx, "pf unlinked interfaces", void pfi_initialize_vnet(void) { + struct pfi_list kifs = LIST_HEAD_INITIALIZER(); + struct epoch_tracker et; + struct pfi_kif *kif; struct ifg_group *ifg; struct ifnet *ifp; - struct pfi_kif *kif; + int nkifs; V_pfi_buffer_max = 64; V_pfi_buffer = malloc(V_pfi_buffer_max * sizeof(*V_pfi_buffer), PFI_MTYPE, M_WAITOK); - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); - PF_RULES_WLOCK(); - V_pfi_all = pfi_kif_attach(kif, IFG_ALL); - PF_RULES_WUNLOCK(); - + nkifs = 1; /* one for V_pfi_all */ IFNET_RLOCK(); CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) - pfi_attach_ifgroup(ifg); + nkifs++; CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) - pfi_attach_ifnet(ifp); + nkifs++; + + for (int n = 0; n < nkifs; n++) { + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + LIST_INSERT_HEAD(&kifs, kif, pfik_list); + } + + NET_EPOCH_ENTER(et); + PF_RULES_WLOCK(); + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + V_pfi_all = pfi_kif_attach(kif, IFG_ALL); + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) { + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + pfi_attach_ifgroup(ifg, kif); + } + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + pfi_attach_ifnet(ifp, kif); + } + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); IFNET_RUNLOCK(); + + MPASS(LIST_EMPTY(&kifs)); } void @@ -296,59 +320,44 @@ pfi_kif_match(struct pfi_kif *rule_kif, struct pfi_kif *packet_kif) { struct ifg_list *p; + NET_EPOCH_ASSERT(); + if (rule_kif == NULL || rule_kif == packet_kif) return (1); if (rule_kif->pfik_group != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next) - if (p->ifgl_group == rule_kif->pfik_group) { - NET_EPOCH_EXIT(et); + if (p->ifgl_group == rule_kif->pfik_group) return (1); - } - NET_EPOCH_EXIT(et); } - return (0); } static void -pfi_attach_ifnet(struct ifnet *ifp) +pfi_attach_ifnet(struct ifnet *ifp, struct pfi_kif *kif) { - struct pfi_kif *kif; - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + PF_RULES_WASSERT(); - PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, ifp->if_xname); - if_ref(ifp); - kif->pfik_ifp = ifp; ifp->if_pf_kif = kif; - pfi_kif_update(kif); - PF_RULES_WUNLOCK(); } static void -pfi_attach_ifgroup(struct ifg_group *ifg) +pfi_attach_ifgroup(struct ifg_group *ifg, struct pfi_kif *kif) { - struct pfi_kif *kif; - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + PF_RULES_WASSERT(); - PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, ifg->ifg_group); - kif->pfik_group = ifg; ifg->ifg_pf_kif = kif; - PF_RULES_WUNLOCK(); } int @@ -389,6 +398,7 @@ pfi_match_addr(struct pfi_dynaddr *dyn, struct pf_addr *a, sa_family_t af) int pfi_dynaddr_setup(struct pf_addr_wrap *aw, sa_family_t af) { + struct epoch_tracker et; struct pfi_dynaddr *dyn; char tblname[PF_TABLE_NAME_SIZE]; struct pf_ruleset *ruleset = NULL; @@ -445,7 +455,9 @@ pfi_dynaddr_setup(struct pf_addr_wrap *aw, sa_family_t af) TAILQ_INSERT_TAIL(&dyn->pfid_kif->pfik_dynaddrs, dyn, entry); aw->p.dyn = dyn; + NET_EPOCH_ENTER(et); pfi_kif_update(dyn->pfid_kif); + NET_EPOCH_EXIT(et); return (0); @@ -467,6 +479,7 @@ pfi_kif_update(struct pfi_kif *kif) struct ifg_list *ifgl; struct pfi_dynaddr *p; + NET_EPOCH_ASSERT(); PF_RULES_WASSERT(); /* update all dynaddr */ @@ -475,13 +488,9 @@ pfi_kif_update(struct pfi_kif *kif) /* again for all groups kif is member of */ if (kif->pfik_ifp != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next) pfi_kif_update((struct pfi_kif *) ifgl->ifgl_group->ifg_pf_kif); - NET_EPOCH_EXIT(et); } } @@ -512,17 +521,15 @@ pfi_table_update(struct pfr_ktable *kt, struct pfi_kif *kif, int net, int flags) int e, size2 = 0; struct ifg_member *ifgm; + NET_EPOCH_ASSERT(); + V_pfi_buffer_cnt = 0; if (kif->pfik_ifp != NULL) pfi_instance_add(kif->pfik_ifp, net, flags); else if (kif->pfik_group != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next) pfi_instance_add(ifgm->ifgm_ifp, net, flags); - NET_EPOCH_EXIT(et); } if ((e = pfr_set_addrs(&kt->pfrkt_t, V_pfi_buffer, V_pfi_buffer_cnt, &size2, @@ -534,12 +541,12 @@ pfi_table_update(struct pfr_ktable *kt, struct pfi_kif *kif, int net, int flags) static void pfi_instance_add(struct ifnet *ifp, int net, int flags) { - struct epoch_tracker et; struct ifaddr *ia; int got4 = 0, got6 = 0; int net2, af; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ia, &ifp->if_addrhead, ifa_link) { if (ia->ifa_addr == NULL) continue; @@ -597,7 +604,6 @@ pfi_instance_add(struct ifnet *ifp, int net, int flags) else pfi_address_add(ia->ifa_addr, af, net2); } - NET_EPOCH_EXIT(et); } static void @@ -732,9 +738,11 @@ pfi_update_status(const char *name, struct pf_status *pfs) void pfi_get_ifaces(const char *name, struct pfi_kif *buf, int *size) { + struct epoch_tracker et; struct pfi_kif *p, *nextp; int n = 0; + NET_EPOCH_ENTER(et); for (p = RB_MIN(pfi_ifhead, &V_pfi_ifs); p; p = nextp) { nextp = RB_NEXT(pfi_ifhead, &V_pfi_ifs, p); if (pfi_skip_if(name, p)) @@ -747,6 +755,7 @@ pfi_get_ifaces(const char *name, struct pfi_kif *buf, int *size) nextp = RB_NEXT(pfi_ifhead, &V_pfi_ifs, p); } *size = n; + NET_EPOCH_EXIT(et); } static int @@ -755,6 +764,8 @@ pfi_skip_if(const char *filter, struct pfi_kif *p) struct ifg_list *i; int n; + NET_EPOCH_ASSERT(); + if (filter == NULL || !*filter) return (0); if (!strcmp(p->pfik_name, filter)) @@ -764,45 +775,43 @@ pfi_skip_if(const char *filter, struct pfi_kif *p) return (1); /* sanity check */ if (filter[n-1] >= '0' && filter[n-1] <= '9') return (1); /* group names may not end in a digit */ - if (p->pfik_ifp != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); - CK_STAILQ_FOREACH(i, &p->pfik_ifp->if_groups, ifgl_next) { - if (!strncmp(i->ifgl_group->ifg_group, filter, - IFNAMSIZ)) { - NET_EPOCH_EXIT(et); - return (0); /* iface is in group "filter" */ - } - } - NET_EPOCH_EXIT(et); - } + if (p->pfik_ifp == NULL) + return (1); + CK_STAILQ_FOREACH(i, &p->pfik_ifp->if_groups, ifgl_next) + if (!strncmp(i->ifgl_group->ifg_group, filter, IFNAMSIZ)) + return (0); /* iface is in group "filter" */ return (1); } int pfi_set_flags(const char *name, int flags) { + struct epoch_tracker et; struct pfi_kif *p; + NET_EPOCH_ENTER(et); RB_FOREACH(p, pfi_ifhead, &V_pfi_ifs) { if (pfi_skip_if(name, p)) continue; p->pfik_flags |= flags; } + NET_EPOCH_EXIT(et); return (0); } int pfi_clear_flags(const char *name, int flags) { + struct epoch_tracker et; struct pfi_kif *p; + NET_EPOCH_ENTER(et); RB_FOREACH(p, pfi_ifhead, &V_pfi_ifs) { if (pfi_skip_if(name, p)) continue; p->pfik_flags &= ~flags; } + NET_EPOCH_EXIT(et); return (0); } @@ -829,22 +838,28 @@ pfi_unmask(void *addr) static void pfi_attach_ifnet_event(void *arg __unused, struct ifnet *ifp) { + struct epoch_tracker et; + struct pfi_kif *kif; if (V_pf_vnet_active == 0) { /* Avoid teardown race in the least expensive way. */ return; } - pfi_attach_ifnet(ifp); -#ifdef ALTQ + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); + pfi_attach_ifnet(ifp, kif); +#ifdef ALTQ pf_altq_ifnet_event(ifp, 0); - PF_RULES_WUNLOCK(); #endif + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_detach_ifnet_event(void *arg __unused, struct ifnet *ifp) { + struct epoch_tracker et; struct pfi_kif *kif = (struct pfi_kif *)ifp->if_pf_kif; if (pfsync_detach_ifnet_ptr) @@ -858,6 +873,7 @@ pfi_detach_ifnet_event(void *arg __unused, struct ifnet *ifp) return; } + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); V_pfi_update++; pfi_kif_update(kif); @@ -871,22 +887,31 @@ pfi_detach_ifnet_event(void *arg __unused, struct ifnet *ifp) pf_altq_ifnet_event(ifp, 1); #endif PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_attach_group_event(void *arg __unused, struct ifg_group *ifg) { + struct epoch_tracker et; + struct pfi_kif *kif; if (V_pf_vnet_active == 0) { /* Avoid teardown race in the least expensive way. */ return; } - pfi_attach_ifgroup(ifg); + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); + PF_RULES_WLOCK(); + pfi_attach_ifgroup(ifg, kif); + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_change_group_event(void *arg __unused, char *gname) { + struct epoch_tracker et; struct pfi_kif *kif; if (V_pf_vnet_active == 0) { @@ -895,11 +920,13 @@ pfi_change_group_event(void *arg __unused, char *gname) } kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, gname); pfi_kif_update(kif); PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void @@ -937,8 +964,12 @@ pfi_ifaddr_event(void *arg __unused, struct ifnet *ifp) } PF_RULES_WLOCK(); if (ifp->if_pf_kif) { + struct epoch_tracker et; + V_pfi_update++; + NET_EPOCH_ENTER(et); pfi_kif_update(ifp->if_pf_kif); + NET_EPOCH_EXIT(et); } PF_RULES_WUNLOCK(); } diff --git a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c index bb1efcc666a1c..e411000f84b3e 100644 --- a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -1591,6 +1591,7 @@ bad: void ipoib_demux(struct ifnet *ifp, struct mbuf *m, u_short proto) { + struct epoch_tracker et; int isr; #ifdef MAC @@ -1632,7 +1633,9 @@ ipoib_demux(struct ifnet *ifp, struct mbuf *m, u_short proto) default: goto discard; } + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); return; discard: -- cgit v1.2.3