9 files changed, 99 insertions, 77 deletions

diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index d964b585e9f6..1373131a8a69 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -1160,9 +1160,8 @@ icmp6_mtudisc_update(ip6cp, validated)
 			rt->rt_rmx.rmx_mtu = mtu;
 		}
 	}
-	if (rt) { /* XXX: need braces to avoid conflict with else in RTFREE. */
-		RTFREE(rt);
-	}
+	if (rt)
+		rtfree(rt);
 }
 
 /*
@@ -2298,7 +2297,7 @@ icmp6_redirect_input(m, off)
 			    "ICMP6 redirect rejected; no route "
 			    "with inet6 gateway found for redirect dst: %s\n",
 			    icmp6_redirect_diag(&src6, &reddst6, &redtgt6)));
-			RTFREE(rt);
+			RTFREE_LOCKED(rt);
 			goto bad;
 		}
 
@@ -2310,7 +2309,7 @@ icmp6_redirect_input(m, off)
 				"%s\n",
 				ip6_sprintf(gw6),
 				icmp6_redirect_diag(&src6, &reddst6, &redtgt6)));
-			RTFREE(rt);
+			RTFREE_LOCKED(rt);
 			goto bad;
 		}
 	} else {
@@ -2320,7 +2319,7 @@ icmp6_redirect_input(m, off)
 			icmp6_redirect_diag(&src6, &reddst6, &redtgt6)));
 		goto bad;
 	}
-	RTFREE(rt);
+	RTFREE_LOCKED(rt);
 	rt = NULL;
     }
 	if (IN6_IS_ADDR_MULTICAST(&reddst6)) {
@@ -2395,8 +2394,7 @@ icmp6_redirect_input(m, off)
 		bcopy(&src6, &ssrc.sin6_addr, sizeof(struct in6_addr));
 		rtredirect((struct sockaddr *)&sdst, (struct sockaddr *)&sgw,
 			   (struct sockaddr *)NULL, RTF_GATEWAY | RTF_HOST,
-			   (struct sockaddr *)&ssrc,
-			   (struct rtentry **)NULL);
+			   (struct sockaddr *)&ssrc);
 	}
 	/* finally update cached route in each socket via pfctlinput */
     {
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index 945ab65379ab..838df1ea8583 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -175,32 +175,35 @@ in6_ifloop_request(int cmd, struct ifaddr *ifa)
 		    e);
 	}
 
-	/*
-	 * Make sure rt_ifa be equal to IFA, the second argument of the
-	 * function.
-	 * We need this because when we refer to rt_ifa->ia6_flags in
-	 * ip6_input, we assume that the rt_ifa points to the address instead
-	 * of the loopback address.
-	 */
-	if (cmd == RTM_ADD && nrt && ifa != nrt->rt_ifa) {
-		IFAFREE(nrt->rt_ifa);
-		IFAREF(ifa);
-		nrt->rt_ifa = ifa;
-	}
-
-	/*
-	 * Report the addition/removal of the address to the routing socket.
-	 * XXX: since we called rtinit for a p2p interface with a destination,
-	 *      we end up reporting twice in such a case.  Should we rather
-	 *      omit the second report?
-	 */
 	if (nrt) {
+		RT_LOCK(nrt);
+		/*
+		 * Make sure rt_ifa be equal to IFA, the second argument of
+		 * the function.  We need this because when we refer to
+		 * rt_ifa->ia6_flags in ip6_input, we assume that the rt_ifa
+		 * points to the address instead of the loopback address.
+		 */
+		if (cmd == RTM_ADD && ifa != nrt->rt_ifa) {
+			IFAFREE(nrt->rt_ifa);
+			IFAREF(ifa);
+			nrt->rt_ifa = ifa;
+		}
+
+		/*
+		 * Report the addition/removal of the address to the routing
+		 * socket.
+		 *
+		 * XXX: since we called rtinit for a p2p interface with a
+		 *      destination, we end up reporting twice in such a case.
+		 *      Should we rather omit the second report?
+		 */
 		rt_newaddrmsg(cmd, ifa, e, nrt);
 		if (cmd == RTM_DELETE) {
-			RTFREE(nrt);
+			rtfree(nrt);
 		} else {
 			/* the cmd must be RTM_ADD here */
 			nrt->rt_refcnt--;
+			RT_UNLOCK(nrt);
 		}
 	}
 }
@@ -223,7 +226,7 @@ in6_ifaddloop(struct ifaddr *ifa)
 	    (rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0)
 		in6_ifloop_request(RTM_ADD, ifa);
 	if (rt)
-		rt->rt_refcnt--;
+		rtfree(rt);
 }
 
 /*
@@ -271,10 +274,13 @@ in6_ifremloop(struct ifaddr *ifa)
 		 * to a shared medium.
 		 */
 		rt = rtalloc1(ifa->ifa_addr, 0, 0);
-		if (rt != NULL && (rt->rt_flags & RTF_HOST) != 0 &&
-		    (rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
-			rt->rt_refcnt--;
-			in6_ifloop_request(RTM_DELETE, ifa);
+		if (rt != NULL) {
+			if ((rt->rt_flags & RTF_HOST) != 0 &&
+			    (rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
+				rtfree(rt);
+				in6_ifloop_request(RTM_DELETE, ifa);
+			} else
+				RT_UNLOCK(rt);
 		}
 	}
 }
diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c
index 88e29c830101..24df8fe02d92 100644
--- a/sys/netinet6/in6_ifattach.c
+++ b/sys/netinet6/in6_ifattach.c
@@ -988,10 +988,14 @@ in6_ifdetach(ifp)
 	sin6.sin6_addr = in6addr_linklocal_allnodes;
 	sin6.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
 	rt = rtalloc1((struct sockaddr *)&sin6, 0, 0UL);
-	if (rt && rt->rt_ifp == ifp) {
-		rtrequest(RTM_DELETE, (struct sockaddr *)rt_key(rt),
-			rt->rt_gateway, rt_mask(rt), rt->rt_flags, 0);
-		rtfree(rt);
+	if (rt) {
+		if (rt->rt_ifp == ifp) {
+			RT_UNLOCK(rt);
+			rtrequest(RTM_DELETE, (struct sockaddr *)rt_key(rt),
+				rt->rt_gateway, rt_mask(rt), rt->rt_flags, 0);
+			RTFREE(rt);
+		} else
+			rtfree(rt);
 	}
 }
 
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index f160661c258b..36130ff40b98 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -569,6 +569,7 @@ in6_selectsrc(dstsock, opts, mopts, ro, laddr, errorp)
 			if (IN6_IS_ADDR_MULTICAST(dst)) {
 				ro->ro_rt = rtalloc1(&((struct route *)ro)
 						     ->ro_dst, 0, 0UL);
+				RT_UNLOCK(ro->ro_rt);
 			} else {
 				rtalloc((struct route *)ro);
 			}
@@ -653,7 +654,7 @@ in6_pcbdetach(inp)
  	ip6_freepcbopts(inp->in6p_outputopts);
  	ip6_freemoptions(inp->in6p_moptions);
 	if (inp->in6p_route.ro_rt)
-		rtfree(inp->in6p_route.ro_rt);
+		RTFREE(inp->in6p_route.ro_rt);
 	/* Check and free IPv4 related resources in case of mapped addr */
 	if (inp->inp_options)
 		(void)m_free(inp->inp_options);
@@ -1038,16 +1039,19 @@ in6_losing(in6p)
 	struct rt_addrinfo info;
 
 	if ((rt = in6p->in6p_route.ro_rt) != NULL) {
+		RT_LOCK(rt);
+		in6p->in6p_route.ro_rt = NULL;
 		bzero((caddr_t)&info, sizeof(info));
 		info.rti_flags = rt->rt_flags;
 		info.rti_info[RTAX_DST] = rt_key(rt);
 		info.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
 		info.rti_info[RTAX_NETMASK] = rt_mask(rt);
 		rt_missmsg(RTM_LOSING, &info, rt->rt_flags, 0);
-		if (rt->rt_flags & RTF_DYNAMIC)
+		if (rt->rt_flags & RTF_DYNAMIC) {
+			RT_UNLOCK(rt);		/* XXX refcnt? */
 			(void)rtrequest1(RTM_DELETE, &info, NULL);
-		in6p->in6p_route.ro_rt = NULL;
-		rtfree(rt);
+		} else
+			rtfree(rt);
 		/*
 		 * A new route can be allocated
 		 * the next time output is attempted.
@@ -1065,7 +1069,7 @@ in6_rtchange(inp, errno)
 	int errno;
 {
 	if (inp->in6p_route.ro_rt) {
-		rtfree(inp->in6p_route.ro_rt);
+		RTFREE(inp->in6p_route.ro_rt);
 		inp->in6p_route.ro_rt = 0;
 		/*
 		 * A new route can be allocated the next time
diff --git a/sys/netinet6/in6_rmx.c b/sys/netinet6/in6_rmx.c
index 14587bd59bbf..e9eba3cfb061 100644
--- a/sys/netinet6/in6_rmx.c
+++ b/sys/netinet6/in6_rmx.c
@@ -82,6 +82,7 @@
 #include <sys/socketvar.h>
 #include <sys/mbuf.h>
 #include <sys/syslog.h>
+#include <sys/callout.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -165,14 +166,17 @@ in6_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
 				rt2->rt_flags & RTF_HOST &&
 				rt2->rt_gateway &&
 				rt2->rt_gateway->sa_family == AF_LINK) {
+				/* NB: must unlock to avoid recursion */
+				RT_UNLOCK(rt2);
 				rtrequest(RTM_DELETE,
 					  (struct sockaddr *)rt_key(rt2),
 					  rt2->rt_gateway,
 					  rt_mask(rt2), rt2->rt_flags, 0);
 				ret = rn_addroute(v_arg, n_arg, head,
 					treenodes);
+				RT_LOCK(rt2);
 			}
-			RTFREE(rt2);
+			RTFREE_LOCKED(rt2);
 		}
 	} else if (ret == NULL && rt->rt_flags & RTF_CLONING) {
 		struct rtentry *rt2;
@@ -198,7 +202,7 @@ in6_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
 			 && rt2->rt_ifp == rt->rt_ifp) {
 				ret = rt2->rt_nodes;
 			}
-			RTFREE(rt2);
+			RTFREE_LOCKED(rt2);
 		}
 	}
 	return ret;
@@ -251,6 +255,8 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head)
 {
 	struct rtentry *rt = (struct rtentry *)rn;
 
+	RT_LOCK_ASSERT(rt);
+
 	if (!(rt->rt_flags & RTF_UP))
 		return;		/* prophylactic measures */
 
@@ -269,10 +275,13 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head)
 		rt->rt_flags |= RTPRF_OURS;
 		rt->rt_rmx.rmx_expire = time_second + rtq_reallyold;
 	} else {
+		/* NB: must unlock to avoid recursion */
+		RT_UNLOCK(rt);
 		rtrequest(RTM_DELETE,
 			  (struct sockaddr *)rt_key(rt),
 			  rt->rt_gateway, rt_mask(rt),
 			  rt->rt_flags, 0);
+		RT_LOCK(rt);
 	}
 }
 
@@ -331,6 +340,7 @@ in6_rtqkill(struct radix_node *rn, void *rock)
 
 #define RTQ_TIMEOUT	60*10	/* run no less than once every ten minutes */
 static int rtq_timeout = RTQ_TIMEOUT;
+static struct callout rtq_timer;
 
 static void
 in6_rtqtimo(void *rock)
@@ -339,17 +349,14 @@ in6_rtqtimo(void *rock)
 	struct rtqk_arg arg;
 	struct timeval atv;
 	static time_t last_adjusted_timeout = 0;
-	int s;
 
 	arg.found = arg.killed = 0;
 	arg.rnh = rnh;
 	arg.nextstop = time_second + rtq_timeout;
 	arg.draining = arg.updating = 0;
-	s = splnet();
 	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in6_rtqkill, &arg);
 	RADIX_NODE_HEAD_UNLOCK(rnh);
-	splx(s);
 
 	/*
 	 * Attempt to be somewhat dynamic about this:
@@ -374,16 +381,14 @@ in6_rtqtimo(void *rock)
 #endif
 		arg.found = arg.killed = 0;
 		arg.updating = 1;
-		s = splnet();
 		RADIX_NODE_HEAD_LOCK(rnh);
 		rnh->rnh_walktree(rnh, in6_rtqkill, &arg);
 		RADIX_NODE_HEAD_UNLOCK(rnh);
-		splx(s);
 	}
 
 	atv.tv_usec = 0;
 	atv.tv_sec = arg.nextstop;
-	timeout(in6_rtqtimo, rock, tvtohz(&atv));
+	callout_reset(&rtq_timer, tvtohz(&atv), in6_rtqtimo, rock);
 }
 
 /*
@@ -393,6 +398,7 @@ struct mtuex_arg {
 	struct radix_node_head *rnh;
 	time_t nextstop;
 };
+static struct callout rtq_mtutimer;
 
 static int
 in6_mtuexpire(struct radix_node *rn, void *rock)
@@ -424,15 +430,12 @@ in6_mtutimo(void *rock)
 	struct radix_node_head *rnh = rock;
 	struct mtuex_arg arg;
 	struct timeval atv;
-	int s;
 
 	arg.rnh = rnh;
 	arg.nextstop = time_second + MTUTIMO_DEFAULT;
-	s = splnet();
 	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in6_mtuexpire, &arg);
 	RADIX_NODE_HEAD_UNLOCK(rnh);
-	splx(s);
 
 	atv.tv_usec = 0;
 	atv.tv_sec = arg.nextstop;
@@ -440,7 +443,7 @@ in6_mtutimo(void *rock)
 		printf("invalid mtu expiration time on routing table\n");
 		arg.nextstop = time_second + 30;	/* last resort */
 	}
-	timeout(in6_mtutimo, rock, tvtohz(&atv));
+	callout_reset(&rtq_mtutimer, tvtohz(&atv), in6_mtutimo, rock);
 }
 
 #if 0
@@ -449,17 +452,15 @@ in6_rtqdrain()
 {
 	struct radix_node_head *rnh = rt_tables[AF_INET6];
 	struct rtqk_arg arg;
-	int s;
+
 	arg.found = arg.killed = 0;
 	arg.rnh = rnh;
 	arg.nextstop = 0;
 	arg.draining = 1;
 	arg.updating = 0;
-	s = splnet();
 	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in6_rtqkill, &arg);
 	RADIX_NODE_HEAD_UNLOCK(rnh);
-	splx(s);
 }
 #endif
 
@@ -481,7 +482,9 @@ in6_inithead(void **head, int off)
 	rnh->rnh_addaddr = in6_addroute;
 	rnh->rnh_matchaddr = in6_matroute;
 	rnh->rnh_close = in6_clsroute;
+	callout_init(&rtq_timer, CALLOUT_MPSAFE);
 	in6_rtqtimo(rnh);	/* kick off timeout first time */
+	callout_init(&rtq_mtutimer, CALLOUT_MPSAFE);
 	in6_mtutimo(rnh);	/* kick off timeout first time */
 	return 1;
 }
diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c
index 3dd2212805ca..84e73d673705 100644
--- a/sys/netinet6/in6_src.c
+++ b/sys/netinet6/in6_src.c
@@ -256,6 +256,7 @@ in6_selectsrc(dstsock, opts, mopts, ro, laddr, errorp)
 			if (IN6_IS_ADDR_MULTICAST(dst)) {
 				ro->ro_rt = rtalloc1(&((struct route *)ro)
 						     ->ro_dst, 0, 0UL);
+				RT_UNLOCK(ro->ro_rt);
 			} else {
 				rtalloc((struct route *)ro);
 			}
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 14e54fda2fc7..ff7cf8bd17a2 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -707,6 +707,7 @@ skip_ipsec2:;
 			ia = ifatoia6(ro->ro_rt->rt_ifa);
 			ifp = ro->ro_rt->rt_ifp;
 			ro->ro_rt->rt_use++;
+			RT_UNLOCK(ro->ro_rt);
 		}
 
 		if ((flags & IPV6_FORWARDING) == 0)
@@ -2080,7 +2081,7 @@ ip6_setmoptions(optname, im6op, m)
 					break;
 				}
 				ifp = ro.ro_rt->rt_ifp;
-				rtfree(ro.ro_rt);
+				RTFREE(ro.ro_rt);
 			}
 		} else
 			ifp = ifnet_byindex(mreq->ipv6mr_interface);
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 8fc4abf01765..7ef59c9b53c1 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -811,17 +811,18 @@ nd6_lookup(addr6, create, ifp)
 	sin6.sin6_scope_id = in6_addr2scopeid(ifp, addr6);
 #endif
 	rt = rtalloc1((struct sockaddr *)&sin6, create, 0UL);
-	if (rt && (rt->rt_flags & RTF_LLINFO) == 0) {
-		/*
-		 * This is the case for the default route.
-		 * If we want to create a neighbor cache for the address, we
-		 * should free the route for the destination and allocate an
-		 * interface route.
-		 */
-		if (create) {
-			RTFREE(rt);
+	if (rt) {
+		if ((rt->rt_flags & RTF_LLINFO) == 0 && create) {
+			/*
+			 * This is the case for the default route.
+			 * If we want to create a neighbor cache for the
+			 * address, we should free the route for the
+			 * destination and allocate an interface route.
+			 */
+			RTFREE_LOCKED(rt);
 			rt = 0;
 		}
+		RT_UNLOCK(rt);
 	}
 	if (!rt) {
 		if (create && ifp) {
@@ -1103,6 +1104,8 @@ nd6_rtrequest(req, rt, info)
 	struct ifnet *ifp = rt->rt_ifp;
 	struct ifaddr *ifa;
 
+	RT_LOCK_ASSERT(rt);
+
 	if ((rt->rt_flags & RTF_GATEWAY))
 		return;
 
@@ -1889,10 +1892,10 @@ nd6_output(ifp, origifp, m0, dst, rt0)
 	 */
 	if (rt) {
 		if ((rt->rt_flags & RTF_UP) == 0) {
-			if ((rt0 = rt = rtalloc1((struct sockaddr *)dst, 1, 0UL)) !=
-				NULL)
-			{
+			rt0 = rt = rtalloc1((struct sockaddr *)dst, 1, 0UL);
+			if (rt != NULL) {
 				rt->rt_refcnt--;
+				RT_UNLOCK(rt);
 				if (rt->rt_ifp != ifp) {
 					/* XXX: loop care? */
 					return nd6_output(ifp, origifp, m0,
@@ -1933,6 +1936,7 @@ nd6_output(ifp, origifp, m0, dst, rt0)
 			lookup: rt->rt_gwroute = rtalloc1(rt->rt_gateway, 1, 0UL);
 				if ((rt = rt->rt_gwroute) == 0)
 					senderr(EHOSTUNREACH);
+				RT_UNLOCK(rt);
 			}
 		}
 	}
diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c
index 7714969d4f95..845c2fd5b7f5 100644
--- a/sys/netinet6/nd6_rtr.c
+++ b/sys/netinet6/nd6_rtr.c
@@ -472,7 +472,6 @@ defrouter_addreq(new)
 {
 	struct sockaddr_in6 def, mask, gate;
 	struct rtentry *newrt = NULL;
-	int s;
 
 	Bzero(&def, sizeof(def));
 	Bzero(&mask, sizeof(mask));
@@ -483,15 +482,15 @@ defrouter_addreq(new)
 	def.sin6_family = mask.sin6_family = gate.sin6_family = AF_INET6;
 	gate.sin6_addr = new->rtaddr;
 
-	s = splnet();
 	(void)rtrequest(RTM_ADD, (struct sockaddr *)&def,
 		(struct sockaddr *)&gate, (struct sockaddr *)&mask,
 		RTF_GATEWAY, &newrt);
 	if (newrt) {
+		RT_LOCK(newrt);
 		nd6_rtmsg(RTM_ADD, newrt); /* tell user process */
 		newrt->rt_refcnt--;
+		RT_UNLOCK(newrt);
 	}
-	splx(s);
 	return;
 }
 
@@ -531,13 +530,12 @@ defrouter_addifreq(ifp)
 		    "defrouter_addifreq: failed to install a route to "
 		    "interface %s (errno = %d)\n",
 		    if_name(ifp), error));
-
-		if (newrt)	/* maybe unnecessary, but do it for safety */
-			newrt->rt_refcnt--;
 	} else {
 		if (newrt) {
+			RT_LOCK(newrt);
 			nd6_rtmsg(RTM_ADD, newrt);
 			newrt->rt_refcnt--;
+			RT_UNLOCK(newrt);
 		}
 	}
 }
@@ -1500,8 +1498,11 @@ nd6_prefix_onlink(pr)
 		    ip6_sprintf(&mask6.sin6_addr), rtflags, error));
 	}
 
-	if (rt != NULL)
+	if (rt != NULL) {
+		RT_LOCK(rt);
 		rt->rt_refcnt--;
+		RT_UNLOCK(rt);
+	}
 
 	return(error);
 }