diff options
45 files changed, 1266 insertions, 743 deletions
diff --git a/sbin/ifconfig/ifconfig.c b/sbin/ifconfig/ifconfig.c index f866223d171e..915fd39be6a0 100644 --- a/sbin/ifconfig/ifconfig.c +++ b/sbin/ifconfig/ifconfig.c @@ -910,7 +910,7 @@ unsetifdescr(const char *val, int value, int s, const struct afswtch *afp) #define IFCAPBITS \ "\020\1RXCSUM\2TXCSUM\3NETCONS\4VLAN_MTU\5VLAN_HWTAGGING\6JUMBO_MTU\7POLLING" \ "\10VLAN_HWCSUM\11TSO4\12TSO6\13LRO\14WOL_UCAST\15WOL_MCAST\16WOL_MAGIC" \ -"\21VLAN_HWFILTER\23VLAN_HWTSO\24LINKSTATE\25NETMAP" \ +"\17TOE4\20TOE6\21VLAN_HWFILTER\23VLAN_HWTSO\24LINKSTATE\25NETMAP" \ "\26RXCSUM_IPV6\27TXCSUM_IPV6" /* @@ -1189,6 +1189,8 @@ static struct cmd basic_cmds[] = { DEF_CMD("-tso4", -IFCAP_TSO4, setifcap), DEF_CMD("tso", IFCAP_TSO, setifcap), DEF_CMD("-tso", -IFCAP_TSO, setifcap), + DEF_CMD("toe", IFCAP_TOE, setifcap), + DEF_CMD("-toe", -IFCAP_TOE, setifcap), DEF_CMD("lro", IFCAP_LRO, setifcap), DEF_CMD("-lro", -IFCAP_LRO, setifcap), DEF_CMD("wol", IFCAP_WOL, setifcap), diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index c3423736197d..55e6a47afc22 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -28,6 +28,7 @@ options SCHED_ULE # ULE scheduler options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols +options TCP_OFFLOAD # TCP offload options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support diff --git a/sys/conf/NOTES b/sys/conf/NOTES index eac921a8954b..c046f6914d78 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -545,6 +545,8 @@ options INET6 #IPv6 communications protocols options ROUTETABLES=2 # max 16. 1 is back compatible. +options TCP_OFFLOAD # TCP offload support. + # In order to enable IPSEC you MUST also add device crypto to # your kernel configuration options IPSEC #IP security (requires device crypto) diff --git a/sys/conf/files b/sys/conf/files index 4e6126db244e..d9038f87f02a 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -2886,7 +2886,7 @@ netinet/tcp_hostcache.c optional inet | inet6 netinet/tcp_input.c optional inet | inet6 netinet/tcp_lro.c optional inet | inet6 netinet/tcp_output.c optional inet | inet6 -netinet/tcp_offload.c optional inet | inet6 +netinet/tcp_offload.c optional tcp_offload inet | tcp_offload inet6 netinet/tcp_reass.c optional inet | inet6 netinet/tcp_sack.c optional inet | inet6 netinet/tcp_subr.c optional inet | inet6 diff --git a/sys/conf/options b/sys/conf/options index 87a8a2a5da7e..46a02047a5dc 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -431,7 +431,7 @@ RADIX_MPATH opt_mpath.h ROUTETABLES opt_route.h SLIP_IFF_OPTS opt_slip.h TCPDEBUG -TCP_OFFLOAD_DISABLE opt_inet.h #Disable code to dispatch tcp offloading +TCP_OFFLOAD opt_inet.h # Enable code to dispatch TCP offloading TCP_SIGNATURE opt_inet.h VLAN_ARRAY opt_vlan.h XBONEHACK diff --git a/sys/contrib/rdma/krping/krping.c b/sys/contrib/rdma/krping/krping.c index d787965cd84b..99d1924704dd 100644 --- a/sys/contrib/rdma/krping/krping.c +++ b/sys/contrib/rdma/krping/krping.c @@ -41,7 +41,6 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/socket.h> -#include <sys/module.h> #include <sys/endian.h> #include <sys/limits.h> #include <sys/proc.h> @@ -53,11 +52,13 @@ __FBSDID("$FreeBSD$"); #include <sys/queue.h> #include <sys/taskqueue.h> #include <sys/syslog.h> +#include <netinet/in.h> #include <vm/vm.h> #include <vm/pmap.h> -#include <contrib/rdma/rdma_cm.h> +#include <linux/types.h> +#include <rdma/rdma_cm.h> #include "getopt.h" #include "krping.h" @@ -83,6 +84,7 @@ static const struct krping_option krping_opts[] = { {"bw", OPT_NOPARAM, 'B'}, {"tx-depth", OPT_INT, 't'}, {"poll", OPT_NOPARAM, 'P'}, + {"memlimit", OPT_INT, 'm'}, {NULL, 0, 0} }; @@ -254,10 +256,14 @@ static void krping_cq_event_handler(struct ib_cq *cq, void *ctx) ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { if (wc.status) { - if (wc.status != IB_WC_WR_FLUSH_ERR) - log(LOG_ERR, "cq completion failed status %d\n", + if (wc.status == IB_WC_WR_FLUSH_ERR) { + DEBUG_LOG("cq flushed\n"); + continue; + } else { + log(LOG_CRIT, "cq completion failed status %d\n", wc.status); - goto error; + goto error; + } } switch (wc.opcode) { @@ -432,8 +438,17 @@ static int krping_setup_buffers(struct krping_cb *cb) } } - cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL, - PAGE_SIZE, 0); + /* RNIC adapters have a limit upto which it can register physical memory + * If DMA-MR memory mode is set then normally driver registers maximum + * supported memory. After that if contigmalloc allocates memory beyond the + * specified RNIC limit then Krping may not work. + */ + if (cb->use_dmamr && cb->memlimit) + cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, cb->memlimit, + PAGE_SIZE, 0); + else + cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL, + PAGE_SIZE, 0); if (!cb->rdma_buf) { log(LOG_ERR, "rdma_buf malloc failed\n"); @@ -458,8 +473,12 @@ static int krping_setup_buffers(struct krping_cb *cb) } if (!cb->server || cb->wlat || cb->rlat || cb->bw) { - cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, - 0, -1UL, PAGE_SIZE, 0); + if (cb->use_dmamr && cb->memlimit) + cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, + 0, cb->memlimit, PAGE_SIZE, 0); + else + cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, + 0, -1UL, PAGE_SIZE, 0); if (!cb->start_buf) { log(LOG_ERR, "start_buf malloc failed\n"); ret = ENOMEM; @@ -1636,6 +1655,8 @@ int krping_doit(char *cmd) cb->state = IDLE; cb->size = 64; cb->txdepth = RPING_SQ_DEPTH; + cb->use_dmamr = 1; + cb->memlimit = 0; mtx_init(&cb->lock, "krping mtx", NULL, MTX_DUPOK|MTX_DEF); while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, @@ -1713,6 +1734,15 @@ int krping_doit(char *cmd) case 'd': debug++; break; + case 'm': + cb->memlimit = optint; + if (cb->memlimit < 1) { + log(LOG_ERR, "Invalid memory limit %ju\n", + cb->memlimit); + ret = EINVAL; + } else + DEBUG_LOG(PFX "memory limit %d\n", (int)optint); + break; default: log(LOG_ERR, "unknown opt %s\n", optarg); ret = EINVAL; diff --git a/sys/contrib/rdma/krping/krping.h b/sys/contrib/rdma/krping/krping.h index 8578e7e7979b..d2348257cf7e 100644 --- a/sys/contrib/rdma/krping/krping.h +++ b/sys/contrib/rdma/krping/krping.h @@ -1,7 +1,7 @@ /* * $FreeBSD$ */ -#include <contrib/rdma/ib_verbs.h> +#include <rdma/ib_verbs.h> #include <netinet/in.h> /* @@ -92,6 +92,8 @@ struct krping_cb { int count; /* ping count */ int size; /* ping data size */ int validate; /* validate ping data */ + uint64_t memlimit; /* limit of the physical memory that + can be registered with dma_mr mode */ /* CM stuff */ struct rdma_cm_id *cm_id; /* connection on client side,*/ diff --git a/sys/contrib/rdma/krping/krping_dev.c b/sys/contrib/rdma/krping/krping_dev.c index 448f19717578..92d954d5e988 100644 --- a/sys/contrib/rdma/krping/krping_dev.c +++ b/sys/contrib/rdma/krping/krping_dev.c @@ -14,7 +14,6 @@ __FBSDID("$FreeBSD$"); #include <sys/types.h> -#include <sys/module.h> #include <sys/systm.h> /* uprintf */ #include <sys/errno.h> #include <sys/param.h> /* defines used in kernel.h */ @@ -51,6 +50,9 @@ typedef struct s_krping { /* vars */ static struct cdev *krping_dev; +#undef MODULE_VERSION +#include <sys/module.h> + static int krping_loader(struct module *m, int what, void *arg) { @@ -175,6 +177,4 @@ krping_write(struct cdev *dev, struct uio *uio, int ioflag) return(err); } -MODULE_DEPEND(krping, rdma_core, 1, 1, 1); -MODULE_DEPEND(krping, rdma_cma, 1, 1, 1); DEV_MODULE(krping,krping_loader,NULL); diff --git a/sys/contrib/rdma/rdma_addr.c b/sys/contrib/rdma/rdma_addr.c index b3f734963c0e..33ec5acb2553 100644 --- a/sys/contrib/rdma/rdma_addr.c +++ b/sys/contrib/rdma/rdma_addr.c @@ -117,7 +117,8 @@ int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev, const unsigned char *dst_dev_addr) { dev_addr->dev_type = RDMA_NODE_RNIC; - memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), MAX_ADDR_LEN); + memset(dev_addr->src_dev_addr, 0, MAX_ADDR_LEN); + memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), dev->if_addrlen); memcpy(dev_addr->broadcast, dev->if_broadcastaddr, MAX_ADDR_LEN); if (dst_dev_addr) memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN); @@ -207,7 +208,7 @@ static int addr_resolve_remote(struct sockaddr_in *src_in, goto put; } ret = arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL, - rt_key(iproute.ro_rt), dmac, &lle); + (struct sockaddr *)dst_in, dmac, &lle); if (ret) { goto put; } diff --git a/sys/contrib/rdma/rdma_cache.c b/sys/contrib/rdma/rdma_cache.c index dced8ebd0110..1e516a1b036f 100644 --- a/sys/contrib/rdma/rdma_cache.c +++ b/sys/contrib/rdma/rdma_cache.c @@ -132,7 +132,7 @@ int ib_find_cached_gid(struct ib_device *device, for (p = 0; p <= end_port(device) - start_port(device); ++p) { cache = device->cache.gid_cache[p]; for (i = 0; i < cache->table_len; ++i) { - if (!memcmp(gid, &cache->table[i], 6)) { /* XXX */ + if (!memcmp(gid, &cache->table[i], sizeof *gid)) { *port_num = p + start_port(device); if (index) *index = i; diff --git a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c index 16b5394a4be6..49ad0745cb2f 100644 --- a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c +++ b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c @@ -880,10 +880,10 @@ act_open_rpl_status_to_errno(int status) case CPL_ERR_CONN_TIMEDOUT: return (ETIMEDOUT); case CPL_ERR_TCAM_FULL: - return (ENOMEM); + return (EAGAIN); case CPL_ERR_CONN_EXIST: log(LOG_ERR, "ACTIVE_OPEN_RPL: 4-tuple in use\n"); - return (EADDRINUSE); + return (EAGAIN); default: return (EIO); } @@ -912,8 +912,7 @@ do_act_open_rpl(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) unsigned int atid = G_TID(ntohl(rpl->atid)); struct toepcb *toep = lookup_atid(&td->tid_maps, atid); struct inpcb *inp = toep->tp_inp; - struct tcpcb *tp = intotcpcb(inp); - int s = rpl->status; + int s = rpl->status, rc; CTR3(KTR_CXGB, "%s: atid %u, status %u ", __func__, atid, s); @@ -923,17 +922,14 @@ do_act_open_rpl(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) if (act_open_has_tid(s)) queue_tid_release(tod, GET_TID(rpl)); - if (s == CPL_ERR_TCAM_FULL || s == CPL_ERR_CONN_EXIST) { - INP_WLOCK(inp); - toe_connect_failed(tod, tp, EAGAIN); - toepcb_release(toep); /* unlocks inp */ - } else { + rc = act_open_rpl_status_to_errno(s); + if (rc != EAGAIN) INP_INFO_WLOCK(&V_tcbinfo); - INP_WLOCK(inp); - toe_connect_failed(tod, tp, act_open_rpl_status_to_errno(s)); - toepcb_release(toep); /* unlocks inp */ + INP_WLOCK(inp); + toe_connect_failed(tod, inp, rc); + toepcb_release(toep); /* unlocks inp */ + if (rc != EAGAIN) INP_INFO_WUNLOCK(&V_tcbinfo); - } m_freem(m); return (0); diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC index c4e75ef054ce..3fcfd714ed5b 100644 --- a/sys/i386/conf/GENERIC +++ b/sys/i386/conf/GENERIC @@ -30,6 +30,7 @@ options SCHED_ULE # ULE scheduler options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols +options TCP_OFFLOAD # TCP offload options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support diff --git a/sys/i386/conf/XEN b/sys/i386/conf/XEN index 8b4a759b0bdb..d9197cb45a5e 100644 --- a/sys/i386/conf/XEN +++ b/sys/i386/conf/XEN @@ -7,7 +7,7 @@ cpu I686_CPU ident XEN makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols -makeoptions WITHOUT_MODULES="aha ahb amd cxgb dpt drm drm2 hptmv ida malo mps mwl nve sound sym trm xfs" +makeoptions WITHOUT_MODULES="aha ahb amd cxgb dpt drm drm2 hptmv ida malo mps mwl nve rdma sound sym trm xfs" options SCHED_ULE # ULE scheduler options PREEMPTION # Enable kernel thread preemption diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 4e83aa2b9225..eb26938e419f 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -313,6 +313,7 @@ SUBDIR= \ ${_ti} \ tl \ tmpfs \ + ${_toecore} \ ${_tpm} \ trm \ ${_twa} \ @@ -395,6 +396,7 @@ _if_gre= if_gre .if (${MK_INET_SUPPORT} != "no" || ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _if_carp= if_carp +_toecore= toecore .endif .if ${MK_IPFILTER} != "no" || defined(ALL_MODULES) diff --git a/sys/modules/rdma/krping/Makefile b/sys/modules/rdma/krping/Makefile index 5e4871b865f1..ae6cc808e693 100644 --- a/sys/modules/rdma/krping/Makefile +++ b/sys/modules/rdma/krping/Makefile @@ -6,5 +6,7 @@ RDMA= ${.CURDIR}/../../../contrib/rdma/krping KMOD= krping SRCS= krping.c krping_dev.c getopt.c SRCS+= bus_if.h device_if.h opt_sched.h pci_if.h pcib_if.h +SRCS+= vnode_if.h +CFLAGS+= -I${.CURDIR}/../../../ofed/include .include <bsd.kmod.mk> diff --git a/sys/modules/toecore/Makefile b/sys/modules/toecore/Makefile new file mode 100644 index 000000000000..0b30ab097521 --- /dev/null +++ b/sys/modules/toecore/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ + +.PATH: ${.CURDIR}/../../netinet + +KMOD= toecore +SRCS= toecore.c +SRCS+= opt_ofed.h + +.include <bsd.kmod.mk> diff --git a/sys/net/if_llatbl.h b/sys/net/if_llatbl.h index 8da08ba6c4f6..39ecf7b1fa3f 100644 --- a/sys/net/if_llatbl.h +++ b/sys/net/if_llatbl.h @@ -205,4 +205,14 @@ lla_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) } int lla_rt_output(struct rt_msghdr *, struct rt_addrinfo *); + +#include <sys/eventhandler.h> +enum { + LLENTRY_RESOLVED, + LLENTRY_TIMEDOUT, + LLENTRY_DELETED, + LLENTRY_EXPIRED, +}; +typedef void (*lle_event_fn)(void *, struct llentry *, int); +EVENTHANDLER_DECLARE(lle_event, lle_event_fn); #endif /* _NET_IF_LLATBL_H_ */ diff --git a/sys/net/if_var.h b/sys/net/if_var.h index 05f872e9eb7a..2ef282282a5a 100644 --- a/sys/net/if_var.h +++ b/sys/net/if_var.h @@ -755,6 +755,8 @@ drbr_inuse(struct ifnet *ifp, struct buf_ring *br) #define IF_MINMTU 72 #define IF_MAXMTU 65535 +#define TOEDEV(ifp) ((ifp)->if_llsoftc) + #endif /* _KERNEL */ /* diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c index 24f5efe71db2..b4c8583748f8 100644 --- a/sys/net/if_vlan.c +++ b/sys/net/if_vlan.c @@ -748,8 +748,8 @@ vlan_modevent(module_t mod, int type, void *data) vlan_trunk_cap_p = NULL; vlan_trunkdev_p = NULL; vlan_tag_p = NULL; - vlan_cookie_p = vlan_cookie; - vlan_setcookie_p = vlan_setcookie; + vlan_cookie_p = NULL; + vlan_setcookie_p = NULL; vlan_devat_p = NULL; VLAN_LOCK_DESTROY(); if (bootverbose) @@ -1510,6 +1510,22 @@ vlan_capabilities(struct ifvlan *ifv) ifp->if_capenable &= ~(p->if_capenable & IFCAP_TSO); ifp->if_hwassist &= ~(p->if_hwassist & CSUM_TSO); } + + /* + * If the parent interface can offload TCP connections over VLANs then + * propagate its TOE capability to the VLAN interface. + * + * All TOE drivers in the tree today can deal with VLANs. If this + * changes then IFCAP_VLAN_TOE should be promoted to a full capability + * with its own bit. + */ +#define IFCAP_VLAN_TOE IFCAP_TOE + if (p->if_capabilities & IFCAP_VLAN_TOE) + ifp->if_capabilities |= p->if_capabilities & IFCAP_TOE; + if (p->if_capenable & IFCAP_VLAN_TOE) { + TOEDEV(ifp) = TOEDEV(p); + ifp->if_capenable |= p->if_capenable & IFCAP_TOE; + } } static void diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c index 13b46bf8c5df..474c09788b08 100644 --- a/sys/netinet/if_ether.c +++ b/sys/netinet/if_ether.c @@ -178,6 +178,16 @@ arptimer(void *arg) ifp = lle->lle_tbl->llt_ifp; CURVNET_SET(ifp->if_vnet); + if ((lle->la_flags & LLE_DELETED) == 0) { + int evt; + + if (lle->la_flags & LLE_VALID) + evt = LLENTRY_EXPIRED; + else + evt = LLENTRY_TIMEDOUT; + EVENTHANDLER_INVOKE(lle_event, lle, evt); + } + callout_stop(&lle->la_timer); /* XXX: LOR avoidance. We still have ref on lle. */ @@ -730,7 +740,7 @@ match: (void)memcpy(&la->ll_addr, ar_sha(ah), ifp->if_addrlen); la->la_flags |= LLE_VALID; - EVENTHANDLER_INVOKE(arp_update_event, la); + EVENTHANDLER_INVOKE(lle_event, la, LLENTRY_RESOLVED); if (!(la->la_flags & LLE_STATIC)) { int canceled; diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h index 1b69436df3ad..ce63d8dbf4de 100644 --- a/sys/netinet/if_ether.h +++ b/sys/netinet/if_ether.h @@ -117,11 +117,6 @@ int arpresolve(struct ifnet *ifp, struct rtentry *rt, struct llentry **lle); void arp_ifinit(struct ifnet *, struct ifaddr *); void arp_ifinit2(struct ifnet *, struct ifaddr *, u_char *); - -#include <sys/eventhandler.h> -typedef void (*llevent_arp_update_fn)(void *, struct llentry *); -EVENTHANDLER_DECLARE(arp_update_event, llevent_arp_update_fn); - #endif #endif diff --git a/sys/netinet/in.c b/sys/netinet/in.c index 04cdbf59f627..f1200383fdba 100644 --- a/sys/netinet/in.c +++ b/sys/netinet/in.c @@ -1527,7 +1527,7 @@ in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3add if (!(lle->la_flags & LLE_IFADDR) || (flags & LLE_IFADDR)) { LLE_WLOCK(lle); lle->la_flags |= LLE_DELETED; - EVENTHANDLER_INVOKE(arp_update_event, lle); + EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED); #ifdef DIAGNOSTIC log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle); #endif diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c index 519ff468d55b..405cdfd4f08c 100644 --- a/sys/netinet/tcp_input.c +++ b/sys/netinet/tcp_input.c @@ -105,6 +105,9 @@ __FBSDID("$FreeBSD$"); #ifdef TCPDEBUG #include <netinet/tcp_debug.h> #endif /* TCPDEBUG */ +#ifdef TCP_OFFLOAD +#include <netinet/tcp_offload.h> +#endif #ifdef IPSEC #include <netipsec/ipsec.h> @@ -995,6 +998,14 @@ relocked: goto dropwithreset; } +#ifdef TCP_OFFLOAD + if (tp->t_flags & TF_TOE) { + tcp_offload_input(tp, m); + m = NULL; /* consumed by the TOE driver */ + goto dropunlock; + } +#endif + /* * We've identified a valid inpcb, but it could be that we need an * inpcbinfo write lock but don't hold it. In this case, attempt to diff --git a/sys/netinet/tcp_offload.c b/sys/netinet/tcp_offload.c index 899a37c7fa34..b77241818206 100644 --- a/sys/netinet/tcp_offload.c +++ b/sys/netinet/tcp_offload.c @@ -1,145 +1,176 @@ /*- - * Copyright (c) 2007, Chelsio Inc. + * Copyright (c) 2012 Chelsio Communications, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Neither the name of the Chelsio Corporation nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. */ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); +#include "opt_inet.h" + #include <sys/param.h> #include <sys/systm.h> #include <sys/types.h> -#include <sys/malloc.h> -#include <sys/kernel.h> -#include <sys/sysctl.h> #include <sys/mbuf.h> #include <sys/socket.h> #include <sys/socketvar.h> - +#include <sys/sockopt.h> #include <net/if.h> -#include <net/if_types.h> -#include <net/if_var.h> #include <net/route.h> -#include <net/vnet.h> - #include <netinet/in.h> -#include <netinet/in_systm.h> #include <netinet/in_pcb.h> #include <netinet/tcp.h> #include <netinet/tcp_var.h> #include <netinet/tcp_offload.h> -#include <netinet/toedev.h> +#define TCPOUTFLAGS +#include <netinet/tcp_fsm.h> +#include <netinet/toecore.h> -uint32_t toedev_registration_count; +int registered_toedevs; +/* + * Provide an opportunity for a TOE driver to offload. + */ int tcp_offload_connect(struct socket *so, struct sockaddr *nam) { struct ifnet *ifp; - struct toedev *tdev; + struct toedev *tod; struct rtentry *rt; - int error; + int error = EOPNOTSUPP; + + INP_WLOCK_ASSERT(sotoinpcb(so)); + KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6, + ("%s: called with sa_family %d", __func__, nam->sa_family)); - if (toedev_registration_count == 0) - return (EINVAL); - - /* - * Look up the route used for the connection to - * determine if it uses an interface capable of - * offloading the connection. - */ - rt = rtalloc1(nam, 0 /*report*/, 0 /*ignflags*/); - if (rt) + if (registered_toedevs == 0) + return (error); + + rt = rtalloc1(nam, 0, 0); + if (rt) RT_UNLOCK(rt); - else + else return (EHOSTUNREACH); ifp = rt->rt_ifp; - if ((ifp->if_capenable & IFCAP_TOE) == 0) { - error = EINVAL; - goto fail; - } - - tdev = TOEDEV(ifp); - if (tdev == NULL) { - error = EPERM; - goto fail; - } - - if (tdev->tod_can_offload(tdev, so) == 0) { - error = EPERM; - goto fail; - } - - return (tdev->tod_connect(tdev, so, rt, nam)); -fail: + + if (nam->sa_family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) + goto done; + if (nam->sa_family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6)) + goto done; + + tod = TOEDEV(ifp); + if (tod != NULL) + error = tod->tod_connect(tod, so, rt, nam); +done: RTFREE(rt); return (error); } +void +tcp_offload_listen_start(struct tcpcb *tp) +{ -/* - * This file contains code as a short-term staging area before it is moved in - * to sys/netinet/tcp_offload.c - */ + INP_WLOCK_ASSERT(tp->t_inpcb); + + EVENTHANDLER_INVOKE(tcp_offload_listen_start, tp); +} + +void +tcp_offload_listen_stop(struct tcpcb *tp) +{ + + INP_WLOCK_ASSERT(tp->t_inpcb); + + EVENTHANDLER_INVOKE(tcp_offload_listen_stop, tp); +} void -tcp_offload_twstart(struct tcpcb *tp) +tcp_offload_input(struct tcpcb *tp, struct mbuf *m) +{ + struct toedev *tod = tp->tod; + + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); + + tod->tod_input(tod, tp, m); +} + +int +tcp_offload_output(struct tcpcb *tp) { + struct toedev *tod = tp->tod; + int error, flags; + + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); - INP_INFO_WLOCK(&V_tcbinfo); - INP_WLOCK(tp->t_inpcb); - tcp_twstart(tp); - INP_INFO_WUNLOCK(&V_tcbinfo); + flags = tcp_outflags[tp->t_state]; + + if (flags & TH_RST) { + /* XXX: avoid repeated calls like we do for FIN */ + error = tod->tod_send_rst(tod, tp); + } else if ((flags & TH_FIN || tp->t_flags & TF_NEEDFIN) && + (tp->t_flags & TF_SENTFIN) == 0) { + error = tod->tod_send_fin(tod, tp); + if (error == 0) + tp->t_flags |= TF_SENTFIN; + } else + error = tod->tod_output(tod, tp); + + return (error); } -struct tcpcb * -tcp_offload_close(struct tcpcb *tp) +void +tcp_offload_rcvd(struct tcpcb *tp) { + struct toedev *tod = tp->tod; - INP_INFO_WLOCK(&V_tcbinfo); - INP_WLOCK(tp->t_inpcb); - tp = tcp_close(tp); - INP_INFO_WUNLOCK(&V_tcbinfo); - if (tp) - INP_WUNLOCK(tp->t_inpcb); + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); - return (tp); + tod->tod_rcvd(tod, tp); } -struct tcpcb * -tcp_offload_drop(struct tcpcb *tp, int error) +void +tcp_offload_ctloutput(struct tcpcb *tp, int sopt_dir, int sopt_name) { + struct toedev *tod = tp->tod; - INP_INFO_WLOCK(&V_tcbinfo); - INP_WLOCK(tp->t_inpcb); - tp = tcp_drop(tp, error); - INP_INFO_WUNLOCK(&V_tcbinfo); - if (tp) - INP_WUNLOCK(tp->t_inpcb); + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); - return (tp); + tod->tod_ctloutput(tod, tp, sopt_dir, sopt_name); } +void +tcp_offload_detach(struct tcpcb *tp) +{ + struct toedev *tod = tp->tod; + + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); + + tod->tod_pcb_detach(tod, tp); +} diff --git a/sys/netinet/tcp_offload.h b/sys/netinet/tcp_offload.h index 313185f6ea6e..a0523665848f 100644 --- a/sys/netinet/tcp_offload.h +++ b/sys/netinet/tcp_offload.h @@ -1,30 +1,30 @@ /*- - * Copyright (c) 2007, Chelsio Inc. + * Copyright (c) 2012 Chelsio Communications, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Neither the name of the Chelsio Corporation nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. * * $FreeBSD$ + * */ #ifndef _NETINET_TCP_OFFLOAD_H_ @@ -34,321 +34,15 @@ #error "no user-serviceable parts inside" #endif -/* - * A driver publishes that it provides offload services - * by setting IFCAP_TOE in the ifnet. The offload connect - * will bypass any further work if the interface that a - * connection would use does not support TCP offload. - * - * The TOE API assumes that the tcp offload engine can offload the - * the entire connection from set up to teardown, with some provision - * being made to allowing the software stack to handle time wait. If - * the device does not meet these criteria, it is the driver's responsibility - * to overload the functions that it needs to in tcp_usrreqs and make - * its own calls to tcp_output if it needs to do so. - * - * There is currently no provision for the device advertising the congestion - * control algorithms it supports as there is currently no API for querying - * an operating system for the protocols that it has loaded. This is a desirable - * future extension. - * - * - * - * It is assumed that individuals deploying TOE will want connections - * to be offloaded without software changes so all connections on an - * interface providing TOE are offloaded unless the SO_NO_OFFLOAD - * flag is set on the socket. - * - * - * The toe_usrreqs structure constitutes the TOE driver's - * interface to the TCP stack for functionality that doesn't - * interact directly with userspace. If one wants to provide - * (optional) functionality to do zero-copy to/from - * userspace one still needs to override soreceive/sosend - * with functions that fault in and pin the user buffers. - * - * + tu_send - * - tells the driver that new data may have been added to the - * socket's send buffer - the driver should not fail if the - * buffer is in fact unchanged - * - the driver is responsible for providing credits (bytes in the send window) - * back to the socket by calling sbdrop() as segments are acknowledged. - * - The driver expects the inpcb lock to be held - the driver is expected - * not to drop the lock. Hence the driver is not allowed to acquire the - * pcbinfo lock during this call. - * - * + tu_rcvd - * - returns credits to the driver and triggers window updates - * to the peer (a credit as used here is a byte in the peer's receive window) - * - the driver is expected to determine how many bytes have been - * consumed and credit that back to the card so that it can grow - * the window again by maintaining its own state between invocations. - * - In principle this could be used to shrink the window as well as - * grow the window, although it is not used for that now. - * - this function needs to correctly handle being called any number of - * times without any bytes being consumed from the receive buffer. - * - The driver expects the inpcb lock to be held - the driver is expected - * not to drop the lock. Hence the driver is not allowed to acquire the - * pcbinfo lock during this call. - * - * + tu_disconnect - * - tells the driver to send FIN to peer - * - driver is expected to send the remaining data and then do a clean half close - * - disconnect implies at least half-close so only send, reset, and detach - * are legal - * - the driver is expected to handle transition through the shutdown - * state machine and allow the stack to support SO_LINGER. - * - The driver expects the inpcb lock to be held - the driver is expected - * not to drop the lock. Hence the driver is not allowed to acquire the - * pcbinfo lock during this call. - * - * + tu_reset - * - closes the connection and sends a RST to peer - * - driver is expectd to trigger an RST and detach the toepcb - * - no further calls are legal after reset - * - The driver expects the inpcb lock to be held - the driver is expected - * not to drop the lock. Hence the driver is not allowed to acquire the - * pcbinfo lock during this call. - * - * The following fields in the tcpcb are expected to be referenced by the driver: - * + iss - * + rcv_nxt - * + rcv_wnd - * + snd_isn - * + snd_max - * + snd_nxt - * + snd_una - * + t_flags - * + t_inpcb - * + t_maxseg - * + t_toe - * - * The following fields in the inpcb are expected to be referenced by the driver: - * + inp_lport - * + inp_fport - * + inp_laddr - * + inp_fport - * + inp_socket - * + inp_ip_tos - * - * The following fields in the socket are expected to be referenced by the - * driver: - * + so_comp - * + so_error - * + so_linger - * + so_options - * + so_rcv - * + so_snd - * + so_state - * + so_timeo - * - * These functions all return 0 on success and can return the following errors - * as appropriate: - * + EPERM: - * + ENOBUFS: memory allocation failed - * + EMSGSIZE: MTU changed during the call - * + EHOSTDOWN: - * + EHOSTUNREACH: - * + ENETDOWN: - * * ENETUNREACH: the peer is no longer reachable - * - * + tu_detach - * - tells driver that the socket is going away so disconnect - * the toepcb and free appropriate resources - * - allows the driver to cleanly handle the case of connection state - * outliving the socket - * - no further calls are legal after detach - * - the driver is expected to provide its own synchronization between - * detach and receiving new data. - * - * + tu_syncache_event - * - even if it is not actually needed, the driver is expected to - * call syncache_add for the initial SYN and then syncache_expand - * for the SYN,ACK - * - tells driver that a connection either has not been added or has - * been dropped from the syncache - * - the driver is expected to maintain state that lives outside the - * software stack so the syncache needs to be able to notify the - * toe driver that the software stack is not going to create a connection - * for a received SYN - * - The driver is responsible for any synchronization required between - * the syncache dropping an entry and the driver processing the SYN,ACK. - * - */ -struct toe_usrreqs { - int (*tu_send)(struct tcpcb *tp); - int (*tu_rcvd)(struct tcpcb *tp); - int (*tu_disconnect)(struct tcpcb *tp); - int (*tu_reset)(struct tcpcb *tp); - void (*tu_detach)(struct tcpcb *tp); - void (*tu_syncache_event)(int event, void *toep); -}; - -/* - * Proxy for struct tcpopt between TOE drivers and TCP functions. - */ -struct toeopt { - u_int64_t to_flags; /* see tcpopt in tcp_var.h */ - u_int16_t to_mss; /* maximum segment size */ - u_int8_t to_wscale; /* window scaling */ - - u_int8_t _pad1; /* explicit pad for 64bit alignment */ - u_int32_t _pad2; /* explicit pad for 64bit alignment */ - u_int64_t _pad3[4]; /* TBD */ -}; - -#define TOE_SC_ENTRY_PRESENT 1 /* 4-tuple already present */ -#define TOE_SC_DROP 2 /* connection was timed out */ - -/* - * Because listen is a one-to-many relationship (a socket can be listening - * on all interfaces on a machine some of which may be using different TCP - * offload devices), listen uses a publish/subscribe mechanism. The TCP - * offload driver registers a listen notification function with the stack. - * When a listen socket is created all TCP offload devices are notified - * so that they can do the appropriate set up to offload connections on the - * port to which the socket is bound. When the listen socket is closed, - * the offload devices are notified so that they will stop listening on that - * port and free any associated resources as well as sending RSTs on any - * connections in the SYN_RCVD state. - * - */ - -typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *); -typedef void (*tcp_offload_listen_stop_fn)(void *, struct tcpcb *); - -EVENTHANDLER_DECLARE(tcp_offload_listen_start, tcp_offload_listen_start_fn); -EVENTHANDLER_DECLARE(tcp_offload_listen_stop, tcp_offload_listen_stop_fn); - -/* - * Check if the socket can be offloaded by the following steps: - * - determine the egress interface - * - check the interface for TOE capability and TOE is enabled - * - check if the device has resources to offload the connection - */ -int tcp_offload_connect(struct socket *so, struct sockaddr *nam); - -/* - * The tcp_output_* routines are wrappers around the toe_usrreqs calls - * which trigger packet transmission. In the non-offloaded case they - * translate to tcp_output. The tcp_offload_* routines notify TOE - * of specific events. I the non-offloaded case they are no-ops. - * - * Listen is a special case because it is a 1 to many relationship - * and there can be more than one offload driver in the system. - */ - -/* - * Connection is offloaded - */ -#define tp_offload(tp) ((tp)->t_flags & TF_TOE) - -/* - * hackish way of allowing this file to also be included by TOE - * which needs to be kept ignorant of socket implementation details - */ -#ifdef _SYS_SOCKETVAR_H_ -/* - * The socket has not been marked as "do not offload" - */ -#define SO_OFFLOADABLE(so) ((so->so_options & SO_NO_OFFLOAD) == 0) - -static __inline int -tcp_output_connect(struct socket *so, struct sockaddr *nam) -{ - struct tcpcb *tp = sototcpcb(so); - int error; - - /* - * If offload has been disabled for this socket or the - * connection cannot be offloaded just call tcp_output - * to start the TCP state machine. - */ -#ifndef TCP_OFFLOAD_DISABLE - if (!SO_OFFLOADABLE(so) || (error = tcp_offload_connect(so, nam)) != 0) -#endif - error = tcp_output(tp); - return (error); -} - -static __inline int -tcp_output_send(struct tcpcb *tp) -{ - -#ifndef TCP_OFFLOAD_DISABLE - if (tp_offload(tp)) - return (tp->t_tu->tu_send(tp)); -#endif - return (tcp_output(tp)); -} - -static __inline int -tcp_output_rcvd(struct tcpcb *tp) -{ - -#ifndef TCP_OFFLOAD_DISABLE - if (tp_offload(tp)) - return (tp->t_tu->tu_rcvd(tp)); -#endif - return (tcp_output(tp)); -} +extern int registered_toedevs; -static __inline int -tcp_output_disconnect(struct tcpcb *tp) -{ +int tcp_offload_connect(struct socket *, struct sockaddr *); +void tcp_offload_listen_start(struct tcpcb *); +void tcp_offload_listen_stop(struct tcpcb *); +void tcp_offload_input(struct tcpcb *, struct mbuf *); +int tcp_offload_output(struct tcpcb *); +void tcp_offload_rcvd(struct tcpcb *); +void tcp_offload_ctloutput(struct tcpcb *, int, int); +void tcp_offload_detach(struct tcpcb *); -#ifndef TCP_OFFLOAD_DISABLE - if (tp_offload(tp)) - return (tp->t_tu->tu_disconnect(tp)); #endif - return (tcp_output(tp)); -} - -static __inline int -tcp_output_reset(struct tcpcb *tp) -{ - -#ifndef TCP_OFFLOAD_DISABLE - if (tp_offload(tp)) - return (tp->t_tu->tu_reset(tp)); -#endif - return (tcp_output(tp)); -} - -static __inline void -tcp_offload_detach(struct tcpcb *tp) -{ - -#ifndef TCP_OFFLOAD_DISABLE - if (tp_offload(tp)) - tp->t_tu->tu_detach(tp); -#endif -} - -static __inline void -tcp_offload_listen_open(struct tcpcb *tp) -{ - -#ifndef TCP_OFFLOAD_DISABLE - if (SO_OFFLOADABLE(tp->t_inpcb->inp_socket)) - EVENTHANDLER_INVOKE(tcp_offload_listen_start, tp); -#endif -} - -static __inline void -tcp_offload_listen_close(struct tcpcb *tp) -{ - -#ifndef TCP_OFFLOAD_DISABLE - EVENTHANDLER_INVOKE(tcp_offload_listen_stop, tp); -#endif -} -#undef SO_OFFLOADABLE -#endif /* _SYS_SOCKETVAR_H_ */ -#undef tp_offload - -void tcp_offload_twstart(struct tcpcb *tp); -struct tcpcb *tcp_offload_close(struct tcpcb *tp); -struct tcpcb *tcp_offload_drop(struct tcpcb *tp, int error); - -#endif /* _NETINET_TCP_OFFLOAD_H_ */ diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c index c507b3fccb91..d81204bced31 100644 --- a/sys/netinet/tcp_output.c +++ b/sys/netinet/tcp_output.c @@ -75,6 +75,9 @@ __FBSDID("$FreeBSD$"); #ifdef TCPDEBUG #include <netinet/tcp_debug.h> #endif +#ifdef TCP_OFFLOAD +#include <netinet/tcp_offload.h> +#endif #ifdef IPSEC #include <netipsec/ipsec.h> @@ -196,6 +199,11 @@ tcp_output(struct tcpcb *tp) INP_WLOCK_ASSERT(tp->t_inpcb); +#ifdef TCP_OFFLOAD + if (tp->t_flags & TF_TOE) + return (tcp_offload_output(tp)); +#endif + /* * Determine length of data that should be transmitted, * and flags that will be used. diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index 1eebaa24cca3..f83ebd3ffeb1 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -85,7 +85,6 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> #include <netinet/tcp_syncache.h> -#include <netinet/tcp_offload.h> #ifdef INET6 #include <netinet6/tcp6_var.h> #endif @@ -96,6 +95,9 @@ __FBSDID("$FreeBSD$"); #ifdef INET6 #include <netinet6/ip6protosw.h> #endif +#ifdef TCP_OFFLOAD +#include <netinet/tcp_offload.h> +#endif #ifdef IPSEC #include <netipsec/ipsec.h> @@ -824,7 +826,7 @@ tcp_drop(struct tcpcb *tp, int errno) if (TCPS_HAVERCVDSYN(tp->t_state)) { tp->t_state = TCPS_CLOSED; - (void) tcp_output_reset(tp); + (void) tcp_output(tp); TCPSTAT_INC(tcps_drops); } else TCPSTAT_INC(tcps_conndrops); @@ -924,8 +926,12 @@ tcp_discardcb(struct tcpcb *tp) /* free the reassembly queue, if any */ tcp_reass_flush(tp); + +#ifdef TCP_OFFLOAD /* Disconnect offload device, if any. */ - tcp_offload_detach(tp); + if (tp->t_flags & TF_TOE) + tcp_offload_detach(tp); +#endif tcp_free_sackholes(tp); @@ -954,9 +960,10 @@ tcp_close(struct tcpcb *tp) INP_INFO_WLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(inp); - /* Notify any offload devices of listener close */ +#ifdef TCP_OFFLOAD if (tp->t_state == TCPS_LISTEN) - tcp_offload_listen_close(tp); + tcp_offload_listen_stop(tp); +#endif in_pcbdrop(inp); TCPSTAT_INC(tcps_closed); KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL")); @@ -1695,7 +1702,7 @@ tcp_mtudisc(struct inpcb *inp, int mtuoffer) tp->snd_recover = tp->snd_max; if (tp->t_flags & TF_SACK_PERMIT) EXIT_FASTRECOVERY(tp->t_flags); - tcp_output_send(tp); + tcp_output(tp); return (inp); } diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c index 21a72f4fea89..cfae7fbee2bb 100644 --- a/sys/netinet/tcp_syncache.c +++ b/sys/netinet/tcp_syncache.c @@ -81,10 +81,12 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> #include <netinet/tcp_syncache.h> -#include <netinet/tcp_offload.h> #ifdef INET6 #include <netinet6/tcp6_var.h> #endif +#ifdef TCP_OFFLOAD +#include <netinet/toecore.h> +#endif #ifdef IPSEC #include <netipsec/ipsec.h> @@ -110,10 +112,8 @@ SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, syncookies_only, CTLFLAG_RW, &VNET_NAME(tcp_syncookiesonly), 0, "Use only TCP SYN cookies"); -#ifdef TCP_OFFLOAD_DISABLE -#define TOEPCB_ISSET(sc) (0) -#else -#define TOEPCB_ISSET(sc) ((sc)->sc_toepcb != NULL) +#ifdef TCP_OFFLOAD +#define ADDED_BY_TOE(sc) ((sc)->sc_tod != NULL) #endif static void syncache_drop(struct syncache *, struct syncache_head *); @@ -332,6 +332,14 @@ syncache_insert(struct syncache *sc, struct syncache_head *sch) TAILQ_INSERT_HEAD(&sch->sch_bucket, sc, sc_hash); sch->sch_length++; +#ifdef TCP_OFFLOAD + if (ADDED_BY_TOE(sc)) { + struct toedev *tod = sc->sc_tod; + + tod->tod_syncache_added(tod, sc->sc_todctx); + } +#endif + /* Reinitialize the bucket row's timer. */ if (sch->sch_length == 1) sch->sch_nextc = ticks + INT_MAX; @@ -356,10 +364,14 @@ syncache_drop(struct syncache *sc, struct syncache_head *sch) TAILQ_REMOVE(&sch->sch_bucket, sc, sc_hash); sch->sch_length--; -#ifndef TCP_OFFLOAD_DISABLE - if (sc->sc_tu) - sc->sc_tu->tu_syncache_event(TOE_SC_DROP, sc->sc_toepcb); -#endif +#ifdef TCP_OFFLOAD + if (ADDED_BY_TOE(sc)) { + struct toedev *tod = sc->sc_tod; + + tod->tod_syncache_removed(tod, sc->sc_todctx); + } +#endif + syncache_free(sc); V_tcp_syncache.cache_count--; } @@ -846,6 +858,18 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m) if (sc->sc_rxmits > 1) tp->snd_cwnd = tp->t_maxseg; +#ifdef TCP_OFFLOAD + /* + * Allow a TOE driver to install its hooks. Note that we hold the + * pcbinfo lock too and that prevents tcp_usr_accept from accepting a + * new connection before the TOE driver has done its thing. + */ + if (ADDED_BY_TOE(sc)) { + struct toedev *tod = sc->sc_tod; + + tod->tod_offload_socket(tod, sc->sc_todctx, so); + } +#endif /* * Copy and activate timers. */ @@ -926,6 +950,13 @@ syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, /* Pull out the entry to unlock the bucket row. */ TAILQ_REMOVE(&sch->sch_bucket, sc, sc_hash); sch->sch_length--; +#ifdef TCP_OFFLOAD + if (ADDED_BY_TOE(sc)) { + struct toedev *tod = sc->sc_tod; + + tod->tod_syncache_removed(tod, sc->sc_todctx); + } +#endif V_tcp_syncache.cache_count--; SCH_UNLOCK(sch); } @@ -934,7 +965,7 @@ syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, * Segment validation: * ACK must match our initial sequence number + 1 (the SYN|ACK). */ - if (th->th_ack != sc->sc_iss + 1 && !TOEPCB_ISSET(sc)) { + if (th->th_ack != sc->sc_iss + 1) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: ACK %u != ISS+1 %u, segment " "rejected\n", s, __func__, th->th_ack, sc->sc_iss); @@ -945,9 +976,8 @@ syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, * The SEQ must fall in the window starting at the received * initial receive sequence number + 1 (the SYN). */ - if ((SEQ_LEQ(th->th_seq, sc->sc_irs) || - SEQ_GT(th->th_seq, sc->sc_irs + sc->sc_wnd)) && - !TOEPCB_ISSET(sc)) { + if (SEQ_LEQ(th->th_seq, sc->sc_irs) || + SEQ_GT(th->th_seq, sc->sc_irs + sc->sc_wnd)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: SEQ %u != IRS+1 %u, segment " "rejected\n", s, __func__, th->th_seq, sc->sc_irs); @@ -964,8 +994,7 @@ syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, * If timestamps were negotiated the reflected timestamp * must be equal to what we actually sent in the SYN|ACK. */ - if ((to->to_flags & TOF_TS) && to->to_tsecr != sc->sc_ts && - !TOEPCB_ISSET(sc)) { + if ((to->to_flags & TOF_TS) && to->to_tsecr != sc->sc_ts) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: TSECR %u != TS %u, " "segment rejected\n", @@ -993,25 +1022,6 @@ failed: return (0); } -int -tcp_offload_syncache_expand(struct in_conninfo *inc, struct toeopt *toeo, - struct tcphdr *th, struct socket **lsop, struct mbuf *m) -{ - struct tcpopt to; - int rc; - - bzero(&to, sizeof(struct tcpopt)); - to.to_mss = toeo->to_mss; - to.to_wscale = toeo->to_wscale; - to.to_flags = toeo->to_flags; - - INP_INFO_WLOCK(&V_tcbinfo); - rc = syncache_expand(inc, &to, th, lsop, m); - INP_INFO_WUNLOCK(&V_tcbinfo); - - return (rc); -} - /* * Given a LISTEN socket and an inbound SYN request, add * this to the syn cache, and send back a segment: @@ -1027,8 +1037,8 @@ tcp_offload_syncache_expand(struct in_conninfo *inc, struct toeopt *toeo, */ static void _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, - struct inpcb *inp, struct socket **lsop, struct mbuf *m, - struct toe_usrreqs *tu, void *toepcb) + struct inpcb *inp, struct socket **lsop, struct mbuf *m, void *tod, + void *todctx) { struct tcpcb *tp; struct socket *so; @@ -1114,11 +1124,6 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, sc = syncache_lookup(inc, &sch); /* returns locked entry */ SCH_LOCK_ASSERT(sch); if (sc != NULL) { -#ifndef TCP_OFFLOAD_DISABLE - if (sc->sc_tu) - sc->sc_tu->tu_syncache_event(TOE_SC_ENTRY_PRESENT, - sc->sc_toepcb); -#endif TCPSTAT_INC(tcps_sc_dupsyn); if (ipopts) { /* @@ -1151,7 +1156,7 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, s, __func__); free(s, M_TCPLOG); } - if (!TOEPCB_ISSET(sc) && syncache_respond(sc) == 0) { + if (syncache_respond(sc) == 0) { sc->sc_rxmits = 0; syncache_timeout(sc, sch, 1); TCPSTAT_INC(tcps_sndacks); @@ -1202,9 +1207,9 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, sc->sc_ip_tos = ip_tos; sc->sc_ip_ttl = ip_ttl; } -#ifndef TCP_OFFLOAD_DISABLE - sc->sc_tu = tu; - sc->sc_toepcb = toepcb; +#ifdef TCP_OFFLOAD + sc->sc_tod = tod; + sc->sc_todctx = todctx; #endif sc->sc_irs = th->th_seq; sc->sc_iss = arc4random(); @@ -1299,7 +1304,7 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, /* * Do a standard 3-way handshake. */ - if (TOEPCB_ISSET(sc) || syncache_respond(sc) == 0) { + if (syncache_respond(sc) == 0) { if (V_tcp_syncookies && V_tcp_syncookiesonly && sc != &scs) syncache_free(sc); else if (sc != &scs) @@ -1480,6 +1485,15 @@ syncache_respond(struct syncache *sc) th->th_sum = in6_cksum_pseudo(ip6, tlen + optlen - hlen, IPPROTO_TCP, 0); ip6->ip6_hlim = in6_selecthlim(NULL, NULL); +#ifdef TCP_OFFLOAD + if (ADDED_BY_TOE(sc)) { + struct toedev *tod = sc->sc_tod; + + error = tod->tod_syncache_respond(tod, sc->sc_todctx, m); + + return (error); + } +#endif error = ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL); } #endif @@ -1491,6 +1505,15 @@ syncache_respond(struct syncache *sc) m->m_pkthdr.csum_flags = CSUM_TCP; th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(tlen + optlen - hlen + IPPROTO_TCP)); +#ifdef TCP_OFFLOAD + if (ADDED_BY_TOE(sc)) { + struct toedev *tod = sc->sc_tod; + + error = tod->tod_syncache_respond(tod, sc->sc_todctx, m); + + return (error); + } +#endif error = ip_output(m, sc->sc_ipopts, NULL, 0, NULL, NULL); } #endif @@ -1505,23 +1528,12 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, } void -tcp_offload_syncache_add(struct in_conninfo *inc, struct toeopt *toeo, - struct tcphdr *th, struct inpcb *inp, struct socket **lsop, - struct toe_usrreqs *tu, void *toepcb) +tcp_offload_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, + struct inpcb *inp, struct socket **lsop, void *tod, void *todctx) { - struct tcpopt to; - - bzero(&to, sizeof(struct tcpopt)); - to.to_mss = toeo->to_mss; - to.to_wscale = toeo->to_wscale; - to.to_flags = toeo->to_flags; - INP_INFO_WLOCK(&V_tcbinfo); - INP_WLOCK(inp); - - _syncache_add(inc, &to, th, inp, lsop, NULL, tu, toepcb); + _syncache_add(inc, to, th, inp, lsop, NULL, tod, todctx); } - /* * The purpose of SYN cookies is to avoid keeping track of all SYN's we * receive and to be able to handle SYN floods from bogus source addresses diff --git a/sys/netinet/tcp_syncache.h b/sys/netinet/tcp_syncache.h index 5783b6d10900..af035daa1b7c 100644 --- a/sys/netinet/tcp_syncache.h +++ b/sys/netinet/tcp_syncache.h @@ -34,8 +34,6 @@ #define _NETINET_TCP_SYNCACHE_H_ #ifdef _KERNEL -struct toeopt; - void syncache_init(void); #ifdef VIMAGE void syncache_destroy(void); @@ -43,14 +41,10 @@ void syncache_destroy(void); void syncache_unreach(struct in_conninfo *, struct tcphdr *); int syncache_expand(struct in_conninfo *, struct tcpopt *, struct tcphdr *, struct socket **, struct mbuf *); -int tcp_offload_syncache_expand(struct in_conninfo *inc, struct toeopt *toeo, - struct tcphdr *th, struct socket **lsop, struct mbuf *m); void syncache_add(struct in_conninfo *, struct tcpopt *, struct tcphdr *, struct inpcb *, struct socket **, struct mbuf *); -void tcp_offload_syncache_add(struct in_conninfo *, struct toeopt *, - struct tcphdr *, struct inpcb *, struct socket **, - struct toe_usrreqs *tu, void *toepcb); - +void tcp_offload_syncache_add(struct in_conninfo *, struct tcpopt *, + struct tcphdr *, struct inpcb *, struct socket **, void *, void *); void syncache_chkrst(struct in_conninfo *, struct tcphdr *); void syncache_badack(struct in_conninfo *); int syncache_pcbcount(void); @@ -75,10 +69,10 @@ struct syncache { u_int8_t sc_requested_s_scale:4, sc_requested_r_scale:4; u_int16_t sc_flags; -#ifndef TCP_OFFLOAD_DISABLE - struct toe_usrreqs *sc_tu; /* TOE operations */ - void *sc_toepcb; /* TOE protocol block */ -#endif +#if defined(TCP_OFFLOAD) || !defined(TCP_OFFLOAD_DISABLE) + struct toedev *sc_tod; /* entry added by this TOE */ + void *sc_todctx; /* TOE driver context */ +#endif struct label *sc_label; /* MAC label reference */ struct ucred *sc_cred; /* cred cache for jail checks */ diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c index 480f9dbe87c4..6495ae0f6bb2 100644 --- a/sys/netinet/tcp_timer.c +++ b/sys/netinet/tcp_timer.c @@ -32,6 +32,7 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); +#include "opt_inet.h" #include "opt_inet6.h" #include "opt_tcpdebug.h" @@ -637,6 +638,11 @@ tcp_timer_activate(struct tcpcb *tp, int timer_type, u_int delta) struct inpcb *inp = tp->t_inpcb; int cpu = INP_CPU(inp); +#ifdef TCP_OFFLOAD + if (tp->t_flags & TF_TOE) + return; +#endif + switch (timer_type) { case TT_DELACK: t_callout = &tp->t_timers->tt_delack; diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 5f7fb0cdf5e8..258b7ef88b16 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -87,7 +87,9 @@ __FBSDID("$FreeBSD$"); #ifdef TCPDEBUG #include <netinet/tcp_debug.h> #endif +#ifdef TCP_OFFLOAD #include <netinet/tcp_offload.h> +#endif /* * TCP protocol interface to socket abstraction. @@ -367,7 +369,10 @@ tcp_usr_listen(struct socket *so, int backlog, struct thread *td) if (error == 0) { tp->t_state = TCPS_LISTEN; solisten_proto(so, backlog); - tcp_offload_listen_open(tp); +#ifdef TCP_OFFLOAD + if ((so->so_options & SO_NO_OFFLOAD) == 0) + tcp_offload_listen_start(tp); +#endif } SOCK_UNLOCK(so); @@ -409,6 +414,10 @@ tcp6_usr_listen(struct socket *so, int backlog, struct thread *td) if (error == 0) { tp->t_state = TCPS_LISTEN; solisten_proto(so, backlog); +#ifdef TCP_OFFLOAD + if ((so->so_options & SO_NO_OFFLOAD) == 0) + tcp_offload_listen_start(tp); +#endif } SOCK_UNLOCK(so); @@ -459,7 +468,14 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) TCPDEBUG1(); if ((error = tcp_connect(tp, nam, td)) != 0) goto out; - error = tcp_output_connect(so, nam); +#ifdef TCP_OFFLOAD + if (registered_toedevs > 0 && + (so->so_options & SO_NO_OFFLOAD) == 0 && + (error = tcp_offload_connect(so, nam)) == 0) + goto out; +#endif + tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); + error = tcp_output(tp); out: TCPDEBUG2(PRU_CONNECT); INP_WUNLOCK(inp); @@ -519,7 +535,13 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) goto out; if ((error = tcp_connect(tp, (struct sockaddr *)&sin, td)) != 0) goto out; - error = tcp_output_connect(so, nam); +#ifdef TCP_OFFLOAD + if (registered_toedevs > 0 && + (so->so_options & SO_NO_OFFLOAD) == 0 && + (error = tcp_offload_connect(so, nam)) == 0) + goto out; +#endif + error = tcp_output(tp); goto out; } #endif @@ -530,7 +552,14 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) goto out; if ((error = tcp6_connect(tp, nam, td)) != 0) goto out; - error = tcp_output_connect(so, nam); +#ifdef TCP_OFFLOAD + if (registered_toedevs > 0 && + (so->so_options & SO_NO_OFFLOAD) == 0 && + (error = tcp_offload_connect(so, nam)) == 0) + goto out; +#endif + tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); + error = tcp_output(tp); out: TCPDEBUG2(PRU_CONNECT); @@ -709,7 +738,7 @@ tcp_usr_shutdown(struct socket *so) socantsendmore(so); tcp_usrclosed(tp); if (!(inp->inp_flags & INP_DROPPED)) - error = tcp_output_disconnect(tp); + error = tcp_output(tp); out: TCPDEBUG2(PRU_SHUTDOWN); @@ -739,7 +768,12 @@ tcp_usr_rcvd(struct socket *so, int flags) } tp = intotcpcb(inp); TCPDEBUG1(); - tcp_output_rcvd(tp); +#ifdef TCP_OFFLOAD + if (tp->t_flags & TF_TOE) + tcp_offload_rcvd(tp); + else +#endif + tcp_output(tp); out: TCPDEBUG2(PRU_RCVD); @@ -835,7 +869,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, if (!(inp->inp_flags & INP_DROPPED)) { if (flags & PRUS_MORETOCOME) tp->t_flags |= TF_MORETOCOME; - error = tcp_output_send(tp); + error = tcp_output(tp); if (flags & PRUS_MORETOCOME) tp->t_flags &= ~TF_MORETOCOME; } @@ -884,7 +918,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, } tp->snd_up = tp->snd_una + so->so_snd.sb_cc; tp->t_flags |= TF_FORCEDATA; - error = tcp_output_send(tp); + error = tcp_output(tp); tp->t_flags &= ~TF_FORCEDATA; } out: @@ -1119,7 +1153,6 @@ tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td) soisconnecting(so); TCPSTAT_INC(tcps_connattempt); tp->t_state = TCPS_SYN_SENT; - tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); tp->iss = tcp_new_isn(tp); tcp_sendseqinit(tp); @@ -1192,7 +1225,6 @@ tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td) soisconnecting(so); TCPSTAT_INC(tcps_connattempt); tp->t_state = TCPS_SYN_SENT; - tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); tp->iss = tcp_new_isn(tp); tcp_sendseqinit(tp); @@ -1323,9 +1355,9 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) tp->t_flags |= TF_SIGNATURE; else tp->t_flags &= ~TF_SIGNATURE; - INP_WUNLOCK(inp); - break; + goto unlock_and_done; #endif /* TCP_SIGNATURE */ + case TCP_NODELAY: case TCP_NOOPT: INP_WUNLOCK(inp); @@ -1351,6 +1383,13 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) tp->t_flags |= opt; else tp->t_flags &= ~opt; +unlock_and_done: +#ifdef TCP_OFFLOAD + if (tp->t_flags & TF_TOE) { + tcp_offload_ctloutput(tp, sopt->sopt_dir, + sopt->sopt_name); + } +#endif INP_WUNLOCK(inp); break; @@ -1369,8 +1408,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) if (TCPS_HAVEESTABLISHED(tp->t_state)) error = tcp_output(tp); } - INP_WUNLOCK(inp); - break; + goto unlock_and_done; case TCP_MAXSEG: INP_WUNLOCK(inp); @@ -1385,8 +1423,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) tp->t_maxseg = optval; else error = EINVAL; - INP_WUNLOCK(inp); - break; + goto unlock_and_done; case TCP_INFO: INP_WUNLOCK(inp); @@ -1438,8 +1475,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) } } CC_LIST_RUNLOCK(); - INP_WUNLOCK(inp); - break; + goto unlock_and_done; case TCP_KEEPIDLE: case TCP_KEEPINTVL: @@ -1483,8 +1519,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) TP_KEEPINIT(tp)); break; } - INP_WUNLOCK(inp); - break; + goto unlock_and_done; case TCP_KEEPCNT: INP_WUNLOCK(inp); @@ -1654,7 +1689,7 @@ tcp_disconnect(struct tcpcb *tp) sbflush(&so->so_rcv); tcp_usrclosed(tp); if (!(inp->inp_flags & INP_DROPPED)) - tcp_output_disconnect(tp); + tcp_output(tp); } } @@ -1677,7 +1712,9 @@ tcp_usrclosed(struct tcpcb *tp) switch (tp->t_state) { case TCPS_LISTEN: - tcp_offload_listen_close(tp); +#ifdef TCP_OFFLOAD + tcp_offload_listen_stop(tp); +#endif /* FALLTHROUGH */ case TCPS_CLOSED: tp->t_state = TCPS_CLOSED; diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h index f5972f26b2b2..248552b13405 100644 --- a/sys/netinet/tcp_var.h +++ b/sys/netinet/tcp_var.h @@ -194,7 +194,7 @@ struct tcpcb { int t_rttlow; /* smallest observerved RTT */ u_int32_t rfbuf_ts; /* recv buffer autoscaling timestamp */ int rfbuf_cnt; /* recv buffer autoscaling byte count */ - struct toe_usrreqs *t_tu; /* offload operations vector */ + struct toedev *tod; /* toedev handling this connection */ int t_sndrexmitpack; /* retransmit packets sent */ int t_rcvoopack; /* out-of-order packets received */ void *t_toe; /* TOE pcb pointer */ diff --git a/sys/netinet/toecore.c b/sys/netinet/toecore.c new file mode 100644 index 000000000000..921cd96a36a5 --- /dev/null +++ b/sys/netinet/toecore.c @@ -0,0 +1,646 @@ +/*- + * Copyright (c) 2012 Chelsio Communications, Inc. + * All rights reserved. + * Written by: Navdeep Parhar <np@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/systm.h> +#include <sys/mbuf.h> +#include <sys/module.h> +#include <sys/types.h> +#include <sys/sockopt.h> +#include <sys/sysctl.h> +#include <sys/socket.h> + +#include <net/ethernet.h> +#include <net/if.h> +#include <net/if_types.h> +#include <net/if_vlan_var.h> +#include <net/if_llatbl.h> +#include <net/route.h> + +#include <netinet/if_ether.h> +#include <netinet/in.h> +#include <netinet/in_pcb.h> +#include <netinet/in_var.h> +#include <netinet6/in6_var.h> +#include <netinet6/in6_pcb.h> +#include <netinet6/nd6.h> +#define TCPSTATES +#include <netinet/tcp.h> +#include <netinet/tcp_fsm.h> +#include <netinet/tcp_timer.h> +#include <netinet/tcp_var.h> +#include <netinet/tcp_syncache.h> +#include <netinet/tcp_offload.h> +#include <netinet/toecore.h> + +static struct mtx toedev_lock; +static TAILQ_HEAD(, toedev) toedev_list; +static eventhandler_tag listen_start_eh; +static eventhandler_tag listen_stop_eh; +static eventhandler_tag lle_event_eh; +static eventhandler_tag route_redirect_eh; + +static int +toedev_connect(struct toedev *tod __unused, struct socket *so __unused, + struct rtentry *rt __unused, struct sockaddr *nam __unused) +{ + + return (ENOTSUP); +} + +static int +toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) +{ + + return (ENOTSUP); +} + +static int +toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) +{ + + return (ENOTSUP); +} + +static void +toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, + struct mbuf *m) +{ + + m_freem(m); + return; +} + +static void +toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) +{ + + return; +} + +static int +toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) +{ + + return (ENOTSUP); +} + +static void +toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) +{ + + return; +} + +static void +toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, + struct sockaddr *sa __unused, uint8_t *lladdr __unused, + uint16_t vtag __unused) +{ + + return; +} + +static void +toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, + struct rtentry *rt0 __unused, struct rtentry *rt1 __unused) +{ + + return; +} + +static void +toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) +{ + + return; +} + +static void +toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) +{ + + return; +} + +static int +toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, + struct mbuf *m) +{ + + m_freem(m); + return (0); +} + +static void +toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, + struct socket *so __unused) +{ + + return; +} + +static void +toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, + int sopt_dir __unused, int sopt_name __unused) +{ + + return; +} + +/* + * Inform one or more TOE devices about a listening socket. + */ +static void +toe_listen_start(struct inpcb *inp, void *arg) +{ + struct toedev *t, *tod; + struct tcpcb *tp; + + INP_WLOCK_ASSERT(inp); + KASSERT(inp->inp_pcbinfo == &V_tcbinfo, + ("%s: inp is not a TCP inp", __func__)); + + if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) + return; + + tp = intotcpcb(inp); + if (tp->t_state != TCPS_LISTEN) + return; + + t = arg; + mtx_lock(&toedev_lock); + TAILQ_FOREACH(tod, &toedev_list, link) { + if (t == NULL || t == tod) + tod->tod_listen_start(tod, tp); + } + mtx_unlock(&toedev_lock); +} + +static void +toe_listen_start_event(void *arg __unused, struct tcpcb *tp) +{ + struct inpcb *inp = tp->t_inpcb; + + INP_WLOCK_ASSERT(inp); + KASSERT(tp->t_state == TCPS_LISTEN, + ("%s: t_state %s", __func__, tcpstates[tp->t_state])); + + toe_listen_start(inp, NULL); +} + +static void +toe_listen_stop_event(void *arg __unused, struct tcpcb *tp) +{ + struct toedev *tod; +#ifdef INVARIANTS + struct inpcb *inp = tp->t_inpcb; +#endif + + INP_WLOCK_ASSERT(inp); + KASSERT(tp->t_state == TCPS_LISTEN, + ("%s: t_state %s", __func__, tcpstates[tp->t_state])); + + mtx_lock(&toedev_lock); + TAILQ_FOREACH(tod, &toedev_list, link) + tod->tod_listen_stop(tod, tp); + mtx_unlock(&toedev_lock); +} + +/* + * Fill up a freshly allocated toedev struct with reasonable defaults. + */ +void +init_toedev(struct toedev *tod) +{ + + tod->tod_softc = NULL; + + /* + * Provide no-op defaults so that the kernel can call any toedev + * function without having to check whether the TOE driver supplied one + * or not. + */ + tod->tod_connect = toedev_connect; + tod->tod_listen_start = toedev_listen_start; + tod->tod_listen_stop = toedev_listen_stop; + tod->tod_input = toedev_input; + tod->tod_rcvd = toedev_rcvd; + tod->tod_output = toedev_output; + tod->tod_send_rst = toedev_output; + tod->tod_send_fin = toedev_output; + tod->tod_pcb_detach = toedev_pcb_detach; + tod->tod_l2_update = toedev_l2_update; + tod->tod_route_redirect = toedev_route_redirect; + tod->tod_syncache_added = toedev_syncache_added; + tod->tod_syncache_removed = toedev_syncache_removed; + tod->tod_syncache_respond = toedev_syncache_respond; + tod->tod_offload_socket = toedev_offload_socket; + tod->tod_ctloutput = toedev_ctloutput; +} + +/* + * Register an active TOE device with the system. This allows it to receive + * notifications from the kernel. + */ +int +register_toedev(struct toedev *tod) +{ + struct toedev *t; + + mtx_lock(&toedev_lock); + TAILQ_FOREACH(t, &toedev_list, link) { + if (t == tod) { + mtx_unlock(&toedev_lock); + return (EEXIST); + } + } + + TAILQ_INSERT_TAIL(&toedev_list, tod, link); + registered_toedevs++; + mtx_unlock(&toedev_lock); + + inp_apply_all(toe_listen_start, tod); + + return (0); +} + +/* + * Remove the TOE device from the global list of active TOE devices. It is the + * caller's responsibility to ensure that the TOE device is quiesced prior to + * this call. + */ +int +unregister_toedev(struct toedev *tod) +{ + struct toedev *t, *t2; + int rc = ENODEV; + + mtx_lock(&toedev_lock); + TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { + if (t == tod) { + TAILQ_REMOVE(&toedev_list, tod, link); + registered_toedevs--; + rc = 0; + break; + } + } + KASSERT(registered_toedevs >= 0, + ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); + mtx_unlock(&toedev_lock); + return (rc); +} + +void +toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, + struct inpcb *inp, void *tod, void *todctx) +{ + struct socket *lso = inp->inp_socket; + + INP_INFO_WLOCK_ASSERT(&V_tcbinfo); + INP_WLOCK_ASSERT(inp); + + tcp_offload_syncache_add(inc, to, th, inp, &lso, tod, todctx); +} + +int +toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, + struct tcphdr *th, struct socket **lsop) +{ + + INP_INFO_WLOCK_ASSERT(&V_tcbinfo); + + return (syncache_expand(inc, to, th, lsop, NULL)); +} + +/* + * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP + * header (presumably for an incoming SYN) is also provided, an existing 4-tuple + * in TIME_WAIT may be assassinated freeing it up for re-use. + * + * Note that the TCP header must have been run through tcp_fields_to_host() or + * equivalent. + */ +int +toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) +{ + struct inpcb *inp; + + if (inc->inc_flags & INC_ISIPV6) { + inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, + inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, + INPLOOKUP_WLOCKPCB, ifp); + } else { + inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, + inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp); + } + if (inp != NULL) { + INP_WLOCK_ASSERT(inp); + + if ((inp->inp_flags & INP_TIMEWAIT) && th != NULL) { + + INP_INFO_WLOCK_ASSERT(&V_tcbinfo); /* for twcheck */ + if (!tcp_twcheck(inp, NULL, th, NULL, 0)) + return (EADDRINUSE); + } else { + INP_WUNLOCK(inp); + return (EADDRINUSE); + } + } + + return (0); +} + +static void +toe_lle_event(void *arg __unused, struct llentry *lle, int evt) +{ + struct toedev *tod; + struct ifnet *ifp; + struct sockaddr *sa; + uint8_t *lladdr; + uint16_t vtag; + + LLE_WLOCK_ASSERT(lle); + + ifp = lle->lle_tbl->llt_ifp; + sa = L3_ADDR(lle); + + KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, + ("%s: lle_event %d for lle %p but sa %p !INET && !INET6", + __func__, evt, lle, sa)); + + /* + * Not interested if the interface's TOE capability is not enabled. + */ + if ((sa->sa_family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || + (sa->sa_family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) + return; + + tod = TOEDEV(ifp); + if (tod == NULL) + return; + + vtag = 0xfff; + if (evt != LLENTRY_RESOLVED) { + + /* + * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean + * this entry is going to be deleted. + */ + + lladdr = NULL; + } else { + + KASSERT(lle->la_flags & LLE_VALID, + ("%s: %p resolved but not valid?", __func__, lle)); + + lladdr = (uint8_t *)&lle->ll_addr; +#ifdef VLAN_TAG + VLAN_TAG(ifp, &vtag); +#endif + } + + tod->tod_l2_update(tod, ifp, sa, lladdr, vtag); +} + +/* + * XXX: implement. + */ +static void +toe_route_redirect_event(void *arg __unused, struct rtentry *rt0, + struct rtentry *rt1, struct sockaddr *sa) +{ + + return; +} + +#ifdef INET6 +/* + * XXX: no checks to verify that sa is really a neighbor because we assume it is + * the result of a route lookup and is on-link on the given ifp. + */ +static int +toe_nd6_resolve(struct ifnet *ifp, struct sockaddr *sa, uint8_t *lladdr) +{ + struct llentry *lle; + struct sockaddr_in6 *sin6 = (void *)sa; + int rc, flags = 0; + +restart: + IF_AFDATA_RLOCK(ifp); + lle = lla_lookup(LLTABLE6(ifp), flags, sa); + IF_AFDATA_RUNLOCK(ifp); + if (lle == NULL) { + IF_AFDATA_LOCK(ifp); + lle = nd6_lookup(&sin6->sin6_addr, ND6_CREATE | ND6_EXCLUSIVE, + ifp); + IF_AFDATA_UNLOCK(ifp); + if (lle == NULL) + return (ENOMEM); /* Couldn't create entry in cache. */ + lle->ln_state = ND6_LLINFO_INCOMPLETE; + nd6_llinfo_settimer_locked(lle, + (long)ND_IFINFO(ifp)->retrans * hz / 1000); + LLE_WUNLOCK(lle); + + nd6_ns_output(ifp, NULL, &sin6->sin6_addr, NULL, 0); + + return (EWOULDBLOCK); + } + + if (lle->ln_state == ND6_LLINFO_STALE) { + if ((flags & LLE_EXCLUSIVE) == 0) { + LLE_RUNLOCK(lle); + flags |= LLE_EXCLUSIVE; + goto restart; + } + + LLE_WLOCK_ASSERT(lle); + + lle->la_asked = 0; + lle->ln_state = ND6_LLINFO_DELAY; + nd6_llinfo_settimer_locked(lle, (long)V_nd6_delay * hz); + } + + if (lle->la_flags & LLE_VALID) { + memcpy(lladdr, &lle->ll_addr, ifp->if_addrlen); + rc = 0; + } else + rc = EWOULDBLOCK; + + if (flags & LLE_EXCLUSIVE) + LLE_WUNLOCK(lle); + else + LLE_RUNLOCK(lle); + + return (rc); +} +#endif + +/* + * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means + * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's + * tod_l2_update will be called later, when the entry is resolved or times out. + */ +int +toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, + uint8_t *lladdr, uint16_t *vtag) +{ +#ifdef INET + struct llentry *lle; +#endif + int rc; + + switch (sa->sa_family) { +#ifdef INET + case AF_INET: + rc = arpresolve(ifp, NULL, NULL, sa, lladdr, &lle); + break; +#endif +#ifdef INET6 + case AF_INET6: + rc = toe_nd6_resolve(ifp, sa, lladdr); + break; +#endif + default: + return (EPROTONOSUPPORT); + } + + if (rc == 0) { +#ifdef VLAN_TAG + if (VLAN_TAG(ifp, vtag) != 0) +#endif + *vtag = 0xfff; + } + + return (rc); +} + +void +toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) +{ + + INP_WLOCK_ASSERT(inp); + + if (!(inp->inp_flags & INP_DROPPED)) { + struct tcpcb *tp = intotcpcb(inp); + + KASSERT(tp->t_flags & TF_TOE, + ("%s: tp %p not offloaded.", __func__, tp)); + + if (err == EAGAIN) { + + /* + * Temporary failure during offload, take this PCB back. + * Detach from the TOE driver and do the rest of what + * TCP's pru_connect would have done if the connection + * wasn't offloaded. + */ + + tod->tod_pcb_detach(tod, tp); + KASSERT(!(tp->t_flags & TF_TOE), + ("%s: tp %p still offloaded.", __func__, tp)); + tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); + (void) tcp_output(tp); + } else { + + INP_INFO_WLOCK_ASSERT(&V_tcbinfo); + tp = tcp_drop(tp, err); + if (tp == NULL) + INP_WLOCK(inp); /* re-acquire */ + } + } + INP_WLOCK_ASSERT(inp); +} + +static int +toecore_load(void) +{ + + mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); + TAILQ_INIT(&toedev_list); + + listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, + toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); + listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, + toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); + lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, + EVENTHANDLER_PRI_ANY); + route_redirect_eh = EVENTHANDLER_REGISTER(route_redirect_event, + toe_route_redirect_event, NULL, EVENTHANDLER_PRI_ANY); + + return (0); +} + +static int +toecore_unload(void) +{ + + mtx_lock(&toedev_lock); + if (!TAILQ_EMPTY(&toedev_list)) { + mtx_unlock(&toedev_lock); + return (EBUSY); + } + + EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); + EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); + EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); + EVENTHANDLER_DEREGISTER(route_redirect_event, route_redirect_eh); + + mtx_unlock(&toedev_lock); + mtx_destroy(&toedev_lock); + + return (0); +} + +static int +toecore_mod_handler(module_t mod, int cmd, void *arg) +{ + + if (cmd == MOD_LOAD) + return (toecore_load()); + + if (cmd == MOD_UNLOAD) + return (toecore_unload()); + + return (EOPNOTSUPP); +} + +static moduledata_t mod_data= { + "toecore", + toecore_mod_handler, + 0 +}; + +MODULE_VERSION(toecore, 1); +DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); diff --git a/sys/netinet/toecore.h b/sys/netinet/toecore.h new file mode 100644 index 000000000000..6ea98518e59b --- /dev/null +++ b/sys/netinet/toecore.h @@ -0,0 +1,130 @@ +/*- + * Copyright (c) 2012 Chelsio Communications, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _NETINET_TOE_H_ +#define _NETINET_TOE_H_ + +#ifndef _KERNEL +#error "no user-serviceable parts inside" +#endif + +struct tcpopt; +struct tcphdr; +struct in_conninfo; + +struct toedev { + TAILQ_ENTRY(toedev) link; /* glue for toedev_list */ + void *tod_softc; /* TOE driver private data */ + + /* + * Active open. If a failure occurs, it is reported back by the driver + * via toe_connect_failed. + */ + int (*tod_connect)(struct toedev *, struct socket *, struct rtentry *, + struct sockaddr *); + + /* Passive open. */ + int (*tod_listen_start)(struct toedev *, struct tcpcb *); + int (*tod_listen_stop)(struct toedev *, struct tcpcb *); + + /* + * The kernel uses this routine to pass on any frame it receives for an + * offloaded connection to the TOE driver. This is an unusual event. + */ + void (*tod_input)(struct toedev *, struct tcpcb *, struct mbuf *); + + /* + * This is called by the kernel during pru_rcvd for an offloaded TCP + * connection and provides an opportunity for the TOE driver to manage + * its rx window and credits. + */ + void (*tod_rcvd)(struct toedev *, struct tcpcb *); + + /* + * Transmit routine. The kernel calls this to have the TOE driver + * evaluate whether there is data to be transmitted, and transmit it. + */ + int (*tod_output)(struct toedev *, struct tcpcb *); + + /* Immediate teardown: send RST to peer. */ + int (*tod_send_rst)(struct toedev *, struct tcpcb *); + + /* Initiate orderly disconnect by sending FIN to the peer. */ + int (*tod_send_fin)(struct toedev *, struct tcpcb *); + + /* Called to indicate that the kernel is done with this TCP PCB. */ + void (*tod_pcb_detach)(struct toedev *, struct tcpcb *); + + /* + * The kernel calls this once it has information about an L2 entry that + * the TOE driver enquired about previously (via toe_l2_resolve). + */ + void (*tod_l2_update)(struct toedev *, struct ifnet *, + struct sockaddr *, uint8_t *, uint16_t); + + /* XXX. Route has been redirected. */ + void (*tod_route_redirect)(struct toedev *, struct ifnet *, + struct rtentry *, struct rtentry *); + + /* Syncache interaction. */ + void (*tod_syncache_added)(struct toedev *, void *); + void (*tod_syncache_removed)(struct toedev *, void *); + int (*tod_syncache_respond)(struct toedev *, void *, struct mbuf *); + void (*tod_offload_socket)(struct toedev *, void *, struct socket *); + + /* TCP socket option */ + void (*tod_ctloutput)(struct toedev *, struct tcpcb *, int, int); +}; + +#include <sys/eventhandler.h> +typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *); +typedef void (*tcp_offload_listen_stop_fn)(void *, struct tcpcb *); +EVENTHANDLER_DECLARE(tcp_offload_listen_start, tcp_offload_listen_start_fn); +EVENTHANDLER_DECLARE(tcp_offload_listen_stop, tcp_offload_listen_stop_fn); + +void init_toedev(struct toedev *); +int register_toedev(struct toedev *); +int unregister_toedev(struct toedev *); + +/* + * General interface for looking up L2 information for an IP address. If an + * answer is not available right away then the TOE driver's tod_l2_update will + * be called later. + */ +int toe_l2_resolve(struct toedev *, struct ifnet *, struct sockaddr *, + uint8_t *, uint16_t *); + +void toe_connect_failed(struct toedev *, struct inpcb *, int); + +void toe_syncache_add(struct in_conninfo *, struct tcpopt *, struct tcphdr *, + struct inpcb *, void *, void *); +int toe_syncache_expand(struct in_conninfo *, struct tcpopt *, struct tcphdr *, + struct socket **); + +int toe_4tuple_check(struct in_conninfo *, struct tcphdr *, struct ifnet *); +#endif diff --git a/sys/netinet/toedev.h b/sys/netinet/toedev.h deleted file mode 100644 index 7edaca126faa..000000000000 --- a/sys/netinet/toedev.h +++ /dev/null @@ -1,162 +0,0 @@ -/*- - * Copyright (c) 2007, Chelsio Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Neither the name of the Chelsio Corporation nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _NETINET_TOEDEV_H_ -#define _NETINET_TOEDEV_H_ - -#ifndef _KERNEL -#error "no user-serviceable parts inside" -#endif - -extern uint32_t toedev_registration_count; - -/* Parameter values for offload_get_phys_egress(). */ -enum { - TOE_OPEN, - TOE_FAILOVER, -}; - -/* Parameter values for toe_failover(). */ -enum { - TOE_ACTIVE_SLAVE, - TOE_LINK_DOWN, - TOE_LINK_UP, - TOE_RELEASE, - TOE_RELEASE_ALL, -}; - -#define TOENAMSIZ 16 - -/* Get the toedev associated with a ifnet. */ -#define TOEDEV(ifp) ((ifp)->if_llsoftc) - -struct offload_id { - unsigned int id; - unsigned long data; -}; - -struct ifnet; -struct rt_entry; -struct tom_info; -struct sysctl_oid; -struct socket; -struct mbuf; - -struct toedev { - TAILQ_ENTRY(toedev) entry; - char tod_name[TOENAMSIZ]; /* TOE device name */ - unsigned int tod_ttid; /* TOE type id */ - unsigned long tod_flags; /* device flags */ - unsigned int tod_mtu; /* max TX offloaded data */ - unsigned int tod_nconn; /* max # of offloaded - * connections - */ - struct ifnet *tod_lldev; /* first interface */ - const struct tom_info *tod_offload_mod; /* TCP offload module */ - - /* - * This TOE device is capable of offloading the connection for socket so - */ - int (*tod_can_offload)(struct toedev *dev, struct socket *so); - - /* - * Establish a connection to nam using the TOE device dev - */ - int (*tod_connect)(struct toedev *dev, struct socket *so, - struct rtentry *rt, struct sockaddr *nam); - /* - * Send an mbuf down to the toe device - */ - int (*tod_send)(struct toedev *dev, struct mbuf *m); - /* - * Receive an array of mbufs from the TOE device dev - */ - int (*tod_recv)(struct toedev *dev, struct mbuf **m, int n); - /* - * Device specific ioctl interface - */ - int (*tod_ctl)(struct toedev *dev, unsigned int req, void *data); - /* - * Update L2 entry in toedev - */ - void (*tod_arp_update)(struct toedev *dev, struct rtentry *neigh); - /* - * Failover from one toe device to another - */ - void (*tod_failover)(struct toedev *dev, struct ifnet *bond_ifp, - struct ifnet *ndev, int event); - void *tod_priv; /* driver private data */ - void *tod_l2opt; /* optional layer 2 data */ - void *tod_l3opt; /* optional layer 3 data */ - void *tod_l4opt; /* optional layer 4 data */ - void *tod_ulp; /* upper lever protocol */ -}; - -struct tom_info { - TAILQ_ENTRY(tom_info) entry; - int (*ti_attach)(struct toedev *dev, - const struct offload_id *entry); - int (*ti_detach)(struct toedev *dev); - const char *ti_name; - const struct offload_id *ti_id_table; -}; - -static __inline void -init_offload_dev(struct toedev *dev) -{ -} - -int register_tom(struct tom_info *t); -int unregister_tom(struct tom_info *t); -int register_toedev(struct toedev *dev, const char *name); -int unregister_toedev(struct toedev *dev); -int activate_offload(struct toedev *dev); -int toe_send(struct toedev *dev, struct mbuf *m); -void toe_arp_update(struct rtentry *rt); -struct ifnet *offload_get_phys_egress(struct ifnet *ifp, - struct socket *so, int context); -int toe_receive_mbuf(struct toedev *dev, struct mbuf **m, int n); - -static __inline void -toe_neigh_update(struct ifnet *ifp) -{ -} - -static __inline void -toe_failover(struct ifnet *bond_ifp, struct ifnet *fail_ifp, int event) -{ -} - -static __inline int -toe_enslave(struct ifnet *bond_ifp, struct ifnet *slave_ifp) -{ - return (0); -} - -#endif /* _NETINET_TOEDEV_H_ */ diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index 473ee3a4f753..36a2bffeeac5 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -509,6 +509,7 @@ nd6_llinfo_timer(void *arg) ln->la_hold = m0; clear_llinfo_pqueue(ln); } + EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_TIMEDOUT); (void)nd6_free(ln, 0); ln = NULL; if (m != NULL) @@ -526,6 +527,7 @@ nd6_llinfo_timer(void *arg) case ND6_LLINFO_STALE: /* Garbage Collection(RFC 2461 5.3) */ if (!ND6_LLINFO_PERMANENT(ln)) { + EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_EXPIRED); (void)nd6_free(ln, 1); ln = NULL; } @@ -553,6 +555,7 @@ nd6_llinfo_timer(void *arg) nd6_ns_output(ifp, dst, dst, ln, 0); LLE_WLOCK(ln); } else { + EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_EXPIRED); (void)nd6_free(ln, 0); ln = NULL; } @@ -1623,6 +1626,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr, */ bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen); ln->la_flags |= LLE_VALID; + EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_RESOLVED); } if (!is_newentry) { @@ -2191,7 +2195,7 @@ nd6_storelladdr(struct ifnet *ifp, struct mbuf *m, *lle = NULL; IF_AFDATA_UNLOCK_ASSERT(ifp); - if (m->m_flags & M_MCAST) { + if (m != NULL && m->m_flags & M_MCAST) { int i; switch (ifp->if_type) { diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c index ba2150fb5000..cec81b022a60 100644 --- a/sys/netinet6/nd6_nbr.c +++ b/sys/netinet6/nd6_nbr.c @@ -755,6 +755,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) */ bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen); ln->la_flags |= LLE_VALID; + EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_RESOLVED); if (is_solicited) { ln->ln_state = ND6_LLINFO_REACHABLE; ln->ln_byhint = 0; @@ -830,6 +831,8 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) if (lladdr != NULL) { bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen); ln->la_flags |= LLE_VALID; + EVENTHANDLER_INVOKE(lle_event, ln, + LLENTRY_RESOLVED); } /* diff --git a/sys/ofed/drivers/infiniband/core/cma.c b/sys/ofed/drivers/infiniband/core/cma.c index 585848b09d24..34419f35efc5 100644 --- a/sys/ofed/drivers/infiniband/core/cma.c +++ b/sys/ofed/drivers/infiniband/core/cma.c @@ -59,10 +59,10 @@ static int tavor_quirk = 0; module_param_named(tavor_quirk, tavor_quirk, int, 0644); MODULE_PARM_DESC(tavor_quirk, "Tavor performance quirk: limit MTU to 1K if > 0"); -int unify_tcp_port_space = 0; +int unify_tcp_port_space = 1; module_param(unify_tcp_port_space, int, 0644); MODULE_PARM_DESC(unify_tcp_port_space, "Unify the host TCP and RDMA port " - "space allocation (default=0)"); + "space allocation (default=1)"); #define CMA_CM_RESPONSE_TIMEOUT 20 #define CMA_MAX_CM_RETRIES 15 @@ -1478,6 +1478,7 @@ static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) struct sockaddr_in *sin; id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, + id_priv->sock, iw_conn_req_handler, id_priv); if (IS_ERR(id_priv->cm_id.iw)) @@ -2055,7 +2056,16 @@ static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id; } } - return rdma_bind_addr(id, src_addr); + if (!cma_any_addr(src_addr)) + return rdma_bind_addr(id, src_addr); + else { + struct sockaddr_in addr_in; + + memset(&addr_in, 0, sizeof addr_in); + addr_in.sin_family = dst_addr->sa_family; + addr_in.sin_len = sizeof addr_in; + return rdma_bind_addr(id, (struct sockaddr *) &addr_in); + } } int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, @@ -2247,6 +2257,7 @@ static int cma_get_tcp_port(struct rdma_id_private *id_priv) sock_release(sock); return ret; } + size = ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr); ret = sock_getname(sock, (struct sockaddr *) &id_priv->id.route.addr.src_addr, @@ -2255,6 +2266,7 @@ static int cma_get_tcp_port(struct rdma_id_private *id_priv) sock_release(sock); return ret; } + id_priv->sock = sock; return 0; } @@ -2604,7 +2616,8 @@ static int cma_connect_iw(struct rdma_id_private *id_priv, int ret; struct iw_cm_conn_param iw_param; - cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); + cm_id = iw_create_cm_id(id_priv->id.device, id_priv->sock, + cma_iw_handler, id_priv); if (IS_ERR(cm_id)) { ret = PTR_ERR(cm_id); goto out; diff --git a/sys/ofed/drivers/infiniband/core/iwcm.c b/sys/ofed/drivers/infiniband/core/iwcm.c index 625fec5a741c..b13e53a33697 100644 --- a/sys/ofed/drivers/infiniband/core/iwcm.c +++ b/sys/ofed/drivers/infiniband/core/iwcm.c @@ -189,6 +189,7 @@ static void rem_ref(struct iw_cm_id *cm_id) static int cm_event_handler(struct iw_cm_id *cm_id, struct iw_cm_event *event); struct iw_cm_id *iw_create_cm_id(struct ib_device *device, + struct socket *so, iw_cm_handler cm_handler, void *context) { @@ -205,6 +206,7 @@ struct iw_cm_id *iw_create_cm_id(struct ib_device *device, cm_id_priv->id.event_handler = cm_event_handler; cm_id_priv->id.add_ref = add_ref; cm_id_priv->id.rem_ref = rem_ref; + cm_id_priv->id.so = so; spin_lock_init(&cm_id_priv->lock); atomic_set(&cm_id_priv->refcount, 1); init_waitqueue_head(&cm_id_priv->connect_wait); @@ -629,6 +631,7 @@ static void cm_conn_req_handler(struct iwcm_id_private *listen_id_priv, spin_unlock_irqrestore(&listen_id_priv->lock, flags); cm_id = iw_create_cm_id(listen_id_priv->id.device, + iw_event->so, listen_id_priv->id.cm_handler, listen_id_priv->id.context); /* If the cm_id could not be created, ignore the request */ diff --git a/sys/ofed/include/linux/net.h b/sys/ofed/include/linux/net.h index 6e2aff39524b..f47acf938fe7 100644 --- a/sys/ofed/include/linux/net.h +++ b/sys/ofed/include/linux/net.h @@ -48,12 +48,12 @@ sock_getname(struct socket *so, struct sockaddr *addr, int *sockaddr_len, int error; nam = NULL; - if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) - return (-ENOTCONN); + if (peer) { + if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) + return (-ENOTCONN); - if (peer) error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, nam); - else + } else error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, nam); if (error) return (-error); diff --git a/sys/ofed/include/linux/workqueue.h b/sys/ofed/include/linux/workqueue.h index f432ea77ade8..ce5759bf01a6 100644 --- a/sys/ofed/include/linux/workqueue.h +++ b/sys/ofed/include/linux/workqueue.h @@ -129,7 +129,7 @@ _create_workqueue_common(char *name, int cpus) wq = kmalloc(sizeof(*wq), M_WAITOK); wq->taskqueue = taskqueue_create((name), M_WAITOK, taskqueue_thread_enqueue, &wq->taskqueue); - taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, (name)); + taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); return (wq); } diff --git a/sys/ofed/include/net/netevent.h b/sys/ofed/include/net/netevent.h index db5b50ef6047..c7bbc5fd5529 100644 --- a/sys/ofed/include/net/netevent.h +++ b/sys/ofed/include/net/netevent.h @@ -42,7 +42,7 @@ enum netevent_notif_type { struct llentry; static inline void -_handle_arp_update_event(void *arg, struct llentry *lle) +_handle_arp_update_event(void *arg, struct llentry *lle, int evt __unused) { struct notifier_block *nb; @@ -54,7 +54,7 @@ static inline int register_netevent_notifier(struct notifier_block *nb) { nb->tags[NETEVENT_NEIGH_UPDATE] = EVENTHANDLER_REGISTER( - arp_update_event, _handle_arp_update_event, nb, 0); + lle_event, _handle_arp_update_event, nb, 0); return (0); } @@ -62,8 +62,7 @@ static inline int unregister_netevent_notifier(struct notifier_block *nb) { - EVENTHANDLER_DEREGISTER(arp_update_event, - nb->tags[NETEVENT_NEIGH_UPDATE]); + EVENTHANDLER_DEREGISTER(lle_event, nb->tags[NETEVENT_NEIGH_UPDATE]); return (0); } diff --git a/sys/ofed/include/rdma/iw_cm.h b/sys/ofed/include/rdma/iw_cm.h index cbb822e8d791..412320e0899a 100644 --- a/sys/ofed/include/rdma/iw_cm.h +++ b/sys/ofed/include/rdma/iw_cm.h @@ -63,6 +63,7 @@ struct iw_cm_event { void *private_data; u8 private_data_len; void *provider_data; + struct socket *so; }; /** @@ -98,6 +99,7 @@ struct iw_cm_id { /* Used by provider to add and remove refs on IW cm_id */ void (*add_ref)(struct iw_cm_id *); void (*rem_ref)(struct iw_cm_id *); + struct socket *so; }; struct iw_cm_conn_param { @@ -139,7 +141,7 @@ struct iw_cm_verbs { * returned IW CM identifier. * @context: User specified context associated with the id. */ -struct iw_cm_id *iw_create_cm_id(struct ib_device *device, +struct iw_cm_id *iw_create_cm_id(struct ib_device *device, struct socket *so, iw_cm_handler cm_handler, void *context); /** diff --git a/usr.bin/netstat/inet.c b/usr.bin/netstat/inet.c index c2d7f9208e61..d7f3aa814846 100644 --- a/usr.bin/netstat/inet.c +++ b/usr.bin/netstat/inet.c @@ -461,7 +461,10 @@ protopr(u_long off, const char *name, int af1, int proto) #endif vchar = ((inp->inp_vflag & INP_IPV4) != 0) ? "4 " : " "; - printf("%-3.3s%-2.2s ", name, vchar); + if (istcp && (tp->t_flags & TF_TOE) != 0) + printf("%-3.3s%-2.2s ", "toe", vchar); + else + printf("%-3.3s%-2.2s ", name, vchar); if (Lflag) { char buf1[15]; diff --git a/usr.bin/sockstat/sockstat.c b/usr.bin/sockstat/sockstat.c index efcb0d64ede5..369ea2b175f0 100644 --- a/usr.bin/sockstat/sockstat.c +++ b/usr.bin/sockstat/sockstat.c @@ -325,6 +325,7 @@ gather_inet(int proto) } inp = &xtp->xt_inp; so = &xtp->xt_socket; + protoname = xtp->xt_tp.t_flags & TF_TOE ? "toe" : "tcp"; break; case IPPROTO_UDP: case IPPROTO_DIVERT: |