diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2014-12-01 13:14:39 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2014-12-01 13:14:39 +0000 |
| commit | 60c317af7299f297b23f477bf4b0bb0e91d95fd9 (patch) | |
| tree | d5fa3f2bfaaf53d3473a85d63e4b7770e9d6c840 | |
| parent | d814fbe9d25f08dafafd5278bfb9fbe40c5527fd (diff) | |
| parent | 5d083d287bf3fa03e14c5ffcd3fbfe203c6c371f (diff) | |
Notes
114 files changed, 1145 insertions, 856 deletions
diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc index 8fc9c5a86a75b..e76b2f88b5d36 100644 --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -38,7 +38,7 @@ # xargs -n1 | sort | uniq -d; # done -# 20141128: new clang import which bumps version from 3.4.1 to 3.5.0. +# 20141201: new clang import which bumps version from 3.4.1 to 3.5.0. OLD_FILES+=usr/include/clang/3.4.1/__wmmintrin_aes.h OLD_FILES+=usr/include/clang/3.4.1/__wmmintrin_pclmul.h OLD_FILES+=usr/include/clang/3.4.1/altivec.h @@ -74,6 +74,8 @@ OLD_FILES+=usr/include/clang/3.4.1/x86intrin.h OLD_FILES+=usr/include/clang/3.4.1/xmmintrin.h OLD_FILES+=usr/include/clang/3.4.1/xopintrin.h OLD_DIRS+=usr/include/clang/3.4.1 +# 20141129: mrouted rc.d scripts removed from base +OLD_FILES+=etc/rc.d/mrouted # 20141126: convert sbin/mdconfig/tests to ATF format tests OLD_FILES+=usr/tests/sbin/mdconfig/legacy_test OLD_FILES+=usr/tests/sbin/mdconfig/mdconfig.test diff --git a/bin/sh/eval.c b/bin/sh/eval.c index c1a9cdbfcd932..d0460a024e3d2 100644 --- a/bin/sh/eval.c +++ b/bin/sh/eval.c @@ -774,15 +774,7 @@ xtracecommand(struct arglist *varlist, struct arglist *arglist) for (sp = arglist->list ; sp ; sp = sp->next) { if (sep != 0) out2c(' '); - /* Disambiguate command looking like assignment. */ - if (sp == arglist->list && - strchr(sp->text, '=') != NULL && - strchr(sp->text, '\'') == NULL) { - out2c('\''); - out2str(sp->text); - out2c('\''); - } else - out2qstr(sp->text); + out2qstr(sp->text); sep = ' '; } out2c('\n'); diff --git a/bin/sh/output.c b/bin/sh/output.c index bf8e17d97ece9..c6d118f115355 100644 --- a/bin/sh/output.c +++ b/bin/sh/output.c @@ -122,8 +122,7 @@ outqstr(const char *p, struct output *file) outstr("''", file); return; } - /* Caller will handle '=' if necessary */ - if (p[strcspn(p, "|&;<>()$`\\\"' \t\n*?[~#")] == '\0' || + if (p[strcspn(p, "|&;<>()$`\\\"' \t\n*?[~#=")] == '\0' || strcmp(p, "[") == 0) { outstr(p, file); return; diff --git a/contrib/binutils/bfd/elf32-arm.c b/contrib/binutils/bfd/elf32-arm.c index 83acfe5647140..47b6566c2e2b1 100644 --- a/contrib/binutils/bfd/elf32-arm.c +++ b/contrib/binutils/bfd/elf32-arm.c @@ -4960,7 +4960,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, + input_section->output_offset + rel->r_offset); - value = abs (relocation); + value = llabs (relocation); if (value >= 0x1000) return bfd_reloc_overflow; @@ -4998,7 +4998,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, + input_section->output_offset + rel->r_offset); - value = abs (relocation); + value = llabs (relocation); if (value >= 0x1000) return bfd_reloc_overflow; @@ -5984,7 +5984,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, /* Calculate the value of the relevant G_n, in encoded constant-with-rotation format. */ - g_n = calculate_group_reloc_mask (abs (signed_value), group, + g_n = calculate_group_reloc_mask (llabs (signed_value), group, &residual); /* Check for overflow if required. */ @@ -5998,7 +5998,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, (*_bfd_error_handler) (_("%B(%A+0x%lx): Overflow whilst splitting 0x%lx for group relocation %s"), input_bfd, input_section, - (long) rel->r_offset, abs (signed_value), howto->name); + (long) rel->r_offset, llabs (signed_value), howto->name); return bfd_reloc_overflow; } @@ -6077,7 +6077,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, /* Calculate the value of the relevant G_{n-1} to obtain the residual at that stage. */ - calculate_group_reloc_mask (abs (signed_value), group - 1, &residual); + calculate_group_reloc_mask (llabs (signed_value), group - 1, &residual); /* Check for overflow. */ if (residual >= 0x1000) @@ -6085,7 +6085,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, (*_bfd_error_handler) (_("%B(%A+0x%lx): Overflow whilst splitting 0x%lx for group relocation %s"), input_bfd, input_section, - (long) rel->r_offset, abs (signed_value), howto->name); + (long) rel->r_offset, llabs (signed_value), howto->name); return bfd_reloc_overflow; } @@ -6160,7 +6160,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, /* Calculate the value of the relevant G_{n-1} to obtain the residual at that stage. */ - calculate_group_reloc_mask (abs (signed_value), group - 1, &residual); + calculate_group_reloc_mask (llabs (signed_value), group - 1, &residual); /* Check for overflow. */ if (residual >= 0x100) @@ -6168,7 +6168,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, (*_bfd_error_handler) (_("%B(%A+0x%lx): Overflow whilst splitting 0x%lx for group relocation %s"), input_bfd, input_section, - (long) rel->r_offset, abs (signed_value), howto->name); + (long) rel->r_offset, llabs (signed_value), howto->name); return bfd_reloc_overflow; } @@ -6243,7 +6243,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, /* Calculate the value of the relevant G_{n-1} to obtain the residual at that stage. */ - calculate_group_reloc_mask (abs (signed_value), group - 1, &residual); + calculate_group_reloc_mask (llabs (signed_value), group - 1, &residual); /* Check for overflow. (The absolute value to go in the place must be divisible by four and, after having been divided by four, must @@ -6253,7 +6253,7 @@ elf32_arm_final_link_relocate (reloc_howto_type * howto, (*_bfd_error_handler) (_("%B(%A+0x%lx): Overflow whilst splitting 0x%lx for group relocation %s"), input_bfd, input_section, - (long) rel->r_offset, abs (signed_value), howto->name); + (long) rel->r_offset, llabs (signed_value), howto->name); return bfd_reloc_overflow; } diff --git a/contrib/gcc/config/arm/lib1funcs.asm b/contrib/gcc/config/arm/lib1funcs.asm index 9245b3ca0c0d2..c29f89496cf2b 100644 --- a/contrib/gcc/config/arm/lib1funcs.asm +++ b/contrib/gcc/config/arm/lib1funcs.asm @@ -980,8 +980,6 @@ LSYM(Lover12): RET - FUNC_END aeabi_ldiv0 - FUNC_END aeabi_idiv0 FUNC_END div0 #endif /* L_divmodsi_tools */ diff --git a/contrib/ofed/librdmacm/examples/build/cmatose/Makefile b/contrib/ofed/librdmacm/examples/build/cmatose/Makefile index c4515ac33ebc0..31d2ae7f7f6bd 100644 --- a/contrib/ofed/librdmacm/examples/build/cmatose/Makefile +++ b/contrib/ofed/librdmacm/examples/build/cmatose/Makefile @@ -6,5 +6,6 @@ PROG= cmatose MAN= SRCS= cmatose.c LDADD+= -libverbs -lrdmacm -lpthread +LDADD+= -lmlx4 .include <bsd.prog.mk> diff --git a/contrib/ofed/librdmacm/examples/build/mckey/Makefile b/contrib/ofed/librdmacm/examples/build/mckey/Makefile index d4ad2954d52bc..4abaf2786d56d 100644 --- a/contrib/ofed/librdmacm/examples/build/mckey/Makefile +++ b/contrib/ofed/librdmacm/examples/build/mckey/Makefile @@ -6,5 +6,6 @@ PROG= mckey MAN= SRCS= mckey.c LDADD+= -libverbs -lrdmacm -lpthread +LDADD+= -lmlx4 .include <bsd.prog.mk> diff --git a/contrib/ofed/librdmacm/examples/build/rping/Makefile b/contrib/ofed/librdmacm/examples/build/rping/Makefile index 05ef562400e9b..b167a543c092d 100644 --- a/contrib/ofed/librdmacm/examples/build/rping/Makefile +++ b/contrib/ofed/librdmacm/examples/build/rping/Makefile @@ -6,5 +6,6 @@ PROG= rping MAN= SRCS= rping.c LDADD+= -libverbs -lrdmacm -lpthread +LDADD+= -lmlx4 .include <bsd.prog.mk> diff --git a/contrib/ofed/librdmacm/examples/build/udaddy/Makefile b/contrib/ofed/librdmacm/examples/build/udaddy/Makefile index 16d941974b654..1e325505bc865 100644 --- a/contrib/ofed/librdmacm/examples/build/udaddy/Makefile +++ b/contrib/ofed/librdmacm/examples/build/udaddy/Makefile @@ -6,5 +6,6 @@ PROG= udaddy MAN= SRCS= udaddy.c LDADD+= -libverbs -lrdmacm -lpthread +LDADD+= -lmlx4 .include <bsd.prog.mk> diff --git a/contrib/ofed/management/infiniband-diags/src/ibnetdiscover.c b/contrib/ofed/management/infiniband-diags/src/ibnetdiscover.c index 2cfaa8ae55efe..5e092f7b0877f 100644 --- a/contrib/ofed/management/infiniband-diags/src/ibnetdiscover.c +++ b/contrib/ofed/management/infiniband-diags/src/ibnetdiscover.c @@ -50,7 +50,7 @@ #include <infiniband/common.h> #include <infiniband/umad.h> #include <infiniband/mad.h> -#include <infiniband/complib/cl_nodenamemap.h> +#include <complib/cl_nodenamemap.h> #include "ibnetdiscover.h" #include "grouping.h" diff --git a/contrib/ofed/management/infiniband-diags/src/ibroute.c b/contrib/ofed/management/infiniband-diags/src/ibroute.c index f2ee170cedc0a..9607aa9999a53 100644 --- a/contrib/ofed/management/infiniband-diags/src/ibroute.c +++ b/contrib/ofed/management/infiniband-diags/src/ibroute.c @@ -49,7 +49,7 @@ #include <infiniband/common.h> #include <infiniband/umad.h> #include <infiniband/mad.h> -#include <infiniband/complib/cl_nodenamemap.h> +#include <complib/cl_nodenamemap.h> #include "ibdiag_common.h" diff --git a/contrib/ofed/management/infiniband-diags/src/ibsendtrap.c b/contrib/ofed/management/infiniband-diags/src/ibsendtrap.c index 66620de62e2bc..2754dc008dea2 100644 --- a/contrib/ofed/management/infiniband-diags/src/ibsendtrap.c +++ b/contrib/ofed/management/infiniband-diags/src/ibsendtrap.c @@ -43,7 +43,7 @@ #include <getopt.h> #include <infiniband/mad.h> -#include <infiniband/iba/ib_types.h> +#include <iba/ib_types.h> #include "ibdiag_common.h" diff --git a/contrib/ofed/management/infiniband-diags/src/ibtracert.c b/contrib/ofed/management/infiniband-diags/src/ibtracert.c index bde0ea7953265..f8ecf76483ccc 100644 --- a/contrib/ofed/management/infiniband-diags/src/ibtracert.c +++ b/contrib/ofed/management/infiniband-diags/src/ibtracert.c @@ -49,7 +49,7 @@ #include <infiniband/common.h> #include <infiniband/umad.h> #include <infiniband/mad.h> -#include <infiniband/complib/cl_nodenamemap.h> +#include <complib/cl_nodenamemap.h> #include "ibdiag_common.h" diff --git a/contrib/ofed/management/infiniband-diags/src/saquery.c b/contrib/ofed/management/infiniband-diags/src/saquery.c index 97663d1d538fb..9f882e73a401b 100644 --- a/contrib/ofed/management/infiniband-diags/src/saquery.c +++ b/contrib/ofed/management/infiniband-diags/src/saquery.c @@ -50,12 +50,12 @@ #include <getopt.h> #include <infiniband/mad.h> -#include <infiniband/opensm/osm_log.h> -#include <infiniband/vendor/osm_vendor_api.h> -#include <infiniband/vendor/osm_vendor_sa_api.h> -#include <infiniband/opensm/osm_mad_pool.h> -#include <infiniband/complib/cl_debug.h> -#include <infiniband/complib/cl_nodenamemap.h> +#include <opensm/osm_log.h> +#include <vendor/osm_vendor_api.h> +#include <vendor/osm_vendor_sa_api.h> +#include <opensm/osm_mad_pool.h> +#include <complib/cl_debug.h> +#include <complib/cl_nodenamemap.h> #include <netinet/in.h> diff --git a/contrib/ofed/management/infiniband-diags/src/smpquery.c b/contrib/ofed/management/infiniband-diags/src/smpquery.c index ed8ec83c9a060..01ccfaf164129 100644 --- a/contrib/ofed/management/infiniband-diags/src/smpquery.c +++ b/contrib/ofed/management/infiniband-diags/src/smpquery.c @@ -50,7 +50,7 @@ #include <infiniband/common.h> #include <infiniband/umad.h> #include <infiniband/mad.h> -#include <infiniband/complib/cl_nodenamemap.h> +#include <complib/cl_nodenamemap.h> #include "ibdiag_common.h" diff --git a/contrib/ofed/usr.bin/Makefile.inc b/contrib/ofed/usr.bin/Makefile.inc index e6a0550b82d31..d05778c68d375 100644 --- a/contrib/ofed/usr.bin/Makefile.inc +++ b/contrib/ofed/usr.bin/Makefile.inc @@ -1,4 +1,9 @@ DIAGPATH= ${.CURDIR}/../../management/infiniband-diags BINDIR?= /usr/bin CFLAGS+= -I${.CURDIR}/../../include/infiniband +CFLAGS+= -I${.CURDIR}/../../include CFLAGS+= -I${.CURDIR}/../../management/opensm/include/ +CFLAGS+= -I${.CURDIR}/../../management/opensm +CFLAGS+= -I${.CURDIR}/../../management/libibcommon/include +CFLAGS+= -I${.CURDIR}/../../management/libibumad/include +CFLAGS+= -I${.CURDIR}/../../management/libibmad/include diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf index 645a2542848b9..4519e87f84582 100644 --- a/etc/defaults/rc.conf +++ b/etc/defaults/rc.conf @@ -390,12 +390,6 @@ gateway_enable="NO" # Set to YES if this host will be a gateway. routed_enable="NO" # Set to YES to enable a routing daemon. routed_program="/sbin/routed" # Name of routing daemon to use if enabled. routed_flags="-q" # Flags for routing daemon. -mrouted_enable="NO" # Do IPv4 multicast routing. -mrouted_program="/usr/local/sbin/mrouted" # Name of IPv4 multicast - # routing daemon. You need to - # install it from package or - # port. -mrouted_flags="" # Flags for multicast routing daemon. arpproxy_all="NO" # replaces obsolete kernel option ARP_PROXYALL. forward_sourceroute="NO" # do source routing (only if gateway_enable is set to "YES") accept_sourceroute="NO" # accept source routed packets to us @@ -69,19 +69,16 @@ fi # and to make the configuration file variables available to rc itself. # . /etc/rc.subr -load_rc_config 'XXX' +load_rc_config # If we receive a SIGALRM, re-source /etc/rc.conf; this allows rc.d # scripts to perform "boot-time configuration" including enabling and # disabling rc.d scripts which appear later in the boot order. -trap "_rc_conf_loaded=false; load_rc_config 'XXX'" ALRM +trap "_rc_conf_loaded=false; load_rc_config" ALRM skip="-s nostart" if [ `/sbin/sysctl -n security.jail.jailed` -eq 1 ]; then skip="$skip -s nojail" - if [ "$early_late_divider" = "FILESYSTEMS" ]; then - early_late_divider=NETWORKING - fi if [ `/sbin/sysctl -n security.jail.vnet` -ne 1 ]; then skip="$skip -s nojailvnet" fi diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index f26c190fe9dd2..4994e9da8aa8f 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -90,7 +90,6 @@ FILES= DAEMON \ mountd \ moused \ mroute6d \ - mrouted \ msgs \ natd \ netif \ diff --git a/etc/rc.d/NETWORKING b/etc/rc.d/NETWORKING index 12dd3b0d71b5c..750d78af67d9b 100755 --- a/etc/rc.d/NETWORKING +++ b/etc/rc.d/NETWORKING @@ -5,7 +5,7 @@ # PROVIDE: NETWORKING NETWORK # REQUIRE: netif netoptions routing ppp ipfw stf -# REQUIRE: defaultroute routed mrouted route6d mroute6d resolv bridge +# REQUIRE: defaultroute routed route6d mroute6d resolv bridge # REQUIRE: static_arp static_ndp local_unbound # This is a dummy dependency, for services which require networking diff --git a/etc/rc.d/ipmon b/etc/rc.d/ipmon index cdd87426695c3..8e42a186f4386 100755 --- a/etc/rc.d/ipmon +++ b/etc/rc.d/ipmon @@ -4,7 +4,7 @@ # # PROVIDE: ipmon -# REQUIRE: FILESYSTEMS hostname sysctl FILESYSTEMS ipfilter +# REQUIRE: FILESYSTEMS hostname sysctl ipfilter # BEFORE: SERVERS # KEYWORD: nojail diff --git a/etc/rc.d/mrouted b/etc/rc.d/mrouted deleted file mode 100755 index d15299f35ce18..0000000000000 --- a/etc/rc.d/mrouted +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -# -# $FreeBSD$ -# - -# PROVIDE: mrouted -# REQUIRE: netif routing FILESYSTEMS -# KEYWORD: nojail - -. /etc/rc.subr - -name="mrouted" -rcvar="mrouted_enable" -command="/usr/local/sbin/${name}" -pidfile="/var/run/${name}.pid" -required_files="/etc/${name}.conf" -extra_commands="reload" - -load_rc_config $name -run_rc_command "$1" diff --git a/etc/rc.d/pflog b/etc/rc.d/pflog index 7647ebf676e7d..3deceabf2ef9a 100755 --- a/etc/rc.d/pflog +++ b/etc/rc.d/pflog @@ -4,7 +4,7 @@ # # PROVIDE: pflog -# REQUIRE: FILESYSTEMS netif FILESYSTEMS +# REQUIRE: FILESYSTEMS netif # KEYWORD: nojail . /etc/rc.subr diff --git a/etc/rc.subr b/etc/rc.subr index 6534f6858f65c..7b1e38737c104 100644 --- a/etc/rc.subr +++ b/etc/rc.subr @@ -1315,9 +1315,6 @@ load_rc_config() { local _name _rcvar_val _var _defval _v _msg _new _d _name=$1 - if [ -z "$_name" ]; then - err 3 'USAGE: load_rc_config name' - fi if ${_rc_conf_loaded:-false}; then : @@ -1333,20 +1330,24 @@ load_rc_config() _rc_conf_loaded=true fi - for _d in /etc ${local_startup%*/rc.d}; do - if [ -f ${_d}/rc.conf.d/"$_name" ]; then - debug "Sourcing ${_d}/rc.conf.d/$_name" - . ${_d}/rc.conf.d/"$_name" - elif [ -d ${_d}/rc.conf.d/"$_name" ] ; then - local _rc - for _rc in ${_d}/rc.conf.d/"$_name"/* ; do - if [ -f "$_rc" ] ; then - debug "Sourcing $_rc" - . "$_rc" - fi - done - fi - done + # If a service name was specified, attempt to load + # service-specific configuration + if [ -n "$_name" ] ; then + for _d in /etc ${local_startup%*/rc.d}; do + if [ -f ${_d}/rc.conf.d/"$_name" ]; then + debug "Sourcing ${_d}/rc.conf.d/$_name" + . ${_d}/rc.conf.d/"$_name" + elif [ -d ${_d}/rc.conf.d/"$_name" ] ; then + local _rc + for _rc in ${_d}/rc.conf.d/"$_name"/* ; do + if [ -f "$_rc" ] ; then + debug "Sourcing $_rc" + . "$_rc" + fi + done + fi + done + fi # Set defaults if defined. for _var in $rcvar $rcvars; do diff --git a/sbin/ifconfig/Makefile b/sbin/ifconfig/Makefile index 885f8a9922bd2..572c0613904cb 100644 --- a/sbin/ifconfig/Makefile +++ b/sbin/ifconfig/Makefile @@ -30,7 +30,7 @@ SRCS+= ifmac.c # MAC support SRCS+= ifmedia.c # SIOC[GS]IFMEDIA support SRCS+= iffib.c # non-default FIB support SRCS+= ifvlan.c # SIOC[GS]ETVLAN support -SRCS+= ifvxlan.c # VXLAN support +#SRCS+= ifvxlan.c # VXLAN support SRCS+= ifgre.c # GRE keys etc SRCS+= ifgif.c # GIF reversed header workaround diff --git a/share/man/man4/tap.4 b/share/man/man4/tap.4 index d68dab30b037a..229de6028eb32 100644 --- a/share/man/man4/tap.4 +++ b/share/man/man4/tap.4 @@ -1,7 +1,7 @@ .\" $FreeBSD$ .\" Based on PR#2411 .\" -.Dd November 4, 2014 +.Dd November 30, 2014 .Dt TAP 4 .Os .Sh NAME @@ -34,6 +34,17 @@ or a terminal for and a character-special device .Dq control interface. +A client program transfers Ethernet frames to or from the +.Nm +.Dq control +interface. +The +.Xr tun 4 +interface provides similar functionality at the network layer: +a client will transfer IP (by default) packets to or from a +.Xr tun 4 +.Dq control +interface. .Pp The network interfaces are named .Dq Li tap0 , @@ -314,4 +325,5 @@ VMware .El .Sh SEE ALSO .Xr inet 4 , -.Xr intro 4 +.Xr intro 4 , +.Xr tun 4 diff --git a/share/man/man4/tun.4 b/share/man/man4/tun.4 index b5139a7cdf0db..0b88b7213b079 100644 --- a/share/man/man4/tun.4 +++ b/share/man/man4/tun.4 @@ -2,7 +2,7 @@ .\" $FreeBSD$ .\" Based on PR#2411 .\" -.Dd February 4, 2007 +.Dd November 30, 2014 .Dt TUN 4 .Os .Sh NAME @@ -35,6 +35,17 @@ or a terminal for and a character-special device .Dq control interface. +A client program transfers IP (by default) packets to or from the +.Nm +.Dq control +interface. +The +.Xr tap 4 +interface provides similar functionality at the Ethernet layer: +a client will transfer Ethernet frames to or from a +.Xr tap 4 +.Dq control +interface. .Pp The network interfaces are named .Dq Li tun0 , @@ -307,6 +318,7 @@ them pile up. .Xr inet 4 , .Xr intro 4 , .Xr pty 4 , +.Xr tap 4 , .Xr ifconfig 8 .Sh AUTHORS This manual page was originally obtained from diff --git a/sys/arm/arm/cpufunc_asm_armv5.S b/sys/arm/arm/cpufunc_asm_armv5.S index 7435b462e51bd..582ea504c14f5 100644 --- a/sys/arm/arm/cpufunc_asm_armv5.S +++ b/sys/arm/arm/cpufunc_asm_armv5.S @@ -194,7 +194,6 @@ ENTRY(armv5_idcache_wbinv_range) END(armv5_idcache_wbinv_range) ENTRY_NP(armv5_idcache_wbinv_all) -armv5_idcache_wbinv_all: .Larmv5_idcache_wbinv_all: /* * We assume that the code here can never be out of sync with the diff --git a/sys/arm/arm/cpufunc_asm_xscale_c3.S b/sys/arm/arm/cpufunc_asm_xscale_c3.S index 6766a327156a4..78fa2d578dcf4 100644 --- a/sys/arm/arm/cpufunc_asm_xscale_c3.S +++ b/sys/arm/arm/cpufunc_asm_xscale_c3.S @@ -144,7 +144,6 @@ __FBSDID("$FreeBSD$"); ENTRY_NP(xscalec3_cache_syncI) -xscalec3_cache_purgeID: EENTRY_NP(xscalec3_cache_purgeID) mcr p15, 0, r0, c7, c5, 0 /* flush I cache (D cleaned below) */ EENTRY_NP(xscalec3_cache_cleanID) diff --git a/sys/arm/arm/fusu.S b/sys/arm/arm/fusu.S index c70215c297dbe..b55f443c2f9f7 100644 --- a/sys/arm/arm/fusu.S +++ b/sys/arm/arm/fusu.S @@ -129,7 +129,7 @@ EENTRY_NP(fuword32) str r1, [r2, #PCB_ONFAULT] mov r0, r3 RET -END(fuword32) +EEND(fuword32) END(fuword) /* @@ -295,7 +295,7 @@ EENTRY_NP(suword32) mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET -END(suword32) +EEND(suword32) END(suword) /* diff --git a/sys/arm/arm/support.S b/sys/arm/arm/support.S index 2a6eec9828e07..1714b0f0d203a 100644 --- a/sys/arm/arm/support.S +++ b/sys/arm/arm/support.S @@ -130,7 +130,7 @@ ENTRY(bzero) .Lnormal0: mov r3, #0x00 b do_memset -EEND(bzero) +END(bzero) /* LINTSTUB: Func: void *memset(void *, int, size_t) */ ENTRY(memset) and r3, r1, #0xff /* We deal with bytes */ diff --git a/sys/arm/include/asm.h b/sys/arm/include/asm.h index 9577001917b26..bd4b6361f3cd1 100644 --- a/sys/arm/include/asm.h +++ b/sys/arm/include/asm.h @@ -58,6 +58,18 @@ #endif /* + * EENTRY()/EEND() mark "extra" entry/exit points from a function. + * The unwind info cannot handle the concept of a nested function, or a function + * with multiple .fnstart directives, but some of our assembler code is written + * with multiple labels to allow entry at several points. The EENTRY() macro + * defines such an extra entry point without a new .fnstart, so that it's + * basically just a label that you can jump to. The EEND() macro does nothing + * at all, except document the exit point associated with the same-named entry. + */ +#define _EENTRY(x) .globl x; .type x,_ASM_TYPE_FUNCTION; x: +#define _EEND(x) /* nothing */ + +/* * gas/arm uses @ as a single comment character and thus cannot be used here * Instead it recognised the # instead of an @ symbols in .type directives * We define a couple of macros so that assembly code will not be dependent @@ -66,22 +78,10 @@ #define _ASM_TYPE_FUNCTION #function #define _ASM_TYPE_OBJECT #object #define GLOBAL(X) .globl x -#define _ENTRY(x) \ - .text; _ALIGN_TEXT; .globl x; .type x,_ASM_TYPE_FUNCTION; x: _FNSTART +#define _ENTRY(x) \ + .text; _ALIGN_TEXT; _EENTRY(x) _FNSTART #define _END(x) .size x, . - x; _FNEND -/* - * EENTRY()/EEND() mark "extra" entry/exit points from a function. - * The unwind info cannot handle the concept of a nested function, or a function - * with multiple .fnstart directives, but some of our assembler code is written - * with multiple labels to allow entry at several points. The EENTRY() macro - * defines such an extra entry point without a new .fnstart, so that it's - * basically just a label that you can jump to. The EEND() macro does nothing - * at all, except document the exit point associated with the same-named entry. - */ -#define _EENTRY(x) .globl x; .type x,_ASM_TYPE_FUNCTION; x: -#define _EEND(x) /* nothing */ - #ifdef GPROF # define _PROF_PROLOGUE \ mov ip, lr; bl __mcount diff --git a/sys/boot/i386/boot2/boot2.c b/sys/boot/i386/boot2/boot2.c index 001126fb83f85..75abc7e895e79 100644 --- a/sys/boot/i386/boot2/boot2.c +++ b/sys/boot/i386/boot2/boot2.c @@ -418,7 +418,7 @@ parse() #if SERIAL } else if (c == 'S') { j = 0; - while ((i = *arg++ - '0') <= 9) + while ((unsigned int)(i = *arg++ - '0') <= 9) j = j * 10 + i; if (j > 0 && i == -'0') { comspeed = j; diff --git a/sys/dev/bxe/bxe.c b/sys/dev/bxe/bxe.c index a27b3e7a56bd9..3509f2d023df1 100644 --- a/sys/dev/bxe/bxe.c +++ b/sys/dev/bxe/bxe.c @@ -3219,7 +3219,7 @@ bxe_tpa_stop(struct bxe_softc *sc, #if __FreeBSD_version >= 800000 /* specify what RSS queue was used for this flow */ m->m_pkthdr.flowid = fp->index; - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); #endif if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); @@ -3454,7 +3454,7 @@ bxe_rxeof(struct bxe_softc *sc, #if __FreeBSD_version >= 800000 /* specify what RSS queue was used for this flow */ m->m_pkthdr.flowid = fp->index; - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); #endif next_rx: @@ -6037,10 +6037,9 @@ bxe_tx_mq_start(struct ifnet *ifp, fp_index = 0; /* default is the first queue */ - /* change the queue if using flow ID */ - if ((m->m_flags & M_FLOWID) != 0) { + /* check if flowid is set */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) fp_index = (m->m_pkthdr.flowid % sc->num_queues); - } fp = &sc->fp[fp_index]; diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c index a7f39a5101097..2b08183b0cd57 100644 --- a/sys/dev/cxgb/cxgb_sge.c +++ b/sys/dev/cxgb/cxgb_sge.c @@ -1733,8 +1733,9 @@ cxgb_transmit(struct ifnet *ifp, struct mbuf *m) m_freem(m); return (0); } - - if (m->m_flags & M_FLOWID) + + /* check if flowid is set */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset; qs = &pi->adapter->sge.qs[qidx]; @@ -2899,9 +2900,10 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget) eop = get_packet(adap, drop_thresh, qs, mh, r); if (eop) { - if (r->rss_hdr.hash_type && !adap->timestamp) - mh->mh_head->m_flags |= M_FLOWID; - mh->mh_head->m_pkthdr.flowid = rss_hash; + if (r->rss_hdr.hash_type && !adap->timestamp) { + M_HASHTYPE_SET(mh->mh_head, M_HASHTYPE_OPAQUE); + mh->mh_head->m_pkthdr.flowid = rss_hash; + } } ethpad = 2; diff --git a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c index 81a446a64a4fc..a4b67088db93d 100644 --- a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c +++ b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c @@ -1199,7 +1199,7 @@ do_rx_data(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) } toep->tp_enqueued += m->m_pkthdr.len; - sbappendstream_locked(so_rcv, m); + sbappendstream_locked(so_rcv, m, 0); sorwakeup_locked(so); SOCKBUF_UNLOCK_ASSERT(so_rcv); diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index f9caf2eec1d81..2c384fd0f1a6e 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -1440,7 +1440,8 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) return (ENETDOWN); } - if (m->m_flags & M_FLOWID) + /* check if flowid is set */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + pi->rsrv_noflowq); br = txq->br; diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index 499f25205998f..58940fe4959c4 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -1734,7 +1734,7 @@ t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) m0->m_data += fl_pktshift; m0->m_pkthdr.rcvif = ifp; - m0->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m0, M_HASHTYPE_OPAQUE); m0->m_pkthdr.flowid = be32toh(rss->hash_val); if (cpl->csum_calc && !cpl->err_vec) { diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 29e5fa243be5b..5dc8434284484 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -1086,7 +1086,7 @@ do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) #ifdef USE_DDP_RX_FLOW_CONTROL toep->rx_credits -= m->m_len; /* adjust for F_RX_FC_DDP */ #endif - sbappendstream_locked(sb, m); + sbappendstream_locked(sb, m, 0); toep->sb_cc = sbused(sb); } socantrcvmore_locked(so); /* unlocks the sockbuf */ @@ -1586,7 +1586,7 @@ do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) ("%s: sb %p has more data (%d) than last time (%d).", __func__, sb, sbused(sb), toep->sb_cc)); toep->rx_credits += toep->sb_cc - sbused(sb); - sbappendstream_locked(sb, m); + sbappendstream_locked(sb, m, 0); toep->sb_cc = sbused(sb); sorwakeup_locked(so); SOCKBUF_UNLOCK_ASSERT(sb); diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c index 89585cf830397..17eb071b3cb78 100644 --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -231,7 +231,7 @@ insert_ddp_data(struct toepcb *toep, uint32_t n) #ifdef USE_DDP_RX_FLOW_CONTROL toep->rx_credits -= n; /* adjust for F_RX_FC_DDP */ #endif - sbappendstream_locked(sb, m); + sbappendstream_locked(sb, m, 0); toep->sb_cc = sbused(sb); } @@ -466,7 +466,7 @@ handle_ddp_data(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt, int len) #ifdef USE_DDP_RX_FLOW_CONTROL toep->rx_credits -= len; /* adjust for F_RX_FC_DDP */ #endif - sbappendstream_locked(sb, m); + sbappendstream_locked(sb, m, 0); toep->sb_cc = sbused(sb); wakeup: KASSERT(toep->ddp_flags & db_flag, @@ -971,8 +971,9 @@ handle_ddp(struct socket *so, struct uio *uio, int flags, int error) */ rc = sbwait(sb); while (toep->ddp_flags & buf_flag) { + /* XXXGL: shouldn't here be sbwait() call? */ sb->sb_flags |= SB_WAIT; - msleep(&sb->sb_cc, &sb->sb_mtx, PSOCK , "sbwait", 0); + msleep(&sb->sb_acc, &sb->sb_mtx, PSOCK , "sbwait", 0); } unwire_ddp_buffer(db); return (rc); diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c index 335aa01d95c9e..75f0fb419d50c 100644 --- a/sys/dev/e1000/if_igb.c +++ b/sys/dev/e1000/if_igb.c @@ -990,7 +990,7 @@ igb_mq_start(struct ifnet *ifp, struct mbuf *m) * If everything is setup correctly, it should be the * same bucket that the current CPU we're on is. */ - if ((m->m_flags & M_FLOWID) != 0) { + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { #ifdef RSS if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), &bucket_id) == 0) { @@ -5166,7 +5166,6 @@ igb_rxeof(struct igb_queue *que, int count, int *done) /* XXX set flowtype once this works right */ rxr->fmp->m_pkthdr.flowid = le32toh(cur->wb.lower.hi_dword.rss); - rxr->fmp->m_flags |= M_FLOWID; switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { case E1000_RXDADV_RSSTYPE_IPV4_TCP: M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_RSS_TCP_IPV4); @@ -5196,11 +5195,11 @@ igb_rxeof(struct igb_queue *que, int count, int *done) default: /* XXX fallthrough */ - M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_NONE); + M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE); } #elif !defined(IGB_LEGACY_TX) rxr->fmp->m_pkthdr.flowid = que->msix; - rxr->fmp->m_flags |= M_FLOWID; + M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE); #endif sendmp = rxr->fmp; /* Make sure to set M_PKTHDR. */ diff --git a/sys/dev/ixgbe/ixgbe.c b/sys/dev/ixgbe/ixgbe.c index 9ac70eeac5e45..2fd126c5396ee 100644 --- a/sys/dev/ixgbe/ixgbe.c +++ b/sys/dev/ixgbe/ixgbe.c @@ -833,7 +833,7 @@ ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) * If everything is setup correctly, it should be the * same bucket that the current CPU we're on is. */ - if ((m->m_flags & M_FLOWID) != 0) { + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { #ifdef RSS if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), &bucket_id) == 0) { @@ -4764,7 +4764,6 @@ ixgbe_rxeof(struct ix_queue *que) #ifdef RSS sendmp->m_pkthdr.flowid = le32toh(cur->wb.lower.hi_dword.rss); - sendmp->m_flags |= M_FLOWID; switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) { case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV4); @@ -4795,11 +4794,12 @@ ixgbe_rxeof(struct ix_queue *que) break; default: /* XXX fallthrough */ - M_HASHTYPE_SET(sendmp, M_HASHTYPE_NONE); + M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); + break; } #else /* RSS */ sendmp->m_pkthdr.flowid = que->msix; - sendmp->m_flags |= M_FLOWID; + M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); #endif /* RSS */ #endif /* FreeBSD_version */ } diff --git a/sys/dev/ixgbe/ixv.c b/sys/dev/ixgbe/ixv.c index 1d77e8b5a27a5..edbdfca859204 100644 --- a/sys/dev/ixgbe/ixv.c +++ b/sys/dev/ixgbe/ixv.c @@ -580,7 +580,7 @@ ixv_mq_start(struct ifnet *ifp, struct mbuf *m) int i = 0, err = 0; /* Which queue to use */ - if ((m->m_flags & M_FLOWID) != 0) + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) i = m->m_pkthdr.flowid % adapter->num_queues; txr = &adapter->tx_rings[i]; @@ -3464,7 +3464,7 @@ ixv_rxeof(struct ix_queue *que, int count) ixv_rx_checksum(staterr, sendmp, ptype); #if __FreeBSD_version >= 800000 sendmp->m_pkthdr.flowid = que->msix; - sendmp->m_flags |= M_FLOWID; + M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); #endif } next_desc: diff --git a/sys/dev/ixl/ixl_txrx.c b/sys/dev/ixl/ixl_txrx.c index 6322f2c9fd0ec..f36b8318961d0 100755 --- a/sys/dev/ixl/ixl_txrx.c +++ b/sys/dev/ixl/ixl_txrx.c @@ -66,8 +66,8 @@ ixl_mq_start(struct ifnet *ifp, struct mbuf *m) struct tx_ring *txr; int err, i; - /* Which queue to use */ - if ((m->m_flags & M_FLOWID) != 0) + /* check if flowid is set */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) i = m->m_pkthdr.flowid % vsi->num_queues; else i = curcpu % vsi->num_queues; @@ -1543,7 +1543,7 @@ ixl_rxeof(struct ixl_queue *que, int count) if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) ixl_rx_checksum(sendmp, status, error, ptype); sendmp->m_pkthdr.flowid = que->msix; - sendmp->m_flags |= M_FLOWID; + M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); } next_desc: bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, diff --git a/sys/dev/mxge/if_mxge.c b/sys/dev/mxge/if_mxge.c index e97ae5c7130ee..fe8c08a72495e 100644 --- a/sys/dev/mxge/if_mxge.c +++ b/sys/dev/mxge/if_mxge.c @@ -2719,7 +2719,7 @@ mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, /* flowid only valid if RSS hashing is enabled */ if (sc->num_slices > 1) { m->m_pkthdr.flowid = (ss - sc->ss); - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); } /* pass the frame up the stack */ (*ifp->if_input)(ifp, m); @@ -2787,7 +2787,7 @@ mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, /* flowid only valid if RSS hashing is enabled */ if (sc->num_slices > 1) { m->m_pkthdr.flowid = (ss - sc->ss); - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); } /* pass the frame up the stack */ (*ifp->if_input)(ifp, m); diff --git a/sys/dev/netmap/netmap_freebsd.c b/sys/dev/netmap/netmap_freebsd.c index 3226708145534..dd968ca515772 100644 --- a/sys/dev/netmap/netmap_freebsd.c +++ b/sys/dev/netmap/netmap_freebsd.c @@ -204,7 +204,7 @@ netmap_catch_tx(struct netmap_generic_adapter *gna, int enable) * of the transmission does not consume resources. * * On FreeBSD, and on multiqueue cards, we can force the queue using - * if ((m->m_flags & M_FLOWID) != 0) + * if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) * i = m->m_pkthdr.flowid % adapter->num_queues; * else * i = curcpu % adapter->num_queues; @@ -240,7 +240,7 @@ generic_xmit_frame(struct ifnet *ifp, struct mbuf *m, m->m_len = m->m_pkthdr.len = len; // inc refcount. All ours, we could skip the atomic atomic_fetchadd_int(PNT_MBUF_REFCNT(m), 1); - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); m->m_pkthdr.flowid = ring_nr; m->m_pkthdr.rcvif = ifp; /* used for tx notification */ ret = NA(ifp)->if_transmit(ifp, m); diff --git a/sys/dev/oce/oce_if.c b/sys/dev/oce/oce_if.c index fe48007e89737..062181b2367bd 100644 --- a/sys/dev/oce/oce_if.c +++ b/sys/dev/oce/oce_if.c @@ -563,7 +563,7 @@ oce_multiq_start(struct ifnet *ifp, struct mbuf *m) int queue_index = 0; int status = 0; - if ((m->m_flags & M_FLOWID) != 0) + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) queue_index = m->m_pkthdr.flowid % sc->nwqs; wq = sc->wq[queue_index]; @@ -1374,7 +1374,7 @@ oce_rx(struct oce_rq *rq, uint32_t rqe_idx, struct oce_nic_rx_cqe *cqe) m->m_pkthdr.flowid = (rq->queue_index - 1); else m->m_pkthdr.flowid = rq->queue_index; - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); #endif /* This deternies if vlan tag is Valid */ if (oce_cqe_vtp_valid(sc, cqe)) { diff --git a/sys/dev/qlxgbe/ql_isr.c b/sys/dev/qlxgbe/ql_isr.c index db00298229273..dee8e198e8f44 100644 --- a/sys/dev/qlxgbe/ql_isr.c +++ b/sys/dev/qlxgbe/ql_isr.c @@ -159,7 +159,7 @@ qla_rx_intr(qla_host_t *ha, qla_sgl_rcv_t *sgc, uint32_t sds_idx) if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); mpf->m_pkthdr.flowid = sgc->rss_hash; - mpf->m_flags |= M_FLOWID; + M_HASHTYPE_SET(mpf, M_HASHTYPE_OPAQUE); (*ifp->if_input)(ifp, mpf); @@ -324,7 +324,7 @@ qla_lro_intr(qla_host_t *ha, qla_sgl_lro_t *sgc, uint32_t sds_idx) mpf->m_pkthdr.csum_data = 0xFFFF; mpf->m_pkthdr.flowid = sgc->rss_hash; - mpf->m_flags |= M_FLOWID; + M_HASHTYPE_SET(mpf, M_HASHTYPE_OPAQUE); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); diff --git a/sys/dev/qlxgbe/ql_os.c b/sys/dev/qlxgbe/ql_os.c index 1684c070fcd8c..04fbf71955293 100644 --- a/sys/dev/qlxgbe/ql_os.c +++ b/sys/dev/qlxgbe/ql_os.c @@ -1140,7 +1140,8 @@ qla_send(qla_host_t *ha, struct mbuf **m_headp) QL_DPRINT8(ha, (ha->pci_dev, "%s: enter\n", __func__)); - if (m_head->m_flags & M_FLOWID) + /* check if flowid is set */ + if (M_HASHTYPE_GET(m_head) != M_HASHTYPE_NONE) txr_idx = m_head->m_pkthdr.flowid & (ha->hw.num_tx_rings - 1); tx_idx = ha->hw.tx_cntxt[txr_idx].txr_next; diff --git a/sys/dev/qlxge/qls_isr.c b/sys/dev/qlxge/qls_isr.c index 14b4839be913b..03b3a3c521529 100644 --- a/sys/dev/qlxge/qls_isr.c +++ b/sys/dev/qlxge/qls_isr.c @@ -190,7 +190,7 @@ qls_rx_comp(qla_host_t *ha, uint32_t rxr_idx, uint32_t cq_idx, q81_rx_t *cq_e) if ((cq_e->flags1 & Q81_RX_FLAGS1_RSS_MATCH_MASK)) { rxr->rss_int++; mp->m_pkthdr.flowid = cq_e->rss; - mp->m_flags |= M_FLOWID; + M_HASHTYPE_SET(mp, M_HASHTYPE_OPAQUE); } if (cq_e->flags0 & (Q81_RX_FLAGS0_TE | Q81_RX_FLAGS0_NU | Q81_RX_FLAGS0_IE)) { diff --git a/sys/dev/qlxge/qls_os.c b/sys/dev/qlxge/qls_os.c index 4f4097935f0c6..1617d24095e1e 100644 --- a/sys/dev/qlxge/qls_os.c +++ b/sys/dev/qlxge/qls_os.c @@ -1136,7 +1136,8 @@ qls_send(qla_host_t *ha, struct mbuf **m_headp) QL_DPRINT8((ha->pci_dev, "%s: enter\n", __func__)); - if (m_head->m_flags & M_FLOWID) + /* check if flowid is set */ + if (M_HASHTYPE_GET(m_head) != M_HASHTYPE_NONE) txr_idx = m_head->m_pkthdr.flowid & (ha->num_tx_rings - 1); tx_idx = ha->tx_ring[txr_idx].txr_next; diff --git a/sys/dev/sfxge/sfxge_rx.c b/sys/dev/sfxge/sfxge_rx.c index 66083d8328579..ccfdfb023d07e 100644 --- a/sys/dev/sfxge/sfxge_rx.c +++ b/sys/dev/sfxge/sfxge_rx.c @@ -302,7 +302,7 @@ sfxge_rx_deliver(struct sfxge_softc *sc, struct sfxge_rx_sw_desc *rx_desc) if (rx_desc->flags & EFX_PKT_TCP) { m->m_pkthdr.flowid = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ, mtod(m, uint8_t *)); - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); } #endif m->m_data += sc->rx_prefix_size; @@ -353,7 +353,7 @@ sfxge_lro_deliver(struct sfxge_lro_state *st, struct sfxge_lro_conn *c) #ifdef SFXGE_HAVE_MQ m->m_pkthdr.flowid = c->conn_hash; - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); #endif m->m_pkthdr.csum_flags = csum_flags; __sfxge_rx_deliver(sc, m); diff --git a/sys/dev/sfxge/sfxge_tx.c b/sys/dev/sfxge/sfxge_tx.c index a12c74756ea50..3fb76d92b6db0 100644 --- a/sys/dev/sfxge/sfxge_tx.c +++ b/sys/dev/sfxge/sfxge_tx.c @@ -631,7 +631,8 @@ sfxge_if_transmit(struct ifnet *ifp, struct mbuf *m) if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_TSO)) { int index = 0; - if (m->m_flags & M_FLOWID) { + /* check if flowid is set */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { uint32_t hash = m->m_pkthdr.flowid; index = sc->rx_indir_table[hash % SFXGE_RX_SCALE_MAX]; diff --git a/sys/dev/virtio/block/virtio_blk.c b/sys/dev/virtio/block/virtio_blk.c index b15cb75a3f40c..a65e23d37b4c2 100644 --- a/sys/dev/virtio/block/virtio_blk.c +++ b/sys/dev/virtio/block/virtio_blk.c @@ -58,7 +58,6 @@ struct vtblk_request { struct virtio_blk_outhdr vbr_hdr; struct bio *vbr_bp; uint8_t vbr_ack; - TAILQ_ENTRY(vtblk_request) vbr_link; }; @@ -132,53 +131,60 @@ static int vtblk_dump(void *, void *, vm_offset_t, off_t, size_t); static void vtblk_strategy(struct bio *); static void vtblk_negotiate_features(struct vtblk_softc *); +static void vtblk_setup_features(struct vtblk_softc *); static int vtblk_maximum_segments(struct vtblk_softc *, struct virtio_blk_config *); static int vtblk_alloc_virtqueue(struct vtblk_softc *); static void vtblk_resize_disk(struct vtblk_softc *, uint64_t); -static void vtblk_set_write_cache(struct vtblk_softc *, int); -static int vtblk_write_cache_enabled(struct vtblk_softc *sc, - struct virtio_blk_config *); -static int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS); static void vtblk_alloc_disk(struct vtblk_softc *, struct virtio_blk_config *); static void vtblk_create_disk(struct vtblk_softc *); -static int vtblk_quiesce(struct vtblk_softc *); -static void vtblk_startio(struct vtblk_softc *); -static struct vtblk_request * vtblk_bio_request(struct vtblk_softc *); -static int vtblk_execute_request(struct vtblk_softc *, +static int vtblk_request_prealloc(struct vtblk_softc *); +static void vtblk_request_free(struct vtblk_softc *); +static struct vtblk_request * + vtblk_request_dequeue(struct vtblk_softc *); +static void vtblk_request_enqueue(struct vtblk_softc *, + struct vtblk_request *); +static struct vtblk_request * + vtblk_request_next_ready(struct vtblk_softc *); +static void vtblk_request_requeue_ready(struct vtblk_softc *, struct vtblk_request *); +static struct vtblk_request * + vtblk_request_next(struct vtblk_softc *); +static struct vtblk_request * + vtblk_request_bio(struct vtblk_softc *); +static int vtblk_request_execute(struct vtblk_softc *, + struct vtblk_request *); +static int vtblk_request_error(struct vtblk_request *); -static void vtblk_vq_intr(void *); +static void vtblk_queue_completed(struct vtblk_softc *, + struct bio_queue *); +static void vtblk_done_completed(struct vtblk_softc *, + struct bio_queue *); +static void vtblk_drain_vq(struct vtblk_softc *, int); +static void vtblk_drain(struct vtblk_softc *); -static void vtblk_stop(struct vtblk_softc *); +static void vtblk_startio(struct vtblk_softc *); +static void vtblk_bio_done(struct vtblk_softc *, struct bio *, int); static void vtblk_read_config(struct vtblk_softc *, struct virtio_blk_config *); -static void vtblk_get_ident(struct vtblk_softc *); -static void vtblk_prepare_dump(struct vtblk_softc *); -static int vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t); -static int vtblk_flush_dump(struct vtblk_softc *); +static void vtblk_ident(struct vtblk_softc *); static int vtblk_poll_request(struct vtblk_softc *, struct vtblk_request *); +static int vtblk_quiesce(struct vtblk_softc *); +static void vtblk_vq_intr(void *); +static void vtblk_stop(struct vtblk_softc *); -static void vtblk_finish_completed(struct vtblk_softc *); -static void vtblk_drain_vq(struct vtblk_softc *, int); -static void vtblk_drain(struct vtblk_softc *); - -static int vtblk_alloc_requests(struct vtblk_softc *); -static void vtblk_free_requests(struct vtblk_softc *); -static struct vtblk_request * vtblk_dequeue_request(struct vtblk_softc *); -static void vtblk_enqueue_request(struct vtblk_softc *, - struct vtblk_request *); - -static struct vtblk_request * vtblk_dequeue_ready(struct vtblk_softc *); -static void vtblk_enqueue_ready(struct vtblk_softc *, - struct vtblk_request *); +static void vtblk_dump_prepare(struct vtblk_softc *); +static int vtblk_dump_write(struct vtblk_softc *, void *, off_t, size_t); +static int vtblk_dump_flush(struct vtblk_softc *); -static int vtblk_request_error(struct vtblk_request *); -static void vtblk_finish_bio(struct bio *, int); +static void vtblk_set_write_cache(struct vtblk_softc *, int); +static int vtblk_write_cache_enabled(struct vtblk_softc *sc, + struct virtio_blk_config *); +static int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS); static void vtblk_setup_sysctl(struct vtblk_softc *); static int vtblk_tunable_int(struct vtblk_softc *, const char *, int); @@ -290,30 +296,18 @@ vtblk_attach(device_t dev) struct virtio_blk_config blkcfg; int error; + virtio_set_feature_desc(dev, vtblk_feature_desc); + sc = device_get_softc(dev); sc->vtblk_dev = dev; - VTBLK_LOCK_INIT(sc, device_get_nameunit(dev)); - bioq_init(&sc->vtblk_bioq); TAILQ_INIT(&sc->vtblk_req_free); TAILQ_INIT(&sc->vtblk_req_ready); - virtio_set_feature_desc(dev, vtblk_feature_desc); - vtblk_negotiate_features(sc); - - if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) - sc->vtblk_flags |= VTBLK_FLAG_INDIRECT; - if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) - sc->vtblk_flags |= VTBLK_FLAG_READONLY; - if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER)) - sc->vtblk_flags |= VTBLK_FLAG_BARRIER; - if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE)) - sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG; - vtblk_setup_sysctl(sc); + vtblk_setup_features(sc); - /* Get local copy of config. */ vtblk_read_config(sc, &blkcfg); /* @@ -352,7 +346,7 @@ vtblk_attach(device_t dev) goto fail; } - error = vtblk_alloc_requests(sc); + error = vtblk_request_prealloc(sc); if (error) { device_printf(dev, "cannot preallocate requests\n"); goto fail; @@ -519,14 +513,14 @@ vtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, VTBLK_LOCK(sc); if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) { - vtblk_prepare_dump(sc); + vtblk_dump_prepare(sc); sc->vtblk_flags |= VTBLK_FLAG_DUMPING; } if (length > 0) - error = vtblk_write_dump(sc, virtual, offset, length); + error = vtblk_dump_write(sc, virtual, offset, length); else if (virtual == NULL && offset == 0) - error = vtblk_flush_dump(sc); + error = vtblk_dump_flush(sc); else { error = EINVAL; sc->vtblk_flags &= ~VTBLK_FLAG_DUMPING; @@ -543,7 +537,7 @@ vtblk_strategy(struct bio *bp) struct vtblk_softc *sc; if ((sc = bp->bio_disk->d_drv1) == NULL) { - vtblk_finish_bio(bp, EINVAL); + vtblk_bio_done(NULL, bp, EINVAL); return; } @@ -553,37 +547,21 @@ vtblk_strategy(struct bio *bp) */ if (sc->vtblk_flags & VTBLK_FLAG_READONLY && (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) { - vtblk_finish_bio(bp, EROFS); + vtblk_bio_done(sc, bp, EROFS); return; } -#ifdef INVARIANTS - /* - * Prevent read/write buffers spanning too many segments from - * getting into the queue. This should only trip if d_maxsize - * was incorrectly set. - */ - if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { - int nsegs, max_nsegs; - - nsegs = sglist_count(bp->bio_data, bp->bio_bcount); - max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS; + VTBLK_LOCK(sc); - KASSERT(nsegs <= max_nsegs, - ("%s: bio %p spanned too many segments: %d, max: %d", - __func__, bp, nsegs, max_nsegs)); + if (sc->vtblk_flags & VTBLK_FLAG_DETACH) { + VTBLK_UNLOCK(sc); + vtblk_bio_done(sc, bp, ENXIO); + return; } -#endif - VTBLK_LOCK(sc); - if (sc->vtblk_flags & VTBLK_FLAG_DETACH) - vtblk_finish_bio(bp, ENXIO); - else { - bioq_insert_tail(&sc->vtblk_bioq, bp); + bioq_insert_tail(&sc->vtblk_bioq, bp); + vtblk_startio(sc); - if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0) - vtblk_startio(sc); - } VTBLK_UNLOCK(sc); } @@ -599,6 +577,25 @@ vtblk_negotiate_features(struct vtblk_softc *sc) sc->vtblk_features = virtio_negotiate_features(dev, features); } +static void +vtblk_setup_features(struct vtblk_softc *sc) +{ + device_t dev; + + dev = sc->vtblk_dev; + + vtblk_negotiate_features(sc); + + if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) + sc->vtblk_flags |= VTBLK_FLAG_INDIRECT; + if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) + sc->vtblk_flags |= VTBLK_FLAG_READONLY; + if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER)) + sc->vtblk_flags |= VTBLK_FLAG_BARRIER; + if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE)) + sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG; +} + static int vtblk_maximum_segments(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) @@ -660,59 +657,6 @@ vtblk_resize_disk(struct vtblk_softc *sc, uint64_t new_capacity) } static void -vtblk_set_write_cache(struct vtblk_softc *sc, int wc) -{ - - /* Set either writeback (1) or writethrough (0) mode. */ - virtio_write_dev_config_1(sc->vtblk_dev, - offsetof(struct virtio_blk_config, writeback), wc); -} - -static int -vtblk_write_cache_enabled(struct vtblk_softc *sc, - struct virtio_blk_config *blkcfg) -{ - int wc; - - if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) { - wc = vtblk_tunable_int(sc, "writecache_mode", - vtblk_writecache_mode); - if (wc >= 0 && wc < VTBLK_CACHE_MAX) - vtblk_set_write_cache(sc, wc); - else - wc = blkcfg->writeback; - } else - wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE); - - return (wc); -} - -static int -vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS) -{ - struct vtblk_softc *sc; - int wc, error; - - sc = oidp->oid_arg1; - wc = sc->vtblk_write_cache; - - error = sysctl_handle_int(oidp, &wc, 0, req); - if (error || req->newptr == NULL) - return (error); - if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0) - return (EPERM); - if (wc < 0 || wc >= VTBLK_CACHE_MAX) - return (EINVAL); - - VTBLK_LOCK(sc); - sc->vtblk_write_cache = wc; - vtblk_set_write_cache(sc, sc->vtblk_write_cache); - VTBLK_UNLOCK(sc); - - return (0); -} - -static void vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) { device_t dev; @@ -728,7 +672,8 @@ vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) dp->d_name = VTBLK_DISK_NAME; dp->d_unit = device_get_unit(dev); dp->d_drv1 = sc; - dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO; + dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO | + DISKFLAG_DIRECT_COMPLETION; dp->d_hba_vendor = virtio_get_vendor(dev); dp->d_hba_device = virtio_get_device(dev); dp->d_hba_subvendor = virtio_get_subvendor(dev); @@ -789,11 +734,7 @@ vtblk_create_disk(struct vtblk_softc *sc) dp = sc->vtblk_disk; - /* - * Retrieving the identification string must be done after - * the virtqueue interrupt is setup otherwise it will hang. - */ - vtblk_get_ident(sc); + vtblk_ident(sc); device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n", (uintmax_t) dp->d_mediasize >> 20, @@ -804,57 +745,107 @@ vtblk_create_disk(struct vtblk_softc *sc) } static int -vtblk_quiesce(struct vtblk_softc *sc) +vtblk_request_prealloc(struct vtblk_softc *sc) { - int error; + struct vtblk_request *req; + int i, nreqs; - error = 0; + nreqs = virtqueue_size(sc->vtblk_vq); - VTBLK_LOCK_ASSERT(sc); + /* + * Preallocate sufficient requests to keep the virtqueue full. Each + * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce + * the number allocated when indirect descriptors are not available. + */ + if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0) + nreqs /= VTBLK_MIN_SEGMENTS; - while (!virtqueue_empty(sc->vtblk_vq)) { - if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq", - VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) { - error = EBUSY; - break; - } + for (i = 0; i < nreqs; i++) { + req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT); + if (req == NULL) + return (ENOMEM); + + MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1); + MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1); + + sc->vtblk_request_count++; + vtblk_request_enqueue(sc, req); } - return (error); + return (0); } static void -vtblk_startio(struct vtblk_softc *sc) +vtblk_request_free(struct vtblk_softc *sc) { - struct virtqueue *vq; struct vtblk_request *req; - int enq; - vq = sc->vtblk_vq; - enq = 0; + MPASS(TAILQ_EMPTY(&sc->vtblk_req_ready)); - VTBLK_LOCK_ASSERT(sc); + while ((req = vtblk_request_dequeue(sc)) != NULL) { + sc->vtblk_request_count--; + free(req, M_DEVBUF); + } - while (!virtqueue_full(vq)) { - if ((req = vtblk_dequeue_ready(sc)) == NULL) - req = vtblk_bio_request(sc); - if (req == NULL) - break; + KASSERT(sc->vtblk_request_count == 0, + ("%s: leaked %d requests", __func__, sc->vtblk_request_count)); +} - if (vtblk_execute_request(sc, req) != 0) { - vtblk_enqueue_ready(sc, req); - break; - } +static struct vtblk_request * +vtblk_request_dequeue(struct vtblk_softc *sc) +{ + struct vtblk_request *req; - enq++; + req = TAILQ_FIRST(&sc->vtblk_req_free); + if (req != NULL) { + TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link); + bzero(req, sizeof(struct vtblk_request)); } - if (enq > 0) - virtqueue_notify(vq); + return (req); +} + +static void +vtblk_request_enqueue(struct vtblk_softc *sc, struct vtblk_request *req) +{ + + TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link); } static struct vtblk_request * -vtblk_bio_request(struct vtblk_softc *sc) +vtblk_request_next_ready(struct vtblk_softc *sc) +{ + struct vtblk_request *req; + + req = TAILQ_FIRST(&sc->vtblk_req_ready); + if (req != NULL) + TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link); + + return (req); +} + +static void +vtblk_request_requeue_ready(struct vtblk_softc *sc, struct vtblk_request *req) +{ + + /* NOTE: Currently, there will be at most one request in the queue. */ + TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link); +} + +static struct vtblk_request * +vtblk_request_next(struct vtblk_softc *sc) +{ + struct vtblk_request *req; + + req = vtblk_request_next_ready(sc); + if (req != NULL) + return (req); + + return (vtblk_request_bio(sc)); +} + +static struct vtblk_request * +vtblk_request_bio(struct vtblk_softc *sc) { struct bio_queue_head *bioq; struct vtblk_request *req; @@ -865,7 +856,7 @@ vtblk_bio_request(struct vtblk_softc *sc) if (bioq_first(bioq) == NULL) return (NULL); - req = vtblk_dequeue_request(sc); + req = vtblk_request_dequeue(sc); if (req == NULL) return (NULL); @@ -890,11 +881,14 @@ vtblk_bio_request(struct vtblk_softc *sc) panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd); } + if (bp->bio_flags & BIO_ORDERED) + req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER; + return (req); } static int -vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req) +vtblk_request_execute(struct vtblk_softc *sc, struct vtblk_request *req) { struct virtqueue *vq; struct sglist *sg; @@ -907,26 +901,20 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req) ordered = 0; writable = 0; - VTBLK_LOCK_ASSERT(sc); - /* - * Wait until the ordered request completes before - * executing subsequent requests. + * Some hosts (such as bhyve) do not implement the barrier feature, + * so we emulate it in the driver by allowing the barrier request + * to be the only one in flight. */ - if (sc->vtblk_req_ordered != NULL) - return (EBUSY); - - if (bp->bio_flags & BIO_ORDERED) { - if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) { - /* - * This request will be executed once all - * the in-flight requests are completed. - */ + if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) { + if (sc->vtblk_req_ordered != NULL) + return (EBUSY); + if (bp->bio_flags & BIO_ORDERED) { if (!virtqueue_empty(vq)) return (EBUSY); ordered = 1; - } else - req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER; + req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER; + } } sglist_reset(sg); @@ -935,7 +923,7 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req) if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { error = sglist_append_bio(sg, bp); if (error || sg->sg_nseg == sg->sg_maxseg) { - panic("%s: data buffer too big bio:%p error:%d", + panic("%s: bio %p data buffer too big %d", __func__, bp, error); } @@ -955,44 +943,156 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req) return (error); } +static int +vtblk_request_error(struct vtblk_request *req) +{ + int error; + + switch (req->vbr_ack) { + case VIRTIO_BLK_S_OK: + error = 0; + break; + case VIRTIO_BLK_S_UNSUPP: + error = ENOTSUP; + break; + default: + error = EIO; + break; + } + + return (error); +} + static void -vtblk_vq_intr(void *xsc) +vtblk_queue_completed(struct vtblk_softc *sc, struct bio_queue *queue) +{ + struct vtblk_request *req; + struct bio *bp; + + while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) { + if (sc->vtblk_req_ordered != NULL) { + MPASS(sc->vtblk_req_ordered == req); + sc->vtblk_req_ordered = NULL; + } + + bp = req->vbr_bp; + bp->bio_error = vtblk_request_error(req); + TAILQ_INSERT_TAIL(queue, bp, bio_queue); + + vtblk_request_enqueue(sc, req); + } +} + +static void +vtblk_done_completed(struct vtblk_softc *sc, struct bio_queue *queue) +{ + struct bio *bp, *tmp; + + TAILQ_FOREACH_SAFE(bp, queue, bio_queue, tmp) { + if (bp->bio_error != 0) + disk_err(bp, "hard error", -1, 1); + vtblk_bio_done(sc, bp, bp->bio_error); + } +} + +static void +vtblk_drain_vq(struct vtblk_softc *sc, int skip_done) { - struct vtblk_softc *sc; struct virtqueue *vq; + struct vtblk_request *req; + int last; - sc = xsc; vq = sc->vtblk_vq; + last = 0; -again: - VTBLK_LOCK(sc); - if (sc->vtblk_flags & VTBLK_FLAG_DETACH) { - VTBLK_UNLOCK(sc); - return; + while ((req = virtqueue_drain(vq, &last)) != NULL) { + if (!skip_done) + vtblk_bio_done(sc, req->vbr_bp, ENXIO); + + vtblk_request_enqueue(sc, req); } - vtblk_finish_completed(sc); + sc->vtblk_req_ordered = NULL; + KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); +} - if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0) - vtblk_startio(sc); - else - wakeup(&sc->vtblk_vq); +static void +vtblk_drain(struct vtblk_softc *sc) +{ + struct bio_queue queue; + struct bio_queue_head *bioq; + struct vtblk_request *req; + struct bio *bp; - if (virtqueue_enable_intr(vq) != 0) { - virtqueue_disable_intr(vq); - VTBLK_UNLOCK(sc); - goto again; + bioq = &sc->vtblk_bioq; + TAILQ_INIT(&queue); + + if (sc->vtblk_vq != NULL) { + vtblk_queue_completed(sc, &queue); + vtblk_done_completed(sc, &queue); + + vtblk_drain_vq(sc, 0); } - VTBLK_UNLOCK(sc); + while ((req = vtblk_request_next_ready(sc)) != NULL) { + vtblk_bio_done(sc, req->vbr_bp, ENXIO); + vtblk_request_enqueue(sc, req); + } + + while (bioq_first(bioq) != NULL) { + bp = bioq_takefirst(bioq); + vtblk_bio_done(sc, bp, ENXIO); + } + + vtblk_request_free(sc); } static void -vtblk_stop(struct vtblk_softc *sc) +vtblk_startio(struct vtblk_softc *sc) { + struct virtqueue *vq; + struct vtblk_request *req; + int enq; - virtqueue_disable_intr(sc->vtblk_vq); - virtio_stop(sc->vtblk_dev); + VTBLK_LOCK_ASSERT(sc); + vq = sc->vtblk_vq; + enq = 0; + + if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND) + return; + + while (!virtqueue_full(vq)) { + req = vtblk_request_next(sc); + if (req == NULL) + break; + + if (vtblk_request_execute(sc, req) != 0) { + vtblk_request_requeue_ready(sc, req); + break; + } + + enq++; + } + + if (enq > 0) + virtqueue_notify(vq); +} + +static void +vtblk_bio_done(struct vtblk_softc *sc, struct bio *bp, int error) +{ + + /* Because of GEOM direct dispatch, we cannot hold any locks. */ + if (sc != NULL) + VTBLK_LOCK_ASSERT_NOTOWNED(sc); + + if (error) { + bp->bio_resid = bp->bio_bcount; + bp->bio_error = error; + bp->bio_flags |= BIO_ERROR; + } + + biodone(bp); } #define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg) \ @@ -1027,7 +1127,7 @@ vtblk_read_config(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) #undef VTBLK_GET_CONFIG static void -vtblk_get_ident(struct vtblk_softc *sc) +vtblk_ident(struct vtblk_softc *sc) { struct bio buf; struct disk *dp; @@ -1040,7 +1140,7 @@ vtblk_get_ident(struct vtblk_softc *sc) if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0) return; - req = vtblk_dequeue_request(sc); + req = vtblk_request_dequeue(sc); if (req == NULL) return; @@ -1060,7 +1160,7 @@ vtblk_get_ident(struct vtblk_softc *sc) error = vtblk_poll_request(sc, req); VTBLK_UNLOCK(sc); - vtblk_enqueue_request(sc, req); + vtblk_request_enqueue(sc, req); if (error) { device_printf(sc->vtblk_dev, @@ -1068,77 +1168,6 @@ vtblk_get_ident(struct vtblk_softc *sc) } } -static void -vtblk_prepare_dump(struct vtblk_softc *sc) -{ - device_t dev; - struct virtqueue *vq; - - dev = sc->vtblk_dev; - vq = sc->vtblk_vq; - - vtblk_stop(sc); - - /* - * Drain all requests caught in-flight in the virtqueue, - * skipping biodone(). When dumping, only one request is - * outstanding at a time, and we just poll the virtqueue - * for the response. - */ - vtblk_drain_vq(sc, 1); - - if (virtio_reinit(dev, sc->vtblk_features) != 0) { - panic("%s: cannot reinit VirtIO block device during dump", - device_get_nameunit(dev)); - } - - virtqueue_disable_intr(vq); - virtio_reinit_complete(dev); -} - -static int -vtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset, - size_t length) -{ - struct bio buf; - struct vtblk_request *req; - - req = &sc->vtblk_dump_request; - req->vbr_ack = -1; - req->vbr_hdr.type = VIRTIO_BLK_T_OUT; - req->vbr_hdr.ioprio = 1; - req->vbr_hdr.sector = offset / 512; - - req->vbr_bp = &buf; - bzero(&buf, sizeof(struct bio)); - - buf.bio_cmd = BIO_WRITE; - buf.bio_data = virtual; - buf.bio_bcount = length; - - return (vtblk_poll_request(sc, req)); -} - -static int -vtblk_flush_dump(struct vtblk_softc *sc) -{ - struct bio buf; - struct vtblk_request *req; - - req = &sc->vtblk_dump_request; - req->vbr_ack = -1; - req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; - req->vbr_hdr.ioprio = 1; - req->vbr_hdr.sector = 0; - - req->vbr_bp = &buf; - bzero(&buf, sizeof(struct bio)); - - buf.bio_cmd = BIO_FLUSH; - - return (vtblk_poll_request(sc, req)); -} - static int vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req) { @@ -1150,7 +1179,7 @@ vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req) if (!virtqueue_empty(vq)) return (EBUSY); - error = vtblk_execute_request(sc, req); + error = vtblk_request_execute(sc, req); if (error) return (error); @@ -1166,212 +1195,188 @@ vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req) return (error); } -static void -vtblk_finish_completed(struct vtblk_softc *sc) +static int +vtblk_quiesce(struct vtblk_softc *sc) { - struct vtblk_request *req; - struct bio *bp; int error; - while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) { - bp = req->vbr_bp; + VTBLK_LOCK_ASSERT(sc); + error = 0; - if (sc->vtblk_req_ordered != NULL) { - /* This should be the only outstanding request. */ - MPASS(sc->vtblk_req_ordered == req); - sc->vtblk_req_ordered = NULL; + while (!virtqueue_empty(sc->vtblk_vq)) { + if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq", + VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) { + error = EBUSY; + break; } - - error = vtblk_request_error(req); - if (error) - disk_err(bp, "hard error", -1, 1); - - vtblk_finish_bio(bp, error); - vtblk_enqueue_request(sc, req); } + + return (error); } static void -vtblk_drain_vq(struct vtblk_softc *sc, int skip_done) +vtblk_vq_intr(void *xsc) { + struct vtblk_softc *sc; struct virtqueue *vq; - struct vtblk_request *req; - int last; + struct bio_queue queue; + sc = xsc; vq = sc->vtblk_vq; - last = 0; - - while ((req = virtqueue_drain(vq, &last)) != NULL) { - if (!skip_done) - vtblk_finish_bio(req->vbr_bp, ENXIO); - - vtblk_enqueue_request(sc, req); - } + TAILQ_INIT(&queue); - sc->vtblk_req_ordered = NULL; - KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); -} - -static void -vtblk_drain(struct vtblk_softc *sc) -{ - struct bio_queue_head *bioq; - struct vtblk_request *req; - struct bio *bp; + VTBLK_LOCK(sc); - bioq = &sc->vtblk_bioq; +again: + if (sc->vtblk_flags & VTBLK_FLAG_DETACH) + goto out; - if (sc->vtblk_vq != NULL) { - vtblk_finish_completed(sc); - vtblk_drain_vq(sc, 0); - } + vtblk_queue_completed(sc, &queue); + vtblk_startio(sc); - while ((req = vtblk_dequeue_ready(sc)) != NULL) { - vtblk_finish_bio(req->vbr_bp, ENXIO); - vtblk_enqueue_request(sc, req); + if (virtqueue_enable_intr(vq) != 0) { + virtqueue_disable_intr(vq); + goto again; } - while (bioq_first(bioq) != NULL) { - bp = bioq_takefirst(bioq); - vtblk_finish_bio(bp, ENXIO); - } + if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND) + wakeup(&sc->vtblk_vq); - vtblk_free_requests(sc); +out: + VTBLK_UNLOCK(sc); + vtblk_done_completed(sc, &queue); } -#ifdef INVARIANTS static void -vtblk_request_invariants(struct vtblk_request *req) +vtblk_stop(struct vtblk_softc *sc) { - int hdr_nsegs, ack_nsegs; - hdr_nsegs = sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)); - ack_nsegs = sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)); - - KASSERT(hdr_nsegs == 1, ("request header crossed page boundary")); - KASSERT(ack_nsegs == 1, ("request ack crossed page boundary")); + virtqueue_disable_intr(sc->vtblk_vq); + virtio_stop(sc->vtblk_dev); } -#endif -static int -vtblk_alloc_requests(struct vtblk_softc *sc) +static void +vtblk_dump_prepare(struct vtblk_softc *sc) { - struct vtblk_request *req; - int i, nreqs; + device_t dev; + struct virtqueue *vq; - nreqs = virtqueue_size(sc->vtblk_vq); + dev = sc->vtblk_dev; + vq = sc->vtblk_vq; + + vtblk_stop(sc); /* - * Preallocate sufficient requests to keep the virtqueue full. Each - * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce - * the number allocated when indirect descriptors are not available. + * Drain all requests caught in-flight in the virtqueue, + * skipping biodone(). When dumping, only one request is + * outstanding at a time, and we just poll the virtqueue + * for the response. */ - if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0) - nreqs /= VTBLK_MIN_SEGMENTS; - - for (i = 0; i < nreqs; i++) { - req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT); - if (req == NULL) - return (ENOMEM); - -#ifdef INVARIANTS - vtblk_request_invariants(req); -#endif + vtblk_drain_vq(sc, 1); - sc->vtblk_request_count++; - vtblk_enqueue_request(sc, req); + if (virtio_reinit(dev, sc->vtblk_features) != 0) { + panic("%s: cannot reinit VirtIO block device during dump", + device_get_nameunit(dev)); } - return (0); + virtqueue_disable_intr(vq); + virtio_reinit_complete(dev); } -static void -vtblk_free_requests(struct vtblk_softc *sc) +static int +vtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset, + size_t length) { + struct bio buf; struct vtblk_request *req; - KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready), - ("%s: ready requests left on queue", __func__)); - - while ((req = vtblk_dequeue_request(sc)) != NULL) { - sc->vtblk_request_count--; - free(req, M_DEVBUF); - } - - KASSERT(sc->vtblk_request_count == 0, - ("%s: leaked %d requests", __func__, sc->vtblk_request_count)); -} + req = &sc->vtblk_dump_request; + req->vbr_ack = -1; + req->vbr_hdr.type = VIRTIO_BLK_T_OUT; + req->vbr_hdr.ioprio = 1; + req->vbr_hdr.sector = offset / 512; -static struct vtblk_request * -vtblk_dequeue_request(struct vtblk_softc *sc) -{ - struct vtblk_request *req; + req->vbr_bp = &buf; + bzero(&buf, sizeof(struct bio)); - req = TAILQ_FIRST(&sc->vtblk_req_free); - if (req != NULL) - TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link); + buf.bio_cmd = BIO_WRITE; + buf.bio_data = virtual; + buf.bio_bcount = length; - return (req); + return (vtblk_poll_request(sc, req)); } -static void -vtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req) +static int +vtblk_dump_flush(struct vtblk_softc *sc) { + struct bio buf; + struct vtblk_request *req; - bzero(req, sizeof(struct vtblk_request)); - TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link); -} + req = &sc->vtblk_dump_request; + req->vbr_ack = -1; + req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; + req->vbr_hdr.ioprio = 1; + req->vbr_hdr.sector = 0; -static struct vtblk_request * -vtblk_dequeue_ready(struct vtblk_softc *sc) -{ - struct vtblk_request *req; + req->vbr_bp = &buf; + bzero(&buf, sizeof(struct bio)); - req = TAILQ_FIRST(&sc->vtblk_req_ready); - if (req != NULL) - TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link); + buf.bio_cmd = BIO_FLUSH; - return (req); + return (vtblk_poll_request(sc, req)); } static void -vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req) +vtblk_set_write_cache(struct vtblk_softc *sc, int wc) { - TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link); + /* Set either writeback (1) or writethrough (0) mode. */ + virtio_write_dev_config_1(sc->vtblk_dev, + offsetof(struct virtio_blk_config, writeback), wc); } static int -vtblk_request_error(struct vtblk_request *req) +vtblk_write_cache_enabled(struct vtblk_softc *sc, + struct virtio_blk_config *blkcfg) { - int error; + int wc; - switch (req->vbr_ack) { - case VIRTIO_BLK_S_OK: - error = 0; - break; - case VIRTIO_BLK_S_UNSUPP: - error = ENOTSUP; - break; - default: - error = EIO; - break; - } + if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) { + wc = vtblk_tunable_int(sc, "writecache_mode", + vtblk_writecache_mode); + if (wc >= 0 && wc < VTBLK_CACHE_MAX) + vtblk_set_write_cache(sc, wc); + else + wc = blkcfg->writeback; + } else + wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE); - return (error); + return (wc); } -static void -vtblk_finish_bio(struct bio *bp, int error) +static int +vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS) { + struct vtblk_softc *sc; + int wc, error; - if (error) { - bp->bio_resid = bp->bio_bcount; - bp->bio_error = error; - bp->bio_flags |= BIO_ERROR; - } + sc = oidp->oid_arg1; + wc = sc->vtblk_write_cache; - biodone(bp); + error = sysctl_handle_int(oidp, &wc, 0, req); + if (error || req->newptr == NULL) + return (error); + if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0) + return (EPERM); + if (wc < 0 || wc >= VTBLK_CACHE_MAX) + return (EINVAL); + + VTBLK_LOCK(sc); + sc->vtblk_write_cache = wc; + vtblk_set_write_cache(sc, sc->vtblk_write_cache); + VTBLK_UNLOCK(sc); + + return (0); } static void diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c index 83cb2d79fe25c..9fba9bdabe0b4 100644 --- a/sys/dev/virtio/network/if_vtnet.c +++ b/sys/dev/virtio/network/if_vtnet.c @@ -1701,7 +1701,7 @@ vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, } m->m_pkthdr.flowid = rxq->vtnrx_id; - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); /* * BMV: FreeBSD does not have the UNNECESSARY and PARTIAL checksum @@ -2347,7 +2347,8 @@ vtnet_txq_mq_start(struct ifnet *ifp, struct mbuf *m) sc = ifp->if_softc; npairs = sc->vtnet_act_vq_pairs; - if (m->m_flags & M_FLOWID) + /* check if flowid is set */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) i = m->m_pkthdr.flowid % npairs; else i = curcpu % npairs; diff --git a/sys/dev/vmware/vmxnet3/if_vmx.c b/sys/dev/vmware/vmxnet3/if_vmx.c index 1d8c12e2fc9b0..5faf716acf4d4 100644 --- a/sys/dev/vmware/vmxnet3/if_vmx.c +++ b/sys/dev/vmware/vmxnet3/if_vmx.c @@ -2059,7 +2059,7 @@ vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, } #else m->m_pkthdr.flowid = rxq->vxrxq_id; - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); #endif if (!rxcd->no_csum) @@ -3002,7 +3002,8 @@ vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) sc = ifp->if_softc; ntxq = sc->vmx_ntxqueues; - if (m->m_flags & M_FLOWID) + /* check if flowid is set */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) i = m->m_pkthdr.flowid % ntxq; else i = curcpu % ntxq; diff --git a/sys/dev/vxge/vxge.c b/sys/dev/vxge/vxge.c index b89ba6cca51ee..9a3cab654f956 100644 --- a/sys/dev/vxge/vxge.c +++ b/sys/dev/vxge/vxge.c @@ -660,7 +660,7 @@ vxge_mq_send(ifnet_t ifp, mbuf_t m_head) if (vdev->config.tx_steering) { i = vxge_vpath_get(vdev, m_head); - } else if ((m_head->m_flags & M_FLOWID) != 0) { + } else if (M_HASHTYPE_GET(m_head) != M_HASHTYPE_NONE) { i = m_head->m_pkthdr.flowid % vdev->no_of_vpath; } @@ -1070,7 +1070,7 @@ vxge_rx_compl(vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, vxge_rx_checksum(ext_info, mbuf_up); #if __FreeBSD_version >= 800000 - mbuf_up->m_flags |= M_FLOWID; + M_HASHTYPE_SET(mbuf_up, M_HASHTYPE_OPAQUE); mbuf_up->m_pkthdr.flowid = vpath->vp_index; #endif /* Post-Read sync for buffers */ diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c index 3a3dbc44c04f7..f82e2457b012e 100644 --- a/sys/kern/subr_taskqueue.c +++ b/sys/kern/subr_taskqueue.c @@ -661,11 +661,11 @@ taskqueue_thread_enqueue(void *context) TASKQUEUE_DEFINE(swi, taskqueue_swi_enqueue, NULL, swi_add(NULL, "task queue", taskqueue_swi_run, NULL, SWI_TQ, - INTR_MPSAFE, &taskqueue_ih)); + INTR_MPSAFE, &taskqueue_ih)); TASKQUEUE_DEFINE(swi_giant, taskqueue_swi_giant_enqueue, NULL, swi_add(NULL, "Giant taskq", taskqueue_swi_giant_run, - NULL, SWI_TQ_GIANT, 0, &taskqueue_giant_ih)); + NULL, SWI_TQ_GIANT, 0, &taskqueue_giant_ih)); TASKQUEUE_DEFINE_THREAD(thread); diff --git a/sys/kern/uipc_debug.c b/sys/kern/uipc_debug.c index 0f233ecebd692..c3081a497ae62 100644 --- a/sys/kern/uipc_debug.c +++ b/sys/kern/uipc_debug.c @@ -401,7 +401,8 @@ db_print_sockbuf(struct sockbuf *sb, const char *sockbufname, int indent) db_printf("sb_sndptroff: %u\n", sb->sb_sndptroff); db_print_indent(indent); - db_printf("sb_cc: %u ", sb->sb_cc); + db_printf("sb_acc: %u ", sb->sb_acc); + db_printf("sb_ccc: %u ", sb->sb_ccc); db_printf("sb_hiwat: %u ", sb->sb_hiwat); db_printf("sb_mbcnt: %u ", sb->sb_mbcnt); db_printf("sb_mbmax: %u\n", sb->sb_mbmax); diff --git a/sys/kern/uipc_domain.c b/sys/kern/uipc_domain.c index 709cc0eb0569d..9eda77c0f2d89 100644 --- a/sys/kern/uipc_domain.c +++ b/sys/kern/uipc_domain.c @@ -152,6 +152,7 @@ protosw_init(struct protosw *pr) DEFAULT(pu->pru_sosend, sosend_generic); DEFAULT(pu->pru_soreceive, soreceive_generic); DEFAULT(pu->pru_sopoll, sopoll_generic); + DEFAULT(pu->pru_ready, pru_ready_notsupp); #undef DEFAULT if (pr->pr_init) (*pr->pr_init)(); diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index 323426898d093..3880456bddf72 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -388,7 +388,7 @@ mb_dupcl(struct mbuf *n, struct mbuf *m) * cleaned too. */ void -m_demote(struct mbuf *m0, int all) +m_demote(struct mbuf *m0, int all, int flags) { struct mbuf *m; @@ -400,7 +400,7 @@ m_demote(struct mbuf *m0, int all) m->m_flags &= ~M_PKTHDR; bzero(&m->m_pkthdr, sizeof(struct pkthdr)); } - m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_NOFREE); + m->m_flags = m->m_flags & (M_EXT | M_RDONLY | M_NOFREE | flags); } } @@ -997,7 +997,7 @@ m_catpkt(struct mbuf *m, struct mbuf *n) M_ASSERTPKTHDR(n); m->m_pkthdr.len += n->m_pkthdr.len; - m_demote(n, 1); + m_demote(n, 1, 0); m_cat(m, n); } diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c index 01e48b55aac07..537f9c867bf8b 100644 --- a/sys/kern/uipc_sockbuf.c +++ b/sys/kern/uipc_sockbuf.c @@ -69,6 +69,123 @@ static struct mbuf *sbcut_internal(struct sockbuf *sb, int len); static void sbflush_internal(struct sockbuf *sb); /* + * Mark ready "count" mbufs starting with "m". + */ +int +sbready(struct sockbuf *sb, struct mbuf *m, int count) +{ + u_int blocker; + + SOCKBUF_LOCK_ASSERT(sb); + KASSERT(sb->sb_fnrdy != NULL, ("%s: sb %p NULL fnrdy", __func__, sb)); + + blocker = (sb->sb_fnrdy == m) ? M_BLOCKED : 0; + + for (int i = 0; i < count; i++, m = m->m_next) { + KASSERT(m->m_flags & M_NOTREADY, + ("%s: m %p !M_NOTREADY", __func__, m)); + m->m_flags &= ~(M_NOTREADY | blocker); + if (blocker) + sb->sb_acc += m->m_len; + } + + if (!blocker) + return (EINPROGRESS); + + /* This one was blocking all the queue. */ + for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) { + KASSERT(m->m_flags & M_BLOCKED, + ("%s: m %p !M_BLOCKED", __func__, m)); + m->m_flags &= ~M_BLOCKED; + sb->sb_acc += m->m_len; + } + + sb->sb_fnrdy = m; + + return (0); +} + +/* + * Adjust sockbuf state reflecting allocation of m. + */ +void +sballoc(struct sockbuf *sb, struct mbuf *m) +{ + + SOCKBUF_LOCK_ASSERT(sb); + + sb->sb_ccc += m->m_len; + + if (sb->sb_fnrdy == NULL) { + if (m->m_flags & M_NOTREADY) + sb->sb_fnrdy = m; + else + sb->sb_acc += m->m_len; + } else + m->m_flags |= M_BLOCKED; + + if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) + sb->sb_ctl += m->m_len; + + sb->sb_mbcnt += MSIZE; + sb->sb_mcnt += 1; + + if (m->m_flags & M_EXT) { + sb->sb_mbcnt += m->m_ext.ext_size; + sb->sb_ccnt += 1; + } +} + +/* + * Adjust sockbuf state reflecting freeing of m. + */ +void +sbfree(struct sockbuf *sb, struct mbuf *m) +{ + +#if 0 /* XXX: not yet: soclose() call path comes here w/o lock. */ + SOCKBUF_LOCK_ASSERT(sb); +#endif + + sb->sb_ccc -= m->m_len; + + if (!(m->m_flags & M_NOTAVAIL)) + sb->sb_acc -= m->m_len; + + if (m == sb->sb_fnrdy) { + struct mbuf *n; + + KASSERT(m->m_flags & M_NOTREADY, + ("%s: m %p !M_NOTREADY", __func__, m)); + + n = m->m_next; + while (n != NULL && !(n->m_flags & M_NOTREADY)) { + n->m_flags &= ~M_BLOCKED; + sb->sb_acc += n->m_len; + n = n->m_next; + } + sb->sb_fnrdy = n; + } + + if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) + sb->sb_ctl -= m->m_len; + + sb->sb_mbcnt -= MSIZE; + sb->sb_mcnt -= 1; + if (m->m_flags & M_EXT) { + sb->sb_mbcnt -= m->m_ext.ext_size; + sb->sb_ccnt -= 1; + } + + if (sb->sb_sndptr == m) { + sb->sb_sndptr = NULL; + sb->sb_sndptroff = 0; + } + if (sb->sb_sndptroff != 0) + sb->sb_sndptroff -= m->m_len; +} + +/* * Socantsendmore indicates that no more data will be sent on the socket; it * would normally be applied to a socket when the user informs the system * that no more data is to be sent, by the protocol code (in case @@ -127,7 +244,7 @@ sbwait(struct sockbuf *sb) SOCKBUF_LOCK_ASSERT(sb); sb->sb_flags |= SB_WAIT; - return (msleep_sbt(&sb->sb_cc, &sb->sb_mtx, + return (msleep_sbt(&sb->sb_acc, &sb->sb_mtx, (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", sb->sb_timeo, 0, 0)); } @@ -184,7 +301,7 @@ sowakeup(struct socket *so, struct sockbuf *sb) sb->sb_flags &= ~SB_SEL; if (sb->sb_flags & SB_WAIT) { sb->sb_flags &= ~SB_WAIT; - wakeup(&sb->sb_cc); + wakeup(&sb->sb_acc); } KNOTE_LOCKED(&sb->sb_sel.si_note, 0); if (sb->sb_upcall != NULL) { @@ -519,7 +636,7 @@ sbappend(struct sockbuf *sb, struct mbuf *m) * that is, a stream protocol (such as TCP). */ void -sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) +sbappendstream_locked(struct sockbuf *sb, struct mbuf *m, int flags) { SOCKBUF_LOCK_ASSERT(sb); @@ -529,8 +646,8 @@ sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) SBLASTMBUFCHK(sb); /* Remove all packet headers and mbuf tags to get a pure data chain. */ - m_demote(m, 1); - + m_demote(m, 1, flags & PRUS_NOTREADY ? M_NOTREADY : 0); + sbcompress(sb, m, sb->sb_mbtail); sb->sb_lastrecord = sb->sb_mb; @@ -543,38 +660,63 @@ sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) * that is, a stream protocol (such as TCP). */ void -sbappendstream(struct sockbuf *sb, struct mbuf *m) +sbappendstream(struct sockbuf *sb, struct mbuf *m, int flags) { SOCKBUF_LOCK(sb); - sbappendstream_locked(sb, m); + sbappendstream_locked(sb, m, flags); SOCKBUF_UNLOCK(sb); } #ifdef SOCKBUF_DEBUG void -sbcheck(struct sockbuf *sb) +sbcheck(struct sockbuf *sb, const char *file, int line) { - struct mbuf *m; - struct mbuf *n = 0; - u_long len = 0, mbcnt = 0; + struct mbuf *m, *n, *fnrdy; + u_long acc, ccc, mbcnt; SOCKBUF_LOCK_ASSERT(sb); + acc = ccc = mbcnt = 0; + fnrdy = NULL; + for (m = sb->sb_mb; m; m = n) { n = m->m_nextpkt; for (; m; m = m->m_next) { - len += m->m_len; + if (m->m_len == 0) { + printf("sb %p empty mbuf %p\n", sb, m); + goto fail; + } + if ((m->m_flags & M_NOTREADY) && fnrdy == NULL) { + if (m != sb->sb_fnrdy) { + printf("sb %p: fnrdy %p != m %p\n", + sb, sb->sb_fnrdy, m); + goto fail; + } + fnrdy = m; + } + if (fnrdy) { + if (!(m->m_flags & M_NOTAVAIL)) { + printf("sb %p: fnrdy %p, m %p is avail\n", + sb, sb->sb_fnrdy, m); + goto fail; + } + } else + acc += m->m_len; + ccc += m->m_len; mbcnt += MSIZE; if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ mbcnt += m->m_ext.ext_size; } } - if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { - printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc, - mbcnt, sb->sb_mbcnt); - panic("sbcheck"); + if (acc != sb->sb_acc || ccc != sb->sb_ccc || mbcnt != sb->sb_mbcnt) { + printf("acc %ld/%u ccc %ld/%u mbcnt %ld/%u\n", + acc, sb->sb_acc, ccc, sb->sb_ccc, mbcnt, sb->sb_mbcnt); + goto fail; } + return; +fail: + panic("%s from %s:%u", __func__, file, line); } #endif @@ -770,8 +912,8 @@ sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) * * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into * an mbuf already in the socket buffer. This can occur if an - * appropriate mbuf exists, there is room, and no merging of data types - * will occur. + * appropriate mbuf exists, there is room, both mbufs are not marked as + * not ready, and no merging of data types will occur. * * (3) The mbuf may be appended to the end of the existing mbuf chain. * @@ -800,13 +942,17 @@ sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) if (n && (n->m_flags & M_EOR) == 0 && M_WRITABLE(n) && ((sb->sb_flags & SB_NOCOALESCE) == 0) && + !(m->m_flags & M_NOTREADY) && + !(n->m_flags & M_NOTREADY) && m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ m->m_len <= M_TRAILINGSPACE(n) && n->m_type == m->m_type) { bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, (unsigned)m->m_len); n->m_len += m->m_len; - sb->sb_cc += m->m_len; + sb->sb_ccc += m->m_len; + if (sb->sb_fnrdy == NULL) + sb->sb_acc += m->m_len; if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) /* XXX: Probably don't need.*/ sb->sb_ctl += m->m_len; @@ -843,13 +989,13 @@ sbflush_internal(struct sockbuf *sb) * Don't call sbcut(sb, 0) if the leading mbuf is non-empty: * we would loop forever. Panic instead. */ - if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) + if (sb->sb_ccc == 0 && (sb->sb_mb == NULL || sb->sb_mb->m_len)) break; - m_freem(sbcut_internal(sb, (int)sb->sb_cc)); + m_freem(sbcut_internal(sb, (int)sb->sb_ccc)); } - if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) - panic("sbflush_internal: cc %u || mb %p || mbcnt %u", - sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); + KASSERT(sb->sb_ccc == 0 && sb->sb_mb == 0 && sb->sb_mbcnt == 0, + ("%s: ccc %u mb %p mbcnt %u", __func__, + sb->sb_ccc, (void *)sb->sb_mb, sb->sb_mbcnt)); } void @@ -875,7 +1021,7 @@ sbflush(struct sockbuf *sb) static struct mbuf * sbcut_internal(struct sockbuf *sb, int len) { - struct mbuf *m, *n, *next, *mfree; + struct mbuf *m, *next, *mfree; next = (m = sb->sb_mb) ? m->m_nextpkt : 0; mfree = NULL; @@ -887,9 +1033,12 @@ sbcut_internal(struct sockbuf *sb, int len) next = m->m_nextpkt; } if (m->m_len > len) { + KASSERT(!(m->m_flags & M_NOTAVAIL), + ("%s: m %p M_NOTAVAIL", __func__, m)); m->m_len -= len; m->m_data += len; - sb->sb_cc -= len; + sb->sb_ccc -= len; + sb->sb_acc -= len; if (sb->sb_sndptroff != 0) sb->sb_sndptroff -= len; if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) @@ -898,10 +1047,20 @@ sbcut_internal(struct sockbuf *sb, int len) } len -= m->m_len; sbfree(sb, m); - n = m->m_next; - m->m_next = mfree; - mfree = m; - m = n; + /* + * Do not put M_NOTREADY buffers to the free list, they + * are referenced from outside. + */ + if (m->m_flags & M_NOTREADY) + m = m->m_next; + else { + struct mbuf *n; + + n = m->m_next; + m->m_next = mfree; + mfree = m; + m = n; + } } if (m) { sb->sb_mb = m; @@ -968,8 +1127,8 @@ sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff) struct mbuf *m, *ret; KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); - KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__)); - KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__)); + KASSERT(off + len <= sb->sb_acc, ("%s: beyond sb", __func__)); + KASSERT(sb->sb_sndptroff <= sb->sb_acc, ("%s: sndptroff broken", __func__)); /* * Is off below stored offset? Happens on retransmits. @@ -1118,7 +1277,7 @@ void sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) { - xsb->sb_cc = sb->sb_cc; + xsb->sb_cc = sb->sb_ccc; xsb->sb_hiwat = sb->sb_hiwat; xsb->sb_mbcnt = sb->sb_mbcnt; xsb->sb_mcnt = sb->sb_mcnt; diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index 240421f694b27..b2091ea284178 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -1706,7 +1706,8 @@ dontblock: */ moff = 0; offset = 0; - while (m != NULL && uio->uio_resid > 0 && error == 0) { + while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0 + && error == 0) { /* * If the type of mbuf has changed since the last mbuf * examined ('type'), end the receive operation. @@ -2044,6 +2045,8 @@ deliver: for (m = sb->sb_mb; m != NULL && m->m_len <= len; m = m->m_next) { + KASSERT(!(m->m_flags & M_NOTAVAIL), + ("%s: m %p not available", __func__, m)); len -= m->m_len; uio->uio_resid -= m->m_len; sbfree(sb, m); @@ -3175,6 +3178,13 @@ pru_send_notsupp(struct socket *so, int flags, struct mbuf *m, return EOPNOTSUPP; } +int +pru_ready_notsupp(struct socket *so, struct mbuf *m, int count) +{ + + return (EOPNOTSUPP); +} + /* * This isn't really a ``null'' operation, but it's the default one and * doesn't do anything destructive. diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c index 00fd8099c2e45..a540cc0dfa859 100644 --- a/sys/kern/uipc_usrreq.c +++ b/sys/kern/uipc_usrreq.c @@ -1048,6 +1048,32 @@ release: } static int +uipc_ready(struct socket *so, struct mbuf *m, int count) +{ + struct unpcb *unp, *unp2; + struct socket *so2; + int error; + + unp = sotounpcb(so); + + UNP_LINK_RLOCK(); + unp2 = unp->unp_conn; + UNP_PCB_LOCK(unp2); + so2 = unp2->unp_socket; + + SOCKBUF_LOCK(&so2->so_rcv); + if ((error = sbready(&so2->so_rcv, m, count)) == 0) + sorwakeup_locked(so2); + else + SOCKBUF_UNLOCK(&so2->so_rcv); + + UNP_PCB_UNLOCK(unp2); + UNP_LINK_RUNLOCK(); + + return (error); +} + +static int uipc_sense(struct socket *so, struct stat *sb) { struct unpcb *unp; @@ -1161,6 +1187,7 @@ static struct pr_usrreqs uipc_usrreqs_stream = { .pru_peeraddr = uipc_peeraddr, .pru_rcvd = uipc_rcvd, .pru_send = uipc_send, + .pru_ready = uipc_ready, .pru_sense = uipc_sense, .pru_shutdown = uipc_shutdown, .pru_sockaddr = uipc_sockaddr, diff --git a/sys/net/flowtable.c b/sys/net/flowtable.c index fe6a52cd3c106..0d584fea3581c 100644 --- a/sys/net/flowtable.c +++ b/sys/net/flowtable.c @@ -688,8 +688,8 @@ flowtable_lookup(sa_family_t sa, struct mbuf *m, struct route *ro) if (fle == NULL) return (EHOSTUNREACH); - if (!(m->m_flags & M_FLOWID)) { - m->m_flags |= M_FLOWID; + if (M_HASHTYPE_GET(m) == M_HASHTYPE_NONE) { + M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); m->m_pkthdr.flowid = fle->f_hash; } diff --git a/sys/net/ieee8023ad_lacp.c b/sys/net/ieee8023ad_lacp.c index 106df68e807a9..75f6366e78888 100644 --- a/sys/net/ieee8023ad_lacp.c +++ b/sys/net/ieee8023ad_lacp.c @@ -835,7 +835,8 @@ lacp_select_tx_port(struct lagg_softc *sc, struct mbuf *m) return (NULL); } - if ((sc->sc_opts & LAGG_OPT_USE_FLOWID) && (m->m_flags & M_FLOWID)) + if ((sc->sc_opts & LAGG_OPT_USE_FLOWID) && + M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) hash = m->m_pkthdr.flowid >> sc->flowid_shift; else hash = lagg_hashmbuf(sc, m, lsc->lsc_hashkey); diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c index 5c8b2e1d2bdf3..45315b6b3b557 100644 --- a/sys/net/if_lagg.c +++ b/sys/net/if_lagg.c @@ -247,14 +247,14 @@ SYSCTL_INT(_net_link_lagg, OID_AUTO, failover_rx_all, CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(lagg_failover_rx_all), 0, "Accept input from any interface in a failover lagg"); -/* Default value for using M_FLOWID */ +/* Default value for using flowid */ static VNET_DEFINE(int, def_use_flowid) = 1; #define V_def_use_flowid VNET(def_use_flowid) SYSCTL_INT(_net_link_lagg, OID_AUTO, default_use_flowid, CTLFLAG_RWTUN, &VNET_NAME(def_use_flowid), 0, "Default setting for using flow id for load sharing"); -/* Default value for using M_FLOWID */ +/* Default value for flowid shift */ static VNET_DEFINE(int, def_flowid_shift) = 16; #define V_def_flowid_shift VNET(def_flowid_shift) SYSCTL_INT(_net_link_lagg, OID_AUTO, default_flowid_shift, CTLFLAG_RWTUN, @@ -2148,7 +2148,8 @@ lagg_lb_start(struct lagg_softc *sc, struct mbuf *m) struct lagg_port *lp = NULL; uint32_t p = 0; - if ((sc->sc_opts & LAGG_OPT_USE_FLOWID) && (m->m_flags & M_FLOWID)) + if ((sc->sc_opts & LAGG_OPT_USE_FLOWID) && + M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) p = m->m_pkthdr.flowid >> sc->flowid_shift; else p = lagg_hashmbuf(sc, m, lb->lb_key); diff --git a/sys/net/if_lagg.h b/sys/net/if_lagg.h index e86ed06fcca73..8d6decd3c069f 100644 --- a/sys/net/if_lagg.h +++ b/sys/net/if_lagg.h @@ -143,9 +143,9 @@ struct lagg_reqopts { int ro_opts; /* Option bitmap */ #define LAGG_OPT_NONE 0x00 -#define LAGG_OPT_USE_FLOWID 0x01 /* use M_FLOWID */ +#define LAGG_OPT_USE_FLOWID 0x01 /* enable use of flowid */ /* Pseudo flags which are used in ro_opts but not stored into sc_opts. */ -#define LAGG_OPT_FLOWIDSHIFT 0x02 /* Set flowid */ +#define LAGG_OPT_FLOWIDSHIFT 0x02 /* set flowid shift */ #define LAGG_OPT_FLOWIDSHIFT_MASK 0x1f /* flowid is uint32_t */ #define LAGG_OPT_LACP_STRICT 0x10 /* LACP strict mode */ #define LAGG_OPT_LACP_TXTEST 0x20 /* LACP debug: txtest */ diff --git a/sys/net/if_vxlan.c b/sys/net/if_vxlan.c index 5badf00051fde..59c76ebf4f1b3 100644 --- a/sys/net/if_vxlan.c +++ b/sys/net/if_vxlan.c @@ -2236,6 +2236,7 @@ vxlan_pick_source_port(struct vxlan_softc *sc, struct mbuf *m) range = sc->vxl_max_port - sc->vxl_min_port + 1; + /* check if flowid is set and not opaque */ if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE && M_HASHTYPE_GET(m) != M_HASHTYPE_OPAQUE) hash = m->m_pkthdr.flowid; diff --git a/sys/net/netisr.c b/sys/net/netisr.c index 049bbf192b538..178c3cbd1479d 100644 --- a/sys/net/netisr.c +++ b/sys/net/netisr.c @@ -682,12 +682,13 @@ netisr_select_cpuid(struct netisr_proto *npp, u_int dispatch_policy, } if (policy == NETISR_POLICY_FLOW) { - if (!(m->m_flags & M_FLOWID) && npp->np_m2flow != NULL) { + if (M_HASHTYPE_GET(m) == M_HASHTYPE_NONE && + npp->np_m2flow != NULL) { m = npp->np_m2flow(m, source); if (m == NULL) return (NULL); } - if (m->m_flags & M_FLOWID) { + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { *cpuidp = netisr_default_flow2cpu(m->m_pkthdr.flowid); return (m); diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h index 04ed0b0bdf831..606795a69bb0d 100644 --- a/sys/netinet/in_pcb.h +++ b/sys/netinet/in_pcb.h @@ -530,8 +530,8 @@ short inp_so_options(const struct inpcb *inp); #define INP_ONESBCAST 0x02000000 /* send all-ones broadcast */ #define INP_DROPPED 0x04000000 /* protocol drop flag */ #define INP_SOCKREF 0x08000000 /* strong socket reference */ -#define INP_SW_FLOWID 0x10000000 /* software generated flow id */ -#define INP_HW_FLOWID 0x20000000 /* hardware generated flow id */ +#define INP_RESERVED_0 0x10000000 /* reserved field */ +#define INP_RESERVED_1 0x20000000 /* reserved field */ #define IN6P_RFC2292 0x40000000 /* used RFC2292 API on the socket */ #define IN6P_MTU 0x80000000 /* receive path MTU */ diff --git a/sys/netinet/in_rss.c b/sys/netinet/in_rss.c index 7a548c2a8d2eb..5d4f3c0dd37f5 100644 --- a/sys/netinet/in_rss.c +++ b/sys/netinet/in_rss.c @@ -568,6 +568,8 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval, const struct ip *ip; const struct tcphdr *th; const struct udphdr *uh; + uint32_t flowid; + uint32_t flowtype; uint8_t proto; int iphlen; int is_frag = 0; @@ -617,12 +619,10 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval, * then we shouldn't just "trust" the 2-tuple hash. We need * a 4-tuple hash. */ - if (m->m_flags & M_FLOWID) { - uint32_t flowid, flowtype; - - flowid = m->m_pkthdr.flowid; - flowtype = M_HASHTYPE_GET(m); + flowid = m->m_pkthdr.flowid; + flowtype = M_HASHTYPE_GET(m); + if (flowtype != M_HASHTYPE_NONE) { switch (proto) { case IPPROTO_UDP: if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) && @@ -743,7 +743,6 @@ rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) /* hash was done; update */ m->m_pkthdr.flowid = hash_val; M_HASHTYPE_SET(m, hash_type); - m->m_flags |= M_FLOWID; *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); } else { /* ret < 0 */ /* no hash was done */ diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index d0229b2dccd15..7b8127e90dae9 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -1196,7 +1196,6 @@ found: if (rss_mbuf_software_hash_v4(m, 0, &rss_hash, &rss_type) == 0) { m->m_pkthdr.flowid = rss_hash; M_HASHTYPE_SET(m, rss_type); - m->m_flags |= M_FLOWID; } /* diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index 5d791e57e99f8..611c53c7e9418 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -147,11 +147,9 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, if (inp != NULL) { INP_LOCK_ASSERT(inp); M_SETFIB(m, inp->inp_inc.inc_fibnum); - if (((flags & IP_NODEFAULTFLOWID) == 0) && - inp->inp_flags & (INP_HW_FLOWID|INP_SW_FLOWID)) { + if ((flags & IP_NODEFAULTFLOWID) == 0) { m->m_pkthdr.flowid = inp->inp_flowid; M_HASHTYPE_SET(m, inp->inp_flowtype); - m->m_flags |= M_FLOWID; } } diff --git a/sys/netinet/sctp_indata.c b/sys/netinet/sctp_indata.c index 2065bb4fdae48..97ca01338e83f 100644 --- a/sys/netinet/sctp_indata.c +++ b/sys/netinet/sctp_indata.c @@ -70,14 +70,14 @@ sctp_calc_rwnd(struct sctp_tcb *stcb, struct sctp_association *asoc) /* * This is really set wrong with respect to a 1-2-m socket. Since - * the sb_cc is the count that everyone as put up. When we re-write + * the sb_ccc is the count that everyone as put up. When we re-write * sctp_soreceive then we will fix this so that ONLY this * associations data is taken into account. */ if (stcb->sctp_socket == NULL) return (calc); - if (stcb->asoc.sb_cc == 0 && + if (stcb->asoc.sb_ccc == 0 && asoc->size_on_reasm_queue == 0 && asoc->size_on_all_streams == 0) { /* Full rwnd granted */ @@ -1363,7 +1363,7 @@ sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc, * When we have NO room in the rwnd we check to make sure * the reader is doing its job... */ - if (stcb->sctp_socket->so_rcv.sb_cc) { + if (stcb->sctp_socket->so_rcv.sb_ccc) { /* some to read, wake-up */ #if defined(__APPLE__) || defined(SCTP_SO_LOCK_TESTING) struct socket *so; diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c index 62de260179ade..52ac81e92fb7f 100644 --- a/sys/netinet/sctp_input.c +++ b/sys/netinet/sctp_input.c @@ -1032,7 +1032,7 @@ sctp_handle_shutdown_ack(struct sctp_shutdown_ack_chunk *cp SCTP_UNUSED, if (stcb->sctp_socket) { if ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) || (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL)) { - stcb->sctp_socket->so_snd.sb_cc = 0; + stcb->sctp_socket->so_snd.sb_ccc = 0; } sctp_ulp_notify(SCTP_NOTIFY_ASSOC_DOWN, stcb, 0, NULL, SCTP_SO_NOT_LOCKED); } diff --git a/sys/netinet/sctp_os_bsd.h b/sys/netinet/sctp_os_bsd.h index 558d4e18357ce..69ecf40655dd8 100644 --- a/sys/netinet/sctp_os_bsd.h +++ b/sys/netinet/sctp_os_bsd.h @@ -399,7 +399,7 @@ typedef struct callout sctp_os_timer_t; #define SCTP_SOWAKEUP(so) wakeup(&(so)->so_timeo) /* clear the socket buffer state */ #define SCTP_SB_CLEAR(sb) \ - (sb).sb_cc = 0; \ + (sb).sb_ccc = 0; \ (sb).sb_mb = NULL; \ (sb).sb_mbcnt = 0; diff --git a/sys/netinet/sctp_output.c b/sys/netinet/sctp_output.c index a122325a28e6b..3139be08091f8 100644 --- a/sys/netinet/sctp_output.c +++ b/sys/netinet/sctp_output.c @@ -7250,7 +7250,7 @@ one_more_time: if ((stcb->sctp_socket != NULL) && \ ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) || (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL))) { - atomic_subtract_int(&stcb->sctp_socket->so_snd.sb_cc, sp->length); + atomic_subtract_int(&stcb->sctp_socket->so_snd.sb_ccc, sp->length); } if (sp->data) { sctp_m_freem(sp->data); @@ -11532,7 +11532,7 @@ jump_out: drp->current_onq = htonl(asoc->size_on_reasm_queue + asoc->size_on_all_streams + asoc->my_rwnd_control_len + - stcb->sctp_socket->so_rcv.sb_cc); + stcb->sctp_socket->so_rcv.sb_ccc); } else { /*- * If my rwnd is 0, possibly from mbuf depletion as well as diff --git a/sys/netinet/sctp_pcb.c b/sys/netinet/sctp_pcb.c index 2fa99b9e6243b..72c3f042f7def 100644 --- a/sys/netinet/sctp_pcb.c +++ b/sys/netinet/sctp_pcb.c @@ -3397,7 +3397,7 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate, int from) if ((asoc->asoc.size_on_reasm_queue > 0) || (asoc->asoc.control_pdapi) || (asoc->asoc.size_on_all_streams > 0) || - (so && (so->so_rcv.sb_cc > 0))) { + (so && (so->so_rcv.sb_ccc > 0))) { /* Left with Data unread */ struct mbuf *op_err; @@ -3625,7 +3625,7 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate, int from) TAILQ_REMOVE(&inp->read_queue, sq, next); sctp_free_remote_addr(sq->whoFrom); if (so) - so->so_rcv.sb_cc -= sq->length; + so->so_rcv.sb_ccc -= sq->length; if (sq->data) { sctp_m_freem(sq->data); sq->data = NULL; @@ -4853,7 +4853,7 @@ sctp_free_assoc(struct sctp_inpcb *inp, struct sctp_tcb *stcb, int from_inpcbfre inp->sctp_flags |= SCTP_PCB_FLAGS_WAS_CONNECTED; if (so) { SOCK_LOCK(so); - if (so->so_rcv.sb_cc == 0) { + if (so->so_rcv.sb_ccc == 0) { so->so_state &= ~(SS_ISCONNECTING | SS_ISDISCONNECTING | SS_ISCONFIRMING | diff --git a/sys/netinet/sctp_pcb.h b/sys/netinet/sctp_pcb.h index c9183b1d82077..25416283f916e 100644 --- a/sys/netinet/sctp_pcb.h +++ b/sys/netinet/sctp_pcb.h @@ -369,7 +369,7 @@ struct sctp_inpcb { } ip_inp; - /* Socket buffer lock protects read_queue and of course sb_cc */ + /* Socket buffer lock protects read_queue and of course sb_ccc */ struct sctp_readhead read_queue; LIST_ENTRY(sctp_inpcb) sctp_list; /* lists all endpoints */ diff --git a/sys/netinet/sctp_structs.h b/sys/netinet/sctp_structs.h index 3f6d935ba1732..127068145525c 100644 --- a/sys/netinet/sctp_structs.h +++ b/sys/netinet/sctp_structs.h @@ -990,7 +990,7 @@ struct sctp_association { uint32_t total_output_queue_size; - uint32_t sb_cc; /* shadow of sb_cc */ + uint32_t sb_ccc; /* shadow of sb_ccc */ uint32_t sb_send_resv; /* amount reserved on a send */ uint32_t my_rwnd_control_len; /* shadow of sb_mbcnt used for rwnd * control */ diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c index e1ad178135e06..9dc7cdced64a0 100644 --- a/sys/netinet/sctp_usrreq.c +++ b/sys/netinet/sctp_usrreq.c @@ -586,7 +586,7 @@ sctp_must_try_again: if (((flags & SCTP_PCB_FLAGS_SOCKET_GONE) == 0) && (atomic_cmpset_int(&inp->sctp_flags, flags, (flags | SCTP_PCB_FLAGS_SOCKET_GONE | SCTP_PCB_FLAGS_CLOSE_IP)))) { if (((so->so_options & SO_LINGER) && (so->so_linger == 0)) || - (so->so_rcv.sb_cc > 0)) { + (so->so_rcv.sb_ccc > 0)) { #ifdef SCTP_LOG_CLOSING sctp_log_closing(inp, NULL, 13); #endif @@ -751,7 +751,7 @@ sctp_disconnect(struct socket *so) } if (((so->so_options & SO_LINGER) && (so->so_linger == 0)) || - (so->so_rcv.sb_cc > 0)) { + (so->so_rcv.sb_ccc > 0)) { if (SCTP_GET_STATE(asoc) != SCTP_STATE_COOKIE_WAIT) { /* Left with Data unread */ @@ -916,7 +916,7 @@ sctp_flush(struct socket *so, int how) inp->sctp_flags |= SCTP_PCB_FLAGS_SOCKET_CANT_READ; SCTP_INP_READ_UNLOCK(inp); SCTP_INP_WUNLOCK(inp); - so->so_rcv.sb_cc = 0; + so->so_rcv.sb_ccc = 0; so->so_rcv.sb_mbcnt = 0; so->so_rcv.sb_mb = NULL; } @@ -925,7 +925,7 @@ sctp_flush(struct socket *so, int how) * First make sure the sb will be happy, we don't use these * except maybe the count */ - so->so_snd.sb_cc = 0; + so->so_snd.sb_ccc = 0; so->so_snd.sb_mbcnt = 0; so->so_snd.sb_mb = NULL; diff --git a/sys/netinet/sctp_var.h b/sys/netinet/sctp_var.h index 31d93dd21bd94..bab8a7709dd55 100644 --- a/sys/netinet/sctp_var.h +++ b/sys/netinet/sctp_var.h @@ -82,9 +82,9 @@ extern struct pr_usrreqs sctp_usrreqs; #define sctp_maxspace(sb) (max((sb)->sb_hiwat,SCTP_MINIMAL_RWND)) -#define sctp_sbspace(asoc, sb) ((long) ((sctp_maxspace(sb) > (asoc)->sb_cc) ? (sctp_maxspace(sb) - (asoc)->sb_cc) : 0)) +#define sctp_sbspace(asoc, sb) ((long) ((sctp_maxspace(sb) > (asoc)->sb_ccc) ? (sctp_maxspace(sb) - (asoc)->sb_ccc) : 0)) -#define sctp_sbspace_failedmsgs(sb) ((long) ((sctp_maxspace(sb) > (sb)->sb_cc) ? (sctp_maxspace(sb) - (sb)->sb_cc) : 0)) +#define sctp_sbspace_failedmsgs(sb) ((long) ((sctp_maxspace(sb) > (sb)->sb_ccc) ? (sctp_maxspace(sb) - (sb)->sb_ccc) : 0)) #define sctp_sbspace_sub(a,b) ((a > b) ? (a - b) : 0) @@ -195,10 +195,10 @@ extern struct pr_usrreqs sctp_usrreqs; } #define sctp_sbfree(ctl, stcb, sb, m) { \ - SCTP_SAVE_ATOMIC_DECREMENT(&(sb)->sb_cc, SCTP_BUF_LEN((m))); \ + SCTP_SAVE_ATOMIC_DECREMENT(&(sb)->sb_ccc, SCTP_BUF_LEN((m))); \ SCTP_SAVE_ATOMIC_DECREMENT(&(sb)->sb_mbcnt, MSIZE); \ if (((ctl)->do_not_ref_stcb == 0) && stcb) {\ - SCTP_SAVE_ATOMIC_DECREMENT(&(stcb)->asoc.sb_cc, SCTP_BUF_LEN((m))); \ + SCTP_SAVE_ATOMIC_DECREMENT(&(stcb)->asoc.sb_ccc, SCTP_BUF_LEN((m))); \ SCTP_SAVE_ATOMIC_DECREMENT(&(stcb)->asoc.my_rwnd_control_len, MSIZE); \ } \ if (SCTP_BUF_TYPE(m) != MT_DATA && SCTP_BUF_TYPE(m) != MT_HEADER && \ @@ -207,10 +207,10 @@ extern struct pr_usrreqs sctp_usrreqs; } #define sctp_sballoc(stcb, sb, m) { \ - atomic_add_int(&(sb)->sb_cc,SCTP_BUF_LEN((m))); \ + atomic_add_int(&(sb)->sb_ccc,SCTP_BUF_LEN((m))); \ atomic_add_int(&(sb)->sb_mbcnt, MSIZE); \ if (stcb) { \ - atomic_add_int(&(stcb)->asoc.sb_cc,SCTP_BUF_LEN((m))); \ + atomic_add_int(&(stcb)->asoc.sb_ccc,SCTP_BUF_LEN((m))); \ atomic_add_int(&(stcb)->asoc.my_rwnd_control_len, MSIZE); \ } \ if (SCTP_BUF_TYPE(m) != MT_DATA && SCTP_BUF_TYPE(m) != MT_HEADER && \ diff --git a/sys/netinet/sctputil.c b/sys/netinet/sctputil.c index 1e87a17b2cba7..e0e0670d5a504 100644 --- a/sys/netinet/sctputil.c +++ b/sys/netinet/sctputil.c @@ -67,9 +67,9 @@ sctp_sblog(struct sockbuf *sb, struct sctp_tcb *stcb, int from, int incr) struct sctp_cwnd_log sctp_clog; sctp_clog.x.sb.stcb = stcb; - sctp_clog.x.sb.so_sbcc = sb->sb_cc; + sctp_clog.x.sb.so_sbcc = sb->sb_ccc; if (stcb) - sctp_clog.x.sb.stcb_sbcc = stcb->asoc.sb_cc; + sctp_clog.x.sb.stcb_sbcc = stcb->asoc.sb_ccc; else sctp_clog.x.sb.stcb_sbcc = 0; sctp_clog.x.sb.incr = incr; @@ -4363,7 +4363,7 @@ sctp_add_to_readq(struct sctp_inpcb *inp, { /* * Here we must place the control on the end of the socket read - * queue AND increment sb_cc so that select will work properly on + * queue AND increment sb_ccc so that select will work properly on * read. */ struct mbuf *m, *prev = NULL; @@ -4489,7 +4489,7 @@ sctp_append_to_readq(struct sctp_inpcb *inp, * the reassembly queue. * * If PDAPI this means we need to add m to the end of the data. - * Increase the length in the control AND increment the sb_cc. + * Increase the length in the control AND increment the sb_ccc. * Otherwise sb is NULL and all we need to do is put it at the end * of the mbuf chain. */ @@ -4701,10 +4701,10 @@ sctp_free_bufspace(struct sctp_tcb *stcb, struct sctp_association *asoc, if (stcb->sctp_socket && (((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL)) || ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE)))) { - if (stcb->sctp_socket->so_snd.sb_cc >= tp1->book_size) { - stcb->sctp_socket->so_snd.sb_cc -= tp1->book_size; + if (stcb->sctp_socket->so_snd.sb_ccc >= tp1->book_size) { + stcb->sctp_socket->so_snd.sb_ccc -= tp1->book_size; } else { - stcb->sctp_socket->so_snd.sb_cc = 0; + stcb->sctp_socket->so_snd.sb_ccc = 0; } } @@ -5254,11 +5254,11 @@ sctp_sorecvmsg(struct socket *so, in_eeor_mode = sctp_is_feature_on(inp, SCTP_PCB_FLAGS_EXPLICIT_EOR); if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_RECV_RWND_LOGGING_ENABLE) { sctp_misc_ints(SCTP_SORECV_ENTER, - rwnd_req, in_eeor_mode, so->so_rcv.sb_cc, uio->uio_resid); + rwnd_req, in_eeor_mode, so->so_rcv.sb_ccc, uio->uio_resid); } if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_RECV_RWND_LOGGING_ENABLE) { sctp_misc_ints(SCTP_SORECV_ENTERPL, - rwnd_req, block_allowed, so->so_rcv.sb_cc, uio->uio_resid); + rwnd_req, block_allowed, so->so_rcv.sb_ccc, uio->uio_resid); } error = sblock(&so->so_rcv, (block_allowed ? SBL_WAIT : 0)); if (error) { @@ -5277,23 +5277,23 @@ restart_nosblocks: (inp->sctp_flags & SCTP_PCB_FLAGS_SOCKET_ALLGONE)) { goto out; } - if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && (so->so_rcv.sb_cc == 0)) { + if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && (so->so_rcv.sb_ccc == 0)) { if (so->so_error) { error = so->so_error; if ((in_flags & MSG_PEEK) == 0) so->so_error = 0; goto out; } else { - if (so->so_rcv.sb_cc == 0) { + if (so->so_rcv.sb_ccc == 0) { /* indicate EOF */ error = 0; goto out; } } } - if ((so->so_rcv.sb_cc <= held_length) && block_allowed) { + if ((so->so_rcv.sb_ccc <= held_length) && block_allowed) { /* we need to wait for data */ - if ((so->so_rcv.sb_cc == 0) && + if ((so->so_rcv.sb_ccc == 0) && ((inp->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) || (inp->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL))) { if ((inp->sctp_flags & SCTP_PCB_FLAGS_CONNECTED) == 0) { @@ -5329,7 +5329,7 @@ restart_nosblocks: } held_length = 0; goto restart_nosblocks; - } else if (so->so_rcv.sb_cc == 0) { + } else if (so->so_rcv.sb_ccc == 0) { if (so->so_error) { error = so->so_error; if ((in_flags & MSG_PEEK) == 0) @@ -5386,11 +5386,11 @@ restart_nosblocks: SCTP_INP_READ_LOCK(inp); } control = TAILQ_FIRST(&inp->read_queue); - if ((control == NULL) && (so->so_rcv.sb_cc != 0)) { + if ((control == NULL) && (so->so_rcv.sb_ccc != 0)) { #ifdef INVARIANTS panic("Huh, its non zero and nothing on control?"); #endif - so->so_rcv.sb_cc = 0; + so->so_rcv.sb_ccc = 0; } SCTP_INP_READ_UNLOCK(inp); hold_rlock = 0; @@ -5511,11 +5511,11 @@ restart_nosblocks: } /* * if we reach here, not suitable replacement is available - * <or> fragment interleave is NOT on. So stuff the sb_cc + * <or> fragment interleave is NOT on. So stuff the sb_ccc * into the our held count, and its time to sleep again. */ - held_length = so->so_rcv.sb_cc; - control->held_length = so->so_rcv.sb_cc; + held_length = so->so_rcv.sb_ccc; + control->held_length = so->so_rcv.sb_ccc; goto restart; } /* Clear the held length since there is something to read */ @@ -5812,10 +5812,10 @@ get_more_data: if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_SB_LOGGING_ENABLE) { sctp_sblog(&so->so_rcv, control->do_not_ref_stcb ? NULL : stcb, SCTP_LOG_SBFREE, cp_len); } - atomic_subtract_int(&so->so_rcv.sb_cc, cp_len); + atomic_subtract_int(&so->so_rcv.sb_ccc, cp_len); if ((control->do_not_ref_stcb == 0) && stcb) { - atomic_subtract_int(&stcb->asoc.sb_cc, cp_len); + atomic_subtract_int(&stcb->asoc.sb_ccc, cp_len); } copied_so_far += cp_len; freed_so_far += cp_len; @@ -5960,7 +5960,7 @@ wait_some_more: (sctp_is_feature_on(inp, SCTP_PCB_FLAGS_FRAG_INTERLEAVE))) { goto release; } - if (so->so_rcv.sb_cc <= control->held_length) { + if (so->so_rcv.sb_ccc <= control->held_length) { error = sbwait(&so->so_rcv); if (error) { goto release; @@ -5987,8 +5987,8 @@ wait_some_more: } goto done_with_control; } - if (so->so_rcv.sb_cc > held_length) { - control->held_length = so->so_rcv.sb_cc; + if (so->so_rcv.sb_ccc > held_length) { + control->held_length = so->so_rcv.sb_ccc; held_length = 0; } goto wait_some_more; @@ -6135,13 +6135,13 @@ out: freed_so_far, ((uio) ? (slen - uio->uio_resid) : slen), stcb->asoc.my_rwnd, - so->so_rcv.sb_cc); + so->so_rcv.sb_ccc); } else { sctp_misc_ints(SCTP_SORECV_DONE, freed_so_far, ((uio) ? (slen - uio->uio_resid) : slen), 0, - so->so_rcv.sb_cc); + so->so_rcv.sb_ccc); } } stage_left: diff --git a/sys/netinet/sctputil.h b/sys/netinet/sctputil.h index 559daa3764faf..a9e72625c5e3c 100644 --- a/sys/netinet/sctputil.h +++ b/sys/netinet/sctputil.h @@ -286,10 +286,10 @@ do { \ } \ if (stcb->sctp_socket && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) || \ (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL))) { \ - if (stcb->sctp_socket->so_snd.sb_cc >= tp1->book_size) { \ - atomic_subtract_int(&((stcb)->sctp_socket->so_snd.sb_cc), tp1->book_size); \ + if (stcb->sctp_socket->so_snd.sb_ccc >= tp1->book_size) { \ + atomic_subtract_int(&((stcb)->sctp_socket->so_snd.sb_ccc), tp1->book_size); \ } else { \ - stcb->sctp_socket->so_snd.sb_cc = 0; \ + stcb->sctp_socket->so_snd.sb_ccc = 0; \ } \ } \ } \ @@ -307,10 +307,10 @@ do { \ } \ if (stcb->sctp_socket && ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) || \ (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL))) { \ - if (stcb->sctp_socket->so_snd.sb_cc >= sp->length) { \ - atomic_subtract_int(&stcb->sctp_socket->so_snd.sb_cc,sp->length); \ + if (stcb->sctp_socket->so_snd.sb_ccc >= sp->length) { \ + atomic_subtract_int(&stcb->sctp_socket->so_snd.sb_ccc,sp->length); \ } else { \ - stcb->sctp_socket->so_snd.sb_cc = 0; \ + stcb->sctp_socket->so_snd.sb_ccc = 0; \ } \ } \ } \ @@ -322,7 +322,7 @@ do { \ if ((stcb->sctp_socket != NULL) && \ ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) || \ (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL))) { \ - atomic_add_int(&stcb->sctp_socket->so_snd.sb_cc,sz); \ + atomic_add_int(&stcb->sctp_socket->so_snd.sb_ccc,sz); \ } \ } while (0) diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c index 468f142105658..e19d05ee318cc 100644 --- a/sys/netinet/tcp_input.c +++ b/sys/netinet/tcp_input.c @@ -884,12 +884,10 @@ findpcb: goto dropwithreset; } INP_WLOCK_ASSERT(inp); - if (!(inp->inp_flags & INP_HW_FLOWID) - && (m->m_flags & M_FLOWID) - && ((inp->inp_socket == NULL) - || !(inp->inp_socket->so_options & SO_ACCEPTCONN))) { - inp->inp_flags |= INP_HW_FLOWID; - inp->inp_flags &= ~INP_SW_FLOWID; + if ((inp->inp_flowtype == M_HASHTYPE_NONE) && + (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) && + ((inp->inp_socket == NULL) || + (inp->inp_socket->so_options & SO_ACCEPTCONN) == 0)) { inp->inp_flowid = m->m_pkthdr.flowid; inp->inp_flowtype = M_HASHTYPE_GET(m); } @@ -1855,7 +1853,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, newsize, so, NULL)) so->so_rcv.sb_flags &= ~SB_AUTOSIZE; m_adj(m, drop_hdrlen); /* delayed header drop */ - sbappendstream_locked(&so->so_rcv, m); + sbappendstream_locked(&so->so_rcv, m, 0); } /* NB: sorwakeup_locked() does an implicit unlock. */ sorwakeup_locked(so); @@ -2882,7 +2880,7 @@ dodata: /* XXX */ if (so->so_rcv.sb_state & SBS_CANTRCVMORE) m_freem(m); else - sbappendstream_locked(&so->so_rcv, m); + sbappendstream_locked(&so->so_rcv, m, 0); /* NB: sorwakeup_locked() does an implicit unlock. */ sorwakeup_locked(so); } else { diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c index dffee00f9b187..17d9a79eb50d5 100644 --- a/sys/netinet/tcp_reass.c +++ b/sys/netinet/tcp_reass.c @@ -262,7 +262,7 @@ present: m_freem(mq); else { mq->m_nextpkt = NULL; - sbappendstream_locked(&so->so_rcv, mq); + sbappendstream_locked(&so->so_rcv, mq, 0); wakeup = 1; } } diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c index 05c50769531b4..c7570e2a26d70 100644 --- a/sys/netinet/tcp_syncache.c +++ b/sys/netinet/tcp_syncache.c @@ -713,9 +713,7 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m) * If there's an mbuf and it has a flowid, then let's initialise the * inp with that particular flowid. */ - if (m != NULL && m->m_flags & M_FLOWID) { - inp->inp_flags |= INP_HW_FLOWID; - inp->inp_flags &= ~INP_SW_FLOWID; + if (m != NULL && M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { inp->inp_flowid = m->m_pkthdr.flowid; inp->inp_flowtype = M_HASHTYPE_GET(m); } diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 2820ffba70ed6..dee1c33186abb 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -821,7 +821,11 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { if (control) m_freem(control); - if (m) + /* + * In case of PRUS_NOTREADY, tcp_usr_ready() is responsible + * for freeing memory. + */ + if (m && (flags & PRUS_NOTREADY) == 0) m_freem(m); error = ECONNRESET; goto out; @@ -843,7 +847,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, m_freem(control); /* empty control, just free it */ } if (!(flags & PRUS_OOB)) { - sbappendstream(&so->so_snd, m); + sbappendstream(&so->so_snd, m, flags); if (nam && tp->t_state < TCPS_SYN_SENT) { /* * Do implied connect if not yet connected, @@ -875,7 +879,8 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, socantsendmore(so); tcp_usrclosed(tp); } - if (!(inp->inp_flags & INP_DROPPED)) { + if (!(inp->inp_flags & INP_DROPPED) && + !(flags & PRUS_NOTREADY)) { if (flags & PRUS_MORETOCOME) tp->t_flags |= TF_MORETOCOME; error = tcp_output(tp); @@ -901,7 +906,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, * of data past the urgent section. * Otherwise, snd_up should be one lower. */ - sbappendstream_locked(&so->so_snd, m); + sbappendstream_locked(&so->so_snd, m, flags); SOCKBUF_UNLOCK(&so->so_snd); if (nam && tp->t_state < TCPS_SYN_SENT) { /* @@ -925,10 +930,12 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, tp->snd_wnd = TTCP_CLIENT_SND_WND; tcp_mss(tp, -1); } - tp->snd_up = tp->snd_una + so->so_snd.sb_cc; - tp->t_flags |= TF_FORCEDATA; - error = tcp_output(tp); - tp->t_flags &= ~TF_FORCEDATA; + tp->snd_up = tp->snd_una + sbavail(&so->so_snd); + if (!(flags & PRUS_NOTREADY)) { + tp->t_flags |= TF_FORCEDATA; + error = tcp_output(tp); + tp->t_flags &= ~TF_FORCEDATA; + } } out: TCPDEBUG2((flags & PRUS_OOB) ? PRU_SENDOOB : @@ -939,6 +946,33 @@ out: return (error); } +static int +tcp_usr_ready(struct socket *so, struct mbuf *m, int count) +{ + struct inpcb *inp; + struct tcpcb *tp; + int error; + + inp = sotoinpcb(so); + INP_WLOCK(inp); + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_WUNLOCK(inp); + for (int i = 0; i < count; i++) + m = m_free(m); + return (ECONNRESET); + } + tp = intotcpcb(inp); + + SOCKBUF_LOCK(&so->so_snd); + error = sbready(&so->so_snd, m, count); + SOCKBUF_UNLOCK(&so->so_snd); + if (error == 0) + error = tcp_output(tp); + INP_WUNLOCK(inp); + + return (error); +} + /* * Abort the TCP. Drop the connection abruptly. */ @@ -1073,6 +1107,7 @@ struct pr_usrreqs tcp_usrreqs = { .pru_rcvd = tcp_usr_rcvd, .pru_rcvoob = tcp_usr_rcvoob, .pru_send = tcp_usr_send, + .pru_ready = tcp_usr_ready, .pru_shutdown = tcp_usr_shutdown, .pru_sockaddr = in_getsockaddr, .pru_sosetlabel = in_pcbsosetlabel, @@ -1095,6 +1130,7 @@ struct pr_usrreqs tcp6_usrreqs = { .pru_rcvd = tcp_usr_rcvd, .pru_rcvoob = tcp_usr_rcvoob, .pru_send = tcp_usr_send, + .pru_ready = tcp_usr_ready, .pru_shutdown = tcp_usr_shutdown, .pru_sockaddr = in6_mapped_sockaddr, .pru_sosetlabel = in_pcbsosetlabel, diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c index c4ef9f07f7084..5fd93b3bcff9f 100644 --- a/sys/netinet/udp_usrreq.c +++ b/sys/netinet/udp_usrreq.c @@ -1106,8 +1106,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, uint8_t pr; uint16_t cscov = 0; uint32_t flowid = 0; - int flowid_type = 0; - int use_flowid = 0; + uint8_t flowtype = M_HASHTYPE_NONE; /* * udp_output() may need to temporarily bind or connect the current @@ -1184,8 +1183,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, error = EINVAL; break; } - flowid_type = *(uint32_t *) CMSG_DATA(cm); - use_flowid = 1; + flowtype = *(uint32_t *) CMSG_DATA(cm); break; #ifdef RSS @@ -1451,10 +1449,9 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, * Once the UDP code decides to set a flowid some other way, * this allows the flowid to be overridden by userland. */ - if (use_flowid) { - m->m_flags |= M_FLOWID; + if (flowtype != M_HASHTYPE_NONE) { m->m_pkthdr.flowid = flowid; - M_HASHTYPE_SET(m, flowid_type); + M_HASHTYPE_SET(m, flowtype); #ifdef RSS } else { uint32_t hash_val, hash_type; @@ -1477,7 +1474,6 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, if (rss_proto_software_hash_v4(faddr, laddr, fport, lport, pr, &hash_val, &hash_type) == 0) { m->m_pkthdr.flowid = hash_val; - m->m_flags |= M_FLOWID; M_HASHTYPE_SET(m, hash_type); } #endif diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c index 2a7b9dcf40f52..3b61849d41877 100644 --- a/sys/netinet6/in6_pcb.c +++ b/sys/netinet6/in6_pcb.c @@ -1252,7 +1252,7 @@ in6_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, * XXXRW: As above, that policy belongs in the pcbgroup code. */ if (in_pcbgroup_enabled(pcbinfo) && - !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) { + M_HASHTYPE_TEST(m, M_HASHTYPE_NONE) == 0) { pcbgroup = in6_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m), m->m_pkthdr.flowid); if (pcbgroup != NULL) diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c index 4245a9fdb9e01..74d7483aa9471 100644 --- a/sys/netinet6/ip6_output.c +++ b/sys/netinet6/ip6_output.c @@ -267,10 +267,10 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, if (inp != NULL) { M_SETFIB(m, inp->inp_inc.inc_fibnum); - if (((flags & IP_NODEFAULTFLOWID) == 0) && - (inp->inp_flags & (INP_HW_FLOWID|INP_SW_FLOWID))) { + if ((flags & IP_NODEFAULTFLOWID) == 0) { + /* unconditionally set flowid */ m->m_pkthdr.flowid = inp->inp_flowid; - m->m_flags |= M_FLOWID; + M_HASHTYPE_SET(m, inp->inp_flowtype); } } diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c index 6c6f27a6e5d30..2b07b8e63fd9b 100644 --- a/sys/netinet6/udp6_usrreq.c +++ b/sys/netinet6/udp6_usrreq.c @@ -843,7 +843,6 @@ udp6_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr6, */ #ifdef RSS m->m_pkthdr.flowid = rss_hash_ip6_2tuple(*faddr, *laddr); - m->m_flags |= M_FLOWID; M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); #endif flags = 0; diff --git a/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c b/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c index a6eba64e16228..2e91d85294348 100644 --- a/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c +++ b/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c @@ -889,7 +889,7 @@ sdp_append(struct sdp_sock *ssk, struct sockbuf *sb, struct mbuf *mb, int cnt) m_adj(mb, SDP_HEAD_SIZE); n->m_pkthdr.len += mb->m_pkthdr.len; n->m_flags |= mb->m_flags & (M_PUSH | M_URG); - m_demote(mb, 1); + m_demote(mb, 1, 0); sbcompress(sb, mb, sb->sb_mbtail); return; } diff --git a/sys/ofed/drivers/net/mlx4/en_rx.c b/sys/ofed/drivers/net/mlx4/en_rx.c index 320462ec02ad9..fa263266d67d3 100644 --- a/sys/ofed/drivers/net/mlx4/en_rx.c +++ b/sys/ofed/drivers/net/mlx4/en_rx.c @@ -604,7 +604,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud } mb->m_pkthdr.flowid = cq->ring; - mb->m_flags |= M_FLOWID; + M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); mb->m_pkthdr.rcvif = dev; if (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_VLAN_PRESENT_MASK) { diff --git a/sys/ofed/drivers/net/mlx4/en_tx.c b/sys/ofed/drivers/net/mlx4/en_tx.c index d1931558bcc9f..d92677fa5ce38 100644 --- a/sys/ofed/drivers/net/mlx4/en_tx.c +++ b/sys/ofed/drivers/net/mlx4/en_tx.c @@ -720,8 +720,11 @@ u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb) up = (vlan_tag >> 13); } - /* hash mbuf */ - queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); + /* check if flowid is set */ + if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) + queue_index = mb->m_pkthdr.flowid; + else + queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); return ((queue_index % rings_p_up) + (up * rings_p_up)); } @@ -1066,15 +1069,11 @@ mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_tx_ring *ring; struct mlx4_en_cq *cq; - int i = 0, err = 0; + int i, err = 0; + + /* Compute which queue to use */ + i = mlx4_en_select_queue(dev, m); - /* Which queue to use */ - if ((m->m_flags & (M_FLOWID | M_VLANTAG)) == M_FLOWID) { - i = m->m_pkthdr.flowid % (priv->tx_ring_num - 1); - } - else { - i = mlx4_en_select_queue(dev, m); - } ring = priv->tx_ring[i]; if (spin_trylock(&ring->tx_lock)) { diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c index 5864a02c4694f..d1413a2c4aba6 100644 --- a/sys/powerpc/aim/machdep.c +++ b/sys/powerpc/aim/machdep.c @@ -757,6 +757,10 @@ db_trap_glue(struct trapframe *frame) || frame->exc == EXC_BPT || frame->exc == EXC_DSI)) { int type = frame->exc; + + /* Ignore DTrace traps. */ + if (*(uint32_t *)frame->srr0 == EXC_DTRACE) + return (0); if (type == EXC_PGM && (frame->srr1 & 0x20000)) { type = T_BREAKPOINT; } diff --git a/sys/powerpc/aim/trap.c b/sys/powerpc/aim/trap.c index cc54e593c00fc..8276e96f19369 100644 --- a/sys/powerpc/aim/trap.c +++ b/sys/powerpc/aim/trap.c @@ -258,7 +258,8 @@ trap(struct trapframe *frame) if (frame->srr1 & EXC_PGM_TRAP) { #ifdef KDTRACE_HOOKS inst = fuword32((const void *)frame->srr0); - if (inst == 0x0FFFDDDD && dtrace_pid_probe_ptr != NULL) { + if (inst == 0x0FFFDDDD && + dtrace_pid_probe_ptr != NULL) { struct reg regs; fill_regs(td, ®s); (*dtrace_pid_probe_ptr)(®s); @@ -301,7 +302,7 @@ trap(struct trapframe *frame) #ifdef KDTRACE_HOOKS case EXC_PGM: if (frame->srr1 & EXC_PGM_TRAP) { - if (*(uint32_t *)frame->srr0 == 0x7c810808) { + if (*(uint32_t *)frame->srr0 == EXC_DTRACE) { if (dtrace_invop_jump_addr != NULL) { dtrace_invop_jump_addr(frame); return; diff --git a/sys/powerpc/aim/trap_subr32.S b/sys/powerpc/aim/trap_subr32.S index 56ca7155ddd06..2aae2a00abbc9 100644 --- a/sys/powerpc/aim/trap_subr32.S +++ b/sys/powerpc/aim/trap_subr32.S @@ -890,8 +890,7 @@ CNAME(dblow): mfcr %r29 /* save CR in r29 */ mfsrr1 %r1 mtcr %r1 - bf 17,2f /* branch if privileged */ -1: + bf 17,1f /* branch if privileged */ /* Unprivileged case */ mtcr %r29 /* put the condition register back */ mfsprg2 %r29 /* ... and r29 */ @@ -900,19 +899,7 @@ CNAME(dblow): li %r1, 0 /* How to get the vector from LR */ bla generictrap /* and we look like a generic trap */ -2: -#ifdef KDTRACE_HOOKS - /* Privileged, so drop to KDB */ - mfsrr0 %r1 - mtsprg3 %r3 - lwz %r1,0(%r1) - /* Check if it's a DTrace trap. */ - li %r3,0x0808 - addis %r3,%r3,0x7c81 - cmplw %cr0,%r3,%r1 - mfsprg3 %r3 - beq %cr0,1b -#endif +1: /* Privileged, so drop to KDB */ GET_CPUINFO(%r1) stw %r28,(PC_DBSAVE+CPUSAVE_R28)(%r1) /* free r28 */ diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S index bedf1f1962c4a..0dc46b34e866c 100644 --- a/sys/powerpc/aim/trap_subr64.S +++ b/sys/powerpc/aim/trap_subr64.S @@ -799,9 +799,8 @@ CNAME(dblow): mfcr %r29 /* save CR in r29 */ mfsrr1 %r1 mtcr %r1 - bf 17,2f /* branch if privileged */ + bf 17,1f /* branch if privileged */ -1: /* Unprivileged case */ mtcr %r29 /* put the condition register back */ mfsprg2 %r29 /* ... and r29 */ @@ -810,19 +809,7 @@ CNAME(dblow): li %r1, 0 /* How to get the vector from LR */ bla generictrap /* and we look like a generic trap */ -2: -#ifdef KDTRACE_HOOKS - /* Privileged, so drop to KDB */ - mfsrr0 %r1 - mtsprg3 %r3 - lwz %r1,0(%r1) - /* Check if it's a DTrace trap. */ - li %r3,0x0808 - addis %r3,%r3,0x7c81 - cmplw %cr0,%r3,%r1 - mfsprg3 %r3 - beq %cr0,1b -#endif +1: GET_CPUINFO(%r1) std %r27,(PC_DBSAVE+CPUSAVE_R27)(%r1) /* free r27 */ std %r28,(PC_DBSAVE+CPUSAVE_R28)(%r1) /* free r28 */ diff --git a/sys/powerpc/include/trap.h b/sys/powerpc/include/trap.h index 89f596659cfb1..211afcf782258 100644 --- a/sys/powerpc/include/trap.h +++ b/sys/powerpc/include/trap.h @@ -120,6 +120,9 @@ #define EXC_PGM_PRIV (1UL << 18) #define EXC_PGM_TRAP (1UL << 17) +/* DTrace trap opcode. */ +#define EXC_DTRACE 0x7c810808 + #ifndef LOCORE struct trapframe; struct pcb; diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h index 190900c43d22d..ac3cfb2d0ced7 100644 --- a/sys/sys/mbuf.h +++ b/sys/sys/mbuf.h @@ -950,7 +950,7 @@ struct mbuf *m_copypacket(struct mbuf *, int); void m_copy_pkthdr(struct mbuf *, struct mbuf *); struct mbuf *m_copyup(struct mbuf *, int, int); struct mbuf *m_defrag(struct mbuf *, int); -void m_demote(struct mbuf *, int); +void m_demote(struct mbuf *, int, int); struct mbuf *m_devget(char *, int, int, struct ifnet *, void (*)(char *, caddr_t, u_int)); struct mbuf *m_dup(struct mbuf *, int); diff --git a/sys/sys/protosw.h b/sys/sys/protosw.h index 2d98a4c25eddd..55db0e339699f 100644 --- a/sys/sys/protosw.h +++ b/sys/sys/protosw.h @@ -208,6 +208,8 @@ struct pr_usrreqs { #define PRUS_OOB 0x1 #define PRUS_EOF 0x2 #define PRUS_MORETOCOME 0x4 +#define PRUS_NOTREADY 0x8 + int (*pru_ready)(struct socket *so, struct mbuf *m, int count); int (*pru_sense)(struct socket *so, struct stat *sb); int (*pru_shutdown)(struct socket *so); int (*pru_flush)(struct socket *so, int direction); @@ -251,6 +253,7 @@ int pru_rcvd_notsupp(struct socket *so, int flags); int pru_rcvoob_notsupp(struct socket *so, struct mbuf *m, int flags); int pru_send_notsupp(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr, struct mbuf *control, struct thread *td); +int pru_ready_notsupp(struct socket *so, struct mbuf *m, int count); int pru_sense_null(struct socket *so, struct stat *sb); int pru_shutdown_notsupp(struct socket *so); int pru_sockaddr_notsupp(struct socket *so, struct sockaddr **nam); diff --git a/sys/sys/sockbuf.h b/sys/sys/sockbuf.h index f9e8da4cde0bc..5bd9bb5562514 100644 --- a/sys/sys/sockbuf.h +++ b/sys/sys/sockbuf.h @@ -89,8 +89,10 @@ struct sockbuf { struct mbuf *sb_lastrecord; /* (c/d) first mbuf of last * record in socket buffer */ struct mbuf *sb_sndptr; /* (c/d) pointer into mbuf chain */ + struct mbuf *sb_fnrdy; /* (c/d) pointer to first not ready buffer */ u_int sb_sndptroff; /* (c/d) byte offset of ptr into chain */ - u_int sb_cc; /* (c/d) actual chars in buffer */ + u_int sb_acc; /* (c/d) available chars in buffer */ + u_int sb_ccc; /* (c/d) claimed chars in buffer */ u_int sb_hiwat; /* (c/d) max actual char count */ u_int sb_mbcnt; /* (c/d) chars of mbufs used */ u_int sb_mcnt; /* (c/d) number of mbufs in buffer */ @@ -120,10 +122,17 @@ struct sockbuf { #define SOCKBUF_LOCK_ASSERT(_sb) mtx_assert(SOCKBUF_MTX(_sb), MA_OWNED) #define SOCKBUF_UNLOCK_ASSERT(_sb) mtx_assert(SOCKBUF_MTX(_sb), MA_NOTOWNED) +/* + * Socket buffer private mbuf(9) flags. + */ +#define M_NOTREADY M_PROTO1 /* m_data not populated yet */ +#define M_BLOCKED M_PROTO2 /* M_NOTREADY in front of m */ +#define M_NOTAVAIL (M_NOTREADY | M_BLOCKED) + void sbappend(struct sockbuf *sb, struct mbuf *m); void sbappend_locked(struct sockbuf *sb, struct mbuf *m); -void sbappendstream(struct sockbuf *sb, struct mbuf *m); -void sbappendstream_locked(struct sockbuf *sb, struct mbuf *m); +void sbappendstream(struct sockbuf *sb, struct mbuf *m, int flags); +void sbappendstream_locked(struct sockbuf *sb, struct mbuf *m, int flags); int sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, struct mbuf *control); int sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, @@ -136,7 +145,6 @@ int sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control); void sbappendrecord(struct sockbuf *sb, struct mbuf *m0); void sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0); -void sbcheck(struct sockbuf *sb); void sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n); struct mbuf * sbcreatecontrol(caddr_t p, int size, int type, int level); @@ -164,10 +172,13 @@ void sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb); int sbwait(struct sockbuf *sb); int sblock(struct sockbuf *sb, int flags); void sbunlock(struct sockbuf *sb); +void sballoc(struct sockbuf *, struct mbuf *); +void sbfree(struct sockbuf *, struct mbuf *); +int sbready(struct sockbuf *, struct mbuf *, int); /* * Return how much data is available to be taken out of socket - * bufffer right now. + * buffer right now. */ static inline u_int sbavail(struct sockbuf *sb) @@ -176,7 +187,7 @@ sbavail(struct sockbuf *sb) #if 0 SOCKBUF_LOCK_ASSERT(sb); #endif - return (sb->sb_cc); + return (sb->sb_acc); } /* @@ -190,59 +201,30 @@ sbused(struct sockbuf *sb) #if 0 SOCKBUF_LOCK_ASSERT(sb); #endif - return (sb->sb_cc); + return (sb->sb_ccc); } /* * How much space is there in a socket buffer (so->so_snd or so->so_rcv)? * This is problematical if the fields are unsigned, as the space might - * still be negative (cc > hiwat or mbcnt > mbmax). Should detect - * overflow and return 0. Should use "lmin" but it doesn't exist now. + * still be negative (ccc > hiwat or mbcnt > mbmax). */ -static __inline -long +static inline long sbspace(struct sockbuf *sb) { - long bleft; - long mleft; + long bleft, mleft; + +#if 0 + SOCKBUF_LOCK_ASSERT(sb); +#endif if (sb->sb_flags & SB_STOP) return(0); - bleft = sb->sb_hiwat - sb->sb_cc; - mleft = sb->sb_mbmax - sb->sb_mbcnt; - return((bleft < mleft) ? bleft : mleft); -} -/* adjust counters in sb reflecting allocation of m */ -#define sballoc(sb, m) { \ - (sb)->sb_cc += (m)->m_len; \ - if ((m)->m_type != MT_DATA && (m)->m_type != MT_OOBDATA) \ - (sb)->sb_ctl += (m)->m_len; \ - (sb)->sb_mbcnt += MSIZE; \ - (sb)->sb_mcnt += 1; \ - if ((m)->m_flags & M_EXT) { \ - (sb)->sb_mbcnt += (m)->m_ext.ext_size; \ - (sb)->sb_ccnt += 1; \ - } \ -} + bleft = sb->sb_hiwat - sb->sb_ccc; + mleft = sb->sb_mbmax - sb->sb_mbcnt; -/* adjust counters in sb reflecting freeing of m */ -#define sbfree(sb, m) { \ - (sb)->sb_cc -= (m)->m_len; \ - if ((m)->m_type != MT_DATA && (m)->m_type != MT_OOBDATA) \ - (sb)->sb_ctl -= (m)->m_len; \ - (sb)->sb_mbcnt -= MSIZE; \ - (sb)->sb_mcnt -= 1; \ - if ((m)->m_flags & M_EXT) { \ - (sb)->sb_mbcnt -= (m)->m_ext.ext_size; \ - (sb)->sb_ccnt -= 1; \ - } \ - if ((sb)->sb_sndptr == (m)) { \ - (sb)->sb_sndptr = NULL; \ - (sb)->sb_sndptroff = 0; \ - } \ - if ((sb)->sb_sndptroff != 0) \ - (sb)->sb_sndptroff -= (m)->m_len; \ + return ((bleft < mleft) ? bleft : mleft); } #define SB_EMPTY_FIXUP(sb) do { \ @@ -254,13 +236,15 @@ sbspace(struct sockbuf *sb) #ifdef SOCKBUF_DEBUG void sblastrecordchk(struct sockbuf *, const char *, int); -#define SBLASTRECORDCHK(sb) sblastrecordchk((sb), __FILE__, __LINE__) - void sblastmbufchk(struct sockbuf *, const char *, int); +void sbcheck(struct sockbuf *, const char *, int); +#define SBLASTRECORDCHK(sb) sblastrecordchk((sb), __FILE__, __LINE__) #define SBLASTMBUFCHK(sb) sblastmbufchk((sb), __FILE__, __LINE__) +#define SBCHECK(sb) sbcheck((sb), __FILE__, __LINE__) #else -#define SBLASTRECORDCHK(sb) /* nothing */ -#define SBLASTMBUFCHK(sb) /* nothing */ +#define SBLASTRECORDCHK(sb) do {} while (0) +#define SBLASTMBUFCHK(sb) do {} while (0) +#define SBCHECK(sb) do {} while (0) #endif /* SOCKBUF_DEBUG */ #endif /* _KERNEL */ diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 50e66c058268b..574eba073f54e 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -146,6 +146,8 @@ static LIST_HEAD(,uma_slab) uma_boot_pages = /* This mutex protects the boot time pages list */ static struct mtx_padalign uma_boot_pages_mtx; +static struct sx uma_drain_lock; + /* Is the VM done starting up? */ static int booted = 0; #define UMA_STARTUP 1 @@ -1876,6 +1878,7 @@ uma_startup2(void) { booted = UMA_STARTUP2; bucket_enable(); + sx_init(&uma_drain_lock, "umadrain"); #ifdef UMA_DEBUG printf("UMA startup2 complete.\n"); #endif @@ -1930,6 +1933,8 @@ uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, { struct uma_zctor_args args; + uma_zone_t res; + bool locked; /* This stuff is essential for the zone ctor */ memset(&args, 0, sizeof(args)); @@ -1943,7 +1948,16 @@ uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, args.flags = flags; args.keg = NULL; - return (zone_alloc_item(zones, &args, M_WAITOK)); + if (booted < UMA_STARTUP2) { + locked = false; + } else { + sx_slock(&uma_drain_lock); + locked = true; + } + res = zone_alloc_item(zones, &args, M_WAITOK); + if (locked) + sx_sunlock(&uma_drain_lock); + return (res); } /* See uma.h */ @@ -1953,6 +1967,8 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, { struct uma_zctor_args args; uma_keg_t keg; + uma_zone_t res; + bool locked; keg = zone_first_keg(master); memset(&args, 0, sizeof(args)); @@ -1966,8 +1982,17 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; args.keg = keg; + if (booted < UMA_STARTUP2) { + locked = false; + } else { + sx_slock(&uma_drain_lock); + locked = true; + } /* XXX Attaches only one keg of potentially many. */ - return (zone_alloc_item(zones, &args, M_WAITOK)); + res = zone_alloc_item(zones, &args, M_WAITOK); + if (locked) + sx_sunlock(&uma_drain_lock); + return (res); } /* See uma.h */ @@ -2085,7 +2110,9 @@ void uma_zdestroy(uma_zone_t zone) { + sx_slock(&uma_drain_lock); zone_free_item(zones, zone, NULL, SKIP_NONE); + sx_sunlock(&uma_drain_lock); } /* See uma.h */ @@ -3205,6 +3232,7 @@ uma_reclaim(void) #ifdef UMA_DEBUG printf("UMA: vm asked us to release pages!\n"); #endif + sx_xlock(&uma_drain_lock); bucket_enable(); zone_foreach(zone_drain); if (vm_page_count_min()) { @@ -3219,6 +3247,7 @@ uma_reclaim(void) zone_drain(slabzone); zone_drain(slabrefzone); bucket_zone_drain(); + sx_xunlock(&uma_drain_lock); } /* See uma.h */ diff --git a/tools/tools/shlib-compat/shlib-compat.py b/tools/tools/shlib-compat/shlib-compat.py index 726c53ffb9671..ca6f036639d18 100755 --- a/tools/tools/shlib-compat/shlib-compat.py +++ b/tools/tools/shlib-compat/shlib-compat.py @@ -60,6 +60,14 @@ class Config(object): origfile = FileConfig() newfile = FileConfig() + exclude_sym_default = [ + '^__bss_start$', + '^_edata$', + '^_end$', + '^_fini$', + '^_init$', + ] + @classmethod def init(cls): cls.version_filter = StrFilter() @@ -338,15 +346,17 @@ class BaseTypeDef(Def): def _pp(self, pp): if self.encoding in self.inttypes: sign = '' if self.encoding == 'DW_ATE_signed' else 'u' - bits = int(self.byte_size) * 8 + bits = int(self.byte_size, 0) * 8 return '%sint%s_t' % (sign, bits) - elif self.encoding == 'DW_ATE_signed_char' and int(self.byte_size) == 1: + elif self.encoding == 'DW_ATE_signed_char' and int(self.byte_size, 0) == 1: return 'char'; + elif self.encoding == 'DW_ATE_boolean' and int(self.byte_size, 0) == 1: + return 'bool'; elif self.encoding == 'DW_ATE_float': - return self._mapval(self.byte_size, { - '16': 'long double', - '8': 'double', - '4': 'float', + return self._mapval(int(self.byte_size, 0), { + 16: 'long double', + 8: 'double', + 4: 'float', }) raise NotImplementedError('Invalid encoding: %s' % self) @@ -374,6 +384,11 @@ class VolatileTypeDef(AnonymousDef): def _pp(self, pp): return 'volatile ' + self.type._pp(pp) +class RestrictTypeDef(AnonymousDef): + _is_alias = True + def _pp(self, pp): + return 'restrict ' + self.type._pp(pp) + class ArrayDef(AnonymousDef): def _pp(self, pp): t = pp.run(self.type) @@ -411,6 +426,11 @@ class ParameterDef(Def): t = pp.run(self.type) return "%s %s" % (t, self._name_opt()) +class VariableDef(Def): + def _pp(self, pp): + t = pp.run(self.type) + return "%s %s" % (t, self._name_opt()) + # TODO class StructForwardDef(Def): pass @@ -485,6 +505,10 @@ class Dwarf(object): result = self._build_optarg_type(raw) return FunctionDef(raw.id, raw.name, params=params, result=result) + def build_variable(self, raw): + type = self._build_optarg_type(raw) + return VariableDef(raw.id, raw.optname, type=type) + def build_subroutine_type(self, raw): params = [ self.build(x) for x in raw.nested ] result = self._build_optarg_type(raw) @@ -547,6 +571,10 @@ class Dwarf(object): type = self._build_optarg_type(raw) return VolatileTypeDef(raw.id, type=type) + def build_restrict_type(self, raw): + type = self._build_optarg_type(raw) + return RestrictTypeDef(raw.id, type=type) + def build_enumeration_type(self, raw): # TODO handle DW_TAG_enumerator ??? return EnumerationTypeDef(raw.id, name=raw.optname, @@ -574,7 +602,7 @@ class Dwarf(object): return int(id) except ValueError: if (id.startswith('<') and id.endswith('>')): - return int(id[1:-1]) + return int(id[1:-1], 0) else: raise ValueError("Invalid dwarf id: %s" % id) @@ -782,7 +810,7 @@ class DwarfdumpParser(Parser): class Tag(object): def __init__(self, unit, data): self.unit = unit - self.id = int(data['id']) + self.id = int(data['id'], 0) self.level = int(data['level']) self.tag = data['tag'] self.args = {} @@ -816,7 +844,7 @@ class DwarfdumpParser(Parser): def __repr__(self): return "Tag(%d, %d, %s)" % (self.level, self.id, self.tag) - re_header = re.compile('<(?P<level>\d+)><(?P<id>\d+\+*\d*)><(?P<tag>\w+)>') + re_header = re.compile('<(?P<level>\d+)><(?P<id>[0xX0-9a-fA-F]+(?:\+(0[xX])?[0-9a-fA-F]+)?)><(?P<tag>\w+)>') re_argname = re.compile('(?P<arg>\w+)<') re_argunknown = re.compile('<Unknown AT value \w+><[^<>]+>') @@ -827,6 +855,10 @@ class DwarfdumpParser(Parser): 'DW_TAG_variable', ]) + external_tags = set([ + 'DW_TAG_variable', + ]) + def __init__(self, libfile): Parser.__init__(self, "%s -di %s" % (Config.dwarfdump, libfile)) self.current_unit = None @@ -888,9 +920,19 @@ class DwarfdumpParser(Parser): while args: args = self.parse_arg(tag, args) tag.unit.tags[tag.id] = tag - if tag.args.has_key('DW_AT_low_pc') and \ - tag.tag not in DwarfdumpParser.skip_tags: - offset = int(tag.args['DW_AT_low_pc'], 16) + def parse_offset(tag): + if tag.args.has_key('DW_AT_low_pc'): + return int(tag.args['DW_AT_low_pc'], 16) + elif tag.args.has_key('DW_AT_location'): + location = tag.args['DW_AT_location'] + if location.startswith('DW_OP_addr'): + return int(location.replace('DW_OP_addr', ''), 16) + return None + offset = parse_offset(tag) + if offset is not None and \ + (tag.tag not in DwarfdumpParser.skip_tags or \ + (tag.args.has_key('DW_AT_external') and \ + tag.tag in DwarfdumpParser.external_tags)): if self.offsetmap.has_key(offset): raise ValueError("Dwarf dump parse error: " + "symbol is aleady defined at offset 0x%x" % offset) @@ -963,10 +1005,15 @@ def cmp_symbols(commonver): names.sort() for symname in names: sym = ver.symbols[symname] - match = sym.origsym.definition == sym.newsym.definition + missing = sym.origsym.definition is None or sym.newsym.definition is None + match = not missing and sym.origsym.definition == sym.newsym.definition if not match: App.result_code = 1 if Config.verbose >= 1 or not match: + if missing: + print '%s: missing definition' % \ + (sym.origsym.name_ver,) + continue print '%s: definitions %smatch' % \ (sym.origsym.name_ver, "" if match else "mis") if Config.dump or (not match and not Config.no_dump): @@ -1035,10 +1082,16 @@ if __name__ == '__main__': help="result output file for original library", metavar="ORIGFILE") parser.add_option('--out-new', action='store', help="result output file for new library", metavar="NEWFILE") + parser.add_option('--dwarfdump', action='store', + help="path to dwarfdump executable", metavar="DWARFDUMP") + parser.add_option('--objdump', action='store', + help="path to objdump executable", metavar="OBJDUMP") parser.add_option('--exclude-ver', action='append', metavar="RE") parser.add_option('--include-ver', action='append', metavar="RE") parser.add_option('--exclude-sym', action='append', metavar="RE") parser.add_option('--include-sym', action='append', metavar="RE") + parser.add_option('--no-exclude-sym-default', action='store_true', + help="don't exclude special symbols like _init, _end, __bss_start") for opt in ['alias', 'cached', 'symbol']: parser.add_option("--w-" + opt, action="store_true", dest="w_" + opt) @@ -1049,6 +1102,10 @@ if __name__ == '__main__': if len(args) != 2: parser.print_help() sys.exit(-1) + if opts.dwarfdump: + Config.dwarfdump = opts.dwarfdump + if opts.objdump: + Config.objdump = opts.objdump if opts.out_orig: Config.origfile.init(opts.out_orig) if opts.out_new: @@ -1071,6 +1128,8 @@ if __name__ == '__main__': opt = getattr(opts, a + k) if opt: getattr(v, a).extend(opt) + if not opts.no_exclude_sym_default: + Config.symbol_filter.exclude.extend(Config.exclude_sym_default) Config.version_filter.compile() Config.symbol_filter.compile() for w in ['w_alias', 'w_cached', 'w_symbol']: diff --git a/tools/tools/shlib-compat/test/Makefile.inc b/tools/tools/shlib-compat/test/Makefile.inc index 14aaef6ea9564..e1a0719a89f4f 100644 --- a/tools/tools/shlib-compat/test/Makefile.inc +++ b/tools/tools/shlib-compat/test/Makefile.inc @@ -11,3 +11,5 @@ DEBUG_FLAGS?= -g VERSION_DEF= ${.CURDIR}/../Versions.def SYMBOL_MAPS= ${.CURDIR}/Symbol.map + +MK_DEBUG_FILES= yes diff --git a/tools/tools/shlib-compat/test/regress.sh b/tools/tools/shlib-compat/test/regress.sh index e113cce87b635..0789fa8b3cf74 100755 --- a/tools/tools/shlib-compat/test/regress.sh +++ b/tools/tools/shlib-compat/test/regress.sh @@ -1,7 +1,7 @@ #!/bin/sh # $FreeBSD$ -run() { ../shlib-compat.py --no-dump -vv libtest$1/libtest$1.so.0.debug libtest$2/libtest$2.so.0.debug; } +run() { ../shlib-compat.py --no-dump -vv libtest$1/libtest$1.so.0.full libtest$2/libtest$2.so.0.full; } echo 1..9 REGRESSION_START($1) REGRESSION_TEST(`1-1', `run 1 1') diff --git a/tools/tools/sysbuild/sysbuild.sh b/tools/tools/sysbuild/sysbuild.sh index 1a32c60e027e5..bb22a697453a6 100644 --- a/tools/tools/sysbuild/sysbuild.sh +++ b/tools/tools/sysbuild/sysbuild.sh @@ -226,8 +226,7 @@ ports_build() ( t=`echo $p | sed 's,/usr/ports/,,'` pn=`cd $p && make package-name` - if [ "x$p" == "x/usr/ports/ports-mgmt/pkg" -o \ - "x$p" == "x/freebsd/ports/ports-mgmt/pkg" ] ; then + if [ "x`basename $p`" == "xpkg" ] ; then log_it "Very Special: $t ($pn)" ( diff --git a/usr.bin/bluetooth/btsockstat/btsockstat.c b/usr.bin/bluetooth/btsockstat/btsockstat.c index f63ee1fe5f7a1..20f4de8076c8d 100644 --- a/usr.bin/bluetooth/btsockstat/btsockstat.c +++ b/usr.bin/bluetooth/btsockstat/btsockstat.c @@ -255,8 +255,8 @@ hcirawpr(kvm_t *kvmd, u_long addr) (unsigned long) pcb.so, (unsigned long) this, pcb.flags, - so.so_rcv.sb_cc, - so.so_snd.sb_cc, + so.so_rcv.sb_ccc, + so.so_snd.sb_ccc, pcb.addr.hci_node); } } /* hcirawpr */ @@ -303,8 +303,8 @@ l2caprawpr(kvm_t *kvmd, u_long addr) "%-8lx %-8lx %6d %6d %-17.17s\n", (unsigned long) pcb.so, (unsigned long) this, - so.so_rcv.sb_cc, - so.so_snd.sb_cc, + so.so_rcv.sb_ccc, + so.so_snd.sb_ccc, bdaddrpr(&pcb.src, NULL, 0)); } } /* l2caprawpr */ @@ -361,8 +361,8 @@ l2cappr(kvm_t *kvmd, u_long addr) fprintf(stdout, "%-8lx %6d %6d %-17.17s/%-5d %-17.17s %-5d %s\n", (unsigned long) this, - so.so_rcv.sb_cc, - so.so_snd.sb_cc, + so.so_rcv.sb_ccc, + so.so_snd.sb_ccc, bdaddrpr(&pcb.src, local, sizeof(local)), pcb.psm, bdaddrpr(&pcb.dst, remote, sizeof(remote)), @@ -467,8 +467,8 @@ rfcommpr(kvm_t *kvmd, u_long addr) fprintf(stdout, "%-8lx %6d %6d %-17.17s %-17.17s %-4d %-4d %s\n", (unsigned long) this, - so.so_rcv.sb_cc, - so.so_snd.sb_cc, + so.so_rcv.sb_ccc, + so.so_snd.sb_ccc, bdaddrpr(&pcb.src, local, sizeof(local)), bdaddrpr(&pcb.dst, remote, sizeof(remote)), pcb.channel, diff --git a/usr.bin/netstat/inet.c b/usr.bin/netstat/inet.c index 10d0698e6beba..c22e49f624051 100644 --- a/usr.bin/netstat/inet.c +++ b/usr.bin/netstat/inet.c @@ -137,7 +137,7 @@ pcblist_sysctl(int proto, const char *name, char **bufp, int istcp __unused) static void sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) { - xsb->sb_cc = sb->sb_cc; + xsb->sb_cc = sb->sb_ccc; xsb->sb_hiwat = sb->sb_hiwat; xsb->sb_mbcnt = sb->sb_mbcnt; xsb->sb_mcnt = sb->sb_mcnt; @@ -479,7 +479,8 @@ protopr(u_long off, const char *name, int af1, int proto) printf("%6u %6u %6u ", tp->t_sndrexmitpack, tp->t_rcvoopack, tp->t_sndzerowin); } else { - printf("%6u %6u ", so->so_rcv.sb_cc, so->so_snd.sb_cc); + printf("%6u %6u ", + so->so_rcv.sb_cc, so->so_snd.sb_cc); } if (numeric_port) { if (inp->inp_vflag & INP_IPV4) { diff --git a/usr.bin/netstat/netgraph.c b/usr.bin/netstat/netgraph.c index 95578af739faf..9a08b6d282c83 100644 --- a/usr.bin/netstat/netgraph.c +++ b/usr.bin/netstat/netgraph.c @@ -119,7 +119,7 @@ netgraphprotopr(u_long off, const char *name, int af1 __unused, if (Aflag) printf("%8lx ", (u_long) this); printf("%-5.5s %6u %6u ", - name, sockb.so_rcv.sb_cc, sockb.so_snd.sb_cc); + name, sockb.so_rcv.sb_ccc, sockb.so_snd.sb_ccc); /* Get info on associated node */ if (ngpcb.node_id == 0 || csock == -1) diff --git a/usr.bin/netstat/unix.c b/usr.bin/netstat/unix.c index 027e812cf7cd6..a5cc037840be7 100644 --- a/usr.bin/netstat/unix.c +++ b/usr.bin/netstat/unix.c @@ -287,7 +287,8 @@ unixdomainpr(struct xunpcb *xunp, struct xsocket *so) } else { printf("%8lx %-6.6s %6u %6u %8lx %8lx %8lx %8lx", (long)so->so_pcb, socktype[so->so_type], so->so_rcv.sb_cc, - so->so_snd.sb_cc, (long)unp->unp_vnode, (long)unp->unp_conn, + so->so_snd.sb_cc, (long)unp->unp_vnode, + (long)unp->unp_conn, (long)LIST_FIRST(&unp->unp_refs), (long)LIST_NEXT(unp, unp_reflink)); } diff --git a/usr.bin/systat/netstat.c b/usr.bin/systat/netstat.c index bb1318da37c80..b5938eb9d911c 100644 --- a/usr.bin/systat/netstat.c +++ b/usr.bin/systat/netstat.c @@ -333,8 +333,8 @@ enter_kvm(struct inpcb *inp, struct socket *so, int state, const char *proto) struct netinfo *p; if ((p = enter(inp, state, proto)) != NULL) { - p->ni_rcvcc = so->so_rcv.sb_cc; - p->ni_sndcc = so->so_snd.sb_cc; + p->ni_rcvcc = so->so_rcv.sb_ccc; + p->ni_sndcc = so->so_snd.sb_ccc; } } |