6 files changed, 156 insertions, 28 deletions

diff --git a/sys/dev/bnxt/bnxt.h b/sys/dev/bnxt/bnxt.h
index a00d31303864..8dce63615f1b 100644
--- a/sys/dev/bnxt/bnxt.h
+++ b/sys/dev/bnxt/bnxt.h
@@ -526,6 +526,14 @@ struct bnxt_func_qcfg {
 	uint16_t alloc_vnics;
 };
 
+struct bnxt_hw_lro {
+	uint16_t enable;
+	uint16_t is_mode_gro;
+	uint16_t max_agg_segs;
+	uint16_t max_aggs;
+	uint32_t min_agg_len;
+};
+
 struct bnxt_softc {
 	device_t	dev;
 	if_ctx_t	ctx;
@@ -586,10 +594,13 @@ struct bnxt_softc {
 
 	struct sysctl_ctx_list	hw_stats;
 	struct sysctl_oid	*hw_stats_oid;
+	struct sysctl_ctx_list	hw_lro_ctx;
+	struct sysctl_oid	*hw_lro_oid;
 
 	struct bnxt_ver_info	*ver_info;
 	struct bnxt_nvram_info	*nvm_info;
 	bool wol;
+	struct bnxt_hw_lro	hw_lro;
 	uint8_t wol_filter_id;
 	uint16_t		rx_coal_usecs;
 	uint16_t		rx_coal_usecs_irq;
diff --git a/sys/dev/bnxt/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_hwrm.c
index 44b358804e75..f7855ec1617d 100644
--- a/sys/dev/bnxt/bnxt_hwrm.c
+++ b/sys/dev/bnxt/bnxt_hwrm.c
@@ -977,29 +977,53 @@ bnxt_cfg_async_cr(struct bnxt_softc *softc)
 	return rc;
 }
 
+void
+bnxt_validate_hw_lro_settings(struct bnxt_softc *softc)
+{
+	softc->hw_lro.enable = min(softc->hw_lro.enable, 1);
+
+        softc->hw_lro.is_mode_gro = min(softc->hw_lro.is_mode_gro, 1);
+
+	softc->hw_lro.max_agg_segs = min(softc->hw_lro.max_agg_segs,
+		HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_MAX);
+
+	softc->hw_lro.max_aggs = min(softc->hw_lro.max_aggs,
+		HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX);
+
+	softc->hw_lro.min_agg_len = min(softc->hw_lro.min_agg_len, BNXT_MAX_MTU);
+}
+
 int
-bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic,
-    uint32_t flags)
+bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc)
 {
 	struct hwrm_vnic_tpa_cfg_input req = {0};
+	uint32_t flags;
 
 	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_TPA_CFG);
 
-	req.flags = htole32(flags);
-	req.vnic_id = htole16(vnic->id);
-	req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS |
-	    HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS |
-	    /* HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_TIMER | */
-	    HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN);
-	/* TODO: Calculate this based on ring size? */
-	req.max_agg_segs = htole16(3);
-	/* Base this in the allocated TPA start size... */
-	req.max_aggs = htole16(7);
-	/*
-	 * TODO: max_agg_timer?
-	 * req.mag_agg_timer = htole32(XXX);
-	 */
-	req.min_agg_len = htole32(0);
+	if (softc->hw_lro.enable) {
+		flags = HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA |
+			HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA |
+			HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN |
+			HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ;
+		
+        	if (softc->hw_lro.is_mode_gro)
+			flags |= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO;
+		else
+			flags |= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE;
+			
+		req.flags = htole32(flags);
+
+		req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS |
+				HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS |
+				HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN);
+
+		req.max_agg_segs = htole16(softc->hw_lro.max_agg_segs);
+		req.max_aggs = htole16(softc->hw_lro.max_aggs);
+		req.min_agg_len = htole32(softc->hw_lro.min_agg_len);
+	}
+
+	req.vnic_id = htole16(softc->vnic_info.id);
 
 	return hwrm_send_message(softc, &req, sizeof(req));
 }
diff --git a/sys/dev/bnxt/bnxt_hwrm.h b/sys/dev/bnxt/bnxt_hwrm.h
index 52450034b1d7..e2934f6b8eb3 100644
--- a/sys/dev/bnxt/bnxt_hwrm.h
+++ b/sys/dev/bnxt/bnxt_hwrm.h
@@ -62,8 +62,8 @@ int bnxt_hwrm_set_filter(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic);
 int bnxt_hwrm_rss_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic,
     uint32_t hash_type);
 int bnxt_cfg_async_cr(struct bnxt_softc *softc);
-int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc,
-    struct bnxt_vnic_info *vnic, uint32_t flags);
+int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc);
+void bnxt_validate_hw_lro_settings(struct bnxt_softc *softc);
 int bnxt_hwrm_nvm_find_dir_entry(struct bnxt_softc *softc, uint16_t type,
     uint16_t *ordinal, uint16_t ext, uint16_t *index, bool use_index,
     uint8_t search_opt, uint32_t *data_length, uint32_t *item_length,
diff --git a/sys/dev/bnxt/bnxt_sysctl.c b/sys/dev/bnxt/bnxt_sysctl.c
index fa423c5b31ea..a66225e6eb0c 100644
--- a/sys/dev/bnxt/bnxt_sysctl.c
+++ b/sys/dev/bnxt/bnxt_sysctl.c
@@ -84,6 +84,16 @@ bnxt_init_sysctl_ctx(struct bnxt_softc *softc)
 		return ENOMEM;
 	}
 
+	sysctl_ctx_init(&softc->hw_lro_ctx);
+	ctx = device_get_sysctl_ctx(softc->dev);
+	softc->hw_lro_oid = SYSCTL_ADD_NODE(ctx,
+	    SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO,
+	    "hw_lro", CTLFLAG_RD, 0, "hardware lro");
+	if (!softc->hw_lro_oid) {
+		sysctl_ctx_free(&softc->hw_lro_ctx);
+		return ENOMEM;
+	}
+
 	return 0;
 }
 
@@ -114,6 +124,13 @@ bnxt_free_sysctl_ctx(struct bnxt_softc *softc)
 		else
 			softc->nvm_info->nvm_oid = NULL;
 	}
+	if (softc->hw_lro_oid != NULL) {
+		orc = sysctl_ctx_free(&softc->hw_lro_ctx);
+		if (orc)
+			rc = orc;
+		else
+			softc->hw_lro_oid = NULL;
+	}
 
 	return rc;
 }
@@ -1210,6 +1227,74 @@ bnxt_create_config_sysctls_pre(struct bnxt_softc *softc)
 	return 0;
 }
 
+#define BNXT_HW_LRO_FN(fn_name, arg)			                   \
+static int						                   \
+fn_name(SYSCTL_HANDLER_ARGS) {				                   \
+	struct bnxt_softc *softc = arg1;		                   \
+	int rc;						                   \
+	int val;					                   \
+							                   \
+	if (softc == NULL)				                   \
+		return EBUSY;				                   \
+							                   \
+	val = softc->hw_lro.arg;			                   \
+	rc = sysctl_handle_int(oidp, &val, 0, req);	                   \
+	if (rc || !req->newptr)				                   \
+		return rc;				                   \
+							                   \
+	if ((if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)) \
+		return EBUSY;				                   \
+							                   \
+	softc->hw_lro.arg = val;			                   \
+	bnxt_validate_hw_lro_settings(softc);		                   \
+	rc = bnxt_hwrm_vnic_tpa_cfg(softc);		                   \
+							                   \
+	return rc;					                   \
+}
+
+BNXT_HW_LRO_FN(bnxt_hw_lro_enable_disable, enable)
+BNXT_HW_LRO_FN(bnxt_hw_lro_set_mode, is_mode_gro)
+BNXT_HW_LRO_FN(bnxt_hw_lro_set_max_agg_segs, max_agg_segs)
+BNXT_HW_LRO_FN(bnxt_hw_lro_set_max_aggs, max_aggs)
+BNXT_HW_LRO_FN(bnxt_hw_lro_set_min_agg_len, min_agg_len)
+
+int
+bnxt_create_hw_lro_sysctls(struct bnxt_softc *softc)
+{
+	struct sysctl_oid *oid = softc->hw_lro_oid;
+
+	if (!oid)
+		return ENOMEM;
+
+	SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+			"enable", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
+			bnxt_hw_lro_enable_disable, "A",
+			"Enable or Disable HW LRO: 0 / 1");
+
+	SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+			"gro_mode", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
+			bnxt_hw_lro_set_mode, "A",
+			"Set mode: 1 = GRO mode, 0 = RSC mode");
+
+	SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+			"max_agg_segs", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
+			bnxt_hw_lro_set_max_agg_segs, "A",
+			"Set Max Agg Seg Value (unit is Log2): "
+			"0 (= 1 seg) / 1 (= 2 segs) /  ... / 31 (= 2^31 segs)");
+	
+        SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+			"max_aggs", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
+			bnxt_hw_lro_set_max_aggs, "A",
+			"Set Max Aggs Value (unit is Log2): "
+			"0 (= 1 agg) / 1 (= 2 aggs) /  ... / 7 (= 2^7 segs)"); 
+
+	SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+			"min_agg_len", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
+			bnxt_hw_lro_set_min_agg_len, "A",
+			"Min Agg Len: 1 to 9000");
+
+	return 0;
+}
 static int
 bnxt_vlan_only_sysctl(SYSCTL_HANDLER_ARGS) {
 	struct bnxt_softc *softc = arg1;
diff --git a/sys/dev/bnxt/bnxt_sysctl.h b/sys/dev/bnxt/bnxt_sysctl.h
index 31985d93b7c3..40456fe17616 100644
--- a/sys/dev/bnxt/bnxt_sysctl.h
+++ b/sys/dev/bnxt/bnxt_sysctl.h
@@ -40,3 +40,4 @@ int bnxt_create_ver_sysctls(struct bnxt_softc *softc);
 int bnxt_create_nvram_sysctls(struct bnxt_nvram_info *ni);
 int bnxt_create_config_sysctls_pre(struct bnxt_softc *softc);
 int bnxt_create_config_sysctls_post(struct bnxt_softc *softc);
+int bnxt_create_hw_lro_sysctls(struct bnxt_softc *softc);
diff --git a/sys/dev/bnxt/if_bnxt.c b/sys/dev/bnxt/if_bnxt.c
index 3cd7728b0675..873217c8c87c 100644
--- a/sys/dev/bnxt/if_bnxt.c
+++ b/sys/dev/bnxt/if_bnxt.c
@@ -826,6 +826,17 @@ bnxt_attach_pre(if_ctx_t ctx)
 	/* iflib will map and release this bar */
 	scctx->isc_msix_bar = pci_msix_table_bar(softc->dev);
 
+        /* 
+         * Default settings for HW LRO (TPA):
+         *  Disable HW LRO by default
+         *  Can be enabled after taking care of 'packet forwarding'
+         */
+	softc->hw_lro.enable = 0;
+	softc->hw_lro.is_mode_gro = 0;
+	softc->hw_lro.max_agg_segs = 5; /* 2^5 = 32 segs */
+	softc->hw_lro.max_aggs = HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX;
+	softc->hw_lro.min_agg_len = 512;
+
 	/* Allocate the default completion ring */
 	softc->def_cp_ring.stats_ctx_id = HWRM_NA_SIGNATURE;
 	softc->def_cp_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
@@ -861,6 +872,10 @@ bnxt_attach_pre(if_ctx_t ctx)
 	if (rc)
 		goto failed;
 
+	rc = bnxt_create_hw_lro_sysctls(softc);
+	if (rc)
+		goto failed;
+
 	/* Initialize the vlan list */
 	SLIST_INIT(&softc->vnic_info.vlan_tags);
 	softc->vnic_info.vlan_tag_list.idi_vaddr = NULL;
@@ -1071,15 +1086,7 @@ bnxt_init(if_ctx_t ctx)
 	if (rc)
 		goto fail;
 
-	/* 
-         * Enable LRO/TPA/GRO 
-         * TBD: 
-         *      Enable / Disable HW_LRO based on
-         *      ifconfig lro / ifconfig -lro setting
-         */
-	rc = bnxt_hwrm_vnic_tpa_cfg(softc, &softc->vnic_info,
-	    (if_getcapenable(iflib_get_ifp(ctx)) & IFCAP_LRO) ?
-	    HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA : 0);
+	rc = bnxt_hwrm_vnic_tpa_cfg(softc);
 	if (rc)
 		goto fail;