13 files changed, 805 insertions, 170 deletions

diff --git a/sys/dev/qlxgbe/ql_dbg.h b/sys/dev/qlxgbe/ql_dbg.h
index 9e2675bb4611..863a7c8f2687 100644
--- a/sys/dev/qlxgbe/ql_dbg.h
+++ b/sys/dev/qlxgbe/ql_dbg.h
@@ -42,17 +42,21 @@ extern void ql_dump_buf16(qla_host_t *ha, const char *str, void *dbuf,
 extern void ql_dump_buf32(qla_host_t *ha, const char *str, void *dbuf,
 		uint32_t len32);
 
-#define INJCT_RX_RXB_INVAL		0x00001
-#define INJCT_RX_MP_NULL		0x00002
-#define INJCT_LRO_RXB_INVAL		0x00003
-#define INJCT_LRO_MP_NULL		0x00004
-#define INJCT_NUM_HNDLE_INVALID		0x00005
-#define INJCT_RDWR_INDREG_FAILURE	0x00006
-#define INJCT_RDWR_OFFCHIPMEM_FAILURE	0x00007
-#define INJCT_MBX_CMD_FAILURE		0x00008
-#define INJCT_HEARTBEAT_FAILURE		0x00009
-#define INJCT_TEMPERATURE_FAILURE	0x0000A
-#define INJCT_M_GETCL_M_GETJCL_FAILURE	0x0000B
+#define INJCT_RX_RXB_INVAL				0x00001
+#define INJCT_RX_MP_NULL				0x00002
+#define INJCT_LRO_RXB_INVAL				0x00003
+#define INJCT_LRO_MP_NULL				0x00004
+#define INJCT_NUM_HNDLE_INVALID				0x00005
+#define INJCT_RDWR_INDREG_FAILURE			0x00006
+#define INJCT_RDWR_OFFCHIPMEM_FAILURE			0x00007
+#define INJCT_MBX_CMD_FAILURE				0x00008
+#define INJCT_HEARTBEAT_FAILURE				0x00009
+#define INJCT_TEMPERATURE_FAILURE			0x0000A
+#define INJCT_M_GETCL_M_GETJCL_FAILURE			0x0000B
+#define INJCT_INV_CONT_OPCODE				0x0000C
+#define INJCT_SGL_RCV_INV_DESC_COUNT			0x0000D
+#define INJCT_SGL_LRO_INV_DESC_COUNT			0x0000E
+#define INJCT_PEER_PORT_FAILURE_ERR_RECOVERY		0x0000F
 
 #ifdef QL_DBG
 
diff --git a/sys/dev/qlxgbe/ql_def.h b/sys/dev/qlxgbe/ql_def.h
index dacc6b19de8b..8915c0ea74f8 100644
--- a/sys/dev/qlxgbe/ql_def.h
+++ b/sys/dev/qlxgbe/ql_def.h
@@ -144,12 +144,12 @@ struct qla_host {
 	volatile uint32_t	qla_watchdog_paused;
 	volatile uint32_t	qla_initiate_recovery;
 	volatile uint32_t	qla_detach_active;
+	volatile uint32_t	offline;
 
 	device_t		pci_dev;
 
-	uint16_t		watchdog_ticks;
+	volatile uint16_t	watchdog_ticks;
 	uint8_t			pci_func;
-	uint8_t			resvd;
 
         /* ioctl related */
         struct cdev             *ioctl_dev;
@@ -182,6 +182,7 @@ struct qla_host {
 
 	/* hardware access lock */
 
+	struct mtx		sp_log_lock;
 	struct mtx		hw_lock;
 	volatile uint32_t	hw_lock_held;
 	uint64_t		hw_lock_failed;
@@ -239,6 +240,9 @@ struct qla_host {
 	volatile const char	*qla_unlock;
 	uint32_t		dbg_level;
 	uint32_t		enable_minidump;
+	uint32_t		enable_driverstate_dump;
+	uint32_t		enable_error_recovery;
+	uint32_t		ms_delay_after_init;
 
 	uint8_t			fw_ver_str[32];
 
@@ -273,4 +277,6 @@ typedef struct qla_host qla_host_t;
 	((((*(uint32_t *) mac1) == (*(uint32_t *) mac2) && \
 	(*(uint16_t *)(mac1 + 4)) == (*(uint16_t *)(mac2 + 4)))) ? 0 : 1)
 
+#define QL_INITIATE_RECOVERY(ha) qla_set_error_recovery(ha)
+
 #endif /* #ifndef _QL_DEF_H_ */
diff --git a/sys/dev/qlxgbe/ql_glbl.h b/sys/dev/qlxgbe/ql_glbl.h
index f8c7c8b165c1..9a5bbb669d4b 100644
--- a/sys/dev/qlxgbe/ql_glbl.h
+++ b/sys/dev/qlxgbe/ql_glbl.h
@@ -47,6 +47,7 @@ extern uint32_t ql_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count);
 extern int ql_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf);
 extern void ql_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf);
 extern int ql_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp);
+extern void qla_set_error_recovery(qla_host_t *ha);
 
 /*
  * from ql_hw.c
@@ -115,5 +116,11 @@ extern unsigned int ql83xx_minidump_len;
 extern void ql_alloc_drvr_state_buffer(qla_host_t *ha);
 extern void ql_free_drvr_state_buffer(qla_host_t *ha);
 extern void ql_capture_drvr_state(qla_host_t *ha);
+extern void ql_sp_log(qla_host_t *ha, uint16_t fmtstr_idx, uint16_t num_params,
+		uint32_t param0, uint32_t param1, uint32_t param2,
+		uint32_t param3, uint32_t param4);
+extern void ql_alloc_sp_log_buffer(qla_host_t *ha);
+extern void ql_free_sp_log_buffer(qla_host_t *ha);
+
 
 #endif /* #ifndef_QL_GLBL_H_ */
diff --git a/sys/dev/qlxgbe/ql_hw.c b/sys/dev/qlxgbe/ql_hw.c
index 102fb337beb7..45aab1222f1a 100644
--- a/sys/dev/qlxgbe/ql_hw.c
+++ b/sys/dev/qlxgbe/ql_hw.c
@@ -49,7 +49,7 @@ __FBSDID("$FreeBSD$");
 
 static void qla_del_rcv_cntxt(qla_host_t *ha);
 static int qla_init_rcv_cntxt(qla_host_t *ha);
-static void qla_del_xmt_cntxt(qla_host_t *ha);
+static int qla_del_xmt_cntxt(qla_host_t *ha);
 static int qla_init_xmt_cntxt(qla_host_t *ha);
 static int qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox,
 	uint32_t *fw_mbox, uint32_t n_fwmbox, uint32_t no_pause);
@@ -647,11 +647,118 @@ qlnx_add_hw_xmt_stats_sysctls(qla_host_t *ha)
 }
 
 static void
+qlnx_add_hw_mbx_cmpl_stats_sysctls(qla_host_t *ha)
+{
+        struct sysctl_ctx_list  *ctx;
+        struct sysctl_oid_list  *node_children;
+
+        ctx = device_get_sysctl_ctx(ha->pci_dev);
+        node_children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev));
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_lt_200ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[0],
+		"mbx_completion_time_lt_200ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_200ms_400ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[1],
+		"mbx_completion_time_200ms_400ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_400ms_600ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[2],
+		"mbx_completion_time_400ms_600ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_600ms_800ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[3],
+		"mbx_completion_time_600ms_800ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_800ms_1000ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[4],
+		"mbx_completion_time_800ms_1000ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_1000ms_1200ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[5],
+		"mbx_completion_time_1000ms_1200ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_1200ms_1400ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[6],
+		"mbx_completion_time_1200ms_1400ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_1400ms_1600ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[7],
+		"mbx_completion_time_1400ms_1600ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_1600ms_1800ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[8],
+		"mbx_completion_time_1600ms_1800ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_1800ms_2000ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[9],
+		"mbx_completion_time_1800ms_2000ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_2000ms_2200ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[10],
+		"mbx_completion_time_2000ms_2200ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_2200ms_2400ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[11],
+		"mbx_completion_time_2200ms_2400ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_2400ms_2600ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[12],
+		"mbx_completion_time_2400ms_2600ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_2600ms_2800ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[13],
+		"mbx_completion_time_2600ms_2800ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_2800ms_3000ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[14],
+		"mbx_completion_time_2800ms_3000ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_3000ms_4000ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[15],
+		"mbx_completion_time_3000ms_4000ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_time_4000ms_5000ms",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[16],
+		"mbx_completion_time_4000ms_5000ms");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_host_mbx_cntrl_timeout",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[17],
+		"mbx_completion_host_mbx_cntrl_timeout");
+
+	SYSCTL_ADD_QUAD(ctx, node_children,
+		OID_AUTO, "mbx_completion_fw_mbx_cntrl_timeout",
+		CTLFLAG_RD, &ha->hw.mbx_comp_msecs[18],
+		"mbx_completion_fw_mbx_cntrl_timeout");
+	return;
+}
+
+static void
 qlnx_add_hw_stats_sysctls(qla_host_t *ha)
 {
 	qlnx_add_hw_mac_stats_sysctls(ha);
 	qlnx_add_hw_rcv_stats_sysctls(ha);
 	qlnx_add_hw_xmt_stats_sysctls(ha);
+	qlnx_add_hw_mbx_cmpl_stats_sysctls(ha);
 
 	return;
 }
@@ -918,6 +1025,30 @@ ql_hw_add_sysctls(qla_host_t *ha)
 		"\t Any change requires ifconfig down/up to take effect\n"
 		"\t Note that LRO may be turned off/on via ifconfig\n");
 
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "sp_log_index", CTLFLAG_RW, &ha->hw.sp_log_index,
+                ha->hw.sp_log_index, "sp_log_index");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "sp_log_stop", CTLFLAG_RW, &ha->hw.sp_log_stop,
+                ha->hw.sp_log_stop, "sp_log_stop");
+
+        ha->hw.sp_log_stop_events = 0;
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "sp_log_stop_events", CTLFLAG_RW,
+		&ha->hw.sp_log_stop_events,
+                ha->hw.sp_log_stop_events, "Slow path event log is stopped"
+		" when OR of the following events occur \n"
+		"\t 0x01 : Heart beat Failure\n"
+		"\t 0x02 : Temperature Failure\n"
+		"\t 0x04 : HW Initialization Failure\n"
+		"\t 0x08 : Interface Initialization Failure\n"
+		"\t 0x10 : Error Recovery Failure\n");
+
 	ha->hw.mdump_active = 0;
         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
                 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
@@ -998,13 +1129,13 @@ ql_hw_link_status(qla_host_t *ha)
 		device_printf(ha->pci_dev, "link Down\n");
 	}
 
-	if (ha->hw.flags.fduplex) {
+	if (ha->hw.fduplex) {
 		device_printf(ha->pci_dev, "Full Duplex\n");
 	} else {
 		device_printf(ha->pci_dev, "Half Duplex\n");
 	}
 
-	if (ha->hw.flags.autoneg) {
+	if (ha->hw.autoneg) {
 		device_printf(ha->pci_dev, "Auto Negotiation Enabled\n");
 	} else {
 		device_printf(ha->pci_dev, "Auto Negotiation Disabled\n");
@@ -1255,19 +1386,39 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox,
 	uint32_t i;
 	uint32_t data;
 	int ret = 0;
+	uint64_t start_usecs;
+	uint64_t end_usecs;
+	uint64_t msecs_200;
+
+	ql_sp_log(ha, 0, 5, no_pause, h_mbox[0], h_mbox[1], h_mbox[2], h_mbox[3]);
+
+	if (ha->offline || ha->qla_initiate_recovery) {
+		ql_sp_log(ha, 1, 2, ha->offline, ha->qla_initiate_recovery, 0, 0, 0);
+		goto exit_qla_mbx_cmd;
+	}
 
-	if (QL_ERR_INJECT(ha, INJCT_MBX_CMD_FAILURE)) {
+	if (((ha->err_inject & 0xFFFF) == INJCT_MBX_CMD_FAILURE) &&
+		(((ha->err_inject & ~0xFFFF) == ((h_mbox[0] & 0xFFFF) << 16))||
+		!(ha->err_inject & ~0xFFFF))) {
 		ret = -3;
-		ha->qla_initiate_recovery = 1;
+		QL_INITIATE_RECOVERY(ha);
 		goto exit_qla_mbx_cmd;
 	}
 
+	start_usecs = qla_get_usec_timestamp();
+
 	if (no_pause)
 		i = 1000;
 	else
 		i = Q8_MBX_MSEC_DELAY;
 
 	while (i) {
+
+		if (ha->qla_initiate_recovery) {
+			ql_sp_log(ha, 2, 1, ha->qla_initiate_recovery, 0, 0, 0, 0);
+			return (-1);
+		}
+
 		data = READ_REG32(ha, Q8_HOST_MBOX_CNTRL);
 		if (data == 0)
 			break;
@@ -1282,8 +1433,10 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox,
 	if (i == 0) {
 		device_printf(ha->pci_dev, "%s: host_mbx_cntrl 0x%08x\n",
 			__func__, data);
+		ql_sp_log(ha, 3, 1, data, 0, 0, 0, 0);
 		ret = -1;
-		ha->qla_initiate_recovery = 1;
+		ha->hw.mbx_comp_msecs[(Q8_MBX_COMP_MSECS - 2)]++;
+		QL_INITIATE_RECOVERY(ha);
 		goto exit_qla_mbx_cmd;
 	}
 
@@ -1297,6 +1450,12 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox,
 
 	i = Q8_MBX_MSEC_DELAY;
 	while (i) {
+
+		if (ha->qla_initiate_recovery) {
+			ql_sp_log(ha, 4, 1, ha->qla_initiate_recovery, 0, 0, 0, 0);
+			return (-1);
+		}
+
 		data = READ_REG32(ha, Q8_FW_MBOX_CNTRL);
 
 		if ((data & 0x3) == 1) {
@@ -1314,18 +1473,44 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox,
 	if (i == 0) {
 		device_printf(ha->pci_dev, "%s: fw_mbx_cntrl 0x%08x\n",
 			__func__, data);
+		ql_sp_log(ha, 5, 1, data, 0, 0, 0, 0);
 		ret = -2;
-		ha->qla_initiate_recovery = 1;
+		ha->hw.mbx_comp_msecs[(Q8_MBX_COMP_MSECS - 1)]++;
+		QL_INITIATE_RECOVERY(ha);
 		goto exit_qla_mbx_cmd;
 	}
 
 	for (i = 0; i < n_fwmbox; i++) {
+
+		if (ha->qla_initiate_recovery) {
+			ql_sp_log(ha, 6, 1, ha->qla_initiate_recovery, 0, 0, 0, 0);
+			return (-1);
+		}
+
 		*fw_mbox++ = READ_REG32(ha, (Q8_FW_MBOX0 + (i << 2)));
 	}
 
 	WRITE_REG32(ha, Q8_FW_MBOX_CNTRL, 0x0);
 	WRITE_REG32(ha, ha->hw.mbx_intr_mask_offset, 0x0);
 
+	end_usecs = qla_get_usec_timestamp();
+
+	if (end_usecs > start_usecs) {
+		msecs_200 = (end_usecs - start_usecs)/(1000 * 200);
+
+		if (msecs_200 < 15) 
+			ha->hw.mbx_comp_msecs[msecs_200]++;
+		else if (msecs_200 < 20)
+			ha->hw.mbx_comp_msecs[15]++;
+		else {
+			device_printf(ha->pci_dev, "%s: [%ld, %ld] %ld\n", __func__,
+				start_usecs, end_usecs, msecs_200);
+			ha->hw.mbx_comp_msecs[16]++;
+		}
+	}
+	ql_sp_log(ha, 7, 5, fw_mbox[0], fw_mbox[1], fw_mbox[2], fw_mbox[3], fw_mbox[4]);
+
+
 exit_qla_mbx_cmd:
 	return (ret);
 }
@@ -1401,7 +1586,8 @@ qla_config_intr_cntxt(qla_host_t *ha, uint32_t start_idx, uint32_t num_intrs,
 	if (qla_mbx_cmd(ha, (uint32_t *)c_intr,
 		(sizeof (q80_config_intr_t) >> 2),
 		ha->hw.mbox, (sizeof (q80_config_intr_rsp_t) >> 2), 0)) {
-		device_printf(dev, "%s: failed0\n", __func__);
+		device_printf(dev, "%s: %s failed0\n", __func__,
+			(create ? "create" : "delete"));
 		return (-1);
 	}
 
@@ -1410,8 +1596,8 @@ qla_config_intr_cntxt(qla_host_t *ha, uint32_t start_idx, uint32_t num_intrs,
 	err = Q8_MBX_RSP_STATUS(c_intr_rsp->regcnt_status);
 
 	if (err) {
-		device_printf(dev, "%s: failed1 [0x%08x, %d]\n", __func__, err,
-			c_intr_rsp->nentries);
+		device_printf(dev, "%s: %s failed1 [0x%08x, %d]\n", __func__,
+			(create ? "create" : "delete"), err, c_intr_rsp->nentries);
 
 		for (i = 0; i < c_intr_rsp->nentries; i++) {
 			device_printf(dev, "%s: [%d]:[0x%x 0x%x 0x%x]\n",
@@ -2015,7 +2201,8 @@ ql_get_stats(qla_host_t *ha)
 
 	cmd |= ((ha->pci_func & 0x1) << 16);
 
-	if (ha->qla_watchdog_pause)
+	if (ha->qla_watchdog_pause || (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) ||
+		ha->offline)
 		goto ql_get_stats_exit;
 
 	if (qla_get_hw_stats(ha, cmd, sizeof (q80_get_stats_rsp_t)) == 0) {
@@ -2032,7 +2219,8 @@ ql_get_stats(qla_host_t *ha)
 //	cmd |= Q8_GET_STATS_CMD_CLEAR;
 	cmd |= (ha->hw.rcv_cntxt_id << 16);
 
-	if (ha->qla_watchdog_pause)
+	if (ha->qla_watchdog_pause || (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) ||
+		ha->offline)
 		goto ql_get_stats_exit;
 
 	if (qla_get_hw_stats(ha, cmd, sizeof (q80_get_stats_rsp_t)) == 0) {
@@ -2043,13 +2231,18 @@ ql_get_stats(qla_host_t *ha)
 			__func__, ha->hw.mbox[0]);
 	}
 
-	if (ha->qla_watchdog_pause)
+	if (ha->qla_watchdog_pause || (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) ||
+		ha->offline)
 		goto ql_get_stats_exit;
 	/*
 	 * Get XMT Statistics
 	 */
-	for (i = 0 ; ((i < ha->hw.num_tx_rings) && (!ha->qla_watchdog_pause));
-		i++) {
+	for (i = 0 ; (i < ha->hw.num_tx_rings); i++) {
+		if (ha->qla_watchdog_pause ||
+			(!(ifp->if_drv_flags & IFF_DRV_RUNNING)) ||
+			ha->offline)
+			goto ql_get_stats_exit;
+
 		cmd = Q8_GET_STATS_CMD_XMT | Q8_GET_STATS_CMD_TYPE_CNTXT;
 //		cmd |= Q8_GET_STATS_CMD_CLEAR;
 		cmd |= (ha->hw.tx_cntxt[i].tx_cntxt_id << 16);
@@ -2679,7 +2872,8 @@ ql_del_hw_if(qla_host_t *ha)
 
 	qla_del_rcv_cntxt(ha);
 
-	qla_del_xmt_cntxt(ha);
+	if(qla_del_xmt_cntxt(ha))
+		goto ql_del_hw_if_exit;
 
 	if (ha->hw.flags.init_intr_cnxt) {
 		for (i = 0; i < ha->hw.num_sds_rings; ) {
@@ -2688,7 +2882,9 @@ ql_del_hw_if(qla_host_t *ha)
 				num_msix = Q8_MAX_INTR_VECTORS;
 			else
 				num_msix = ha->hw.num_sds_rings - i;
-			qla_config_intr_cntxt(ha, i, num_msix, 0);
+
+			if (qla_config_intr_cntxt(ha, i, num_msix, 0))
+				break;
 
 			i += num_msix;
 		}
@@ -2696,6 +2892,7 @@ ql_del_hw_if(qla_host_t *ha)
 		ha->hw.flags.init_intr_cnxt = 0;
 	}
 
+ql_del_hw_if_exit:
 	if (ha->hw.enable_soft_lro) {
 		qla_drain_soft_lro(ha);
 		qla_free_soft_lro(ha);
@@ -3328,19 +3525,22 @@ qla_del_xmt_cntxt_i(qla_host_t *ha, uint32_t txr_idx)
 
 	return (0);
 }
-static void
+static int
 qla_del_xmt_cntxt(qla_host_t *ha)
 {
 	uint32_t i;
+	int ret = 0;
 
 	if (!ha->hw.flags.init_tx_cnxt)
-		return;
+		return (ret);
 
 	for (i = 0; i < ha->hw.num_tx_rings; i++) {
-		if (qla_del_xmt_cntxt_i(ha, i))
+		if ((ret = qla_del_xmt_cntxt_i(ha, i)) != 0)
 			break;
 	}
 	ha->hw.flags.init_tx_cnxt = 0;
+
+	return (ret);
 }
 
 static int
@@ -3350,8 +3550,10 @@ qla_init_xmt_cntxt(qla_host_t *ha)
 
 	for (i = 0; i < ha->hw.num_tx_rings; i++) {
 		if (qla_init_xmt_cntxt_i(ha, i) != 0) {
-			for (j = 0; j < i; j++)
-				qla_del_xmt_cntxt_i(ha, j);
+			for (j = 0; j < i; j++) {
+				if (qla_del_xmt_cntxt_i(ha, j))
+					break;
+			}
 			return (-1);
 		}
 	}
@@ -3627,21 +3829,22 @@ ql_hw_tx_done_locked(qla_host_t *ha, uint32_t txr_idx)
 void
 ql_update_link_state(qla_host_t *ha)
 {
-	uint32_t link_state;
+	uint32_t link_state = 0;
 	uint32_t prev_link_state;
 
-	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) {
-		ha->hw.link_up = 0;
-		return;
-	}
-	link_state = READ_REG32(ha, Q8_LINK_STATE);
-
 	prev_link_state =  ha->hw.link_up;
 
-	if (ha->pci_func == 0) 
-		ha->hw.link_up = (((link_state & 0xF) == 1)? 1 : 0);
-	else
-		ha->hw.link_up = ((((link_state >> 4)& 0xF) == 1)? 1 : 0);
+	if (ha->ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		link_state = READ_REG32(ha, Q8_LINK_STATE);
+
+		if (ha->pci_func == 0) {
+			link_state = (((link_state & 0xF) == 1)? 1 : 0);
+		} else {
+			link_state = ((((link_state >> 4)& 0xF) == 1)? 1 : 0);
+		}
+	}
+
+	atomic_store_rel_8(&ha->hw.link_up, (uint8_t)link_state);
 
 	if (prev_link_state !=  ha->hw.link_up) {
 		if (ha->hw.link_up) {
@@ -3669,8 +3872,14 @@ ql_hw_check_health(qla_host_t *ha)
 
 	if (((val & 0xFFFF) == 2) || ((val & 0xFFFF) == 3) ||
 		(QL_ERR_INJECT(ha, INJCT_TEMPERATURE_FAILURE))) {
-		device_printf(ha->pci_dev, "%s: Temperature Alert [0x%08x]\n",
-			__func__, val);
+		device_printf(ha->pci_dev, "%s: Temperature Alert"
+			" at ts_usecs %ld ts_reg = 0x%08x\n",
+			__func__, qla_get_usec_timestamp(), val);
+
+		if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_TEMP_FAILURE)
+			ha->hw.sp_log_stop = -1;
+
+		QL_INITIATE_RECOVERY(ha);
 		return -1;
 	}
 
@@ -3691,9 +3900,25 @@ ql_hw_check_health(qla_host_t *ha)
 			__func__, val);
 	if (ha->hw.hbeat_failure < 2) /* we ignore the first failure */
 		return 0;
-	else 
-		device_printf(ha->pci_dev, "%s: Heartbeat Failue [0x%08x]\n",
-			__func__, val);
+	else {
+		uint32_t peg_halt_status1;
+		uint32_t peg_halt_status2;
+
+		peg_halt_status1 = READ_REG32(ha, Q8_PEG_HALT_STATUS1);
+		peg_halt_status2 = READ_REG32(ha, Q8_PEG_HALT_STATUS2);
+
+		device_printf(ha->pci_dev,
+			"%s: Heartbeat Failue at ts_usecs = %ld "
+			"fw_heart_beat = 0x%08x "
+			"peg_halt_status1 = 0x%08x "
+			"peg_halt_status2 = 0x%08x\n",
+			__func__, qla_get_usec_timestamp(), val,
+			peg_halt_status1, peg_halt_status2);
+
+		if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_HBEAT_FAILURE)
+			ha->hw.sp_log_stop = -1;
+	}
+	QL_INITIATE_RECOVERY(ha);
 
 	return -1;
 }
@@ -4429,8 +4654,8 @@ ql_minidump(qla_host_t *ha)
 
 	if (ha->hw.mdump_done)
 		return;
-
-		ha->hw.mdump_start_seq_index = ql_stop_sequence(ha);
+	ha->hw.mdump_usec_ts = qla_get_usec_timestamp();
+	ha->hw.mdump_start_seq_index = ql_stop_sequence(ha);
 
 	bzero(ha->hw.mdump_buffer, ha->hw.mdump_buffer_size);
 	bzero(ha->hw.mdump_template, ha->hw.mdump_template_size);
diff --git a/sys/dev/qlxgbe/ql_hw.h b/sys/dev/qlxgbe/ql_hw.h
index e8f53c86407d..f59af522d0e3 100644
--- a/sys/dev/qlxgbe/ql_hw.h
+++ b/sys/dev/qlxgbe/ql_hw.h
@@ -1600,26 +1600,26 @@ typedef struct _qla_hw {
 		uint32_t
 			unicast_mac	:1,
 			bcast_mac	:1,
-			loopback_mode	:2,
 			init_tx_cnxt	:1,
 			init_rx_cnxt	:1,
 			init_intr_cnxt	:1,
-			fduplex		:1,
-			autoneg		:1,
 			fdt_valid	:1;
 	} flags;
 
 
-	uint16_t	link_speed;
-	uint16_t	cable_length;
-	uint32_t	cable_oui;
-	uint8_t		link_up;
-	uint8_t		module_type;
-	uint8_t		link_faults;
+	volatile uint16_t	link_speed;
+	volatile uint16_t	cable_length;
+	volatile uint32_t	cable_oui;
+	volatile uint8_t	link_up;
+	volatile uint8_t	module_type;
+	volatile uint8_t	link_faults;
+	volatile uint8_t	loopback_mode;
+	volatile uint8_t	fduplex;
+	volatile uint8_t	autoneg;
 
-	uint8_t		mac_rcv_mode;
+	volatile uint8_t	mac_rcv_mode;
 
-	uint32_t	max_mtu;
+	volatile uint32_t	max_mtu;
 
 	uint8_t		mac_addr[ETHER_ADDR_LEN];
 
@@ -1703,9 +1703,25 @@ typedef struct _qla_hw {
 	uint32_t	mdump_buffer_size;
 	void		*mdump_template;
 	uint32_t	mdump_template_size;
+	uint64_t	mdump_usec_ts;
 
+#define Q8_MBX_COMP_MSECS	(19)
+	uint64_t	mbx_comp_msecs[Q8_MBX_COMP_MSECS];
 	/* driver state related */
 	void		*drvr_state;
+
+	/* slow path trace */
+	uint32_t	sp_log_stop_events;
+#define Q8_SP_LOG_STOP_HBEAT_FAILURE		0x001
+#define Q8_SP_LOG_STOP_TEMP_FAILURE		0x002
+#define Q8_SP_LOG_STOP_HW_INIT_FAILURE		0x004
+#define Q8_SP_LOG_STOP_IF_START_FAILURE		0x008
+#define Q8_SP_LOG_STOP_ERR_RECOVERY_FAILURE	0x010
+
+	uint32_t	sp_log_stop;
+	uint32_t	sp_log_index;
+	uint32_t	sp_log_num_entries;
+	void		*sp_log;
 } qla_hw_t;
 
 #define QL_UPDATE_RDS_PRODUCER_INDEX(ha, prod_reg, val) \
diff --git a/sys/dev/qlxgbe/ql_inline.h b/sys/dev/qlxgbe/ql_inline.h
index 986855a73939..41093de21811 100644
--- a/sys/dev/qlxgbe/ql_inline.h
+++ b/sys/dev/qlxgbe/ql_inline.h
@@ -166,7 +166,7 @@ qla_lock(qla_host_t *ha, const char *str, uint32_t timeout_ms,
 	while (1) {
 		mtx_lock(&ha->hw_lock);
 
-		if (ha->qla_detach_active) {
+		if (ha->qla_detach_active || ha->offline) {
 			mtx_unlock(&ha->hw_lock);
 			break;
 		}
@@ -191,7 +191,10 @@ qla_lock(qla_host_t *ha, const char *str, uint32_t timeout_ms,
 		}
 	}
 
-	//device_printf(ha->pci_dev, "%s: %s ret = %d\n", __func__, str,ret);
+//	if (!ha->enable_error_recovery)
+//		device_printf(ha->pci_dev, "%s: %s ret = %d\n", __func__,
+//			str,ret);
+
 	return (ret);
 }
 
@@ -202,7 +205,9 @@ qla_unlock(qla_host_t *ha, const char *str)
 	ha->hw_lock_held = 0;
 	ha->qla_unlock = str;
 	mtx_unlock(&ha->hw_lock);
-	//device_printf(ha->pci_dev, "%s: %s\n", __func__, str);
+
+//	if (!ha->enable_error_recovery)
+//		device_printf(ha->pci_dev, "%s: %s\n", __func__, str);
 
 	return;
 }
diff --git a/sys/dev/qlxgbe/ql_ioctl.c b/sys/dev/qlxgbe/ql_ioctl.c
index 7675d54823c4..377c28ecbcce 100644
--- a/sys/dev/qlxgbe/ql_ioctl.c
+++ b/sys/dev/qlxgbe/ql_ioctl.c
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include "ql_ver.h"
 #include "ql_dbg.h"
 
+static int ql_slowpath_log(qla_host_t *ha, qla_sp_log_t *log);
 static int ql_drvr_state(qla_host_t *ha, qla_driver_state_t *drvr_state);
 static uint32_t ql_drvr_state_size(qla_host_t *ha);
 static int ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
@@ -224,6 +225,7 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
 	case QLA_RD_FW_DUMP:
 
 		if (ha->hw.mdump_init == 0) {
+			device_printf(pci_dev, "%s: minidump not initialized\n", __func__);
 			rval = EINVAL;
 			break;
 		}
@@ -233,45 +235,85 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
 		if ((fw_dump->minidump == NULL) ||
 			(fw_dump->minidump_size != (ha->hw.mdump_buffer_size +
 				ha->hw.mdump_template_size))) {
+			device_printf(pci_dev,
+				"%s: minidump buffer [%p] size = [%d, %d] invalid\n", __func__,
+				fw_dump->minidump, fw_dump->minidump_size,
+				(ha->hw.mdump_buffer_size + ha->hw.mdump_template_size));
 			rval = EINVAL;
 			break;
 		}
 
-		if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) {
-			if (!ha->hw.mdump_done)
-				ha->qla_initiate_recovery = 1;
-			QLA_UNLOCK(ha, __func__);
-		} else {
+		if ((ha->pci_func & 0x1)) {
+			device_printf(pci_dev, "%s: mindump allowed only on Port0\n", __func__);
 			rval = ENXIO;
 			break;
 		}
+
+		fw_dump->saved = 1;
+
+		if (ha->offline) {
+
+			if (ha->enable_minidump)
+				ql_minidump(ha);
+
+			fw_dump->saved = 0;
+			fw_dump->usec_ts = ha->hw.mdump_usec_ts;
+
+			if (!ha->hw.mdump_done) {
+				device_printf(pci_dev,
+					"%s: port offline minidump failed\n", __func__);
+				rval = ENXIO;
+				break;
+			}
+		} else {
+
+			if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) {
+				if (!ha->hw.mdump_done) {
+					fw_dump->saved = 0;
+					QL_INITIATE_RECOVERY(ha);
+					device_printf(pci_dev, "%s: recovery initiated "
+						" to trigger minidump\n",
+						__func__);
+				}
+				QLA_UNLOCK(ha, __func__);
+			} else {
+				device_printf(pci_dev, "%s: QLA_LOCK() failed0\n", __func__);
+				rval = ENXIO;
+				break;
+			}
 	
 #define QLNX_DUMP_WAIT_SECS	30
 
-		count = QLNX_DUMP_WAIT_SECS * 1000;
+			count = QLNX_DUMP_WAIT_SECS * 1000;
 
-		while (count) {
-			if (ha->hw.mdump_done)
-				break;
-			qla_mdelay(__func__, 100);
-			count -= 100;
-		}
+			while (count) {
+				if (ha->hw.mdump_done)
+					break;
+				qla_mdelay(__func__, 100);
+				count -= 100;
+			}
 
-		if (!ha->hw.mdump_done) {
-			rval = ENXIO;
-			break;
-		}
+			if (!ha->hw.mdump_done) {
+				device_printf(pci_dev,
+					"%s: port not offline minidump failed\n", __func__);
+				rval = ENXIO;
+				break;
+			}
+			fw_dump->usec_ts = ha->hw.mdump_usec_ts;
 			
-		if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) {
-			ha->hw.mdump_done = 0;
-			QLA_UNLOCK(ha, __func__);
-		} else {
-			rval = ENXIO;
-			break;
+			if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) {
+				ha->hw.mdump_done = 0;
+				QLA_UNLOCK(ha, __func__);
+			} else {
+				device_printf(pci_dev, "%s: QLA_LOCK() failed1\n", __func__);
+				rval = ENXIO;
+				break;
+			}
 		}
 
 		if ((rval = copyout(ha->hw.mdump_template,
 			fw_dump->minidump, ha->hw.mdump_template_size))) {
+			device_printf(pci_dev, "%s: template copyout failed\n", __func__);
 			rval = ENXIO;
 			break;
 		}
@@ -279,14 +321,20 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
 		if ((rval = copyout(ha->hw.mdump_buffer,
 				((uint8_t *)fw_dump->minidump +
 					ha->hw.mdump_template_size),
-				ha->hw.mdump_buffer_size)))
+				ha->hw.mdump_buffer_size))) {
+			device_printf(pci_dev, "%s: minidump copyout failed\n", __func__);
 			rval = ENXIO;
+		}
 		break;
 
 	case QLA_RD_DRVR_STATE:
 		rval = ql_drvr_state(ha, (qla_driver_state_t *)data);
 		break;
 
+	case QLA_RD_SLOWPATH_LOG:
+		rval = ql_slowpath_log(ha, (qla_sp_log_t *)data);
+		break;
+
 	case QLA_RD_PCI_IDS:
 		pci_ids = (qla_rd_pci_ids_t *)data;
 		pci_ids->ven_id = pci_get_vendor(pci_dev);
@@ -304,12 +352,12 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
 }
 
 
+
 static int
 ql_drvr_state(qla_host_t *ha, qla_driver_state_t *state)
 {
 	int rval = 0;
 	uint32_t drvr_state_size;
-	qla_drvr_state_hdr_t *hdr;
 
 	drvr_state_size = ql_drvr_state_size(ha);
 
@@ -324,10 +372,7 @@ ql_drvr_state(qla_host_t *ha, qla_driver_state_t *state)
 	if (ha->hw.drvr_state == NULL)
 		return (ENOMEM);
 
-	hdr = ha->hw.drvr_state;
-
-	if (!hdr->drvr_version_major)
-		ql_capture_drvr_state(ha);
+	ql_capture_drvr_state(ha);
 
 	rval = copyout(ha->hw.drvr_state, state->buffer, drvr_state_size);
 
@@ -416,21 +461,25 @@ ql_capture_drvr_state(qla_host_t *ha)
 {
 	uint8_t *state_buffer;
 	uint8_t *ptr;
-	uint32_t drvr_state_size;
 	qla_drvr_state_hdr_t *hdr;
 	uint32_t size;
 	int i;
 
-	drvr_state_size = ql_drvr_state_size(ha);
-
 	state_buffer =  ha->hw.drvr_state;
 
 	if (state_buffer == NULL)
 		return;
-	
-	bzero(state_buffer, drvr_state_size);
 
 	hdr = (qla_drvr_state_hdr_t *)state_buffer;
+	
+	hdr->saved = 0;
+
+	if (hdr->drvr_version_major) {
+		hdr->saved = 1;
+		return;
+	}
+
+	hdr->usec_ts = qla_get_usec_timestamp();
 
 	hdr->drvr_version_major = QLA_VERSION_MAJOR;
 	hdr->drvr_version_minor = QLA_VERSION_MINOR;
@@ -512,6 +561,9 @@ ql_alloc_drvr_state_buffer(qla_host_t *ha)
 
 	ha->hw.drvr_state =  malloc(drvr_state_size, M_QLA83XXBUF, M_NOWAIT);	
 
+	if (ha->hw.drvr_state != NULL)
+		bzero(ha->hw.drvr_state, drvr_state_size);
+
 	return;
 }
 
@@ -523,3 +575,93 @@ ql_free_drvr_state_buffer(qla_host_t *ha)
 	return;
 }
 
+void
+ql_sp_log(qla_host_t *ha, uint16_t fmtstr_idx, uint16_t num_params,
+	uint32_t param0, uint32_t param1, uint32_t param2, uint32_t param3,
+	uint32_t param4)
+{
+	qla_sp_log_entry_t *sp_e, *sp_log;
+
+	if (((sp_log = ha->hw.sp_log) == NULL) || ha->hw.sp_log_stop)
+		return;
+
+	mtx_lock(&ha->sp_log_lock);
+
+	sp_e = &sp_log[ha->hw.sp_log_index];
+
+	bzero(sp_e, sizeof (qla_sp_log_entry_t));
+
+	sp_e->fmtstr_idx = fmtstr_idx;
+	sp_e->num_params = num_params;
+
+	sp_e->usec_ts = qla_get_usec_timestamp();
+
+	sp_e->params[0] = param0;
+	sp_e->params[1] = param1;
+	sp_e->params[2] = param2;
+	sp_e->params[3] = param3;
+	sp_e->params[4] = param4;
+
+	ha->hw.sp_log_index = (ha->hw.sp_log_index + 1) & (NUM_LOG_ENTRIES - 1);
+
+	if (ha->hw.sp_log_num_entries < NUM_LOG_ENTRIES)
+		ha->hw.sp_log_num_entries++;
+
+	mtx_unlock(&ha->sp_log_lock);
+
+	return;
+}
+
+void
+ql_alloc_sp_log_buffer(qla_host_t *ha)
+{
+	uint32_t size;
+
+	size = (sizeof(qla_sp_log_entry_t)) * NUM_LOG_ENTRIES;
+
+	ha->hw.sp_log =  malloc(size, M_QLA83XXBUF, M_NOWAIT);	
+
+	if (ha->hw.sp_log != NULL)
+		bzero(ha->hw.sp_log, size);
+
+	ha->hw.sp_log_index = 0;
+	ha->hw.sp_log_num_entries = 0;
+
+	return;
+}
+
+void
+ql_free_sp_log_buffer(qla_host_t *ha)
+{
+	if (ha->hw.sp_log != NULL)
+		free(ha->hw.sp_log, M_QLA83XXBUF);
+	return;
+}
+
+static int
+ql_slowpath_log(qla_host_t *ha, qla_sp_log_t *log)
+{
+	int rval = 0;
+	uint32_t size;
+
+	if ((ha->hw.sp_log == NULL) || (log->buffer == NULL))
+		return (EINVAL);
+
+	size = (sizeof(qla_sp_log_entry_t) * NUM_LOG_ENTRIES);
+
+	mtx_lock(&ha->sp_log_lock);
+
+	rval = copyout(ha->hw.sp_log, log->buffer, size);
+
+	if (!rval) {
+		log->next_idx = ha->hw.sp_log_index;
+		log->num_entries = ha->hw.sp_log_num_entries;
+	}
+	device_printf(ha->pci_dev,
+		"%s: exit [rval = %d][%p, next_idx = %d, %d entries, %d bytes]\n",
+		__func__, rval, log->buffer, log->next_idx, log->num_entries, size);
+	mtx_unlock(&ha->sp_log_lock);
+
+	return (rval);
+}
+
diff --git a/sys/dev/qlxgbe/ql_ioctl.h b/sys/dev/qlxgbe/ql_ioctl.h
index f985958a74e6..3a8150b637ab 100644
--- a/sys/dev/qlxgbe/ql_ioctl.h
+++ b/sys/dev/qlxgbe/ql_ioctl.h
@@ -73,6 +73,17 @@ struct qla_rd_pci_ids {
 };
 typedef struct qla_rd_pci_ids qla_rd_pci_ids_t;
 
+#define NUM_LOG_ENTRY_PARAMS	5
+#define NUM_LOG_ENTRIES		512
+
+struct qla_sp_log_entry {
+	uint32_t fmtstr_idx;
+	uint32_t num_params;
+	uint64_t usec_ts;
+	uint32_t params[NUM_LOG_ENTRY_PARAMS];
+};
+typedef struct qla_sp_log_entry qla_sp_log_entry_t;
+
 /*
  * structure encapsulating the value to read/write from/to offchip (MS) memory
  */
@@ -88,6 +99,8 @@ typedef struct qla_offchip_mem_val qla_offchip_mem_val_t;
 
 struct qla_rd_fw_dump {
 	uint16_t pci_func;
+	uint16_t saved;
+	uint64_t usec_ts;
 	uint32_t minidump_size;
 	void *minidump;
 };
@@ -122,6 +135,8 @@ struct qla_drvr_state_hdr {
 	uint32_t	drvr_version_build;
 
 	uint8_t         mac_addr[ETHER_ADDR_LEN];
+	uint16_t	saved;
+	uint64_t	usec_ts;
         uint16_t        link_speed;
         uint16_t        cable_length;
         uint32_t        cable_oui;
@@ -159,6 +174,13 @@ struct qla_driver_state {
 };
 typedef struct qla_driver_state qla_driver_state_t;
 
+struct qla_sp_log {
+	uint32_t	next_idx; /* index of next entry in slowpath trace log */
+	uint32_t	num_entries; /* number of entries in slowpath trace log */
+	void		*buffer;
+};
+typedef struct qla_sp_log qla_sp_log_t;
+
 /*
  * Read/Write Register
  */
@@ -204,5 +226,58 @@ typedef struct qla_driver_state qla_driver_state_t;
  */
 #define QLA_RD_DRVR_STATE	_IOWR('q', 9, qla_driver_state_t)
 
+/*
+ * Read Slowpath Log
+ */
+#define QLA_RD_SLOWPATH_LOG	_IOWR('q', 10, qla_sp_log_t)
+
+/*
+ * Format Strings For Slowpath Trace Logs
+ */
+#define SP_TLOG_FMT_STR_0	\
+	"qla_mbx_cmd [%ld]: enter no_pause = %d [0x%08x 0x%08x 0x%08x 0x%08x]\n"
+
+#define SP_TLOG_FMT_STR_1	\
+	"qla_mbx_cmd [%ld]: offline = 0x%08x qla_initiate_recovery = 0x%08x exit1\n"
+
+#define SP_TLOG_FMT_STR_2	\
+	"qla_mbx_cmd [%ld]: qla_initiate_recovery = 0x%08x exit2\n"
+
+#define SP_TLOG_FMT_STR_3	\
+	"qla_mbx_cmd [%ld]: timeout exit3 [host_mbx_cntrl = 0x%08x]\n"
+
+#define SP_TLOG_FMT_STR_4	\
+	"qla_mbx_cmd [%ld]: qla_initiate_recovery = 0x%08x exit4\n"
+
+#define SP_TLOG_FMT_STR_5	\
+	"qla_mbx_cmd [%ld]: timeout exit5 [fw_mbx_cntrl = 0x%08x]\n"
+
+#define SP_TLOG_FMT_STR_6	\
+	"qla_mbx_cmd [%ld]: qla_initiate_recovery = 0x%08x exit6\n"
+
+#define SP_TLOG_FMT_STR_7	\
+	"qla_mbx_cmd [%ld]: exit [0x%08x 0x%08x 0x%08x 0x%08x 0x%08x]\n"
+
+#define SP_TLOG_FMT_STR_8	\
+	"qla_ioctl [%ld]: SIOCSIFADDR if_drv_flags = 0x%08x [0x%08x] ipv4 = 0x%08x\n"
+
+#define SP_TLOG_FMT_STR_9	\
+	"qla_ioctl [%ld]: SIOCSIFMTU if_drv_flags = 0x%08x [0x%08x] max_frame_size = 0x%08x if_mtu = 0x%08x\n"
+
+#define SP_TLOG_FMT_STR_10	\
+	"qla_ioctl [%ld]: SIOCSIFFLAGS if_drv_flags = 0x%08x [0x%08x] ha->if_flags = 0x%08x ifp->if_flags = 0x%08x\n"
+
+#define SP_TLOG_FMT_STR_11	\
+	"qla_ioctl [%ld]: SIOCSIFCAP if_drv_flags = 0x%08x [0x%08x] mask = 0x%08x ifp->if_capenable = 0x%08x\n"
+
+#define SP_TLOG_FMT_STR_12	\
+	"qla_set_multi [%ld]: if_drv_flags = 0x%08x [0x%08x] add_multi = 0x%08x mcnt = 0x%08x\n"
+
+#define SP_TLOG_FMT_STR_13	\
+	"qla_stop [%ld]: \n"
+
+#define SP_TLOG_FMT_STR_14	\
+	"qla_init_locked [%ld]: \n"
+
 
 #endif /* #ifndef _QL_IOCTL_H_ */
diff --git a/sys/dev/qlxgbe/ql_isr.c b/sys/dev/qlxgbe/ql_isr.c
index 79bc3b56bcf5..d147df3e68ce 100644
--- a/sys/dev/qlxgbe/ql_isr.c
+++ b/sys/dev/qlxgbe/ql_isr.c
@@ -49,7 +49,7 @@ static void
 qla_rcv_error(qla_host_t *ha)
 {
 	ha->stop_rcv = 1;
-	ha->qla_initiate_recovery = 1;
+	QL_INITIATE_RECOVERY(ha);
 }
 
 
@@ -389,7 +389,7 @@ qla_rcv_cont_sds(qla_host_t *ha, uint32_t sds_idx, uint32_t comp_idx,
 
 		opcode = Q8_STAT_DESC_OPCODE((sdesc->data[1]));
 
-		if (!opcode) {
+		if (!opcode || QL_ERR_INJECT(ha, INJCT_INV_CONT_OPCODE)) {
 			device_printf(ha->pci_dev, "%s: opcode=0 %p %p\n",
 				__func__, (void *)sdesc->data[0],
 				(void *)sdesc->data[1]);
@@ -558,8 +558,9 @@ ql_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count)
 				sdesc0 = (q80_stat_desc_t *)
 					&hw->sds[sds_idx].sds_ring_base[c_idx];
 
-				if (Q8_STAT_DESC_OPCODE((sdesc0->data[1])) !=
-						Q8_STAT_DESC_OPCODE_CONT) {
+				if ((Q8_STAT_DESC_OPCODE((sdesc0->data[1])) !=
+						Q8_STAT_DESC_OPCODE_CONT) ||
+				QL_ERR_INJECT(ha, INJCT_SGL_RCV_INV_DESC_COUNT)) {
 					desc_count = 0;
 					break;
 				}
@@ -620,8 +621,9 @@ ql_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count)
 				sdesc0 = (q80_stat_desc_t *)
 					&hw->sds[sds_idx].sds_ring_base[c_idx];
 
-				if (Q8_STAT_DESC_OPCODE((sdesc0->data[1])) !=
-						Q8_STAT_DESC_OPCODE_CONT) {
+				if ((Q8_STAT_DESC_OPCODE((sdesc0->data[1])) !=
+						Q8_STAT_DESC_OPCODE_CONT) ||
+				QL_ERR_INJECT(ha, INJCT_SGL_LRO_INV_DESC_COUNT)) {
 					desc_count = 0;
 					break;
 				}
@@ -822,7 +824,13 @@ ql_mbx_isr(void *arg)
 		data = READ_REG32(ha, (Q8_FW_MBOX0 + 12));
 
 		prev_link_state =  ha->hw.link_up;
-		ha->hw.link_up = (((data & 0xFF) == 0) ? 0 : 1);
+
+		data = (((data & 0xFF) == 0) ? 0 : 1);
+		atomic_store_rel_8(&ha->hw.link_up, (uint8_t)data);
+
+		device_printf(ha->pci_dev,
+			"%s: AEN[0x8001] data = 0x%08x, prev_link_state = 0x%08x\n",
+			__func__, data, prev_link_state);
 
 		if (prev_link_state !=  ha->hw.link_up) {
 			if (ha->hw.link_up)
@@ -833,17 +841,18 @@ ql_mbx_isr(void *arg)
 
 
 		ha->hw.module_type = ((data >> 8) & 0xFF);
-		ha->hw.flags.fduplex = (((data & 0xFF0000) == 0) ? 0 : 1);
-		ha->hw.flags.autoneg = (((data & 0xFF000000) == 0) ? 0 : 1);
+		ha->hw.fduplex = (((data & 0xFF0000) == 0) ? 0 : 1);
+		ha->hw.autoneg = (((data & 0xFF000000) == 0) ? 0 : 1);
 		
 		data = READ_REG32(ha, (Q8_FW_MBOX0 + 16));
-		ha->hw.flags.loopback_mode = data & 0x03;
+		ha->hw.loopback_mode = data & 0x03;
 
 		ha->hw.link_faults = (data >> 3) & 0xFF;
 
 		break;
 
         case 0x8100:
+		device_printf(ha->pci_dev, "%s: AEN[0x%08x]\n", __func__, data);
 		ha->hw.imd_compl=1;
 		break;
 
@@ -854,6 +863,9 @@ ql_mbx_isr(void *arg)
                 ha->hw.aen_mb2 = READ_REG32(ha, (Q8_FW_MBOX0 + 8));
                 ha->hw.aen_mb3 = READ_REG32(ha, (Q8_FW_MBOX0 + 12));
                 ha->hw.aen_mb4 = READ_REG32(ha, (Q8_FW_MBOX0 + 16));
+		device_printf(ha->pci_dev, "%s: AEN[0x%08x 0x%08x 0x%08x 0%08x 0x%08x]\n",
+			__func__, data, ha->hw.aen_mb1, ha->hw.aen_mb2,
+			ha->hw.aen_mb3, ha->hw.aen_mb4);
                 break;
 
         case 0x8110:
diff --git a/sys/dev/qlxgbe/ql_misc.c b/sys/dev/qlxgbe/ql_misc.c
index 74d126905f8e..18528200841e 100644
--- a/sys/dev/qlxgbe/ql_misc.c
+++ b/sys/dev/qlxgbe/ql_misc.c
@@ -77,7 +77,7 @@ ql_rdwr_indreg32(qla_host_t *ha, uint32_t addr, uint32_t *val, uint32_t rd)
 	if (!count || QL_ERR_INJECT(ha, INJCT_RDWR_INDREG_FAILURE)) {
 		device_printf(ha->pci_dev, "%s: [0x%08x, 0x%08x, %d] failed\n",
 			__func__, addr, *val, rd);
-		ha->qla_initiate_recovery = 1;
+		QL_INITIATE_RECOVERY(ha);
 		return -1;
 	}
 
@@ -210,7 +210,7 @@ exit_ql_rdwr_offchip_mem:
 		(uint32_t)(addr), val->data_lo, val->data_hi, val->data_ulo,
 		val->data_uhi, rd, step);
 
-	ha->qla_initiate_recovery = 1;
+	QL_INITIATE_RECOVERY(ha);
 
 	return (-1);
 }
@@ -773,12 +773,13 @@ ql_init_hw(qla_host_t *ha)
 			}
 			qla_mdelay(__func__, 100);
 		}
-		return (-1);
+		ret = -1;
+		goto ql_init_hw_exit;
 	}
 
 	
 	val = READ_REG32(ha, Q8_CMDPEG_STATE);
-	if (!cold || (val != 0xFF01)) {
+	if (!cold || (val != 0xFF01) || ha->qla_initiate_recovery) {
         	ret = qla_init_from_flash(ha);
 		qla_mdelay(__func__, 100);
 	}
@@ -794,6 +795,13 @@ qla_init_exit:
 		ha->hw.flags.fdt_valid = 1;
 	}
 
+ql_init_hw_exit:
+
+	if (ret) {
+		if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_HW_INIT_FAILURE)
+			ha->hw.sp_log_stop = -1;
+	}
+
         return (ret);
 }
 
@@ -1280,6 +1288,7 @@ qla_ld_fw_init(qla_host_t *ha)
 
 	hdr = (q8_tmplt_hdr_t *)ql83xx_resetseq;
 
+	device_printf(ha->pci_dev, "%s: reset sequence\n", __func__);
 	if (qla_tmplt_16bit_checksum(ha, (uint16_t *)ql83xx_resetseq,
 		(uint32_t)hdr->size)) {
 		device_printf(ha->pci_dev, "%s: reset seq checksum failed\n",
@@ -1290,7 +1299,7 @@ qla_ld_fw_init(qla_host_t *ha)
 
 	buf = ql83xx_resetseq + hdr->stop_seq_off;
 
-//	device_printf(ha->pci_dev, "%s: stop sequence\n", __func__);
+	device_printf(ha->pci_dev, "%s: stop sequence\n", __func__);
 	if (qla_tmplt_execute(ha, buf, index , &end_idx, hdr->nentries)) {
 		device_printf(ha->pci_dev, "%s: stop seq failed\n", __func__);
 		return -1;
@@ -1300,7 +1309,7 @@ qla_ld_fw_init(qla_host_t *ha)
 
 	buf = ql83xx_resetseq + hdr->init_seq_off;
 
-//	device_printf(ha->pci_dev, "%s: init sequence\n", __func__);
+	device_printf(ha->pci_dev, "%s: init sequence\n", __func__);
 	if (qla_tmplt_execute(ha, buf, index , &end_idx, hdr->nentries)) {
 		device_printf(ha->pci_dev, "%s: init seq failed\n", __func__);
 		return -1;
@@ -1322,7 +1331,7 @@ qla_ld_fw_init(qla_host_t *ha)
 	index = end_idx;
 	buf = ql83xx_resetseq + hdr->start_seq_off;
 
-//	device_printf(ha->pci_dev, "%s: start sequence\n", __func__);
+	device_printf(ha->pci_dev, "%s: start sequence\n", __func__);
 	if (qla_tmplt_execute(ha, buf, index , &end_idx, hdr->nentries)) {
 		device_printf(ha->pci_dev, "%s: init seq failed\n", __func__);
 		return -1;
diff --git a/sys/dev/qlxgbe/ql_os.c b/sys/dev/qlxgbe/ql_os.c
index 0accee7bf0d0..d4f3c2948f07 100644
--- a/sys/dev/qlxgbe/ql_os.c
+++ b/sys/dev/qlxgbe/ql_os.c
@@ -190,7 +190,31 @@ qla_add_sysctls(qla_host_t *ha)
 		SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 		OID_AUTO, "enable_minidump", CTLFLAG_RW,
 		&ha->enable_minidump, ha->enable_minidump,
-		"Minidump retrival is enabled only when this is set");
+		"Minidump retrival prior to error recovery "
+		"is enabled only when this is set");
+
+	ha->enable_driverstate_dump = 1;
+	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+		SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+		OID_AUTO, "enable_driverstate_dump", CTLFLAG_RW,
+		&ha->enable_driverstate_dump, ha->enable_driverstate_dump,
+		"Driver State retrival prior to error recovery "
+		"is enabled only when this is set");
+
+	ha->enable_error_recovery = 1;
+	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+		SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+		OID_AUTO, "enable_error_recovery", CTLFLAG_RW,
+		&ha->enable_error_recovery, ha->enable_error_recovery,
+		"when set error recovery is enabled on fatal errors "
+		"otherwise the port is turned offline");
+
+	ha->ms_delay_after_init = 1000;
+	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+		SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+		OID_AUTO, "ms_delay_after_init", CTLFLAG_RW,
+		&ha->ms_delay_after_init, ha->ms_delay_after_init,
+		"millisecond delay after hw_init");
 
 	ha->std_replenish = QL_STD_REPLENISH_THRES;
         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
@@ -249,49 +273,47 @@ qla_watchdog(void *arg)
 	ha->qla_watchdog_exited = 0;
 
 	if (!ha->qla_watchdog_pause) {
-		if (ql_hw_check_health(ha) || ha->qla_initiate_recovery ||
-			(ha->msg_from_peer == QL_PEER_MSG_RESET)) {
+                if (!ha->offline &&
+                        (ql_hw_check_health(ha) || ha->qla_initiate_recovery ||
+                        (ha->msg_from_peer == QL_PEER_MSG_RESET))) {
 
-			if (!(ha->dbg_level & 0x8000)) {
+	        	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+			ql_update_link_state(ha);
+
+			if (ha->enable_error_recovery) {
 				ha->qla_watchdog_paused = 1;
 				ha->qla_watchdog_pause = 1;
-				ha->qla_initiate_recovery = 0;
 				ha->err_inject = 0;
 				device_printf(ha->pci_dev,
 					"%s: taskqueue_enqueue(err_task) \n",
 					__func__);
 				taskqueue_enqueue(ha->err_tq, &ha->err_task);
-				return;
+			} else {
+				if (ifp != NULL)
+					ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+				ha->offline = 1;
 			}
+			return;
 
-		} else if (ha->qla_interface_up) {
-
-			ha->watchdog_ticks++;
+		} else {
+			if (ha->qla_interface_up) {
 
-			if (ha->watchdog_ticks > 1000)
-				ha->watchdog_ticks = 0;
+				ha->watchdog_ticks++;
 
-                        if (!ha->watchdog_ticks && QL_RUNNING(ifp)) {
-                                taskqueue_enqueue(ha->stats_tq, &ha->stats_task);
-                        }
+				if (ha->watchdog_ticks > 1000)
+					ha->watchdog_ticks = 0;
 
-                        if (ha->async_event) {
-                                taskqueue_enqueue(ha->async_event_tq,
-                                        &ha->async_event_task);
-                        }
+				if (!ha->watchdog_ticks && QL_RUNNING(ifp)) {
+					taskqueue_enqueue(ha->stats_tq,
+						&ha->stats_task);
+				}
 
-#if 0
-			for (i = 0; ((i < ha->hw.num_sds_rings) &&
-					!ha->watchdog_ticks); i++) {
-				qla_tx_fp_t *fp = &ha->tx_fp[i];
+				if (ha->async_event) {
+					taskqueue_enqueue(ha->async_event_tq,
+						&ha->async_event_task);
+				}
 
-				if (fp->fp_taskqueue != NULL)
-					taskqueue_enqueue(fp->fp_taskqueue,
-						&fp->fp_task);
 			}
-#endif
-			ha->qla_watchdog_paused = 0;
-		} else {
 			ha->qla_watchdog_paused = 0;
 		}
 	} else {
@@ -345,6 +367,7 @@ qla_pci_attach(device_t dev)
 					ha->reg_rid);
 
 	mtx_init(&ha->hw_lock, "qla83xx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
+	mtx_init(&ha->sp_log_lock, "qla83xx_sp_log_lock", MTX_NETWORK_LOCK, MTX_DEF);
 	ha->flags.lock_init = 1;
 
 	qla_add_sysctls(ha);
@@ -493,6 +516,7 @@ qla_pci_attach(device_t dev)
 		goto qla_pci_attach_err;
 	}
 	ql_alloc_drvr_state_buffer(ha);
+	ql_alloc_sp_log_buffer(ha);
 	/* create the o.s ethernet interface */
 	qla_init_ifnet(dev, ha);
 
@@ -538,6 +562,7 @@ qla_pci_attach_err:
 
 	if (ha->flags.lock_init) {
 		mtx_destroy(&ha->hw_lock);
+		mtx_destroy(&ha->sp_log_lock);
 	}
 
 	QL_DPRINT2(ha, (dev, "%s: exit ENXIO\n", __func__));
@@ -576,6 +601,7 @@ qla_pci_detach(device_t dev)
 
 	if (ha->flags.lock_init) {
 		mtx_destroy(&ha->hw_lock);
+		mtx_destroy(&ha->sp_log_lock);
 	}
 
 	QL_DPRINT2(ha, (dev, "%s: exit\n", __func__));
@@ -617,17 +643,17 @@ qla_release(qla_host_t *ha)
 	dev = ha->pci_dev;
 
         if (ha->async_event_tq) {
-                taskqueue_drain(ha->async_event_tq, &ha->async_event_task);
+                taskqueue_drain_all(ha->async_event_tq);
                 taskqueue_free(ha->async_event_tq);
         }
 
 	if (ha->err_tq) {
-		taskqueue_drain(ha->err_tq, &ha->err_task);
+		taskqueue_drain_all(ha->err_tq);
 		taskqueue_free(ha->err_tq);
 	}
 
 	if (ha->stats_tq) {
-		taskqueue_drain(ha->stats_tq, &ha->stats_task);
+		taskqueue_drain_all(ha->stats_tq);
 		taskqueue_free(ha->stats_tq);
 	}
 
@@ -647,6 +673,7 @@ qla_release(qla_host_t *ha)
 		ether_ifdetach(ha->ifp);
 
 	ql_free_drvr_state_buffer(ha);
+	ql_free_sp_log_buffer(ha);
 	ql_free_dma(ha); 
 	qla_free_parent_dma_tag(ha);
 
@@ -677,10 +704,6 @@ qla_release(qla_host_t *ha)
 	if (ha->msix_count)
 		pci_release_msi(dev);
 
-//	if (ha->flags.lock_init) {
-//		mtx_destroy(&ha->hw_lock);
-//	}
-
         if (ha->pci_reg)
                 (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid,
 				ha->pci_reg);
@@ -874,6 +897,7 @@ qla_init_ifnet(device_t dev, qla_host_t *ha)
 
 	ifp->if_capabilities |= IFCAP_HWCSUM |
 				IFCAP_TSO4 |
+				IFCAP_TSO6 |
 				IFCAP_JUMBO_MTU |
 				IFCAP_VLAN_HWTAGGING |
 				IFCAP_VLAN_MTU |
@@ -902,6 +926,8 @@ qla_init_locked(qla_host_t *ha)
 {
 	struct ifnet *ifp = ha->ifp;
 
+	ql_sp_log(ha, 14, 0, 0, 0, 0, 0, 0);
+
 	qla_stop(ha);
 
 	if (qla_alloc_xmt_bufs(ha) != 0) 
@@ -921,13 +947,17 @@ qla_init_locked(qla_host_t *ha)
  	if (ql_init_hw_if(ha) == 0) {
 		ifp = ha->ifp;
 		ifp->if_drv_flags |= IFF_DRV_RUNNING;
-		ha->qla_watchdog_pause = 0;
 		ha->hw_vlan_tx_frames = 0;
 		ha->tx_tso_frames = 0;
 		ha->qla_interface_up = 1;
 		ql_update_link_state(ha);
+	} else {
+		if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_IF_START_FAILURE)
+			ha->hw.sp_log_stop = -1;
 	}
 
+	ha->qla_watchdog_pause = 0;
+
 	return;
 }
 
@@ -981,6 +1011,10 @@ qla_set_multi(qla_host_t *ha, uint32_t add_multi)
 		QLA_LOCK_NO_SLEEP) != 0)
 		return (-1);
 
+	ql_sp_log(ha, 12, 4, ifp->if_drv_flags,
+		(ifp->if_drv_flags & IFF_DRV_RUNNING),
+		add_multi, (uint32_t)mcnt, 0);
+
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 
 		if (!add_multi) {
@@ -1011,6 +1045,8 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 	qla_host_t *ha;
 
 	ha = (qla_host_t *)ifp->if_softc;
+	if (ha->offline || ha->qla_initiate_recovery)
+		return (ret);
 
 	switch (cmd) {
 	case SIOCSIFADDR:
@@ -1027,6 +1063,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 
 			ifp->if_flags |= IFF_UP;
 
+			ql_sp_log(ha, 8, 3, ifp->if_drv_flags,
+				(ifp->if_drv_flags & IFF_DRV_RUNNING),
+				ntohl(IA_SIN(ifa)->sin_addr.s_addr), 0, 0);
+
 			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 				qla_init_locked(ha);
 			}
@@ -1060,6 +1100,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			ha->max_frame_size =
 				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 
+			ql_sp_log(ha, 9, 4, ifp->if_drv_flags,
+				(ifp->if_drv_flags & IFF_DRV_RUNNING),
+				ha->max_frame_size, ifp->if_mtu, 0);
+
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				qla_init_locked(ha);
 			}
@@ -1085,6 +1129,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		if (ret)
 			break;
 
+		ql_sp_log(ha, 10, 4, ifp->if_drv_flags,
+			(ifp->if_drv_flags & IFF_DRV_RUNNING),
+			ha->if_flags, ifp->if_flags, 0);
+
 		if (ifp->if_flags & IFF_UP) {
 
 			ha->max_frame_size = ifp->if_mtu +
@@ -1160,6 +1208,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			if (ret)
 				break;
 
+			ql_sp_log(ha, 11, 4, ifp->if_drv_flags,
+				(ifp->if_drv_flags & IFF_DRV_RUNNING),
+				mask, ifp->if_capenable, 0);
+
 			qla_init_locked(ha);
 
 			QLA_UNLOCK(ha, __func__);
@@ -1531,7 +1583,7 @@ qla_destroy_fp_taskqueues(qla_host_t *ha)
                 qla_tx_fp_t *fp = &ha->tx_fp[i];
 
                 if (fp->fp_taskqueue != NULL) {
-                        taskqueue_drain(fp->fp_taskqueue, &fp->fp_task);
+                        taskqueue_drain_all(fp->fp_taskqueue);
                         taskqueue_free(fp->fp_taskqueue);
                         fp->fp_taskqueue = NULL;
                 }
@@ -1548,7 +1600,7 @@ qla_drain_fp_taskqueues(qla_host_t *ha)
                 qla_tx_fp_t *fp = &ha->tx_fp[i];
 
                 if (fp->fp_taskqueue != NULL) {
-                        taskqueue_drain(fp->fp_taskqueue, &fp->fp_task);
+                        taskqueue_drain_all(fp->fp_taskqueue);
                 }
         }
         return;
@@ -1633,6 +1685,8 @@ qla_stop(qla_host_t *ha)
 	device_t	dev;
 	int i = 0;
 
+	ql_sp_log(ha, 13, 0, 0, 0, 0, 0, 0);
+
 	dev = ha->pci_dev;
 
 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
@@ -2041,27 +2095,45 @@ qla_send_msg_to_peer(qla_host_t *ha, uint32_t msg_to_peer)
 	}
 }
 
+void
+qla_set_error_recovery(qla_host_t *ha)
+{
+	struct ifnet *ifp = ha->ifp;
+
+	if (!cold && ha->enable_error_recovery) {
+		if (ifp)
+			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+		ha->qla_initiate_recovery = 1;
+	} else
+		ha->offline = 1;
+	return;
+}
+
 static void
 qla_error_recovery(void *context, int pending)
 {
 	qla_host_t *ha = context;
-	uint32_t msecs_100 = 100;
+	uint32_t msecs_100 = 400;
 	struct ifnet *ifp = ha->ifp;
 	int i = 0;
 
-device_printf(ha->pci_dev, "%s: \n", __func__);
+	device_printf(ha->pci_dev, "%s: enter\n", __func__);
 	ha->hw.imd_compl = 1;
 
+	taskqueue_drain_all(ha->stats_tq);
+	taskqueue_drain_all(ha->async_event_tq);
+
 	if (QLA_LOCK(ha, __func__, -1, 0) != 0)
 		return;
 
-device_printf(ha->pci_dev, "%s: enter\n", __func__);
+	device_printf(ha->pci_dev, "%s: ts_usecs = %ld start\n",
+		__func__, qla_get_usec_timestamp());
 
 	if (ha->qla_interface_up) {
 
 		qla_mdelay(__func__, 300);
 
-	        ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+	        //ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 
 		for (i = 0; i < ha->hw.num_sds_rings; i++) {
 	        	qla_tx_fp_t *fp;
@@ -2078,7 +2150,6 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__);
 		}
 	}
 
-
 	qla_drain_fp_taskqueues(ha);
 
 	if ((ha->pci_func & 0x1) == 0) {
@@ -2096,21 +2167,32 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__);
 		if (ha->enable_minidump)
 			ql_minidump(ha);
 
-		(void) ql_init_hw(ha);
+		if (ha->enable_driverstate_dump)
+			ql_capture_drvr_state(ha);
 
+		if (ql_init_hw(ha)) {
+			device_printf(ha->pci_dev,
+				"%s: ts_usecs = %ld exit: ql_init_hw failed\n",
+				__func__, qla_get_usec_timestamp());
+			ha->offline = 1;
+			goto qla_error_recovery_exit;
+		}
+			
 		if (ha->qla_interface_up) {
 			qla_free_xmt_bufs(ha);
 			qla_free_rcv_bufs(ha);
 		}
 
-		qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
+		if (!QL_ERR_INJECT(ha, INJCT_PEER_PORT_FAILURE_ERR_RECOVERY))
+			qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
 
 	} else {
 		if (ha->msg_from_peer == QL_PEER_MSG_RESET) {
 
 			ha->msg_from_peer = 0;
 
-			qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
+			if (!QL_ERR_INJECT(ha, INJCT_PEER_PORT_FAILURE_ERR_RECOVERY))
+				qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
 		} else {
 			qla_send_msg_to_peer(ha, QL_PEER_MSG_RESET);
 		}
@@ -2119,9 +2201,24 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__);
 			qla_mdelay(__func__, 100);
 		ha->msg_from_peer = 0;
 
-		(void) ql_init_hw(ha);
+		if (ha->enable_driverstate_dump)
+			ql_capture_drvr_state(ha);
 
-		qla_mdelay(__func__, 1000);
+		if (msecs_100 == 0) {
+			device_printf(ha->pci_dev,
+				"%s: ts_usecs = %ld exit: QL_PEER_MSG_ACK not received\n",
+				__func__, qla_get_usec_timestamp());
+			ha->offline = 1;
+			goto qla_error_recovery_exit;
+		}
+
+		if (ql_init_hw(ha)) {
+			device_printf(ha->pci_dev,
+				"%s: ts_usecs = %ld exit: ql_init_hw failed\n",
+				__func__, qla_get_usec_timestamp());
+			ha->offline = 1;
+			goto qla_error_recovery_exit;
+		}
 
 		if (ha->qla_interface_up) {
 			qla_free_xmt_bufs(ha);
@@ -2129,14 +2226,22 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__);
 		}
 	}
 
+	qla_mdelay(__func__, ha->ms_delay_after_init);
+
+	*((uint32_t *)&ha->hw.flags) = 0;
+	ha->qla_initiate_recovery = 0;
+
 	if (ha->qla_interface_up) {
 
 		if (qla_alloc_xmt_bufs(ha) != 0) {
+			ha->offline = 1;
 			goto qla_error_recovery_exit;
 		}
+
 		qla_confirm_9kb_enable(ha);
 
 		if (qla_alloc_rcv_bufs(ha) != 0) {
+			ha->offline = 1;
 			goto qla_error_recovery_exit;
 		}
 
@@ -2146,18 +2251,38 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__);
 			ifp = ha->ifp;
 			ifp->if_drv_flags |= IFF_DRV_RUNNING;
 			ha->qla_watchdog_pause = 0;
+			ql_update_link_state(ha);
+		} else {
+			ha->offline = 1;
+
+			if (ha->hw.sp_log_stop_events &
+				Q8_SP_LOG_STOP_IF_START_FAILURE)
+				ha->hw.sp_log_stop = -1;
 		}
-	} else
+	} else {
 		ha->qla_watchdog_pause = 0;
+	}
 
 qla_error_recovery_exit:
 
-device_printf(ha->pci_dev, "%s: exit\n", __func__);
+	if (ha->offline ) {
+		device_printf(ha->pci_dev, "%s: ts_usecs = %ld port offline\n",
+			__func__, qla_get_usec_timestamp());
+		if (ha->hw.sp_log_stop_events &
+			Q8_SP_LOG_STOP_ERR_RECOVERY_FAILURE)
+			ha->hw.sp_log_stop = -1;
+	}
+
 
         QLA_UNLOCK(ha, __func__);
 
-	callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
-		qla_watchdog, ha);
+	if (!ha->offline)
+		callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
+			qla_watchdog, ha);
+
+	device_printf(ha->pci_dev,
+		"%s: ts_usecs = %ld exit\n",
+		__func__, qla_get_usec_timestamp());
 	return;
 }
 
@@ -2187,6 +2312,7 @@ qla_stats(void *context, int pending)
         ha = context;
 
 	ql_get_stats(ha);
+
 	return;
 }
 
diff --git a/sys/dev/qlxgbe/ql_os.h b/sys/dev/qlxgbe/ql_os.h
index e8000cd52b97..443f3e2500b9 100644
--- a/sys/dev/qlxgbe/ql_os.h
+++ b/sys/dev/qlxgbe/ql_os.h
@@ -119,6 +119,14 @@ static __inline int qla_sec_to_hz(int sec)
 	return (tvtohz(&t));
 }
 
+static __inline uint64_t qla_get_usec_timestamp(void)
+{
+	struct timeval tv;
+
+	microuptime(&tv);
+
+	return ((uint64_t)(((uint64_t)tv.tv_sec) * 1000000 + tv.tv_usec));
+}
 
 #define qla_host_to_le16(x)	htole16(x)
 #define qla_host_to_le32(x)	htole32(x)
diff --git a/sys/dev/qlxgbe/ql_ver.h b/sys/dev/qlxgbe/ql_ver.h
index 24a9d30d1901..9887ac6c4adc 100644
--- a/sys/dev/qlxgbe/ql_ver.h
+++ b/sys/dev/qlxgbe/ql_ver.h
@@ -36,6 +36,6 @@
 
 #define QLA_VERSION_MAJOR 	3
 #define QLA_VERSION_MINOR	10
-#define QLA_VERSION_BUILD       35
+#define QLA_VERSION_BUILD       36
 
 #endif /* #ifndef _QL_VER_H_ */