diff options
Diffstat (limited to 'sys/dev/qlnx/qlnxe/qlnx_os.c')
| -rw-r--r-- | sys/dev/qlnx/qlnxe/qlnx_os.c | 8359 |
1 files changed, 8359 insertions, 0 deletions
diff --git a/sys/dev/qlnx/qlnxe/qlnx_os.c b/sys/dev/qlnx/qlnxe/qlnx_os.c new file mode 100644 index 000000000000..4ad190374f87 --- /dev/null +++ b/sys/dev/qlnx/qlnxe/qlnx_os.c @@ -0,0 +1,8359 @@ +/* + * Copyright (c) 2017-2018 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * File: qlnx_os.c + * Author : David C Somayajulu, Cavium, Inc., San Jose, CA 95131. + */ + +#include "opt_inet.h" + +#include <sys/cdefs.h> +#include "qlnx_os.h" +#include "bcm_osal.h" +#include "reg_addr.h" +#include "ecore_gtt_reg_addr.h" +#include "ecore.h" +#include "ecore_chain.h" +#include "ecore_status.h" +#include "ecore_hw.h" +#include "ecore_rt_defs.h" +#include "ecore_init_ops.h" +#include "ecore_int.h" +#include "ecore_cxt.h" +#include "ecore_spq.h" +#include "ecore_init_fw_funcs.h" +#include "ecore_sp_commands.h" +#include "ecore_dev_api.h" +#include "ecore_l2_api.h" +#include "ecore_mcp.h" +#include "ecore_hw_defs.h" +#include "mcp_public.h" +#include "ecore_iro.h" +#include "nvm_cfg.h" +#include "ecore_dbg_fw_funcs.h" +#include "ecore_iov_api.h" +#include "ecore_vf_api.h" + +#include "qlnx_ioctl.h" +#include "qlnx_def.h" +#include "qlnx_ver.h" + +#ifdef QLNX_ENABLE_IWARP +#include "qlnx_rdma.h" +#endif /* #ifdef QLNX_ENABLE_IWARP */ + +#ifdef CONFIG_ECORE_SRIOV +#include <sys/nv.h> +#include <sys/iov_schema.h> +#include <dev/pci/pci_iov.h> +#endif /* #ifdef CONFIG_ECORE_SRIOV */ + +#include <sys/smp.h> + +/* + * static functions + */ +/* + * ioctl related functions + */ +static void qlnx_add_sysctls(qlnx_host_t *ha); + +/* + * main driver + */ +static void qlnx_release(qlnx_host_t *ha); +static void qlnx_fp_isr(void *arg); +static void qlnx_init_ifnet(device_t dev, qlnx_host_t *ha); +static void qlnx_init(void *arg); +static void qlnx_init_locked(qlnx_host_t *ha); +static int qlnx_set_multi(qlnx_host_t *ha, uint32_t add_multi); +static int qlnx_set_promisc(qlnx_host_t *ha, int enabled); +static int qlnx_set_allmulti(qlnx_host_t *ha, int enabled); +static int qlnx_ioctl(if_t ifp, u_long cmd, caddr_t data); +static int qlnx_media_change(if_t ifp); +static void qlnx_media_status(if_t ifp, struct ifmediareq *ifmr); +static void qlnx_stop(qlnx_host_t *ha); +static int qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct mbuf **m_headp); +static int qlnx_get_ifq_snd_maxlen(qlnx_host_t *ha); +static uint32_t qlnx_get_optics(qlnx_host_t *ha, + struct qlnx_link_output *if_link); +static int qlnx_transmit(if_t ifp, struct mbuf *mp); +static int qlnx_transmit_locked(if_t ifp, struct qlnx_fastpath *fp, + struct mbuf *mp); +static void qlnx_qflush(if_t ifp); + +static int qlnx_alloc_parent_dma_tag(qlnx_host_t *ha); +static void qlnx_free_parent_dma_tag(qlnx_host_t *ha); +static int qlnx_alloc_tx_dma_tag(qlnx_host_t *ha); +static void qlnx_free_tx_dma_tag(qlnx_host_t *ha); +static int qlnx_alloc_rx_dma_tag(qlnx_host_t *ha); +static void qlnx_free_rx_dma_tag(qlnx_host_t *ha); + +static int qlnx_get_mfw_version(qlnx_host_t *ha, uint32_t *mfw_ver); +static int qlnx_get_flash_size(qlnx_host_t *ha, uint32_t *flash_size); + +static int qlnx_nic_setup(struct ecore_dev *cdev, + struct ecore_pf_params *func_params); +static int qlnx_nic_start(struct ecore_dev *cdev); +static int qlnx_slowpath_start(qlnx_host_t *ha); +static int qlnx_slowpath_stop(qlnx_host_t *ha); +static int qlnx_init_hw(qlnx_host_t *ha); +static void qlnx_set_id(struct ecore_dev *cdev, char name[NAME_SIZE], + char ver_str[VER_SIZE]); +static void qlnx_unload(qlnx_host_t *ha); +static int qlnx_load(qlnx_host_t *ha); +static void qlnx_hw_set_multi(qlnx_host_t *ha, uint8_t *mta, uint32_t mcnt, + uint32_t add_mac); +static void qlnx_dump_buf8(qlnx_host_t *ha, const char *msg, void *dbuf, + uint32_t len); +static int qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq); +static void qlnx_reuse_rx_data(struct qlnx_rx_queue *rxq); +static void qlnx_update_rx_prod(struct ecore_hwfn *p_hwfn, + struct qlnx_rx_queue *rxq); +static int qlnx_set_rx_accept_filter(qlnx_host_t *ha, uint8_t filter); +static int qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords, + int hwfn_index); +static int qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords, + int hwfn_index); +static void qlnx_timer(void *arg); +static int qlnx_alloc_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp); +static void qlnx_free_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp); +static void qlnx_trigger_dump(qlnx_host_t *ha); +static uint16_t qlnx_num_tx_compl(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq); +static void qlnx_tx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq); +static int qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget, + int lro_enable); +static void qlnx_fp_taskqueue(void *context, int pending); +static void qlnx_sample_storm_stats(qlnx_host_t *ha); +static int qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size, + struct qlnx_agg_info *tpa); +static void qlnx_free_tpa_mbuf(qlnx_host_t *ha, struct qlnx_agg_info *tpa); + +static uint64_t qlnx_get_counter(if_t ifp, ift_counter cnt); + +/* + * Hooks to the Operating Systems + */ +static int qlnx_pci_probe (device_t); +static int qlnx_pci_attach (device_t); +static int qlnx_pci_detach (device_t); + +#ifndef QLNX_VF + +#ifdef CONFIG_ECORE_SRIOV + +static int qlnx_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params); +static void qlnx_iov_uninit(device_t dev); +static int qlnx_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params); +static void qlnx_initialize_sriov(qlnx_host_t *ha); +static void qlnx_pf_taskqueue(void *context, int pending); +static int qlnx_create_pf_taskqueues(qlnx_host_t *ha); +static void qlnx_destroy_pf_taskqueues(qlnx_host_t *ha); +static void qlnx_inform_vf_link_state(struct ecore_hwfn *p_hwfn, qlnx_host_t *ha); + +#endif /* #ifdef CONFIG_ECORE_SRIOV */ + +static device_method_t qlnx_pci_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, qlnx_pci_probe), + DEVMETHOD(device_attach, qlnx_pci_attach), + DEVMETHOD(device_detach, qlnx_pci_detach), + +#ifdef CONFIG_ECORE_SRIOV + DEVMETHOD(pci_iov_init, qlnx_iov_init), + DEVMETHOD(pci_iov_uninit, qlnx_iov_uninit), + DEVMETHOD(pci_iov_add_vf, qlnx_iov_add_vf), +#endif /* #ifdef CONFIG_ECORE_SRIOV */ + { 0, 0 } +}; + +static driver_t qlnx_pci_driver = { + "ql", qlnx_pci_methods, sizeof (qlnx_host_t), +}; + +MODULE_VERSION(if_qlnxe,1); +DRIVER_MODULE(if_qlnxe, pci, qlnx_pci_driver, 0, 0); + +MODULE_DEPEND(if_qlnxe, pci, 1, 1, 1); +MODULE_DEPEND(if_qlnxe, ether, 1, 1, 1); + +#else + +static device_method_t qlnxv_pci_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, qlnx_pci_probe), + DEVMETHOD(device_attach, qlnx_pci_attach), + DEVMETHOD(device_detach, qlnx_pci_detach), + { 0, 0 } +}; + +static driver_t qlnxv_pci_driver = { + "ql", qlnxv_pci_methods, sizeof (qlnx_host_t), +}; + +MODULE_VERSION(if_qlnxev,1); +DRIVER_MODULE(if_qlnxev, pci, qlnxv_pci_driver, 0, 0); + +MODULE_DEPEND(if_qlnxev, pci, 1, 1, 1); +MODULE_DEPEND(if_qlnxev, ether, 1, 1, 1); + +#endif /* #ifdef QLNX_VF */ + +MALLOC_DEFINE(M_QLNXBUF, "qlnxbuf", "Buffers for qlnx driver"); + +char qlnx_ver_str[VER_SIZE]; +char qlnx_name_str[NAME_SIZE]; + +/* + * Some PCI Configuration Space Related Defines + */ + +#ifndef PCI_VENDOR_QLOGIC +#define PCI_VENDOR_QLOGIC 0x1077 +#endif + +/* 40G Adapter QLE45xxx*/ +#ifndef QLOGIC_PCI_DEVICE_ID_1634 +#define QLOGIC_PCI_DEVICE_ID_1634 0x1634 +#endif + +/* 100G Adapter QLE45xxx*/ +#ifndef QLOGIC_PCI_DEVICE_ID_1644 +#define QLOGIC_PCI_DEVICE_ID_1644 0x1644 +#endif + +/* 25G Adapter QLE45xxx*/ +#ifndef QLOGIC_PCI_DEVICE_ID_1656 +#define QLOGIC_PCI_DEVICE_ID_1656 0x1656 +#endif + +/* 50G Adapter QLE45xxx*/ +#ifndef QLOGIC_PCI_DEVICE_ID_1654 +#define QLOGIC_PCI_DEVICE_ID_1654 0x1654 +#endif + +/* 10G/25G/40G Adapter QLE41xxx*/ +#ifndef QLOGIC_PCI_DEVICE_ID_8070 +#define QLOGIC_PCI_DEVICE_ID_8070 0x8070 +#endif + +/* SRIOV Device (All Speeds) Adapter QLE41xxx*/ +#ifndef QLOGIC_PCI_DEVICE_ID_8090 +#define QLOGIC_PCI_DEVICE_ID_8090 0x8090 +#endif + +SYSCTL_NODE(_hw, OID_AUTO, qlnxe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "qlnxe driver parameters"); + +/* Number of Queues: 0 (Auto) or 1 to 32 (fixed queue number) */ +static int qlnxe_queue_count = QLNX_DEFAULT_RSS; + +SYSCTL_INT(_hw_qlnxe, OID_AUTO, queue_count, CTLFLAG_RDTUN, + &qlnxe_queue_count, 0, "Multi-Queue queue count"); + +/* + * Note on RDMA personality setting + * + * Read the personality configured in NVRAM + * If the personality is ETH_ONLY, ETH_IWARP or ETH_ROCE and + * the configured personality in sysctl is QLNX_PERSONALITY_DEFAULT + * use the personality in NVRAM. + + * Otherwise use t the personality configured in sysctl. + * + */ +#define QLNX_PERSONALITY_DEFAULT 0x0 /* use personality in NVRAM */ +#define QLNX_PERSONALITY_ETH_ONLY 0x1 /* Override with ETH_ONLY */ +#define QLNX_PERSONALITY_ETH_IWARP 0x2 /* Override with ETH_IWARP */ +#define QLNX_PERSONALITY_ETH_ROCE 0x3 /* Override with ETH_ROCE */ +#define QLNX_PERSONALITY_BITS_PER_FUNC 4 +#define QLNX_PERSONALIY_MASK 0xF + +/* RDMA configuration; 64bit field allows setting for 16 physical functions*/ +static uint64_t qlnxe_rdma_configuration = 0x22222222; + +SYSCTL_U64(_hw_qlnxe, OID_AUTO, rdma_configuration, CTLFLAG_RDTUN, + &qlnxe_rdma_configuration, 0, "RDMA Configuration"); + +int +qlnx_vf_device(qlnx_host_t *ha) +{ + uint16_t device_id; + + device_id = ha->device_id; + + if (device_id == QLOGIC_PCI_DEVICE_ID_8090) + return 0; + + return -1; +} + +static int +qlnx_valid_device(qlnx_host_t *ha) +{ + uint16_t device_id; + + device_id = ha->device_id; + +#ifndef QLNX_VF + if ((device_id == QLOGIC_PCI_DEVICE_ID_1634) || + (device_id == QLOGIC_PCI_DEVICE_ID_1644) || + (device_id == QLOGIC_PCI_DEVICE_ID_1656) || + (device_id == QLOGIC_PCI_DEVICE_ID_1654) || + (device_id == QLOGIC_PCI_DEVICE_ID_8070)) + return 0; +#else + if (device_id == QLOGIC_PCI_DEVICE_ID_8090) + return 0; + +#endif /* #ifndef QLNX_VF */ + return -1; +} + +#ifdef QLNX_ENABLE_IWARP +static int +qlnx_rdma_supported(struct qlnx_host *ha) +{ + uint16_t device_id; + + device_id = pci_get_device(ha->pci_dev); + + if ((device_id == QLOGIC_PCI_DEVICE_ID_1634) || + (device_id == QLOGIC_PCI_DEVICE_ID_1656) || + (device_id == QLOGIC_PCI_DEVICE_ID_1654) || + (device_id == QLOGIC_PCI_DEVICE_ID_8070)) + return (0); + + return (-1); +} +#endif /* #ifdef QLNX_ENABLE_IWARP */ + +/* + * Name: qlnx_pci_probe + * Function: Validate the PCI device to be a QLA80XX device + */ +static int +qlnx_pci_probe(device_t dev) +{ + snprintf(qlnx_ver_str, sizeof(qlnx_ver_str), "v%d.%d.%d", + QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); + snprintf(qlnx_name_str, sizeof(qlnx_name_str), "qlnx"); + + if (pci_get_vendor(dev) != PCI_VENDOR_QLOGIC) { + return (ENXIO); + } + + switch (pci_get_device(dev)) { +#ifndef QLNX_VF + + case QLOGIC_PCI_DEVICE_ID_1644: + device_set_descf(dev, "%s v%d.%d.%d", + "Qlogic 100GbE PCI CNA Adapter-Ethernet Function", + QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, + QLNX_VERSION_BUILD); + break; + + case QLOGIC_PCI_DEVICE_ID_1634: + device_set_descf(dev, "%s v%d.%d.%d", + "Qlogic 40GbE PCI CNA Adapter-Ethernet Function", + QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, + QLNX_VERSION_BUILD); + break; + + case QLOGIC_PCI_DEVICE_ID_1656: + device_set_descf(dev, "%s v%d.%d.%d", + "Qlogic 25GbE PCI CNA Adapter-Ethernet Function", + QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, + QLNX_VERSION_BUILD); + break; + + case QLOGIC_PCI_DEVICE_ID_1654: + device_set_descf(dev, "%s v%d.%d.%d", + "Qlogic 50GbE PCI CNA Adapter-Ethernet Function", + QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, + QLNX_VERSION_BUILD); + break; + + case QLOGIC_PCI_DEVICE_ID_8070: + device_set_descf(dev, "%s v%d.%d.%d", + "Qlogic 10GbE/25GbE/40GbE PCI CNA (AH)" + " Adapter-Ethernet Function", + QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, + QLNX_VERSION_BUILD); + break; + +#else + case QLOGIC_PCI_DEVICE_ID_8090: + device_set_descf(dev, "%s v%d.%d.%d", + "Qlogic SRIOV PCI CNA (AH) " + "Adapter-Ethernet Function", + QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, + QLNX_VERSION_BUILD); + break; + +#endif /* #ifndef QLNX_VF */ + + default: + return (ENXIO); + } + +#ifdef QLNX_ENABLE_IWARP + qlnx_rdma_init(); +#endif /* #ifdef QLNX_ENABLE_IWARP */ + + return (BUS_PROBE_DEFAULT); +} + +static uint16_t +qlnx_num_tx_compl(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq) +{ + u16 hw_bd_cons; + u16 ecore_cons_idx; + + hw_bd_cons = le16toh(*txq->hw_cons_ptr); + + ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl); + + return (hw_bd_cons - ecore_cons_idx); +} + +static void +qlnx_sp_intr(void *arg) +{ + struct ecore_hwfn *p_hwfn; + qlnx_host_t *ha; + int i; + + p_hwfn = arg; + + if (p_hwfn == NULL) { + printf("%s: spurious slowpath intr\n", __func__); + return; + } + + ha = (qlnx_host_t *)p_hwfn->p_dev; + + QL_DPRINT2(ha, "enter\n"); + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + if (&ha->cdev.hwfns[i] == p_hwfn) { + taskqueue_enqueue(ha->sp_taskqueue[i], &ha->sp_task[i]); + break; + } + } + QL_DPRINT2(ha, "exit\n"); + + return; +} + +static void +qlnx_sp_taskqueue(void *context, int pending) +{ + struct ecore_hwfn *p_hwfn; + + p_hwfn = context; + + if (p_hwfn != NULL) { + qlnx_sp_isr(p_hwfn); + } + return; +} + +static int +qlnx_create_sp_taskqueues(qlnx_host_t *ha) +{ + int i; + uint8_t tq_name[32]; + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i]; + + bzero(tq_name, sizeof (tq_name)); + snprintf(tq_name, sizeof (tq_name), "ql_sp_tq_%d", i); + + TASK_INIT(&ha->sp_task[i], 0, qlnx_sp_taskqueue, p_hwfn); + + ha->sp_taskqueue[i] = taskqueue_create(tq_name, M_NOWAIT, + taskqueue_thread_enqueue, &ha->sp_taskqueue[i]); + + if (ha->sp_taskqueue[i] == NULL) + return (-1); + + taskqueue_start_threads(&ha->sp_taskqueue[i], 1, PI_NET, "%s", + tq_name); + + QL_DPRINT1(ha, "%p\n", ha->sp_taskqueue[i]); + } + + return (0); +} + +static void +qlnx_destroy_sp_taskqueues(qlnx_host_t *ha) +{ + int i; + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + if (ha->sp_taskqueue[i] != NULL) { + taskqueue_drain(ha->sp_taskqueue[i], &ha->sp_task[i]); + taskqueue_free(ha->sp_taskqueue[i]); + } + } + return; +} + +static void +qlnx_fp_taskqueue(void *context, int pending) +{ + struct qlnx_fastpath *fp; + qlnx_host_t *ha; + if_t ifp; + + fp = context; + + if (fp == NULL) + return; + + ha = (qlnx_host_t *)fp->edev; + + ifp = ha->ifp; + + if(if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + if (!drbr_empty(ifp, fp->tx_br)) { + if(mtx_trylock(&fp->tx_mtx)) { +#ifdef QLNX_TRACE_PERF_DATA + tx_pkts = fp->tx_pkts_transmitted; + tx_compl = fp->tx_pkts_completed; +#endif + + qlnx_transmit_locked(ifp, fp, NULL); + +#ifdef QLNX_TRACE_PERF_DATA + fp->tx_pkts_trans_fp += + (fp->tx_pkts_transmitted - tx_pkts); + fp->tx_pkts_compl_fp += + (fp->tx_pkts_completed - tx_compl); +#endif + mtx_unlock(&fp->tx_mtx); + } + } + } + + QL_DPRINT2(ha, "exit \n"); + return; +} + +static int +qlnx_create_fp_taskqueues(qlnx_host_t *ha) +{ + int i; + uint8_t tq_name[32]; + struct qlnx_fastpath *fp; + + for (i = 0; i < ha->num_rss; i++) { + fp = &ha->fp_array[i]; + + bzero(tq_name, sizeof (tq_name)); + snprintf(tq_name, sizeof (tq_name), "ql_fp_tq_%d", i); + + TASK_INIT(&fp->fp_task, 0, qlnx_fp_taskqueue, fp); + + fp->fp_taskqueue = taskqueue_create(tq_name, M_NOWAIT, + taskqueue_thread_enqueue, + &fp->fp_taskqueue); + + if (fp->fp_taskqueue == NULL) + return (-1); + + taskqueue_start_threads(&fp->fp_taskqueue, 1, PI_NET, "%s", + tq_name); + + QL_DPRINT1(ha, "%p\n",fp->fp_taskqueue); + } + + return (0); +} + +static void +qlnx_destroy_fp_taskqueues(qlnx_host_t *ha) +{ + int i; + struct qlnx_fastpath *fp; + + for (i = 0; i < ha->num_rss; i++) { + fp = &ha->fp_array[i]; + + if (fp->fp_taskqueue != NULL) { + taskqueue_drain(fp->fp_taskqueue, &fp->fp_task); + taskqueue_free(fp->fp_taskqueue); + fp->fp_taskqueue = NULL; + } + } + return; +} + +static void +qlnx_drain_fp_taskqueues(qlnx_host_t *ha) +{ + int i; + struct qlnx_fastpath *fp; + + for (i = 0; i < ha->num_rss; i++) { + fp = &ha->fp_array[i]; + + if (fp->fp_taskqueue != NULL) { + QLNX_UNLOCK(ha); + taskqueue_drain(fp->fp_taskqueue, &fp->fp_task); + QLNX_LOCK(ha); + } + } + return; +} + +static void +qlnx_get_params(qlnx_host_t *ha) +{ + if ((qlnxe_queue_count < 0) || (qlnxe_queue_count > QLNX_MAX_RSS)) { + device_printf(ha->pci_dev, "invalid queue_count value (%d)\n", + qlnxe_queue_count); + qlnxe_queue_count = 0; + } + return; +} + +static void +qlnx_error_recovery_taskqueue(void *context, int pending) +{ + qlnx_host_t *ha; + + ha = context; + + QL_DPRINT2(ha, "enter\n"); + + QLNX_LOCK(ha); + qlnx_stop(ha); + QLNX_UNLOCK(ha); + +#ifdef QLNX_ENABLE_IWARP + qlnx_rdma_dev_remove(ha); +#endif /* #ifdef QLNX_ENABLE_IWARP */ + + qlnx_slowpath_stop(ha); + qlnx_slowpath_start(ha); + +#ifdef QLNX_ENABLE_IWARP + qlnx_rdma_dev_add(ha); +#endif /* #ifdef QLNX_ENABLE_IWARP */ + + qlnx_init(ha); + + callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha); + + QL_DPRINT2(ha, "exit\n"); + + return; +} + +static int +qlnx_create_error_recovery_taskqueue(qlnx_host_t *ha) +{ + uint8_t tq_name[32]; + + bzero(tq_name, sizeof (tq_name)); + snprintf(tq_name, sizeof (tq_name), "ql_err_tq"); + + TASK_INIT(&ha->err_task, 0, qlnx_error_recovery_taskqueue, ha); + + ha->err_taskqueue = taskqueue_create(tq_name, M_NOWAIT, + taskqueue_thread_enqueue, &ha->err_taskqueue); + + if (ha->err_taskqueue == NULL) + return (-1); + + taskqueue_start_threads(&ha->err_taskqueue, 1, PI_NET, "%s", tq_name); + + QL_DPRINT1(ha, "%p\n",ha->err_taskqueue); + + return (0); +} + +static void +qlnx_destroy_error_recovery_taskqueue(qlnx_host_t *ha) +{ + if (ha->err_taskqueue != NULL) { + taskqueue_drain(ha->err_taskqueue, &ha->err_task); + taskqueue_free(ha->err_taskqueue); + } + + ha->err_taskqueue = NULL; + + return; +} + +/* + * Name: qlnx_pci_attach + * Function: attaches the device to the operating system + */ +static int +qlnx_pci_attach(device_t dev) +{ + qlnx_host_t *ha = NULL; + uint32_t rsrc_len_reg __unused = 0; + uint32_t rsrc_len_dbells = 0; + uint32_t rsrc_len_msix __unused = 0; + int i; + uint32_t mfw_ver; + uint32_t num_sp_msix = 0; + uint32_t num_rdma_irqs = 0; + + if ((ha = device_get_softc(dev)) == NULL) { + device_printf(dev, "cannot get softc\n"); + return (ENOMEM); + } + + memset(ha, 0, sizeof (qlnx_host_t)); + + ha->device_id = pci_get_device(dev); + + if (qlnx_valid_device(ha) != 0) { + device_printf(dev, "device is not valid device\n"); + return (ENXIO); + } + ha->pci_func = pci_get_function(dev); + + ha->pci_dev = dev; + + sx_init(&ha->hw_lock, "qlnx_hw_lock"); + + ha->flags.lock_init = 1; + + pci_enable_busmaster(dev); + + /* + * map the PCI BARs + */ + + ha->reg_rid = PCIR_BAR(0); + ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid, + RF_ACTIVE); + + if (ha->pci_reg == NULL) { + device_printf(dev, "unable to map BAR0\n"); + goto qlnx_pci_attach_err; + } + + rsrc_len_reg = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY, + ha->reg_rid); + + ha->dbells_rid = PCIR_BAR(2); + rsrc_len_dbells = (uint32_t) bus_get_resource_count(dev, + SYS_RES_MEMORY, + ha->dbells_rid); + if (rsrc_len_dbells) { + ha->pci_dbells = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &ha->dbells_rid, RF_ACTIVE); + + if (ha->pci_dbells == NULL) { + device_printf(dev, "unable to map BAR1\n"); + goto qlnx_pci_attach_err; + } + ha->dbells_phys_addr = (uint64_t) + bus_get_resource_start(dev, SYS_RES_MEMORY, ha->dbells_rid); + + ha->dbells_size = rsrc_len_dbells; + } else { + if (qlnx_vf_device(ha) != 0) { + device_printf(dev, " BAR1 size is zero\n"); + goto qlnx_pci_attach_err; + } + } + + ha->msix_rid = PCIR_BAR(4); + ha->msix_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &ha->msix_rid, RF_ACTIVE); + + if (ha->msix_bar == NULL) { + device_printf(dev, "unable to map BAR2\n"); + goto qlnx_pci_attach_err; + } + + rsrc_len_msix = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY, + ha->msix_rid); + + ha->dbg_level = 0x0000; + + QL_DPRINT1(ha, "\n\t\t\t" + "pci_dev = %p pci_reg = %p, reg_len = 0x%08x reg_rid = 0x%08x" + "\n\t\t\tdbells = %p, dbells_len = 0x%08x dbells_rid = 0x%08x" + "\n\t\t\tmsix = %p, msix_len = 0x%08x msix_rid = 0x%08x" + " msix_avail = 0x%x " + "\n\t\t\t[ncpus = %d]\n", + ha->pci_dev, ha->pci_reg, rsrc_len_reg, + ha->reg_rid, ha->pci_dbells, rsrc_len_dbells, ha->dbells_rid, + ha->msix_bar, rsrc_len_msix, ha->msix_rid, pci_msix_count(dev), + mp_ncpus); + /* + * allocate dma tags + */ + + if (qlnx_alloc_parent_dma_tag(ha)) + goto qlnx_pci_attach_err; + + if (qlnx_alloc_tx_dma_tag(ha)) + goto qlnx_pci_attach_err; + + if (qlnx_alloc_rx_dma_tag(ha)) + goto qlnx_pci_attach_err; + + + if (qlnx_init_hw(ha) != 0) + goto qlnx_pci_attach_err; + + ha->flags.hw_init = 1; + + qlnx_get_params(ha); + + if((pci_get_device(dev) == QLOGIC_PCI_DEVICE_ID_1644) && + (qlnxe_queue_count == QLNX_DEFAULT_RSS)) { + qlnxe_queue_count = QLNX_MAX_RSS; + } + + /* + * Allocate MSI-x vectors + */ + if (qlnx_vf_device(ha) != 0) { + if (qlnxe_queue_count == 0) + ha->num_rss = QLNX_DEFAULT_RSS; + else + ha->num_rss = qlnxe_queue_count; + + num_sp_msix = ha->cdev.num_hwfns; + } else { + uint8_t max_rxq; + uint8_t max_txq; + + ecore_vf_get_num_rxqs(&ha->cdev.hwfns[0], &max_rxq); + ecore_vf_get_num_rxqs(&ha->cdev.hwfns[0], &max_txq); + + if (max_rxq < max_txq) + ha->num_rss = max_rxq; + else + ha->num_rss = max_txq; + + if (ha->num_rss > QLNX_MAX_VF_RSS) + ha->num_rss = QLNX_MAX_VF_RSS; + + num_sp_msix = 0; + } + + if (ha->num_rss > mp_ncpus) + ha->num_rss = mp_ncpus; + + ha->num_tc = QLNX_MAX_TC; + + ha->msix_count = pci_msix_count(dev); + +#ifdef QLNX_ENABLE_IWARP + + num_rdma_irqs = qlnx_rdma_get_num_irqs(ha); + +#endif /* #ifdef QLNX_ENABLE_IWARP */ + + if (!ha->msix_count || + (ha->msix_count < (num_sp_msix + 1 + num_rdma_irqs))) { + device_printf(dev, "%s: msix_count[%d] not enough\n", __func__, + ha->msix_count); + goto qlnx_pci_attach_err; + } + + if (ha->msix_count > (ha->num_rss + num_sp_msix + num_rdma_irqs)) + ha->msix_count = ha->num_rss + num_sp_msix + num_rdma_irqs; + else + ha->num_rss = ha->msix_count - (num_sp_msix + num_rdma_irqs); + + QL_DPRINT1(ha, "\n\t\t\t" + "pci_reg = %p, reg_len = 0x%08x reg_rid = 0x%08x" + "\n\t\t\tdbells = %p, dbells_len = 0x%08x dbells_rid = 0x%08x" + "\n\t\t\tmsix = %p, msix_len = 0x%08x msix_rid = 0x%08x" + " msix_avail = 0x%x msix_alloc = 0x%x" + "\n\t\t\t[ncpus = %d][num_rss = 0x%x] [num_tc = 0x%x]\n", + ha->pci_reg, rsrc_len_reg, + ha->reg_rid, ha->pci_dbells, rsrc_len_dbells, ha->dbells_rid, + ha->msix_bar, rsrc_len_msix, ha->msix_rid, pci_msix_count(dev), + ha->msix_count, mp_ncpus, ha->num_rss, ha->num_tc); + + if (pci_alloc_msix(dev, &ha->msix_count)) { + device_printf(dev, "%s: pci_alloc_msix[%d] failed\n", __func__, + ha->msix_count); + ha->msix_count = 0; + goto qlnx_pci_attach_err; + } + + /* + * Initialize slow path interrupt and task queue + */ + + if (num_sp_msix) { + if (qlnx_create_sp_taskqueues(ha) != 0) + goto qlnx_pci_attach_err; + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i]; + + ha->sp_irq_rid[i] = i + 1; + ha->sp_irq[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &ha->sp_irq_rid[i], + (RF_ACTIVE | RF_SHAREABLE)); + if (ha->sp_irq[i] == NULL) { + device_printf(dev, + "could not allocate mbx interrupt\n"); + goto qlnx_pci_attach_err; + } + + if (bus_setup_intr(dev, ha->sp_irq[i], + (INTR_TYPE_NET | INTR_MPSAFE), NULL, + qlnx_sp_intr, p_hwfn, &ha->sp_handle[i])) { + device_printf(dev, + "could not setup slow path interrupt\n"); + goto qlnx_pci_attach_err; + } + + QL_DPRINT1(ha, "p_hwfn [%p] sp_irq_rid %d" + " sp_irq %p sp_handle %p\n", p_hwfn, + ha->sp_irq_rid[i], ha->sp_irq[i], ha->sp_handle[i]); + } + } + + /* + * initialize fast path interrupt + */ + if (qlnx_create_fp_taskqueues(ha) != 0) + goto qlnx_pci_attach_err; + + for (i = 0; i < ha->num_rss; i++) { + ha->irq_vec[i].rss_idx = i; + ha->irq_vec[i].ha = ha; + ha->irq_vec[i].irq_rid = (1 + num_sp_msix) + i; + + ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &ha->irq_vec[i].irq_rid, + (RF_ACTIVE | RF_SHAREABLE)); + + if (ha->irq_vec[i].irq == NULL) { + device_printf(dev, + "could not allocate interrupt[%d] irq_rid = %d\n", + i, ha->irq_vec[i].irq_rid); + goto qlnx_pci_attach_err; + } + + if (qlnx_alloc_tx_br(ha, &ha->fp_array[i])) { + device_printf(dev, "could not allocate tx_br[%d]\n", i); + goto qlnx_pci_attach_err; + } + } + + if (qlnx_vf_device(ha) != 0) { + callout_init(&ha->qlnx_callout, 1); + ha->flags.callout_init = 1; + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + if (qlnx_grc_dumpsize(ha, &ha->grcdump_size[i], i) != 0) + goto qlnx_pci_attach_err; + if (ha->grcdump_size[i] == 0) + goto qlnx_pci_attach_err; + + ha->grcdump_size[i] = ha->grcdump_size[i] << 2; + QL_DPRINT1(ha, "grcdump_size[%d] = 0x%08x\n", + i, ha->grcdump_size[i]); + + ha->grcdump[i] = qlnx_zalloc(ha->grcdump_size[i]); + if (ha->grcdump[i] == NULL) { + device_printf(dev, "grcdump alloc[%d] failed\n", i); + goto qlnx_pci_attach_err; + } + + if (qlnx_idle_chk_size(ha, &ha->idle_chk_size[i], i) != 0) + goto qlnx_pci_attach_err; + if (ha->idle_chk_size[i] == 0) + goto qlnx_pci_attach_err; + + ha->idle_chk_size[i] = ha->idle_chk_size[i] << 2; + QL_DPRINT1(ha, "idle_chk_size[%d] = 0x%08x\n", + i, ha->idle_chk_size[i]); + + ha->idle_chk[i] = qlnx_zalloc(ha->idle_chk_size[i]); + + if (ha->idle_chk[i] == NULL) { + device_printf(dev, "idle_chk alloc failed\n"); + goto qlnx_pci_attach_err; + } + } + + if (qlnx_create_error_recovery_taskqueue(ha) != 0) + goto qlnx_pci_attach_err; + } + + if (qlnx_slowpath_start(ha) != 0) + goto qlnx_pci_attach_err; + else + ha->flags.slowpath_start = 1; + + if (qlnx_vf_device(ha) != 0) { + if (qlnx_get_flash_size(ha, &ha->flash_size) != 0) { + qlnx_mdelay(__func__, 1000); + qlnx_trigger_dump(ha); + + goto qlnx_pci_attach_err0; + } + + if (qlnx_get_mfw_version(ha, &mfw_ver) != 0) { + qlnx_mdelay(__func__, 1000); + qlnx_trigger_dump(ha); + + goto qlnx_pci_attach_err0; + } + } else { + struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[0]; + ecore_mcp_get_mfw_ver(p_hwfn, NULL, &mfw_ver, NULL); + } + + snprintf(ha->mfw_ver, sizeof(ha->mfw_ver), "%d.%d.%d.%d", + ((mfw_ver >> 24) & 0xFF), ((mfw_ver >> 16) & 0xFF), + ((mfw_ver >> 8) & 0xFF), (mfw_ver & 0xFF)); + snprintf(ha->stormfw_ver, sizeof(ha->stormfw_ver), "%d.%d.%d.%d", + FW_MAJOR_VERSION, FW_MINOR_VERSION, FW_REVISION_VERSION, + FW_ENGINEERING_VERSION); + + QL_DPRINT1(ha, "STORM_FW version %s MFW version %s\n", + ha->stormfw_ver, ha->mfw_ver); + + qlnx_init_ifnet(dev, ha); + + /* + * add sysctls + */ + qlnx_add_sysctls(ha); + +qlnx_pci_attach_err0: + /* + * create ioctl device interface + */ + if (qlnx_vf_device(ha) != 0) { + if (qlnx_make_cdev(ha)) { + device_printf(dev, "%s: ql_make_cdev failed\n", __func__); + goto qlnx_pci_attach_err; + } + +#ifdef QLNX_ENABLE_IWARP + qlnx_rdma_dev_add(ha); +#endif /* #ifdef QLNX_ENABLE_IWARP */ + } + +#ifndef QLNX_VF +#ifdef CONFIG_ECORE_SRIOV + + if (qlnx_vf_device(ha) != 0) + qlnx_initialize_sriov(ha); + +#endif /* #ifdef CONFIG_ECORE_SRIOV */ +#endif /* #ifdef QLNX_VF */ + + QL_DPRINT2(ha, "success\n"); + + return (0); + +qlnx_pci_attach_err: + + qlnx_release(ha); + + return (ENXIO); +} + +/* + * Name: qlnx_pci_detach + * Function: Unhooks the device from the operating system + */ +static int +qlnx_pci_detach(device_t dev) +{ + qlnx_host_t *ha = NULL; + + if ((ha = device_get_softc(dev)) == NULL) { + device_printf(dev, "%s: cannot get softc\n", __func__); + return (ENOMEM); + } + + if (qlnx_vf_device(ha) != 0) { +#ifdef CONFIG_ECORE_SRIOV + int ret; + + ret = pci_iov_detach(dev); + if (ret) { + device_printf(dev, "%s: SRIOV in use\n", __func__); + return (ret); + } + +#endif /* #ifdef CONFIG_ECORE_SRIOV */ + +#ifdef QLNX_ENABLE_IWARP + if (qlnx_rdma_dev_remove(ha) != 0) + return (EBUSY); +#endif /* #ifdef QLNX_ENABLE_IWARP */ + } + + QLNX_LOCK(ha); + qlnx_stop(ha); + QLNX_UNLOCK(ha); + + qlnx_release(ha); + + return (0); +} + +#ifdef QLNX_ENABLE_IWARP + +static uint8_t +qlnx_get_personality(uint8_t pci_func) +{ + uint8_t personality; + + personality = (qlnxe_rdma_configuration >> + (pci_func * QLNX_PERSONALITY_BITS_PER_FUNC)) & + QLNX_PERSONALIY_MASK; + return (personality); +} + +static void +qlnx_set_personality(qlnx_host_t *ha) +{ + uint8_t personality; + + personality = qlnx_get_personality(ha->pci_func); + + switch (personality) { + case QLNX_PERSONALITY_DEFAULT: + device_printf(ha->pci_dev, "%s: DEFAULT\n", + __func__); + ha->personality = ECORE_PCI_DEFAULT; + break; + + case QLNX_PERSONALITY_ETH_ONLY: + device_printf(ha->pci_dev, "%s: ETH_ONLY\n", + __func__); + ha->personality = ECORE_PCI_ETH; + break; + + case QLNX_PERSONALITY_ETH_IWARP: + device_printf(ha->pci_dev, "%s: ETH_IWARP\n", + __func__); + ha->personality = ECORE_PCI_ETH_IWARP; + break; + + case QLNX_PERSONALITY_ETH_ROCE: + device_printf(ha->pci_dev, "%s: ETH_ROCE\n", + __func__); + ha->personality = ECORE_PCI_ETH_ROCE; + break; + } + + return; +} + +#endif /* #ifdef QLNX_ENABLE_IWARP */ + +static int +qlnx_init_hw(qlnx_host_t *ha) +{ + int rval = 0; + struct ecore_hw_prepare_params params; + + ha->cdev.ha = ha; + ecore_init_struct(&ha->cdev); + + /* ha->dp_module = ECORE_MSG_PROBE | + ECORE_MSG_INTR | + ECORE_MSG_SP | + ECORE_MSG_LINK | + ECORE_MSG_SPQ | + ECORE_MSG_RDMA; + ha->dp_level = ECORE_LEVEL_VERBOSE;*/ + //ha->dp_module = ECORE_MSG_RDMA | ECORE_MSG_INTR | ECORE_MSG_LL2; + ha->dp_level = ECORE_LEVEL_NOTICE; + //ha->dp_level = ECORE_LEVEL_VERBOSE; + + ecore_init_dp(&ha->cdev, ha->dp_module, ha->dp_level, ha->pci_dev); + + ha->cdev.regview = ha->pci_reg; + + ha->personality = ECORE_PCI_DEFAULT; + + if (qlnx_vf_device(ha) == 0) { + ha->cdev.b_is_vf = true; + + if (ha->pci_dbells != NULL) { + ha->cdev.doorbells = ha->pci_dbells; + ha->cdev.db_phys_addr = ha->dbells_phys_addr; + ha->cdev.db_size = ha->dbells_size; + } else { + ha->pci_dbells = ha->pci_reg; + } + } else { + ha->cdev.doorbells = ha->pci_dbells; + ha->cdev.db_phys_addr = ha->dbells_phys_addr; + ha->cdev.db_size = ha->dbells_size; + +#ifdef QLNX_ENABLE_IWARP + + if (qlnx_rdma_supported(ha) == 0) + qlnx_set_personality(ha); + +#endif /* #ifdef QLNX_ENABLE_IWARP */ + } + QL_DPRINT2(ha, "%s: %s\n", __func__, + (ha->personality == ECORE_PCI_ETH_IWARP ? "iwarp": "ethernet")); + + bzero(¶ms, sizeof (struct ecore_hw_prepare_params)); + + params.personality = ha->personality; + + params.drv_resc_alloc = false; + params.chk_reg_fifo = false; + params.initiate_pf_flr = true; + params.epoch = 0; + + ecore_hw_prepare(&ha->cdev, ¶ms); + + qlnx_set_id(&ha->cdev, qlnx_name_str, qlnx_ver_str); + + QL_DPRINT1(ha, "ha = %p cdev = %p p_hwfn = %p\n", + ha, &ha->cdev, &ha->cdev.hwfns[0]); + + return (rval); +} + +static void +qlnx_release(qlnx_host_t *ha) +{ + device_t dev; + int i; + + dev = ha->pci_dev; + + QL_DPRINT2(ha, "enter\n"); + + for (i = 0; i < QLNX_MAX_HW_FUNCS; i++) { + if (ha->idle_chk[i] != NULL) { + free(ha->idle_chk[i], M_QLNXBUF); + ha->idle_chk[i] = NULL; + } + + if (ha->grcdump[i] != NULL) { + free(ha->grcdump[i], M_QLNXBUF); + ha->grcdump[i] = NULL; + } + } + + if (ha->flags.callout_init) + callout_drain(&ha->qlnx_callout); + + if (ha->flags.slowpath_start) { + qlnx_slowpath_stop(ha); + } + + if (ha->flags.hw_init) + ecore_hw_remove(&ha->cdev); + + qlnx_del_cdev(ha); + + if (ha->ifp != NULL) + ether_ifdetach(ha->ifp); + + qlnx_free_tx_dma_tag(ha); + + qlnx_free_rx_dma_tag(ha); + + qlnx_free_parent_dma_tag(ha); + + if (qlnx_vf_device(ha) != 0) { + qlnx_destroy_error_recovery_taskqueue(ha); + } + + for (i = 0; i < ha->num_rss; i++) { + struct qlnx_fastpath *fp = &ha->fp_array[i]; + + if (ha->irq_vec[i].handle) { + (void)bus_teardown_intr(dev, ha->irq_vec[i].irq, + ha->irq_vec[i].handle); + } + + if (ha->irq_vec[i].irq) { + (void)bus_release_resource(dev, SYS_RES_IRQ, + ha->irq_vec[i].irq_rid, + ha->irq_vec[i].irq); + } + + qlnx_free_tx_br(ha, fp); + } + qlnx_destroy_fp_taskqueues(ha); + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + if (ha->sp_handle[i]) + (void)bus_teardown_intr(dev, ha->sp_irq[i], + ha->sp_handle[i]); + + if (ha->sp_irq[i]) + (void) bus_release_resource(dev, SYS_RES_IRQ, + ha->sp_irq_rid[i], ha->sp_irq[i]); + } + + qlnx_destroy_sp_taskqueues(ha); + + if (ha->msix_count) + pci_release_msi(dev); + + if (ha->flags.lock_init) { + sx_destroy(&ha->hw_lock); + } + + if (ha->pci_reg) + (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid, + ha->pci_reg); + + if (ha->dbells_size && ha->pci_dbells) + (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->dbells_rid, + ha->pci_dbells); + + if (ha->msix_bar) + (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->msix_rid, + ha->msix_bar); + + QL_DPRINT2(ha, "exit\n"); + return; +} + +static void +qlnx_trigger_dump(qlnx_host_t *ha) +{ + int i; + + if (ha->ifp != NULL) + if_setdrvflagbits(ha->ifp, 0, (IFF_DRV_OACTIVE | IFF_DRV_RUNNING)); + + QL_DPRINT2(ha, "enter\n"); + + if (qlnx_vf_device(ha) == 0) + return; + + ha->error_recovery = 1; + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + qlnx_grc_dump(ha, &ha->grcdump_dwords[i], i); + qlnx_idle_chk(ha, &ha->idle_chk_dwords[i], i); + } + + QL_DPRINT2(ha, "exit\n"); + + return; +} + +static int +qlnx_trigger_dump_sysctl(SYSCTL_HANDLER_ARGS) +{ + int err, ret = 0; + qlnx_host_t *ha; + + err = sysctl_handle_int(oidp, &ret, 0, req); + + if (err || !req->newptr) + return (err); + + if (ret == 1) { + ha = (qlnx_host_t *)arg1; + qlnx_trigger_dump(ha); + } + return (err); +} + +static int +qlnx_set_tx_coalesce(SYSCTL_HANDLER_ARGS) +{ + int err, i, ret = 0, usecs = 0; + qlnx_host_t *ha; + struct ecore_hwfn *p_hwfn; + struct qlnx_fastpath *fp; + + err = sysctl_handle_int(oidp, &usecs, 0, req); + + if (err || !req->newptr || !usecs || (usecs > 255)) + return (err); + + ha = (qlnx_host_t *)arg1; + + if (qlnx_vf_device(ha) == 0) + return (-1); + + for (i = 0; i < ha->num_rss; i++) { + p_hwfn = &ha->cdev.hwfns[(i % ha->cdev.num_hwfns)]; + + fp = &ha->fp_array[i]; + + if (fp->txq[0]->handle != NULL) { + ret = ecore_set_queue_coalesce(p_hwfn, 0, + (uint16_t)usecs, fp->txq[0]->handle); + } + } + + if (!ret) + ha->tx_coalesce_usecs = (uint8_t)usecs; + + return (err); +} + +static int +qlnx_set_rx_coalesce(SYSCTL_HANDLER_ARGS) +{ + int err, i, ret = 0, usecs = 0; + qlnx_host_t *ha; + struct ecore_hwfn *p_hwfn; + struct qlnx_fastpath *fp; + + err = sysctl_handle_int(oidp, &usecs, 0, req); + + if (err || !req->newptr || !usecs || (usecs > 255)) + return (err); + + ha = (qlnx_host_t *)arg1; + + if (qlnx_vf_device(ha) == 0) + return (-1); + + for (i = 0; i < ha->num_rss; i++) { + p_hwfn = &ha->cdev.hwfns[(i % ha->cdev.num_hwfns)]; + + fp = &ha->fp_array[i]; + + if (fp->rxq->handle != NULL) { + ret = ecore_set_queue_coalesce(p_hwfn, (uint16_t)usecs, + 0, fp->rxq->handle); + } + } + + if (!ret) + ha->rx_coalesce_usecs = (uint8_t)usecs; + + return (err); +} + +static void +qlnx_add_sp_stats_sysctls(qlnx_host_t *ha) +{ + struct sysctl_ctx_list *ctx; + struct sysctl_oid_list *children; + struct sysctl_oid *ctx_oid; + + ctx = device_get_sysctl_ctx(ha->pci_dev); + children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev)); + + ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "spstat", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "spstat"); + children = SYSCTL_CHILDREN(ctx_oid); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "sp_interrupts", + CTLFLAG_RD, &ha->sp_interrupts, + "No. of slowpath interrupts"); + + return; +} + +static void +qlnx_add_fp_stats_sysctls(qlnx_host_t *ha) +{ + struct sysctl_ctx_list *ctx; + struct sysctl_oid_list *children; + struct sysctl_oid_list *node_children; + struct sysctl_oid *ctx_oid; + int i, j; + uint8_t name_str[16]; + + ctx = device_get_sysctl_ctx(ha->pci_dev); + children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev)); + + ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fpstat", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "fpstat"); + children = SYSCTL_CHILDREN(ctx_oid); + + for (i = 0; i < ha->num_rss; i++) { + bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); + snprintf(name_str, sizeof(name_str), "%d", i); + + ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name_str, + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, name_str); + node_children = SYSCTL_CHILDREN(ctx_oid); + + /* Tx Related */ + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_processed", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_processed, + "No. of packets processed for transmission"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_freed", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_freed, + "No. of freed packets"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_transmitted", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_transmitted, + "No. of transmitted packets"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_completed", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_completed, + "No. of transmit completions"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_non_tso_pkts", + CTLFLAG_RD, &ha->fp_array[i].tx_non_tso_pkts, + "No. of non LSO transmited packets"); + +#ifdef QLNX_TRACE_PERF_DATA + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_trans_ctx", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_trans_ctx, + "No. of transmitted packets in transmit context"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_compl_ctx", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_ctx, + "No. of transmit completions in transmit context"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_trans_fp", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_trans_fp, + "No. of transmitted packets in taskqueue"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_compl_fp", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_fp, + "No. of transmit completions in taskqueue"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_pkts_compl_intr", + CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_intr, + "No. of transmit completions in interrupt ctx"); +#endif + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_tso_pkts", + CTLFLAG_RD, &ha->fp_array[i].tx_tso_pkts, + "No. of LSO transmited packets"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_lso_wnd_min_len", + CTLFLAG_RD, &ha->fp_array[i].tx_lso_wnd_min_len, + "tx_lso_wnd_min_len"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_defrag", + CTLFLAG_RD, &ha->fp_array[i].tx_defrag, + "tx_defrag"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tx_nsegs_gt_elem_left", + CTLFLAG_RD, &ha->fp_array[i].tx_nsegs_gt_elem_left, + "tx_nsegs_gt_elem_left"); + + SYSCTL_ADD_UINT(ctx, node_children, + OID_AUTO, "tx_tso_max_nsegs", + CTLFLAG_RD, &ha->fp_array[i].tx_tso_max_nsegs, + ha->fp_array[i].tx_tso_max_nsegs, "tx_tso_max_nsegs"); + + SYSCTL_ADD_UINT(ctx, node_children, + OID_AUTO, "tx_tso_min_nsegs", + CTLFLAG_RD, &ha->fp_array[i].tx_tso_min_nsegs, + ha->fp_array[i].tx_tso_min_nsegs, "tx_tso_min_nsegs"); + + SYSCTL_ADD_UINT(ctx, node_children, + OID_AUTO, "tx_tso_max_pkt_len", + CTLFLAG_RD, &ha->fp_array[i].tx_tso_max_pkt_len, + ha->fp_array[i].tx_tso_max_pkt_len, + "tx_tso_max_pkt_len"); + + SYSCTL_ADD_UINT(ctx, node_children, + OID_AUTO, "tx_tso_min_pkt_len", + CTLFLAG_RD, &ha->fp_array[i].tx_tso_min_pkt_len, + ha->fp_array[i].tx_tso_min_pkt_len, + "tx_tso_min_pkt_len"); + + for (j = 0; j < QLNX_FP_MAX_SEGS; j++) { + bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); + snprintf(name_str, sizeof(name_str), + "tx_pkts_nseg_%02d", (j+1)); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, name_str, CTLFLAG_RD, + &ha->fp_array[i].tx_pkts[j], name_str); + } + +#ifdef QLNX_TRACE_PERF_DATA + for (j = 0; j < 18; j++) { + bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); + snprintf(name_str, sizeof(name_str), + "tx_pkts_hist_%02d", (j+1)); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, name_str, CTLFLAG_RD, + &ha->fp_array[i].tx_pkts_hist[j], name_str); + } + for (j = 0; j < 5; j++) { + bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); + snprintf(name_str, sizeof(name_str), + "tx_comInt_%02d", (j+1)); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, name_str, CTLFLAG_RD, + &ha->fp_array[i].tx_comInt[j], name_str); + } + for (j = 0; j < 18; j++) { + bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); + snprintf(name_str, sizeof(name_str), + "tx_pkts_q_%02d", (j+1)); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, name_str, CTLFLAG_RD, + &ha->fp_array[i].tx_pkts_q[j], name_str); + } +#endif + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_nsegs_gt_elem_left", + CTLFLAG_RD, &ha->fp_array[i].err_tx_nsegs_gt_elem_left, + "err_tx_nsegs_gt_elem_left"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_dmamap_create", + CTLFLAG_RD, &ha->fp_array[i].err_tx_dmamap_create, + "err_tx_dmamap_create"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_defrag_dmamap_load", + CTLFLAG_RD, &ha->fp_array[i].err_tx_defrag_dmamap_load, + "err_tx_defrag_dmamap_load"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_non_tso_max_seg", + CTLFLAG_RD, &ha->fp_array[i].err_tx_non_tso_max_seg, + "err_tx_non_tso_max_seg"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_dmamap_load", + CTLFLAG_RD, &ha->fp_array[i].err_tx_dmamap_load, + "err_tx_dmamap_load"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_defrag", + CTLFLAG_RD, &ha->fp_array[i].err_tx_defrag, + "err_tx_defrag"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_free_pkt_null", + CTLFLAG_RD, &ha->fp_array[i].err_tx_free_pkt_null, + "err_tx_free_pkt_null"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_tx_cons_idx_conflict", + CTLFLAG_RD, &ha->fp_array[i].err_tx_cons_idx_conflict, + "err_tx_cons_idx_conflict"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "lro_cnt_64", + CTLFLAG_RD, &ha->fp_array[i].lro_cnt_64, + "lro_cnt_64"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "lro_cnt_128", + CTLFLAG_RD, &ha->fp_array[i].lro_cnt_128, + "lro_cnt_128"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "lro_cnt_256", + CTLFLAG_RD, &ha->fp_array[i].lro_cnt_256, + "lro_cnt_256"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "lro_cnt_512", + CTLFLAG_RD, &ha->fp_array[i].lro_cnt_512, + "lro_cnt_512"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "lro_cnt_1024", + CTLFLAG_RD, &ha->fp_array[i].lro_cnt_1024, + "lro_cnt_1024"); + + /* Rx Related */ + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "rx_pkts", + CTLFLAG_RD, &ha->fp_array[i].rx_pkts, + "No. of received packets"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tpa_start", + CTLFLAG_RD, &ha->fp_array[i].tpa_start, + "No. of tpa_start packets"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tpa_cont", + CTLFLAG_RD, &ha->fp_array[i].tpa_cont, + "No. of tpa_cont packets"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "tpa_end", + CTLFLAG_RD, &ha->fp_array[i].tpa_end, + "No. of tpa_end packets"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_m_getcl", + CTLFLAG_RD, &ha->fp_array[i].err_m_getcl, + "err_m_getcl"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_m_getjcl", + CTLFLAG_RD, &ha->fp_array[i].err_m_getjcl, + "err_m_getjcl"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_rx_hw_errors", + CTLFLAG_RD, &ha->fp_array[i].err_rx_hw_errors, + "err_rx_hw_errors"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "err_rx_alloc_errors", + CTLFLAG_RD, &ha->fp_array[i].err_rx_alloc_errors, + "err_rx_alloc_errors"); + } + + return; +} + +static void +qlnx_add_hw_stats_sysctls(qlnx_host_t *ha) +{ + struct sysctl_ctx_list *ctx; + struct sysctl_oid_list *children; + struct sysctl_oid *ctx_oid; + + ctx = device_get_sysctl_ctx(ha->pci_dev); + children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev)); + + ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "hwstat", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "hwstat"); + children = SYSCTL_CHILDREN(ctx_oid); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "no_buff_discards", + CTLFLAG_RD, &ha->hw_stats.common.no_buff_discards, + "No. of packets discarded due to lack of buffer"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "packet_too_big_discard", + CTLFLAG_RD, &ha->hw_stats.common.packet_too_big_discard, + "No. of packets discarded because packet was too big"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "ttl0_discard", + CTLFLAG_RD, &ha->hw_stats.common.ttl0_discard, + "ttl0_discard"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_ucast_bytes", + CTLFLAG_RD, &ha->hw_stats.common.rx_ucast_bytes, + "rx_ucast_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mcast_bytes", + CTLFLAG_RD, &ha->hw_stats.common.rx_mcast_bytes, + "rx_mcast_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_bcast_bytes", + CTLFLAG_RD, &ha->hw_stats.common.rx_bcast_bytes, + "rx_bcast_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_ucast_pkts", + CTLFLAG_RD, &ha->hw_stats.common.rx_ucast_pkts, + "rx_ucast_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mcast_pkts", + CTLFLAG_RD, &ha->hw_stats.common.rx_mcast_pkts, + "rx_mcast_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_bcast_pkts", + CTLFLAG_RD, &ha->hw_stats.common.rx_bcast_pkts, + "rx_bcast_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "mftag_filter_discards", + CTLFLAG_RD, &ha->hw_stats.common.mftag_filter_discards, + "mftag_filter_discards"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "mac_filter_discards", + CTLFLAG_RD, &ha->hw_stats.common.mac_filter_discards, + "mac_filter_discards"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_ucast_bytes", + CTLFLAG_RD, &ha->hw_stats.common.tx_ucast_bytes, + "tx_ucast_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_mcast_bytes", + CTLFLAG_RD, &ha->hw_stats.common.tx_mcast_bytes, + "tx_mcast_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_bcast_bytes", + CTLFLAG_RD, &ha->hw_stats.common.tx_bcast_bytes, + "tx_bcast_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_ucast_pkts", + CTLFLAG_RD, &ha->hw_stats.common.tx_ucast_pkts, + "tx_ucast_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_mcast_pkts", + CTLFLAG_RD, &ha->hw_stats.common.tx_mcast_pkts, + "tx_mcast_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_bcast_pkts", + CTLFLAG_RD, &ha->hw_stats.common.tx_bcast_pkts, + "tx_bcast_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_err_drop_pkts", + CTLFLAG_RD, &ha->hw_stats.common.tx_err_drop_pkts, + "tx_err_drop_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tpa_coalesced_pkts", + CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_pkts, + "tpa_coalesced_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tpa_coalesced_events", + CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_events, + "tpa_coalesced_events"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tpa_aborts_num", + CTLFLAG_RD, &ha->hw_stats.common.tpa_aborts_num, + "tpa_aborts_num"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tpa_not_coalesced_pkts", + CTLFLAG_RD, &ha->hw_stats.common.tpa_not_coalesced_pkts, + "tpa_not_coalesced_pkts"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tpa_coalesced_bytes", + CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_bytes, + "tpa_coalesced_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_64_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_64_byte_packets, + "rx_64_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_65_to_127_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_65_to_127_byte_packets, + "rx_65_to_127_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_128_to_255_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_128_to_255_byte_packets, + "rx_128_to_255_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_256_to_511_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_256_to_511_byte_packets, + "rx_256_to_511_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_512_to_1023_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_512_to_1023_byte_packets, + "rx_512_to_1023_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_1024_to_1518_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_1024_to_1518_byte_packets, + "rx_1024_to_1518_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_1519_to_1522_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.rx_1519_to_1522_byte_packets, + "rx_1519_to_1522_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_1523_to_2047_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.rx_1519_to_2047_byte_packets, + "rx_1523_to_2047_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_2048_to_4095_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.rx_2048_to_4095_byte_packets, + "rx_2048_to_4095_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_4096_to_9216_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.rx_4096_to_9216_byte_packets, + "rx_4096_to_9216_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_9217_to_16383_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.rx_9217_to_16383_byte_packets, + "rx_9217_to_16383_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_crc_errors", + CTLFLAG_RD, &ha->hw_stats.common.rx_crc_errors, + "rx_crc_errors"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mac_crtl_frames", + CTLFLAG_RD, &ha->hw_stats.common.rx_mac_crtl_frames, + "rx_mac_crtl_frames"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_pause_frames", + CTLFLAG_RD, &ha->hw_stats.common.rx_pause_frames, + "rx_pause_frames"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_pfc_frames", + CTLFLAG_RD, &ha->hw_stats.common.rx_pfc_frames, + "rx_pfc_frames"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_align_errors", + CTLFLAG_RD, &ha->hw_stats.common.rx_align_errors, + "rx_align_errors"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_carrier_errors", + CTLFLAG_RD, &ha->hw_stats.common.rx_carrier_errors, + "rx_carrier_errors"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_oversize_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_oversize_packets, + "rx_oversize_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_jabbers", + CTLFLAG_RD, &ha->hw_stats.common.rx_jabbers, + "rx_jabbers"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_undersize_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_undersize_packets, + "rx_undersize_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_fragments", + CTLFLAG_RD, &ha->hw_stats.common.rx_fragments, + "rx_fragments"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_64_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_64_byte_packets, + "tx_64_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_65_to_127_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_65_to_127_byte_packets, + "tx_65_to_127_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_128_to_255_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_128_to_255_byte_packets, + "tx_128_to_255_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_256_to_511_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_256_to_511_byte_packets, + "tx_256_to_511_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_512_to_1023_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_512_to_1023_byte_packets, + "tx_512_to_1023_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_1024_to_1518_byte_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_1024_to_1518_byte_packets, + "tx_1024_to_1518_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_1519_to_2047_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.tx_1519_to_2047_byte_packets, + "tx_1519_to_2047_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_2048_to_4095_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.tx_2048_to_4095_byte_packets, + "tx_2048_to_4095_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_4096_to_9216_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.tx_4096_to_9216_byte_packets, + "tx_4096_to_9216_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_9217_to_16383_byte_packets", + CTLFLAG_RD, &ha->hw_stats.bb.tx_9217_to_16383_byte_packets, + "tx_9217_to_16383_byte_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_pause_frames", + CTLFLAG_RD, &ha->hw_stats.common.tx_pause_frames, + "tx_pause_frames"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_pfc_frames", + CTLFLAG_RD, &ha->hw_stats.common.tx_pfc_frames, + "tx_pfc_frames"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_lpi_entry_count", + CTLFLAG_RD, &ha->hw_stats.bb.tx_lpi_entry_count, + "tx_lpi_entry_count"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_total_collisions", + CTLFLAG_RD, &ha->hw_stats.bb.tx_total_collisions, + "tx_total_collisions"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "brb_truncates", + CTLFLAG_RD, &ha->hw_stats.common.brb_truncates, + "brb_truncates"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "brb_discards", + CTLFLAG_RD, &ha->hw_stats.common.brb_discards, + "brb_discards"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mac_bytes", + CTLFLAG_RD, &ha->hw_stats.common.rx_mac_bytes, + "rx_mac_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mac_uc_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_mac_uc_packets, + "rx_mac_uc_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mac_mc_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_mac_mc_packets, + "rx_mac_mc_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mac_bc_packets", + CTLFLAG_RD, &ha->hw_stats.common.rx_mac_bc_packets, + "rx_mac_bc_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "rx_mac_frames_ok", + CTLFLAG_RD, &ha->hw_stats.common.rx_mac_frames_ok, + "rx_mac_frames_ok"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_mac_bytes", + CTLFLAG_RD, &ha->hw_stats.common.tx_mac_bytes, + "tx_mac_bytes"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_mac_uc_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_mac_uc_packets, + "tx_mac_uc_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_mac_mc_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_mac_mc_packets, + "tx_mac_mc_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_mac_bc_packets", + CTLFLAG_RD, &ha->hw_stats.common.tx_mac_bc_packets, + "tx_mac_bc_packets"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "tx_mac_ctrl_frames", + CTLFLAG_RD, &ha->hw_stats.common.tx_mac_ctrl_frames, + "tx_mac_ctrl_frames"); + return; +} + +static void +qlnx_add_sysctls(qlnx_host_t *ha) +{ + device_t dev = ha->pci_dev; + struct sysctl_ctx_list *ctx; + struct sysctl_oid_list *children; + + ctx = device_get_sysctl_ctx(dev); + children = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); + + qlnx_add_fp_stats_sysctls(ha); + qlnx_add_sp_stats_sysctls(ha); + + if (qlnx_vf_device(ha) != 0) + qlnx_add_hw_stats_sysctls(ha); + + SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "Driver_Version", + CTLFLAG_RD, qlnx_ver_str, 0, + "Driver Version"); + + SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "STORMFW_Version", + CTLFLAG_RD, ha->stormfw_ver, 0, + "STORM Firmware Version"); + + SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "MFW_Version", + CTLFLAG_RD, ha->mfw_ver, 0, + "Management Firmware Version"); + + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "personality", CTLFLAG_RD, + &ha->personality, ha->personality, + "\tpersonality = 0 => Ethernet Only\n" + "\tpersonality = 3 => Ethernet and RoCE\n" + "\tpersonality = 4 => Ethernet and iWARP\n" + "\tpersonality = 6 => Default in Shared Memory\n"); + + ha->dbg_level = 0; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "debug", CTLFLAG_RW, + &ha->dbg_level, ha->dbg_level, "Debug Level"); + + ha->dp_level = 0x01; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "dp_level", CTLFLAG_RW, + &ha->dp_level, ha->dp_level, "DP Level"); + + ha->dbg_trace_lro_cnt = 0; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "dbg_trace_lro_cnt", CTLFLAG_RW, + &ha->dbg_trace_lro_cnt, ha->dbg_trace_lro_cnt, + "Trace LRO Counts"); + + ha->dbg_trace_tso_pkt_len = 0; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "dbg_trace_tso_pkt_len", CTLFLAG_RW, + &ha->dbg_trace_tso_pkt_len, ha->dbg_trace_tso_pkt_len, + "Trace TSO packet lengths"); + + ha->dp_module = 0; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "dp_module", CTLFLAG_RW, + &ha->dp_module, ha->dp_module, "DP Module"); + + ha->err_inject = 0; + + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "err_inject", CTLFLAG_RW, + &ha->err_inject, ha->err_inject, "Error Inject"); + + ha->storm_stats_enable = 0; + + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "storm_stats_enable", CTLFLAG_RW, + &ha->storm_stats_enable, ha->storm_stats_enable, + "Enable Storm Statistics Gathering"); + + ha->storm_stats_index = 0; + + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "storm_stats_index", CTLFLAG_RD, + &ha->storm_stats_index, ha->storm_stats_index, + "Enable Storm Statistics Gathering Current Index"); + + ha->grcdump_taken = 0; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "grcdump_taken", CTLFLAG_RD, + &ha->grcdump_taken, ha->grcdump_taken, + "grcdump_taken"); + + ha->idle_chk_taken = 0; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "idle_chk_taken", CTLFLAG_RD, + &ha->idle_chk_taken, ha->idle_chk_taken, + "idle_chk_taken"); + + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "rx_coalesce_usecs", CTLFLAG_RD, + &ha->rx_coalesce_usecs, ha->rx_coalesce_usecs, + "rx_coalesce_usecs"); + + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "tx_coalesce_usecs", CTLFLAG_RD, + &ha->tx_coalesce_usecs, ha->tx_coalesce_usecs, + "tx_coalesce_usecs"); + + SYSCTL_ADD_PROC(ctx, children, + OID_AUTO, "trigger_dump", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, + (void *)ha, 0, qlnx_trigger_dump_sysctl, "I", "trigger_dump"); + + SYSCTL_ADD_PROC(ctx, children, + OID_AUTO, "set_rx_coalesce_usecs", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, + (void *)ha, 0, qlnx_set_rx_coalesce, "I", + "rx interrupt coalesce period microseconds"); + + SYSCTL_ADD_PROC(ctx, children, + OID_AUTO, "set_tx_coalesce_usecs", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, + (void *)ha, 0, qlnx_set_tx_coalesce, "I", + "tx interrupt coalesce period microseconds"); + + ha->rx_pkt_threshold = 128; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "rx_pkt_threshold", CTLFLAG_RW, + &ha->rx_pkt_threshold, ha->rx_pkt_threshold, + "No. of Rx Pkts to process at a time"); + + ha->rx_jumbo_buf_eq_mtu = 0; + SYSCTL_ADD_UINT(ctx, children, + OID_AUTO, "rx_jumbo_buf_eq_mtu", CTLFLAG_RW, + &ha->rx_jumbo_buf_eq_mtu, ha->rx_jumbo_buf_eq_mtu, + "== 0 => Rx Jumbo buffers are capped to 4Kbytes\n" + "otherwise Rx Jumbo buffers are set to >= MTU size\n"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "err_illegal_intr", CTLFLAG_RD, + &ha->err_illegal_intr, "err_illegal_intr"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "err_fp_null", CTLFLAG_RD, + &ha->err_fp_null, "err_fp_null"); + + SYSCTL_ADD_QUAD(ctx, children, + OID_AUTO, "err_get_proto_invalid_type", CTLFLAG_RD, + &ha->err_get_proto_invalid_type, "err_get_proto_invalid_type"); + return; +} + +/***************************************************************************** + * Operating System Network Interface Functions + *****************************************************************************/ + +static void +qlnx_init_ifnet(device_t dev, qlnx_host_t *ha) +{ + uint16_t device_id; + if_t ifp; + + ifp = ha->ifp = if_alloc(IFT_ETHER); + if_initname(ifp, device_get_name(dev), device_get_unit(dev)); + + device_id = pci_get_device(ha->pci_dev); + + if (device_id == QLOGIC_PCI_DEVICE_ID_1634) + if_setbaudrate(ifp, IF_Gbps(40)); + else if ((device_id == QLOGIC_PCI_DEVICE_ID_1656) || + (device_id == QLOGIC_PCI_DEVICE_ID_8070)) + if_setbaudrate(ifp, IF_Gbps(25)); + else if (device_id == QLOGIC_PCI_DEVICE_ID_1654) + if_setbaudrate(ifp, IF_Gbps(50)); + else if (device_id == QLOGIC_PCI_DEVICE_ID_1644) + if_setbaudrate(ifp, IF_Gbps(100)); + + if_setinitfn(ifp, qlnx_init); + if_setsoftc(ifp, ha); + if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); + if_setioctlfn(ifp, qlnx_ioctl); + if_settransmitfn(ifp, qlnx_transmit); + if_setqflushfn(ifp, qlnx_qflush); + + if_setsendqlen(ifp, qlnx_get_ifq_snd_maxlen(ha)); + if_setsendqready(ifp); + + if_setgetcounterfn(ifp, qlnx_get_counter); + + ha->max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; + + memcpy(ha->primary_mac, qlnx_get_mac_addr(ha), ETH_ALEN); + + if (!ha->primary_mac[0] && !ha->primary_mac[1] && + !ha->primary_mac[2] && !ha->primary_mac[3] && + !ha->primary_mac[4] && !ha->primary_mac[5]) { + uint32_t rnd; + + rnd = arc4random(); + + ha->primary_mac[0] = 0x00; + ha->primary_mac[1] = 0x0e; + ha->primary_mac[2] = 0x1e; + ha->primary_mac[3] = rnd & 0xFF; + ha->primary_mac[4] = (rnd >> 8) & 0xFF; + ha->primary_mac[5] = (rnd >> 16) & 0xFF; + } + + if_setcapabilities(ifp, IFCAP_HWCSUM); + if_setcapabilitiesbit(ifp, IFCAP_JUMBO_MTU, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWCSUM, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTSO, 0); + if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); + if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0); + if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); + if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE, 0); + if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); + + if_sethwtsomax(ifp, QLNX_MAX_TSO_FRAME_SIZE - + (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); + if_sethwtsomaxsegcount(ifp, QLNX_MAX_SEGMENTS - 1); /* hdr */ + if_sethwtsomaxsegsize(ifp, QLNX_MAX_TX_MBUF_SIZE); + + if_setcapenable(ifp, if_getcapabilities(ifp)); + + if_sethwassist(ifp, CSUM_IP); + if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); + if_sethwassistbits(ifp, CSUM_TCP_IPV6 | CSUM_UDP_IPV6, 0); + if_sethwassistbits(ifp, CSUM_TSO, 0); + + if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); + + ifmedia_init(&ha->media, IFM_IMASK, qlnx_media_change,\ + qlnx_media_status); + + if (device_id == QLOGIC_PCI_DEVICE_ID_1634) { + ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_LR4), 0, NULL); + ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_SR4), 0, NULL); + ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_CR4), 0, NULL); + } else if ((device_id == QLOGIC_PCI_DEVICE_ID_1656) || + (device_id == QLOGIC_PCI_DEVICE_ID_8070)) { + ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_25G_SR), 0, NULL); + ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_25G_CR), 0, NULL); + } else if (device_id == QLOGIC_PCI_DEVICE_ID_1654) { + ifmedia_add(&ha->media, (IFM_ETHER | IFM_50G_KR2), 0, NULL); + ifmedia_add(&ha->media, (IFM_ETHER | IFM_50G_CR2), 0, NULL); + } else if (device_id == QLOGIC_PCI_DEVICE_ID_1644) { + ifmedia_add(&ha->media, + (IFM_ETHER | QLNX_IFM_100G_LR4), 0, NULL); + ifmedia_add(&ha->media, + (IFM_ETHER | QLNX_IFM_100G_SR4), 0, NULL); + ifmedia_add(&ha->media, + (IFM_ETHER | QLNX_IFM_100G_CR4), 0, NULL); + } + + ifmedia_add(&ha->media, (IFM_ETHER | IFM_FDX), 0, NULL); + ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL); + + ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO)); + + ether_ifattach(ifp, ha->primary_mac); + bcopy(if_getlladdr(ha->ifp), ha->primary_mac, ETHER_ADDR_LEN); + + QL_DPRINT2(ha, "exit\n"); + + return; +} + +static void +qlnx_init_locked(qlnx_host_t *ha) +{ + if_t ifp = ha->ifp; + + QL_DPRINT1(ha, "Driver Initialization start \n"); + + qlnx_stop(ha); + + if (qlnx_load(ha) == 0) { + if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); + if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); + +#ifdef QLNX_ENABLE_IWARP + if (qlnx_vf_device(ha) != 0) { + qlnx_rdma_dev_open(ha); + } +#endif /* #ifdef QLNX_ENABLE_IWARP */ + } + + return; +} + +static void +qlnx_init(void *arg) +{ + qlnx_host_t *ha; + + ha = (qlnx_host_t *)arg; + + QL_DPRINT2(ha, "enter\n"); + + QLNX_LOCK(ha); + qlnx_init_locked(ha); + QLNX_UNLOCK(ha); + + QL_DPRINT2(ha, "exit\n"); + + return; +} + +static int +qlnx_config_mcast_mac_addr(qlnx_host_t *ha, uint8_t *mac_addr, uint32_t add_mac) +{ + struct ecore_filter_mcast *mcast; + struct ecore_dev *cdev; + int rc; + + cdev = &ha->cdev; + + mcast = &ha->ecore_mcast; + bzero(mcast, sizeof(struct ecore_filter_mcast)); + + if (add_mac) + mcast->opcode = ECORE_FILTER_ADD; + else + mcast->opcode = ECORE_FILTER_REMOVE; + + mcast->num_mc_addrs = 1; + memcpy(mcast->mac, mac_addr, ETH_ALEN); + + rc = ecore_filter_mcast_cmd(cdev, mcast, ECORE_SPQ_MODE_CB, NULL); + + return (rc); +} + +static int +qlnx_hw_add_mcast(qlnx_host_t *ha, uint8_t *mta) +{ + int i; + + for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { + if (QL_MAC_CMP(ha->mcast[i].addr, mta) == 0) + return 0; /* its been already added */ + } + + for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { + if ((ha->mcast[i].addr[0] == 0) && + (ha->mcast[i].addr[1] == 0) && + (ha->mcast[i].addr[2] == 0) && + (ha->mcast[i].addr[3] == 0) && + (ha->mcast[i].addr[4] == 0) && + (ha->mcast[i].addr[5] == 0)) { + if (qlnx_config_mcast_mac_addr(ha, mta, 1)) + return (-1); + + bcopy(mta, ha->mcast[i].addr, ETH_ALEN); + ha->nmcast++; + + return 0; + } + } + return 0; +} + +static int +qlnx_hw_del_mcast(qlnx_host_t *ha, uint8_t *mta) +{ + int i; + + for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { + if (QL_MAC_CMP(ha->mcast[i].addr, mta) == 0) { + if (qlnx_config_mcast_mac_addr(ha, mta, 0)) + return (-1); + + ha->mcast[i].addr[0] = 0; + ha->mcast[i].addr[1] = 0; + ha->mcast[i].addr[2] = 0; + ha->mcast[i].addr[3] = 0; + ha->mcast[i].addr[4] = 0; + ha->mcast[i].addr[5] = 0; + + ha->nmcast--; + + return 0; + } + } + return 0; +} + +/* + * Name: qls_hw_set_multi + * Function: Sets the Multicast Addresses provided the host O.S into the + * hardware (for the given interface) + */ +static void +qlnx_hw_set_multi(qlnx_host_t *ha, uint8_t *mta, uint32_t mcnt, + uint32_t add_mac) +{ + int i; + + for (i = 0; i < mcnt; i++) { + if (add_mac) { + if (qlnx_hw_add_mcast(ha, mta)) + break; + } else { + if (qlnx_hw_del_mcast(ha, mta)) + break; + } + + mta += ETHER_HDR_LEN; + } + return; +} + +static u_int +qlnx_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int mcnt) +{ + uint8_t *mta = arg; + + if (mcnt == QLNX_MAX_NUM_MULTICAST_ADDRS) + return (0); + + bcopy(LLADDR(sdl), &mta[mcnt * ETHER_HDR_LEN], ETHER_HDR_LEN); + + return (1); +} + +static int +qlnx_set_multi(qlnx_host_t *ha, uint32_t add_multi) +{ + uint8_t mta[QLNX_MAX_NUM_MULTICAST_ADDRS * ETHER_HDR_LEN]; + if_t ifp = ha->ifp; + u_int mcnt; + + if (qlnx_vf_device(ha) == 0) + return (0); + + mcnt = if_foreach_llmaddr(ifp, qlnx_copy_maddr, mta); + + QLNX_LOCK(ha); + qlnx_hw_set_multi(ha, mta, mcnt, add_multi); + QLNX_UNLOCK(ha); + + return (0); +} + +static int +qlnx_set_promisc(qlnx_host_t *ha, int enabled) +{ + int rc = 0; + uint8_t filter; + + if (qlnx_vf_device(ha) == 0) + return (0); + + filter = ha->filter; + if (enabled) { + filter |= ECORE_ACCEPT_MCAST_UNMATCHED; + filter |= ECORE_ACCEPT_UCAST_UNMATCHED; + } else { + filter &= ~ECORE_ACCEPT_MCAST_UNMATCHED; + filter &= ~ECORE_ACCEPT_UCAST_UNMATCHED; + } + + rc = qlnx_set_rx_accept_filter(ha, filter); + return (rc); +} + +static int +qlnx_set_allmulti(qlnx_host_t *ha, int enabled) +{ + int rc = 0; + uint8_t filter; + + if (qlnx_vf_device(ha) == 0) + return (0); + + filter = ha->filter; + if (enabled) { + filter |= ECORE_ACCEPT_MCAST_UNMATCHED; + } else { + filter &= ~ECORE_ACCEPT_MCAST_UNMATCHED; + } + rc = qlnx_set_rx_accept_filter(ha, filter); + + return (rc); +} + +static int +qlnx_ioctl(if_t ifp, u_long cmd, caddr_t data) +{ + int ret = 0, mask; + int flags; + struct ifreq *ifr = (struct ifreq *)data; +#ifdef INET + struct ifaddr *ifa = (struct ifaddr *)data; +#endif + qlnx_host_t *ha; + + ha = (qlnx_host_t *)if_getsoftc(ifp); + + switch (cmd) { + case SIOCSIFADDR: + QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx)\n", cmd); + +#ifdef INET + if (ifa->ifa_addr->sa_family == AF_INET) { + if_setflagbits(ifp, IFF_UP, 0); + if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { + QLNX_LOCK(ha); + qlnx_init_locked(ha); + QLNX_UNLOCK(ha); + } + QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n", + cmd, ntohl(IA_SIN(ifa)->sin_addr.s_addr)); + + arp_ifinit(ifp, ifa); + break; + } +#endif + ether_ioctl(ifp, cmd, data); + break; + + case SIOCSIFMTU: + QL_DPRINT4(ha, "SIOCSIFMTU (0x%lx)\n", cmd); + + if (ifr->ifr_mtu > QLNX_MAX_MTU) { + ret = EINVAL; + } else { + QLNX_LOCK(ha); + if_setmtu(ifp, ifr->ifr_mtu); + ha->max_frame_size = + if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + qlnx_init_locked(ha); + } + + QLNX_UNLOCK(ha); + } + + break; + + case SIOCSIFFLAGS: + QL_DPRINT4(ha, "SIOCSIFFLAGS (0x%lx)\n", cmd); + + QLNX_LOCK(ha); + flags = if_getflags(ifp); + + if (flags & IFF_UP) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + if ((flags ^ ha->if_flags) & + IFF_PROMISC) { + ret = qlnx_set_promisc(ha, flags & IFF_PROMISC); + } else if ((if_getflags(ifp) ^ ha->if_flags) & + IFF_ALLMULTI) { + ret = qlnx_set_allmulti(ha, flags & IFF_ALLMULTI); + } + } else { + ha->max_frame_size = if_getmtu(ifp) + + ETHER_HDR_LEN + ETHER_CRC_LEN; + qlnx_init_locked(ha); + } + } else { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) + qlnx_stop(ha); + } + + ha->if_flags = if_getflags(ifp); + QLNX_UNLOCK(ha); + break; + + case SIOCADDMULTI: + QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCADDMULTI", cmd); + + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + if (qlnx_set_multi(ha, 1)) + ret = EINVAL; + } + break; + + case SIOCDELMULTI: + QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCDELMULTI", cmd); + + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + if (qlnx_set_multi(ha, 0)) + ret = EINVAL; + } + break; + + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + QL_DPRINT4(ha, "SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n", cmd); + + ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd); + break; + + case SIOCSIFCAP: + + mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); + + QL_DPRINT4(ha, "SIOCSIFCAP (0x%lx)\n", cmd); + + if (mask & IFCAP_HWCSUM) + if_togglecapenable(ifp, IFCAP_HWCSUM); + if (mask & IFCAP_TSO4) + if_togglecapenable(ifp, IFCAP_TSO4); + if (mask & IFCAP_TSO6) + if_togglecapenable(ifp, IFCAP_TSO6); + if (mask & IFCAP_VLAN_HWTAGGING) + if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); + if (mask & IFCAP_VLAN_HWTSO) + if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); + if (mask & IFCAP_LRO) + if_togglecapenable(ifp, IFCAP_LRO); + + QLNX_LOCK(ha); + + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) + qlnx_init_locked(ha); + + QLNX_UNLOCK(ha); + + VLAN_CAPABILITIES(ifp); + break; + + case SIOCGI2C: + { + struct ifi2creq i2c; + struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[0]; + struct ecore_ptt *p_ptt; + + ret = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); + + if (ret) + break; + + if ((i2c.len > sizeof (i2c.data)) || + (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2)) { + ret = EINVAL; + break; + } + + p_ptt = ecore_ptt_acquire(p_hwfn); + + if (!p_ptt) { + QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); + ret = ERESTART; + break; + } + + ret = ecore_mcp_phy_sfp_read(p_hwfn, p_ptt, + (ha->pci_func & 0x1), i2c.dev_addr, i2c.offset, + i2c.len, &i2c.data[0]); + + ecore_ptt_release(p_hwfn, p_ptt); + + if (ret) { + ret = ENODEV; + break; + } + + ret = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); + + QL_DPRINT8(ha, "SIOCGI2C copyout ret = %d \ + len = %d addr = 0x%02x offset = 0x%04x \ + data[0..7]=0x%02x 0x%02x 0x%02x 0x%02x 0x%02x \ + 0x%02x 0x%02x 0x%02x\n", + ret, i2c.len, i2c.dev_addr, i2c.offset, + i2c.data[0], i2c.data[1], i2c.data[2], i2c.data[3], + i2c.data[4], i2c.data[5], i2c.data[6], i2c.data[7]); + break; + } + + default: + QL_DPRINT4(ha, "default (0x%lx)\n", cmd); + ret = ether_ioctl(ifp, cmd, data); + break; + } + + return (ret); +} + +static int +qlnx_media_change(if_t ifp) +{ + qlnx_host_t *ha; + struct ifmedia *ifm; + int ret = 0; + + ha = (qlnx_host_t *)if_getsoftc(ifp); + + QL_DPRINT2(ha, "enter\n"); + + ifm = &ha->media; + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + ret = EINVAL; + + QL_DPRINT2(ha, "exit\n"); + + return (ret); +} + +static void +qlnx_media_status(if_t ifp, struct ifmediareq *ifmr) +{ + qlnx_host_t *ha; + + ha = (qlnx_host_t *)if_getsoftc(ifp); + + QL_DPRINT2(ha, "enter\n"); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (ha->link_up) { + ifmr->ifm_status |= IFM_ACTIVE; + ifmr->ifm_active |= + (IFM_FDX | qlnx_get_optics(ha, &ha->if_link)); + + if (ha->if_link.link_partner_caps & + (QLNX_LINK_CAP_Pause | QLNX_LINK_CAP_Asym_Pause)) + ifmr->ifm_active |= + (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE); + } + + QL_DPRINT2(ha, "exit (%s)\n", (ha->link_up ? "link_up" : "link_down")); + + return; +} + +static void +qlnx_free_tx_pkt(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq) +{ + u16 idx; + struct mbuf *mp; + bus_dmamap_t map; + int i; +// struct eth_tx_bd *tx_data_bd; + struct eth_tx_1st_bd *first_bd; + int nbds = 0; + + idx = txq->sw_tx_cons; + mp = txq->sw_tx_ring[idx].mp; + map = txq->sw_tx_ring[idx].map; + + if ((mp == NULL) || QL_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_MBUF_NULL)){ + QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_MBUF_NULL); + + QL_DPRINT1(ha, "(mp == NULL) " + " tx_idx = 0x%x" + " ecore_prod_idx = 0x%x" + " ecore_cons_idx = 0x%x" + " hw_bd_cons = 0x%x" + " txq_db_last = 0x%x" + " elem_left = 0x%x\n", + fp->rss_id, + ecore_chain_get_prod_idx(&txq->tx_pbl), + ecore_chain_get_cons_idx(&txq->tx_pbl), + le16toh(*txq->hw_cons_ptr), + txq->tx_db.raw, + ecore_chain_get_elem_left(&txq->tx_pbl)); + + fp->err_tx_free_pkt_null++; + + //DEBUG + qlnx_trigger_dump(ha); + + return; + } else { + QLNX_INC_OPACKETS((ha->ifp)); + QLNX_INC_OBYTES((ha->ifp), (mp->m_pkthdr.len)); + + bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(ha->tx_tag, map); + + fp->tx_pkts_freed++; + fp->tx_pkts_completed++; + + m_freem(mp); + } + + first_bd = (struct eth_tx_1st_bd *)ecore_chain_consume(&txq->tx_pbl); + nbds = first_bd->data.nbds; + +// BD_SET_UNMAP_ADDR_LEN(first_bd, 0, 0); + + for (i = 1; i < nbds; i++) { + /* tx_data_bd = */ ecore_chain_consume(&txq->tx_pbl); +// BD_SET_UNMAP_ADDR_LEN(tx_data_bd, 0, 0); + } + txq->sw_tx_ring[idx].flags = 0; + txq->sw_tx_ring[idx].mp = NULL; + txq->sw_tx_ring[idx].map = (bus_dmamap_t)0; + + return; +} + +static void +qlnx_tx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq) +{ + u16 hw_bd_cons; + u16 ecore_cons_idx; + uint16_t diff; + uint16_t idx, idx2; + + hw_bd_cons = le16toh(*txq->hw_cons_ptr); + + while (hw_bd_cons != + (ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl))) { + diff = hw_bd_cons - ecore_cons_idx; + if ((diff > TX_RING_SIZE) || + QL_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_DIFF)){ + QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_DIFF); + + QL_DPRINT1(ha, "(diff = 0x%x) " + " tx_idx = 0x%x" + " ecore_prod_idx = 0x%x" + " ecore_cons_idx = 0x%x" + " hw_bd_cons = 0x%x" + " txq_db_last = 0x%x" + " elem_left = 0x%x\n", + diff, + fp->rss_id, + ecore_chain_get_prod_idx(&txq->tx_pbl), + ecore_chain_get_cons_idx(&txq->tx_pbl), + le16toh(*txq->hw_cons_ptr), + txq->tx_db.raw, + ecore_chain_get_elem_left(&txq->tx_pbl)); + + fp->err_tx_cons_idx_conflict++; + + //DEBUG + qlnx_trigger_dump(ha); + } + + idx = (txq->sw_tx_cons + 1) & (TX_RING_SIZE - 1); + idx2 = (txq->sw_tx_cons + 2) & (TX_RING_SIZE - 1); + prefetch(txq->sw_tx_ring[idx].mp); + prefetch(txq->sw_tx_ring[idx2].mp); + + qlnx_free_tx_pkt(ha, fp, txq); + + txq->sw_tx_cons = (txq->sw_tx_cons + 1) & (TX_RING_SIZE - 1); + } + return; +} + +static int +qlnx_transmit_locked(if_t ifp, struct qlnx_fastpath *fp, struct mbuf *mp) +{ + int ret = 0; + struct qlnx_tx_queue *txq; + qlnx_host_t * ha; + uint16_t elem_left; + + txq = fp->txq[0]; + ha = (qlnx_host_t *)fp->edev; + + if ((!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) || (!ha->link_up)) { + if(mp != NULL) + ret = drbr_enqueue(ifp, fp->tx_br, mp); + return (ret); + } + + if(mp != NULL) + ret = drbr_enqueue(ifp, fp->tx_br, mp); + + mp = drbr_peek(ifp, fp->tx_br); + + while (mp != NULL) { + if (qlnx_send(ha, fp, &mp)) { + if (mp != NULL) { + drbr_putback(ifp, fp->tx_br, mp); + } else { + fp->tx_pkts_processed++; + drbr_advance(ifp, fp->tx_br); + } + goto qlnx_transmit_locked_exit; + + } else { + drbr_advance(ifp, fp->tx_br); + fp->tx_pkts_transmitted++; + fp->tx_pkts_processed++; + } + + mp = drbr_peek(ifp, fp->tx_br); + } + +qlnx_transmit_locked_exit: + if((qlnx_num_tx_compl(ha,fp, fp->txq[0]) > QLNX_TX_COMPL_THRESH) || + ((int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl)) + < QLNX_TX_ELEM_MAX_THRESH)) + (void)qlnx_tx_int(ha, fp, fp->txq[0]); + + QL_DPRINT2(ha, "%s: exit ret = %d\n", __func__, ret); + return ret; +} + +static int +qlnx_transmit(if_t ifp, struct mbuf *mp) +{ + qlnx_host_t *ha = (qlnx_host_t *)if_getsoftc(ifp); + struct qlnx_fastpath *fp; + int rss_id = 0, ret = 0; + +#ifdef QLNX_TRACEPERF_DATA + uint64_t tx_pkts = 0, tx_compl = 0; +#endif + + QL_DPRINT2(ha, "enter\n"); + + if (M_HASHTYPE_GET(mp) != M_HASHTYPE_NONE) + rss_id = (mp->m_pkthdr.flowid % ECORE_RSS_IND_TABLE_SIZE) % + ha->num_rss; + + fp = &ha->fp_array[rss_id]; + + if (fp->tx_br == NULL) { + ret = EINVAL; + goto qlnx_transmit_exit; + } + + if (mtx_trylock(&fp->tx_mtx)) { +#ifdef QLNX_TRACEPERF_DATA + tx_pkts = fp->tx_pkts_transmitted; + tx_compl = fp->tx_pkts_completed; +#endif + + ret = qlnx_transmit_locked(ifp, fp, mp); + +#ifdef QLNX_TRACEPERF_DATA + fp->tx_pkts_trans_ctx += (fp->tx_pkts_transmitted - tx_pkts); + fp->tx_pkts_compl_ctx += (fp->tx_pkts_completed - tx_compl); +#endif + mtx_unlock(&fp->tx_mtx); + } else { + if (mp != NULL && (fp->fp_taskqueue != NULL)) { + ret = drbr_enqueue(ifp, fp->tx_br, mp); + taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task); + } + } + +qlnx_transmit_exit: + + QL_DPRINT2(ha, "exit ret = %d\n", ret); + return ret; +} + +static void +qlnx_qflush(if_t ifp) +{ + int rss_id; + struct qlnx_fastpath *fp; + struct mbuf *mp; + qlnx_host_t *ha; + + ha = (qlnx_host_t *)if_getsoftc(ifp); + + QL_DPRINT2(ha, "enter\n"); + + for (rss_id = 0; rss_id < ha->num_rss; rss_id++) { + fp = &ha->fp_array[rss_id]; + + if (fp == NULL) + continue; + + if (fp->tx_br) { + mtx_lock(&fp->tx_mtx); + + while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) { + fp->tx_pkts_freed++; + m_freem(mp); + } + mtx_unlock(&fp->tx_mtx); + } + } + QL_DPRINT2(ha, "exit\n"); + + return; +} + +static void +qlnx_txq_doorbell_wr32(qlnx_host_t *ha, void *reg_addr, uint32_t value) +{ + uint32_t offset; + + offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)ha->pci_dbells); + + bus_write_4(ha->pci_dbells, offset, value); + bus_barrier(ha->pci_reg, 0, 0, BUS_SPACE_BARRIER_READ); + bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); + + return; +} + +static uint32_t +qlnx_tcp_offset(qlnx_host_t *ha, struct mbuf *mp) +{ + struct ether_vlan_header *eh = NULL; + struct ip *ip = NULL; + struct ip6_hdr *ip6 = NULL; + struct tcphdr *th = NULL; + uint32_t ehdrlen = 0, ip_hlen = 0, offset = 0; + uint16_t etype = 0; + uint8_t buf[sizeof(struct ip6_hdr)]; + + eh = mtod(mp, struct ether_vlan_header *); + + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + etype = ntohs(eh->evl_proto); + } else { + ehdrlen = ETHER_HDR_LEN; + etype = ntohs(eh->evl_encap_proto); + } + + switch (etype) { + case ETHERTYPE_IP: + ip = (struct ip *)(mp->m_data + ehdrlen); + + ip_hlen = sizeof (struct ip); + + if (mp->m_len < (ehdrlen + ip_hlen)) { + m_copydata(mp, ehdrlen, sizeof(struct ip), buf); + ip = (struct ip *)buf; + } + + th = (struct tcphdr *)(ip + 1); + offset = ip_hlen + ehdrlen + (th->th_off << 2); + break; + + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); + + ip_hlen = sizeof(struct ip6_hdr); + + if (mp->m_len < (ehdrlen + ip_hlen)) { + m_copydata(mp, ehdrlen, sizeof (struct ip6_hdr), + buf); + ip6 = (struct ip6_hdr *)buf; + } + th = (struct tcphdr *)(ip6 + 1); + offset = ip_hlen + ehdrlen + (th->th_off << 2); + break; + + default: + break; + } + + return (offset); +} + +static __inline int +qlnx_tso_check(struct qlnx_fastpath *fp, bus_dma_segment_t *segs, int nsegs, + uint32_t offset) +{ + int i; + uint32_t sum, nbds_in_hdr = 1; + uint32_t window; + bus_dma_segment_t *s_seg; + + /* If the header spans multiple segments, skip those segments */ + + if (nsegs < ETH_TX_LSO_WINDOW_BDS_NUM) + return (0); + + i = 0; + + while ((i < nsegs) && (offset >= segs->ds_len)) { + offset = offset - segs->ds_len; + segs++; + i++; + nbds_in_hdr++; + } + + window = ETH_TX_LSO_WINDOW_BDS_NUM - nbds_in_hdr; + + nsegs = nsegs - i; + + while (nsegs >= window) { + sum = 0; + s_seg = segs; + + for (i = 0; i < window; i++){ + sum += s_seg->ds_len; + s_seg++; + } + + if (sum < ETH_TX_LSO_WINDOW_MIN_LEN) { + fp->tx_lso_wnd_min_len++; + return (-1); + } + + nsegs = nsegs - 1; + segs++; + } + + return (0); +} + +static int +qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp) +{ + bus_dma_segment_t *segs; + bus_dmamap_t map = 0; + uint32_t nsegs = 0; + int ret = -1; + struct mbuf *m_head = *m_headp; + uint16_t idx = 0; + uint16_t elem_left; + + uint8_t nbd = 0; + struct qlnx_tx_queue *txq; + + struct eth_tx_1st_bd *first_bd; + struct eth_tx_2nd_bd *second_bd; + struct eth_tx_3rd_bd *third_bd; + struct eth_tx_bd *tx_data_bd; + + int seg_idx = 0; + uint32_t nbds_in_hdr = 0; + uint32_t offset = 0; + +#ifdef QLNX_TRACE_PERF_DATA + uint16_t bd_used; +#endif + + QL_DPRINT8(ha, "enter[%d]\n", fp->rss_id); + + if (!ha->link_up) + return (-1); + + first_bd = NULL; + second_bd = NULL; + third_bd = NULL; + tx_data_bd = NULL; + + txq = fp->txq[0]; + + if ((int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl)) < + QLNX_TX_ELEM_MIN_THRESH) { + fp->tx_nsegs_gt_elem_left++; + fp->err_tx_nsegs_gt_elem_left++; + + return (ENOBUFS); + } + + idx = txq->sw_tx_prod; + + map = txq->sw_tx_ring[idx].map; + segs = txq->segs; + + ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs, + BUS_DMA_NOWAIT); + + if (ha->dbg_trace_tso_pkt_len) { + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { + if (!fp->tx_tso_min_pkt_len) { + fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len; + fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len; + } else { + if (fp->tx_tso_min_pkt_len > m_head->m_pkthdr.len) + fp->tx_tso_min_pkt_len = + m_head->m_pkthdr.len; + if (fp->tx_tso_max_pkt_len < m_head->m_pkthdr.len) + fp->tx_tso_max_pkt_len = + m_head->m_pkthdr.len; + } + } + } + + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) + offset = qlnx_tcp_offset(ha, m_head); + + if ((ret == EFBIG) || + ((nsegs > QLNX_MAX_SEGMENTS_NON_TSO) && ( + (!(m_head->m_pkthdr.csum_flags & CSUM_TSO)) || + ((m_head->m_pkthdr.csum_flags & CSUM_TSO) && + qlnx_tso_check(fp, segs, nsegs, offset))))) { + struct mbuf *m; + + QL_DPRINT8(ha, "EFBIG [%d]\n", m_head->m_pkthdr.len); + + fp->tx_defrag++; + + m = m_defrag(m_head, M_NOWAIT); + if (m == NULL) { + fp->err_tx_defrag++; + fp->tx_pkts_freed++; + m_freem(m_head); + *m_headp = NULL; + QL_DPRINT1(ha, "m_defrag() = NULL [%d]\n", ret); + return (ENOBUFS); + } + + m_head = m; + *m_headp = m_head; + + if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, + segs, &nsegs, BUS_DMA_NOWAIT))) { + fp->err_tx_defrag_dmamap_load++; + + QL_DPRINT1(ha, + "bus_dmamap_load_mbuf_sg failed0 [%d, %d]\n", + ret, m_head->m_pkthdr.len); + + fp->tx_pkts_freed++; + m_freem(m_head); + *m_headp = NULL; + + return (ret); + } + + if ((nsegs > QLNX_MAX_SEGMENTS_NON_TSO) && + !(m_head->m_pkthdr.csum_flags & CSUM_TSO)) { + fp->err_tx_non_tso_max_seg++; + + QL_DPRINT1(ha, + "(%d) nsegs too many for non-TSO [%d, %d]\n", + ret, nsegs, m_head->m_pkthdr.len); + + fp->tx_pkts_freed++; + m_freem(m_head); + *m_headp = NULL; + + return (ret); + } + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) + offset = qlnx_tcp_offset(ha, m_head); + + } else if (ret) { + fp->err_tx_dmamap_load++; + + QL_DPRINT1(ha, "bus_dmamap_load_mbuf_sg failed1 [%d, %d]\n", + ret, m_head->m_pkthdr.len); + fp->tx_pkts_freed++; + m_freem(m_head); + *m_headp = NULL; + return (ret); + } + + QL_ASSERT(ha, (nsegs != 0), ("qlnx_send: empty packet")); + + if (ha->dbg_trace_tso_pkt_len) { + if (nsegs < QLNX_FP_MAX_SEGS) + fp->tx_pkts[(nsegs - 1)]++; + else + fp->tx_pkts[(QLNX_FP_MAX_SEGS - 1)]++; + } + +#ifdef QLNX_TRACE_PERF_DATA + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { + if(m_head->m_pkthdr.len <= 2048) + fp->tx_pkts_hist[0]++; + else if((m_head->m_pkthdr.len > 2048) && + (m_head->m_pkthdr.len <= 4096)) + fp->tx_pkts_hist[1]++; + else if((m_head->m_pkthdr.len > 4096) && + (m_head->m_pkthdr.len <= 8192)) + fp->tx_pkts_hist[2]++; + else if((m_head->m_pkthdr.len > 8192) && + (m_head->m_pkthdr.len <= 12288 )) + fp->tx_pkts_hist[3]++; + else if((m_head->m_pkthdr.len > 11288) && + (m_head->m_pkthdr.len <= 16394)) + fp->tx_pkts_hist[4]++; + else if((m_head->m_pkthdr.len > 16384) && + (m_head->m_pkthdr.len <= 20480)) + fp->tx_pkts_hist[5]++; + else if((m_head->m_pkthdr.len > 20480) && + (m_head->m_pkthdr.len <= 24576)) + fp->tx_pkts_hist[6]++; + else if((m_head->m_pkthdr.len > 24576) && + (m_head->m_pkthdr.len <= 28672)) + fp->tx_pkts_hist[7]++; + else if((m_head->m_pkthdr.len > 28762) && + (m_head->m_pkthdr.len <= 32768)) + fp->tx_pkts_hist[8]++; + else if((m_head->m_pkthdr.len > 32768) && + (m_head->m_pkthdr.len <= 36864)) + fp->tx_pkts_hist[9]++; + else if((m_head->m_pkthdr.len > 36864) && + (m_head->m_pkthdr.len <= 40960)) + fp->tx_pkts_hist[10]++; + else if((m_head->m_pkthdr.len > 40960) && + (m_head->m_pkthdr.len <= 45056)) + fp->tx_pkts_hist[11]++; + else if((m_head->m_pkthdr.len > 45056) && + (m_head->m_pkthdr.len <= 49152)) + fp->tx_pkts_hist[12]++; + else if((m_head->m_pkthdr.len > 49512) && + m_head->m_pkthdr.len <= 53248)) + fp->tx_pkts_hist[13]++; + else if((m_head->m_pkthdr.len > 53248) && + (m_head->m_pkthdr.len <= 57344)) + fp->tx_pkts_hist[14]++; + else if((m_head->m_pkthdr.len > 53248) && + (m_head->m_pkthdr.len <= 57344)) + fp->tx_pkts_hist[15]++; + else if((m_head->m_pkthdr.len > 57344) && + (m_head->m_pkthdr.len <= 61440)) + fp->tx_pkts_hist[16]++; + else + fp->tx_pkts_hist[17]++; + } + + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { + elem_left = ecore_chain_get_elem_left(&txq->tx_pbl); + bd_used = TX_RING_SIZE - elem_left; + + if(bd_used <= 100) + fp->tx_pkts_q[0]++; + else if((bd_used > 100) && (bd_used <= 500)) + fp->tx_pkts_q[1]++; + else if((bd_used > 500) && (bd_used <= 1000)) + fp->tx_pkts_q[2]++; + else if((bd_used > 1000) && (bd_used <= 2000)) + fp->tx_pkts_q[3]++; + else if((bd_used > 3000) && (bd_used <= 4000)) + fp->tx_pkts_q[4]++; + else if((bd_used > 4000) && (bd_used <= 5000)) + fp->tx_pkts_q[5]++; + else if((bd_used > 6000) && (bd_used <= 7000)) + fp->tx_pkts_q[6]++; + else if((bd_used > 7000) && (bd_used <= 8000)) + fp->tx_pkts_q[7]++; + else if((bd_used > 8000) && (bd_used <= 9000)) + fp->tx_pkts_q[8]++; + else if((bd_used > 9000) && (bd_used <= 10000)) + fp->tx_pkts_q[9]++; + else if((bd_used > 10000) && (bd_used <= 11000)) + fp->tx_pkts_q[10]++; + else if((bd_used > 11000) && (bd_used <= 12000)) + fp->tx_pkts_q[11]++; + else if((bd_used > 12000) && (bd_used <= 13000)) + fp->tx_pkts_q[12]++; + else if((bd_used > 13000) && (bd_used <= 14000)) + fp->tx_pkts_q[13]++; + else if((bd_used > 14000) && (bd_used <= 15000)) + fp->tx_pkts_q[14]++; + else if((bd_used > 15000) && (bd_used <= 16000)) + fp->tx_pkts_q[15]++; + else + fp->tx_pkts_q[16]++; + } + +#endif /* end of QLNX_TRACE_PERF_DATA */ + + if ((nsegs + QLNX_TX_ELEM_RESERVE) > + (int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl))) { + QL_DPRINT1(ha, "(%d, 0x%x) insuffient BDs" + " in chain[%d] trying to free packets\n", + nsegs, elem_left, fp->rss_id); + + fp->tx_nsegs_gt_elem_left++; + + (void)qlnx_tx_int(ha, fp, txq); + + if ((nsegs + QLNX_TX_ELEM_RESERVE) > (int)(elem_left = + ecore_chain_get_elem_left(&txq->tx_pbl))) { + QL_DPRINT1(ha, + "(%d, 0x%x) insuffient BDs in chain[%d]\n", + nsegs, elem_left, fp->rss_id); + + fp->err_tx_nsegs_gt_elem_left++; + fp->tx_ring_full = 1; + if (ha->storm_stats_enable) + ha->storm_stats_gather = 1; + return (ENOBUFS); + } + } + + bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE); + + txq->sw_tx_ring[idx].mp = m_head; + + first_bd = (struct eth_tx_1st_bd *)ecore_chain_produce(&txq->tx_pbl); + + memset(first_bd, 0, sizeof(*first_bd)); + + first_bd->data.bd_flags.bitfields = + 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; + + BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, segs->ds_len); + + nbd++; + + if (m_head->m_pkthdr.csum_flags & CSUM_IP) { + first_bd->data.bd_flags.bitfields |= + (1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT); + } + + if (m_head->m_pkthdr.csum_flags & + (CSUM_UDP | CSUM_TCP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) { + first_bd->data.bd_flags.bitfields |= + (1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT); + } + + if (m_head->m_flags & M_VLANTAG) { + first_bd->data.vlan = m_head->m_pkthdr.ether_vtag; + first_bd->data.bd_flags.bitfields |= + (1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT); + } + + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { + first_bd->data.bd_flags.bitfields |= + (1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT); + first_bd->data.bd_flags.bitfields |= + (1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT); + + nbds_in_hdr = 1; + + if (offset == segs->ds_len) { + BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, offset); + segs++; + seg_idx++; + + second_bd = (struct eth_tx_2nd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(second_bd, 0, sizeof(*second_bd)); + nbd++; + + if (seg_idx < nsegs) { + BD_SET_UNMAP_ADDR_LEN(second_bd, \ + (segs->ds_addr), (segs->ds_len)); + segs++; + seg_idx++; + } + + third_bd = (struct eth_tx_3rd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(third_bd, 0, sizeof(*third_bd)); + third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz; + third_bd->data.bitfields |= + (nbds_in_hdr<<ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT); + nbd++; + + if (seg_idx < nsegs) { + BD_SET_UNMAP_ADDR_LEN(third_bd, \ + (segs->ds_addr), (segs->ds_len)); + segs++; + seg_idx++; + } + + for (; seg_idx < nsegs; seg_idx++) { + tx_data_bd = (struct eth_tx_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(tx_data_bd, 0, sizeof(*tx_data_bd)); + BD_SET_UNMAP_ADDR_LEN(tx_data_bd, \ + segs->ds_addr,\ + segs->ds_len); + segs++; + nbd++; + } + + } else if (offset < segs->ds_len) { + BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, offset); + + second_bd = (struct eth_tx_2nd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(second_bd, 0, sizeof(*second_bd)); + BD_SET_UNMAP_ADDR_LEN(second_bd, \ + (segs->ds_addr + offset),\ + (segs->ds_len - offset)); + nbd++; + segs++; + + third_bd = (struct eth_tx_3rd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(third_bd, 0, sizeof(*third_bd)); + + BD_SET_UNMAP_ADDR_LEN(third_bd, \ + segs->ds_addr,\ + segs->ds_len); + third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz; + third_bd->data.bitfields |= + (nbds_in_hdr<<ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT); + segs++; + nbd++; + + for (seg_idx = 2; seg_idx < nsegs; seg_idx++) { + tx_data_bd = (struct eth_tx_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(tx_data_bd, 0, sizeof(*tx_data_bd)); + BD_SET_UNMAP_ADDR_LEN(tx_data_bd, \ + segs->ds_addr,\ + segs->ds_len); + segs++; + nbd++; + } + + } else { + offset = offset - segs->ds_len; + segs++; + + for (seg_idx = 1; seg_idx < nsegs; seg_idx++) { + if (offset) + nbds_in_hdr++; + + tx_data_bd = (struct eth_tx_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(tx_data_bd, 0, sizeof(*tx_data_bd)); + + if (second_bd == NULL) { + second_bd = (struct eth_tx_2nd_bd *) + tx_data_bd; + } else if (third_bd == NULL) { + third_bd = (struct eth_tx_3rd_bd *) + tx_data_bd; + } + + if (offset && (offset < segs->ds_len)) { + BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\ + segs->ds_addr, offset); + + tx_data_bd = (struct eth_tx_bd *) + ecore_chain_produce(&txq->tx_pbl); + + memset(tx_data_bd, 0, + sizeof(*tx_data_bd)); + + if (second_bd == NULL) { + second_bd = + (struct eth_tx_2nd_bd *)tx_data_bd; + } else if (third_bd == NULL) { + third_bd = + (struct eth_tx_3rd_bd *)tx_data_bd; + } + BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\ + (segs->ds_addr + offset), \ + (segs->ds_len - offset)); + nbd++; + offset = 0; + } else { + if (offset) + offset = offset - segs->ds_len; + BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\ + segs->ds_addr, segs->ds_len); + } + segs++; + nbd++; + } + + if (third_bd == NULL) { + third_bd = (struct eth_tx_3rd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(third_bd, 0, sizeof(*third_bd)); + } + + third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz; + third_bd->data.bitfields |= + (nbds_in_hdr<<ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT); + } + fp->tx_tso_pkts++; + } else { + segs++; + for (seg_idx = 1; seg_idx < nsegs; seg_idx++) { + tx_data_bd = (struct eth_tx_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(tx_data_bd, 0, sizeof(*tx_data_bd)); + BD_SET_UNMAP_ADDR_LEN(tx_data_bd, segs->ds_addr,\ + segs->ds_len); + segs++; + nbd++; + } + first_bd->data.bitfields = + (m_head->m_pkthdr.len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) + << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; + first_bd->data.bitfields = + htole16(first_bd->data.bitfields); + fp->tx_non_tso_pkts++; + } + + first_bd->data.nbds = nbd; + + if (ha->dbg_trace_tso_pkt_len) { + if (fp->tx_tso_max_nsegs < nsegs) + fp->tx_tso_max_nsegs = nsegs; + + if ((nsegs < fp->tx_tso_min_nsegs) || (!fp->tx_tso_min_nsegs)) + fp->tx_tso_min_nsegs = nsegs; + } + + txq->sw_tx_ring[idx].nsegs = nsegs; + txq->sw_tx_prod = (txq->sw_tx_prod + 1) & (TX_RING_SIZE - 1); + + txq->tx_db.data.bd_prod = + htole16(ecore_chain_get_prod_idx(&txq->tx_pbl)); + + qlnx_txq_doorbell_wr32(ha, txq->doorbell_addr, txq->tx_db.raw); + + QL_DPRINT8(ha, "exit[%d]\n", fp->rss_id); + return (0); +} + +static void +qlnx_stop(qlnx_host_t *ha) +{ + if_t ifp = ha->ifp; + int i; + + if_setdrvflagbits(ifp, 0, (IFF_DRV_OACTIVE | IFF_DRV_RUNNING)); + + /* + * We simply lock and unlock each fp->tx_mtx to + * propagate the if_drv_flags + * state to each tx thread + */ + QL_DPRINT1(ha, "QLNX STATE = %d\n",ha->state); + + if (ha->state == QLNX_STATE_OPEN) { + for (i = 0; i < ha->num_rss; i++) { + struct qlnx_fastpath *fp = &ha->fp_array[i]; + + mtx_lock(&fp->tx_mtx); + mtx_unlock(&fp->tx_mtx); + + if (fp->fp_taskqueue != NULL) + taskqueue_enqueue(fp->fp_taskqueue, + &fp->fp_task); + } + } +#ifdef QLNX_ENABLE_IWARP + if (qlnx_vf_device(ha) != 0) { + qlnx_rdma_dev_close(ha); + } +#endif /* #ifdef QLNX_ENABLE_IWARP */ + + qlnx_unload(ha); + + return; +} + +static int +qlnx_get_ifq_snd_maxlen(qlnx_host_t *ha) +{ + return(TX_RING_SIZE - 1); +} + +uint8_t * +qlnx_get_mac_addr(qlnx_host_t *ha) +{ + struct ecore_hwfn *p_hwfn; + unsigned char mac[ETHER_ADDR_LEN]; + uint8_t p_is_forced; + + p_hwfn = &ha->cdev.hwfns[0]; + + if (qlnx_vf_device(ha) != 0) + return (p_hwfn->hw_info.hw_mac_addr); + + ecore_vf_read_bulletin(p_hwfn, &p_is_forced); + if (ecore_vf_bulletin_get_forced_mac(p_hwfn, mac, &p_is_forced) == + true) { + device_printf(ha->pci_dev, "%s: p_is_forced = %d" + " mac_addr = %02x:%02x:%02x:%02x:%02x:%02x\n", __func__, + p_is_forced, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + memcpy(ha->primary_mac, mac, ETH_ALEN); + } + + return (ha->primary_mac); +} + +static uint32_t +qlnx_get_optics(qlnx_host_t *ha, struct qlnx_link_output *if_link) +{ + uint32_t ifm_type = 0; + + switch (if_link->media_type) { + case MEDIA_MODULE_FIBER: + case MEDIA_UNSPECIFIED: + if (if_link->speed == (100 * 1000)) + ifm_type = QLNX_IFM_100G_SR4; + else if (if_link->speed == (40 * 1000)) + ifm_type = IFM_40G_SR4; + else if (if_link->speed == (25 * 1000)) + ifm_type = QLNX_IFM_25G_SR; + else if (if_link->speed == (10 * 1000)) + ifm_type = (IFM_10G_LR | IFM_10G_SR); + else if (if_link->speed == (1 * 1000)) + ifm_type = (IFM_1000_SX | IFM_1000_LX); + + break; + + case MEDIA_DA_TWINAX: + if (if_link->speed == (100 * 1000)) + ifm_type = QLNX_IFM_100G_CR4; + else if (if_link->speed == (40 * 1000)) + ifm_type = IFM_40G_CR4; + else if (if_link->speed == (25 * 1000)) + ifm_type = QLNX_IFM_25G_CR; + else if (if_link->speed == (10 * 1000)) + ifm_type = IFM_10G_TWINAX; + + break; + + default : + ifm_type = IFM_UNKNOWN; + break; + } + return (ifm_type); +} + +/***************************************************************************** + * Interrupt Service Functions + *****************************************************************************/ + +static int +qlnx_rx_jumbo_chain(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct mbuf *mp_head, uint16_t len) +{ + struct mbuf *mp, *mpf, *mpl; + struct sw_rx_data *sw_rx_data; + struct qlnx_rx_queue *rxq; + uint16_t len_in_buffer; + + rxq = fp->rxq; + mpf = mpl = mp = NULL; + + while (len) { + rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; + mp = sw_rx_data->data; + + if (mp == NULL) { + QL_DPRINT1(ha, "mp = NULL\n"); + fp->err_rx_mp_null++; + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + if (mpf != NULL) + m_freem(mpf); + + return (-1); + } + bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, + BUS_DMASYNC_POSTREAD); + + if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { + QL_DPRINT1(ha, "New buffer allocation failed, dropping" + " incoming packet and reusing its buffer\n"); + + qlnx_reuse_rx_data(rxq); + fp->err_rx_alloc_errors++; + + if (mpf != NULL) + m_freem(mpf); + + return (-1); + } + ecore_chain_consume(&rxq->rx_bd_ring); + + if (len > rxq->rx_buf_size) + len_in_buffer = rxq->rx_buf_size; + else + len_in_buffer = len; + + len = len - len_in_buffer; + + mp->m_flags &= ~M_PKTHDR; + mp->m_next = NULL; + mp->m_len = len_in_buffer; + + if (mpf == NULL) + mpf = mpl = mp; + else { + mpl->m_next = mp; + mpl = mp; + } + } + + if (mpf != NULL) + mp_head->m_next = mpf; + + return (0); +} + +static void +qlnx_tpa_start(qlnx_host_t *ha, + struct qlnx_fastpath *fp, + struct qlnx_rx_queue *rxq, + struct eth_fast_path_rx_tpa_start_cqe *cqe) +{ + uint32_t agg_index; + if_t ifp = ha->ifp; + struct mbuf *mp; + struct mbuf *mpf = NULL, *mpl = NULL, *mpc = NULL; + struct sw_rx_data *sw_rx_data; + dma_addr_t addr; + bus_dmamap_t map; + struct eth_rx_bd *rx_bd; + int i; + uint8_t hash_type; + + agg_index = cqe->tpa_agg_index; + + QL_DPRINT7(ha, "[rss_id = %d]: enter\n \ + \t type = 0x%x\n \ + \t bitfields = 0x%x\n \ + \t seg_len = 0x%x\n \ + \t pars_flags = 0x%x\n \ + \t vlan_tag = 0x%x\n \ + \t rss_hash = 0x%x\n \ + \t len_on_first_bd = 0x%x\n \ + \t placement_offset = 0x%x\n \ + \t tpa_agg_index = 0x%x\n \ + \t header_len = 0x%x\n \ + \t ext_bd_len_list[0] = 0x%x\n \ + \t ext_bd_len_list[1] = 0x%x\n \ + \t ext_bd_len_list[2] = 0x%x\n \ + \t ext_bd_len_list[3] = 0x%x\n \ + \t ext_bd_len_list[4] = 0x%x\n", + fp->rss_id, cqe->type, cqe->bitfields, cqe->seg_len, + cqe->pars_flags.flags, cqe->vlan_tag, + cqe->rss_hash, cqe->len_on_first_bd, cqe->placement_offset, + cqe->tpa_agg_index, cqe->header_len, + cqe->ext_bd_len_list[0], cqe->ext_bd_len_list[1], + cqe->ext_bd_len_list[2], cqe->ext_bd_len_list[3], + cqe->ext_bd_len_list[4]); + + if (agg_index >= ETH_TPA_MAX_AGGS_NUM) { + fp->err_rx_tpa_invalid_agg_num++; + return; + } + + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; + bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD); + mp = sw_rx_data->data; + + QL_DPRINT7(ha, "[rss_id = %d]: mp = %p \n ", fp->rss_id, mp); + + if (mp == NULL) { + QL_DPRINT7(ha, "[%d]: mp = NULL\n", fp->rss_id); + fp->err_rx_mp_null++; + rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + return; + } + + if ((le16toh(cqe->pars_flags.flags)) & CQE_FLAGS_ERR) { + QL_DPRINT7(ha, "[%d]: CQE in CONS = %u has error," + " flags = %x, dropping incoming packet\n", fp->rss_id, + rxq->sw_rx_cons, le16toh(cqe->pars_flags.flags)); + + fp->err_rx_hw_errors++; + + qlnx_reuse_rx_data(rxq); + + QLNX_INC_IERRORS(ifp); + + return; + } + + if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { + QL_DPRINT7(ha, "[%d]: New buffer allocation failed," + " dropping incoming packet and reusing its buffer\n", + fp->rss_id); + + fp->err_rx_alloc_errors++; + QLNX_INC_IQDROPS(ifp); + + /* + * Load the tpa mbuf into the rx ring and save the + * posted mbuf + */ + + map = sw_rx_data->map; + addr = sw_rx_data->dma_addr; + + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod]; + + sw_rx_data->data = rxq->tpa_info[agg_index].rx_buf.data; + sw_rx_data->dma_addr = rxq->tpa_info[agg_index].rx_buf.dma_addr; + sw_rx_data->map = rxq->tpa_info[agg_index].rx_buf.map; + + rxq->tpa_info[agg_index].rx_buf.data = mp; + rxq->tpa_info[agg_index].rx_buf.dma_addr = addr; + rxq->tpa_info[agg_index].rx_buf.map = map; + + rx_bd = (struct eth_rx_bd *) + ecore_chain_produce(&rxq->rx_bd_ring); + + rx_bd->addr.hi = htole32(U64_HI(sw_rx_data->dma_addr)); + rx_bd->addr.lo = htole32(U64_LO(sw_rx_data->dma_addr)); + + bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, + BUS_DMASYNC_PREREAD); + + rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1); + rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + ecore_chain_consume(&rxq->rx_bd_ring); + + /* Now reuse any buffers posted in ext_bd_len_list */ + for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) { + if (cqe->ext_bd_len_list[i] == 0) + break; + + qlnx_reuse_rx_data(rxq); + } + + rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; + return; + } + + if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) { + QL_DPRINT7(ha, "[%d]: invalid aggregation state," + " dropping incoming packet and reusing its buffer\n", + fp->rss_id); + + QLNX_INC_IQDROPS(ifp); + + /* if we already have mbuf head in aggregation free it */ + if (rxq->tpa_info[agg_index].mpf) { + m_freem(rxq->tpa_info[agg_index].mpf); + rxq->tpa_info[agg_index].mpl = NULL; + } + rxq->tpa_info[agg_index].mpf = mp; + rxq->tpa_info[agg_index].mpl = NULL; + + rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + ecore_chain_consume(&rxq->rx_bd_ring); + + /* Now reuse any buffers posted in ext_bd_len_list */ + for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) { + if (cqe->ext_bd_len_list[i] == 0) + break; + + qlnx_reuse_rx_data(rxq); + } + rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; + + return; + } + + /* + * first process the ext_bd_len_list + * if this fails then we simply drop the packet + */ + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) { + QL_DPRINT7(ha, "[%d]: 4\n ", fp->rss_id); + + if (cqe->ext_bd_len_list[i] == 0) + break; + + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; + bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, + BUS_DMASYNC_POSTREAD); + + mpc = sw_rx_data->data; + + if (mpc == NULL) { + QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id); + fp->err_rx_mp_null++; + if (mpf != NULL) + m_freem(mpf); + mpf = mpl = NULL; + rxq->tpa_info[agg_index].agg_state = + QLNX_AGG_STATE_ERROR; + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + continue; + } + + if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { + QL_DPRINT7(ha, "[%d]: New buffer allocation failed," + " dropping incoming packet and reusing its" + " buffer\n", fp->rss_id); + + qlnx_reuse_rx_data(rxq); + + if (mpf != NULL) + m_freem(mpf); + mpf = mpl = NULL; + + rxq->tpa_info[agg_index].agg_state = + QLNX_AGG_STATE_ERROR; + + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + continue; + } + + mpc->m_flags &= ~M_PKTHDR; + mpc->m_next = NULL; + mpc->m_len = cqe->ext_bd_len_list[i]; + + if (mpf == NULL) { + mpf = mpl = mpc; + } else { + mpl->m_len = ha->rx_buf_size; + mpl->m_next = mpc; + mpl = mpc; + } + + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + } + + if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) { + QL_DPRINT7(ha, "[%d]: invalid aggregation state, dropping" + " incoming packet and reusing its buffer\n", + fp->rss_id); + + QLNX_INC_IQDROPS(ifp); + + rxq->tpa_info[agg_index].mpf = mp; + rxq->tpa_info[agg_index].mpl = NULL; + + return; + } + + rxq->tpa_info[agg_index].placement_offset = cqe->placement_offset; + + if (mpf != NULL) { + mp->m_len = ha->rx_buf_size; + mp->m_next = mpf; + rxq->tpa_info[agg_index].mpf = mp; + rxq->tpa_info[agg_index].mpl = mpl; + } else { + mp->m_len = cqe->len_on_first_bd + cqe->placement_offset; + rxq->tpa_info[agg_index].mpf = mp; + rxq->tpa_info[agg_index].mpl = mp; + mp->m_next = NULL; + } + + mp->m_flags |= M_PKTHDR; + + /* assign packet to this interface interface */ + mp->m_pkthdr.rcvif = ifp; + + /* assume no hardware checksum has complated */ + mp->m_pkthdr.csum_flags = 0; + + //mp->m_pkthdr.flowid = fp->rss_id; + mp->m_pkthdr.flowid = cqe->rss_hash; + + hash_type = cqe->bitfields & + (ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK << + ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT); + + switch (hash_type) { + case RSS_HASH_TYPE_IPV4: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV4); + break; + + case RSS_HASH_TYPE_TCP_IPV4: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV4); + break; + + case RSS_HASH_TYPE_IPV6: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV6); + break; + + case RSS_HASH_TYPE_TCP_IPV6: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV6); + break; + + default: + M_HASHTYPE_SET(mp, M_HASHTYPE_OPAQUE); + break; + } + + mp->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID | + CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + + mp->m_pkthdr.csum_data = 0xFFFF; + + if (CQE_HAS_VLAN(cqe->pars_flags.flags)) { + mp->m_pkthdr.ether_vtag = le16toh(cqe->vlan_tag); + mp->m_flags |= M_VLANTAG; + } + + rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_START; + + QL_DPRINT7(ha, "[%d]: 5\n\tagg_state = %d\n\t mpf = %p mpl = %p\n", + fp->rss_id, rxq->tpa_info[agg_index].agg_state, + rxq->tpa_info[agg_index].mpf, rxq->tpa_info[agg_index].mpl); + + return; +} + +static void +qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_rx_queue *rxq, + struct eth_fast_path_rx_tpa_cont_cqe *cqe) +{ + struct sw_rx_data *sw_rx_data; + int i; + struct mbuf *mpf = NULL, *mpl = NULL, *mpc = NULL; + struct mbuf *mp; + uint32_t agg_index; + + QL_DPRINT7(ha, "[%d]: enter\n \ + \t type = 0x%x\n \ + \t tpa_agg_index = 0x%x\n \ + \t len_list[0] = 0x%x\n \ + \t len_list[1] = 0x%x\n \ + \t len_list[2] = 0x%x\n \ + \t len_list[3] = 0x%x\n \ + \t len_list[4] = 0x%x\n \ + \t len_list[5] = 0x%x\n", + fp->rss_id, cqe->type, cqe->tpa_agg_index, + cqe->len_list[0], cqe->len_list[1], cqe->len_list[2], + cqe->len_list[3], cqe->len_list[4], cqe->len_list[5]); + + agg_index = cqe->tpa_agg_index; + + if (agg_index >= ETH_TPA_MAX_AGGS_NUM) { + QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id); + fp->err_rx_tpa_invalid_agg_num++; + return; + } + + for (i = 0; i < ETH_TPA_CQE_CONT_LEN_LIST_SIZE; i++) { + QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id); + + if (cqe->len_list[i] == 0) + break; + + if (rxq->tpa_info[agg_index].agg_state != + QLNX_AGG_STATE_START) { + qlnx_reuse_rx_data(rxq); + continue; + } + + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; + bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, + BUS_DMASYNC_POSTREAD); + + mpc = sw_rx_data->data; + + if (mpc == NULL) { + QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id); + + fp->err_rx_mp_null++; + if (mpf != NULL) + m_freem(mpf); + mpf = mpl = NULL; + rxq->tpa_info[agg_index].agg_state = + QLNX_AGG_STATE_ERROR; + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + continue; + } + + if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { + QL_DPRINT7(ha, "[%d]: New buffer allocation failed," + " dropping incoming packet and reusing its" + " buffer\n", fp->rss_id); + + qlnx_reuse_rx_data(rxq); + + if (mpf != NULL) + m_freem(mpf); + mpf = mpl = NULL; + + rxq->tpa_info[agg_index].agg_state = + QLNX_AGG_STATE_ERROR; + + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + continue; + } + + mpc->m_flags &= ~M_PKTHDR; + mpc->m_next = NULL; + mpc->m_len = cqe->len_list[i]; + + if (mpf == NULL) { + mpf = mpl = mpc; + } else { + mpl->m_len = ha->rx_buf_size; + mpl->m_next = mpc; + mpl = mpc; + } + + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + } + + QL_DPRINT7(ha, "[%d]: 2\n" "\tmpf = %p mpl = %p\n", + fp->rss_id, mpf, mpl); + + if (mpf != NULL) { + mp = rxq->tpa_info[agg_index].mpl; + mp->m_len = ha->rx_buf_size; + mp->m_next = mpf; + rxq->tpa_info[agg_index].mpl = mpl; + } + + return; +} + +static int +qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_rx_queue *rxq, + struct eth_fast_path_rx_tpa_end_cqe *cqe) +{ + struct sw_rx_data *sw_rx_data; + int i; + struct mbuf *mpf = NULL, *mpl = NULL, *mpc = NULL; + struct mbuf *mp; + uint32_t agg_index; + uint32_t len = 0; + if_t ifp = ha->ifp; + + QL_DPRINT7(ha, "[%d]: enter\n \ + \t type = 0x%x\n \ + \t tpa_agg_index = 0x%x\n \ + \t total_packet_len = 0x%x\n \ + \t num_of_bds = 0x%x\n \ + \t end_reason = 0x%x\n \ + \t num_of_coalesced_segs = 0x%x\n \ + \t ts_delta = 0x%x\n \ + \t len_list[0] = 0x%x\n \ + \t len_list[1] = 0x%x\n \ + \t len_list[2] = 0x%x\n \ + \t len_list[3] = 0x%x\n", + fp->rss_id, cqe->type, cqe->tpa_agg_index, + cqe->total_packet_len, cqe->num_of_bds, + cqe->end_reason, cqe->num_of_coalesced_segs, cqe->ts_delta, + cqe->len_list[0], cqe->len_list[1], cqe->len_list[2], + cqe->len_list[3]); + + agg_index = cqe->tpa_agg_index; + + if (agg_index >= ETH_TPA_MAX_AGGS_NUM) { + QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id); + + fp->err_rx_tpa_invalid_agg_num++; + return (0); + } + + for (i = 0; i < ETH_TPA_CQE_END_LEN_LIST_SIZE; i++) { + QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id); + + if (cqe->len_list[i] == 0) + break; + + if (rxq->tpa_info[agg_index].agg_state != + QLNX_AGG_STATE_START) { + QL_DPRINT7(ha, "[%d]: 2\n ", fp->rss_id); + + qlnx_reuse_rx_data(rxq); + continue; + } + + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; + bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, + BUS_DMASYNC_POSTREAD); + + mpc = sw_rx_data->data; + + if (mpc == NULL) { + QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id); + + fp->err_rx_mp_null++; + if (mpf != NULL) + m_freem(mpf); + mpf = mpl = NULL; + rxq->tpa_info[agg_index].agg_state = + QLNX_AGG_STATE_ERROR; + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + continue; + } + + if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { + QL_DPRINT7(ha, "[%d]: New buffer allocation failed," + " dropping incoming packet and reusing its" + " buffer\n", fp->rss_id); + + qlnx_reuse_rx_data(rxq); + + if (mpf != NULL) + m_freem(mpf); + mpf = mpl = NULL; + + rxq->tpa_info[agg_index].agg_state = + QLNX_AGG_STATE_ERROR; + + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + + continue; + } + + mpc->m_flags &= ~M_PKTHDR; + mpc->m_next = NULL; + mpc->m_len = cqe->len_list[i]; + + if (mpf == NULL) { + mpf = mpl = mpc; + } else { + mpl->m_len = ha->rx_buf_size; + mpl->m_next = mpc; + mpl = mpc; + } + + ecore_chain_consume(&rxq->rx_bd_ring); + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + } + + QL_DPRINT7(ha, "[%d]: 5\n ", fp->rss_id); + + if (mpf != NULL) { + QL_DPRINT7(ha, "[%d]: 6\n ", fp->rss_id); + + mp = rxq->tpa_info[agg_index].mpl; + mp->m_len = ha->rx_buf_size; + mp->m_next = mpf; + } + + if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_START) { + QL_DPRINT7(ha, "[%d]: 7\n ", fp->rss_id); + + if (rxq->tpa_info[agg_index].mpf != NULL) + m_freem(rxq->tpa_info[agg_index].mpf); + rxq->tpa_info[agg_index].mpf = NULL; + rxq->tpa_info[agg_index].mpl = NULL; + rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_NONE; + return (0); + } + + mp = rxq->tpa_info[agg_index].mpf; + m_adj(mp, rxq->tpa_info[agg_index].placement_offset); + mp->m_pkthdr.len = cqe->total_packet_len; + + if (mp->m_next == NULL) + mp->m_len = mp->m_pkthdr.len; + else { + /* compute the total packet length */ + mpf = mp; + while (mpf != NULL) { + len += mpf->m_len; + mpf = mpf->m_next; + } + + if (cqe->total_packet_len > len) { + mpl = rxq->tpa_info[agg_index].mpl; + mpl->m_len += (cqe->total_packet_len - len); + } + } + + QLNX_INC_IPACKETS(ifp); + QLNX_INC_IBYTES(ifp, (cqe->total_packet_len)); + + QL_DPRINT7(ha, "[%d]: 8 csum_data = 0x%x csum_flags = 0x%" PRIu64 "\n \ + m_len = 0x%x m_pkthdr_len = 0x%x\n", + fp->rss_id, mp->m_pkthdr.csum_data, + (uint64_t)mp->m_pkthdr.csum_flags, mp->m_len, mp->m_pkthdr.len); + + if_input(ifp, mp); + + rxq->tpa_info[agg_index].mpf = NULL; + rxq->tpa_info[agg_index].mpl = NULL; + rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_NONE; + + return (cqe->num_of_coalesced_segs); +} + +static int +qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget, + int lro_enable) +{ + uint16_t hw_comp_cons, sw_comp_cons; + int rx_pkt = 0; + struct qlnx_rx_queue *rxq = fp->rxq; + if_t ifp = ha->ifp; + struct ecore_dev *cdev = &ha->cdev; + struct ecore_hwfn *p_hwfn; + +#ifdef QLNX_SOFT_LRO + struct lro_ctrl *lro; + + lro = &rxq->lro; +#endif /* #ifdef QLNX_SOFT_LRO */ + + hw_comp_cons = le16toh(*rxq->hw_cons_ptr); + sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); + + p_hwfn = &ha->cdev.hwfns[(fp->rss_id % cdev->num_hwfns)]; + + /* Memory barrier to prevent the CPU from doing speculative reads of CQE + * / BD in the while-loop before reading hw_comp_cons. If the CQE is + * read before it is written by FW, then FW writes CQE and SB, and then + * the CPU reads the hw_comp_cons, it will use an old CQE. + */ + + /* Loop to complete all indicated BDs */ + while (sw_comp_cons != hw_comp_cons) { + union eth_rx_cqe *cqe; + struct eth_fast_path_rx_reg_cqe *fp_cqe; + struct sw_rx_data *sw_rx_data; + register struct mbuf *mp; + enum eth_rx_cqe_type cqe_type; + uint16_t len, pad, len_on_first_bd; + uint8_t *data; + uint8_t hash_type; + + /* Get the CQE from the completion ring */ + cqe = (union eth_rx_cqe *) + ecore_chain_consume(&rxq->rx_comp_ring); + cqe_type = cqe->fast_path_regular.type; + + if (cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH) { + QL_DPRINT3(ha, "Got a slowath CQE\n"); + + ecore_eth_cqe_completion(p_hwfn, + (struct eth_slow_path_rx_cqe *)cqe); + goto next_cqe; + } + + if (cqe_type != ETH_RX_CQE_TYPE_REGULAR) { + switch (cqe_type) { + case ETH_RX_CQE_TYPE_TPA_START: + qlnx_tpa_start(ha, fp, rxq, + &cqe->fast_path_tpa_start); + fp->tpa_start++; + break; + + case ETH_RX_CQE_TYPE_TPA_CONT: + qlnx_tpa_cont(ha, fp, rxq, + &cqe->fast_path_tpa_cont); + fp->tpa_cont++; + break; + + case ETH_RX_CQE_TYPE_TPA_END: + rx_pkt += qlnx_tpa_end(ha, fp, rxq, + &cqe->fast_path_tpa_end); + fp->tpa_end++; + break; + + default: + break; + } + + goto next_cqe; + } + + /* Get the data from the SW ring */ + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; + mp = sw_rx_data->data; + + if (mp == NULL) { + QL_DPRINT1(ha, "mp = NULL\n"); + fp->err_rx_mp_null++; + rxq->sw_rx_cons = + (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + goto next_cqe; + } + bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, + BUS_DMASYNC_POSTREAD); + + /* non GRO */ + fp_cqe = &cqe->fast_path_regular;/* MK CR TPA check assembly */ + len = le16toh(fp_cqe->pkt_len); + pad = fp_cqe->placement_offset; +#if 0 + QL_DPRINT3(ha, "CQE type = %x, flags = %x, vlan = %x," + " len %u, parsing flags = %d pad = %d\n", + cqe_type, fp_cqe->bitfields, + le16toh(fp_cqe->vlan_tag), + len, le16toh(fp_cqe->pars_flags.flags), pad); +#endif + data = mtod(mp, uint8_t *); + data = data + pad; + + if (0) + qlnx_dump_buf8(ha, __func__, data, len); + + /* For every Rx BD consumed, we allocate a new BD so the BD ring + * is always with a fixed size. If allocation fails, we take the + * consumed BD and return it to the ring in the PROD position. + * The packet that was received on that BD will be dropped (and + * not passed to the upper stack). + */ + /* If this is an error packet then drop it */ + if ((le16toh(cqe->fast_path_regular.pars_flags.flags)) & + CQE_FLAGS_ERR) { + QL_DPRINT1(ha, "CQE in CONS = %u has error, flags = %x," + " dropping incoming packet\n", sw_comp_cons, + le16toh(cqe->fast_path_regular.pars_flags.flags)); + fp->err_rx_hw_errors++; + + qlnx_reuse_rx_data(rxq); + + QLNX_INC_IERRORS(ifp); + + goto next_cqe; + } + + if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { + QL_DPRINT1(ha, "New buffer allocation failed, dropping" + " incoming packet and reusing its buffer\n"); + qlnx_reuse_rx_data(rxq); + + fp->err_rx_alloc_errors++; + + QLNX_INC_IQDROPS(ifp); + + goto next_cqe; + } + + ecore_chain_consume(&rxq->rx_bd_ring); + + len_on_first_bd = fp_cqe->len_on_first_bd; + m_adj(mp, pad); + mp->m_pkthdr.len = len; + + if ((len > 60 ) && (len > len_on_first_bd)) { + mp->m_len = len_on_first_bd; + + if (qlnx_rx_jumbo_chain(ha, fp, mp, + (len - len_on_first_bd)) != 0) { + m_freem(mp); + + QLNX_INC_IQDROPS(ifp); + + goto next_cqe; + } + + } else if (len_on_first_bd < len) { + fp->err_rx_jumbo_chain_pkts++; + } else { + mp->m_len = len; + } + + mp->m_flags |= M_PKTHDR; + + /* assign packet to this interface interface */ + mp->m_pkthdr.rcvif = ifp; + + /* assume no hardware checksum has complated */ + mp->m_pkthdr.csum_flags = 0; + + mp->m_pkthdr.flowid = fp_cqe->rss_hash; + + hash_type = fp_cqe->bitfields & + (ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK << + ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT); + + switch (hash_type) { + case RSS_HASH_TYPE_IPV4: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV4); + break; + + case RSS_HASH_TYPE_TCP_IPV4: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV4); + break; + + case RSS_HASH_TYPE_IPV6: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV6); + break; + + case RSS_HASH_TYPE_TCP_IPV6: + M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV6); + break; + + default: + M_HASHTYPE_SET(mp, M_HASHTYPE_OPAQUE); + break; + } + + if (CQE_L3_PACKET(fp_cqe->pars_flags.flags)) { + mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; + } + + if (!(CQE_IP_HDR_ERR(fp_cqe->pars_flags.flags))) { + mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; + } + + if (CQE_L4_HAS_CSUM(fp_cqe->pars_flags.flags)) { + mp->m_pkthdr.csum_data = 0xFFFF; + mp->m_pkthdr.csum_flags |= + (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + } + + if (CQE_HAS_VLAN(fp_cqe->pars_flags.flags)) { + mp->m_pkthdr.ether_vtag = le16toh(fp_cqe->vlan_tag); + mp->m_flags |= M_VLANTAG; + } + + QLNX_INC_IPACKETS(ifp); + QLNX_INC_IBYTES(ifp, len); + +#ifdef QLNX_SOFT_LRO + if (lro_enable) + tcp_lro_queue_mbuf(lro, mp); + else + if_input(ifp, mp); +#else + + if_input(ifp, mp); + +#endif /* #ifdef QLNX_SOFT_LRO */ + + rx_pkt++; + + rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + +next_cqe: /* don't consume bd rx buffer */ + ecore_chain_recycle_consumed(&rxq->rx_comp_ring); + sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); + + /* CR TPA - revisit how to handle budget in TPA perhaps + increase on "end" */ + if (rx_pkt == budget) + break; + } /* repeat while sw_comp_cons != hw_comp_cons... */ + + /* Update producers */ + qlnx_update_rx_prod(p_hwfn, rxq); + + return rx_pkt; +} + +/* + * fast path interrupt + */ + +static void +qlnx_fp_isr(void *arg) +{ + qlnx_ivec_t *ivec = arg; + qlnx_host_t *ha; + struct qlnx_fastpath *fp = NULL; + int idx; + + ha = ivec->ha; + + if (ha->state != QLNX_STATE_OPEN) { + return; + } + + idx = ivec->rss_idx; + + if ((idx = ivec->rss_idx) >= ha->num_rss) { + QL_DPRINT1(ha, "illegal interrupt[%d]\n", idx); + ha->err_illegal_intr++; + return; + } + fp = &ha->fp_array[idx]; + + if (fp == NULL) { + ha->err_fp_null++; + } else { + int rx_int = 0; +#ifdef QLNX_SOFT_LRO + int total_rx_count = 0; +#endif + int lro_enable, tc; + struct qlnx_tx_queue *txq; + uint16_t elem_left; + + lro_enable = if_getcapenable(ha->ifp) & IFCAP_LRO; + + ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0); + + do { + for (tc = 0; tc < ha->num_tc; tc++) { + txq = fp->txq[tc]; + + if((int)(elem_left = + ecore_chain_get_elem_left(&txq->tx_pbl)) < + QLNX_TX_ELEM_THRESH) { + if (mtx_trylock(&fp->tx_mtx)) { +#ifdef QLNX_TRACE_PERF_DATA + tx_compl = fp->tx_pkts_completed; +#endif + + qlnx_tx_int(ha, fp, fp->txq[tc]); +#ifdef QLNX_TRACE_PERF_DATA + fp->tx_pkts_compl_intr += + (fp->tx_pkts_completed - tx_compl); + if ((fp->tx_pkts_completed - tx_compl) <= 32) + fp->tx_comInt[0]++; + else if (((fp->tx_pkts_completed - tx_compl) > 32) && + ((fp->tx_pkts_completed - tx_compl) <= 64)) + fp->tx_comInt[1]++; + else if(((fp->tx_pkts_completed - tx_compl) > 64) && + ((fp->tx_pkts_completed - tx_compl) <= 128)) + fp->tx_comInt[2]++; + else if(((fp->tx_pkts_completed - tx_compl) > 128)) + fp->tx_comInt[3]++; +#endif + mtx_unlock(&fp->tx_mtx); + } + } + } + + rx_int = qlnx_rx_int(ha, fp, ha->rx_pkt_threshold, + lro_enable); + + if (rx_int) { + fp->rx_pkts += rx_int; +#ifdef QLNX_SOFT_LRO + total_rx_count += rx_int; +#endif + } + + } while (rx_int); + +#ifdef QLNX_SOFT_LRO + { + struct lro_ctrl *lro; + + lro = &fp->rxq->lro; + + if (lro_enable && total_rx_count) { + +#ifdef QLNX_TRACE_LRO_CNT + if (lro->lro_mbuf_count & ~1023) + fp->lro_cnt_1024++; + else if (lro->lro_mbuf_count & ~511) + fp->lro_cnt_512++; + else if (lro->lro_mbuf_count & ~255) + fp->lro_cnt_256++; + else if (lro->lro_mbuf_count & ~127) + fp->lro_cnt_128++; + else if (lro->lro_mbuf_count & ~63) + fp->lro_cnt_64++; +#endif /* #ifdef QLNX_TRACE_LRO_CNT */ + + tcp_lro_flush_all(lro); + } + } +#endif /* #ifdef QLNX_SOFT_LRO */ + + ecore_sb_update_sb_idx(fp->sb_info); + rmb(); + ecore_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1); + } + + return; +} + +/* + * slow path interrupt processing function + * can be invoked in polled mode or in interrupt mode via taskqueue. + */ +void +qlnx_sp_isr(void *arg) +{ + struct ecore_hwfn *p_hwfn; + qlnx_host_t *ha; + + p_hwfn = arg; + + ha = (qlnx_host_t *)p_hwfn->p_dev; + + ha->sp_interrupts++; + + QL_DPRINT2(ha, "enter\n"); + + ecore_int_sp_dpc(p_hwfn); + + QL_DPRINT2(ha, "exit\n"); + + return; +} + +/***************************************************************************** + * Support Functions for DMA'able Memory + *****************************************************************************/ + +static void +qlnx_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + *((bus_addr_t *)arg) = 0; + + if (error) { + printf("%s: bus_dmamap_load failed (%d)\n", __func__, error); + return; + } + + *((bus_addr_t *)arg) = segs[0].ds_addr; + + return; +} + +static int +qlnx_alloc_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf) +{ + int ret = 0; + bus_addr_t b_addr; + + ret = bus_dma_tag_create( + ha->parent_tag,/* parent */ + dma_buf->alignment, + ((bus_size_t)(1ULL << 32)),/* boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + dma_buf->size, /* maxsize */ + 1, /* nsegments */ + dma_buf->size, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &dma_buf->dma_tag); + + if (ret) { + QL_DPRINT1(ha, "could not create dma tag\n"); + goto qlnx_alloc_dmabuf_exit; + } + ret = bus_dmamem_alloc(dma_buf->dma_tag, + (void **)&dma_buf->dma_b, + (BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT), + &dma_buf->dma_map); + if (ret) { + bus_dma_tag_destroy(dma_buf->dma_tag); + QL_DPRINT1(ha, "bus_dmamem_alloc failed\n"); + goto qlnx_alloc_dmabuf_exit; + } + + ret = bus_dmamap_load(dma_buf->dma_tag, + dma_buf->dma_map, + dma_buf->dma_b, + dma_buf->size, + qlnx_dmamap_callback, + &b_addr, BUS_DMA_NOWAIT); + + if (ret || !b_addr) { + bus_dma_tag_destroy(dma_buf->dma_tag); + bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, + dma_buf->dma_map); + ret = -1; + goto qlnx_alloc_dmabuf_exit; + } + + dma_buf->dma_addr = b_addr; + +qlnx_alloc_dmabuf_exit: + + return ret; +} + +static void +qlnx_free_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf) +{ + bus_dmamap_unload(dma_buf->dma_tag, dma_buf->dma_map); + bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map); + bus_dma_tag_destroy(dma_buf->dma_tag); + return; +} + +void * +qlnx_dma_alloc_coherent(void *ecore_dev, bus_addr_t *phys, uint32_t size) +{ + qlnx_dma_t dma_buf; + qlnx_dma_t *dma_p; + qlnx_host_t *ha __unused; + + ha = (qlnx_host_t *)ecore_dev; + + size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); + + memset(&dma_buf, 0, sizeof (qlnx_dma_t)); + + dma_buf.size = size + PAGE_SIZE; + dma_buf.alignment = 8; + + if (qlnx_alloc_dmabuf((qlnx_host_t *)ecore_dev, &dma_buf) != 0) + return (NULL); + bzero((uint8_t *)dma_buf.dma_b, dma_buf.size); + + *phys = dma_buf.dma_addr; + + dma_p = (qlnx_dma_t *)((uint8_t *)dma_buf.dma_b + size); + + memcpy(dma_p, &dma_buf, sizeof(qlnx_dma_t)); + + QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n", + (void *)dma_buf.dma_map, (void *)dma_buf.dma_tag, + dma_buf.dma_b, (void *)dma_buf.dma_addr, size); + + return (dma_buf.dma_b); +} + +void +qlnx_dma_free_coherent(void *ecore_dev, void *v_addr, bus_addr_t phys, + uint32_t size) +{ + qlnx_dma_t dma_buf, *dma_p; + qlnx_host_t *ha; + + ha = (qlnx_host_t *)ecore_dev; + + if (v_addr == NULL) + return; + + size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); + + dma_p = (qlnx_dma_t *)((uint8_t *)v_addr + size); + + QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n", + (void *)dma_p->dma_map, (void *)dma_p->dma_tag, + dma_p->dma_b, (void *)dma_p->dma_addr, size); + + dma_buf = *dma_p; + + if (!ha->qlnxr_debug) + qlnx_free_dmabuf((qlnx_host_t *)ecore_dev, &dma_buf); + return; +} + +static int +qlnx_alloc_parent_dma_tag(qlnx_host_t *ha) +{ + int ret; + device_t dev; + + dev = ha->pci_dev; + + /* + * Allocate parent DMA Tag + */ + ret = bus_dma_tag_create( + bus_get_dma_tag(dev), /* parent */ + 1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ + 0, /* nsegments */ + BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &ha->parent_tag); + + if (ret) { + QL_DPRINT1(ha, "could not create parent dma tag\n"); + return (-1); + } + + ha->flags.parent_tag = 1; + + return (0); +} + +static void +qlnx_free_parent_dma_tag(qlnx_host_t *ha) +{ + if (ha->parent_tag != NULL) { + bus_dma_tag_destroy(ha->parent_tag); + ha->parent_tag = NULL; + } + return; +} + +static int +qlnx_alloc_tx_dma_tag(qlnx_host_t *ha) +{ + if (bus_dma_tag_create(NULL, /* parent */ + 1, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + QLNX_MAX_TSO_FRAME_SIZE, /* maxsize */ + QLNX_MAX_SEGMENTS, /* nsegments */ + QLNX_MAX_TX_MBUF_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &ha->tx_tag)) { + QL_DPRINT1(ha, "tx_tag alloc failed\n"); + return (-1); + } + + return (0); +} + +static void +qlnx_free_tx_dma_tag(qlnx_host_t *ha) +{ + if (ha->tx_tag != NULL) { + bus_dma_tag_destroy(ha->tx_tag); + ha->tx_tag = NULL; + } + return; +} + +static int +qlnx_alloc_rx_dma_tag(qlnx_host_t *ha) +{ + if (bus_dma_tag_create(NULL, /* parent */ + 1, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MJUM9BYTES, /* maxsize */ + 1, /* nsegments */ + MJUM9BYTES, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &ha->rx_tag)) { + QL_DPRINT1(ha, " rx_tag alloc failed\n"); + + return (-1); + } + return (0); +} + +static void +qlnx_free_rx_dma_tag(qlnx_host_t *ha) +{ + if (ha->rx_tag != NULL) { + bus_dma_tag_destroy(ha->rx_tag); + ha->rx_tag = NULL; + } + return; +} + +/********************************* + * Exported functions + *********************************/ +uint32_t +qlnx_pci_bus_get_bar_size(void *ecore_dev, uint8_t bar_id) +{ + uint32_t bar_size; + + bar_id = bar_id * 2; + + bar_size = bus_get_resource_count(((qlnx_host_t *)ecore_dev)->pci_dev, + SYS_RES_MEMORY, + PCIR_BAR(bar_id)); + + return (bar_size); +} + +uint32_t +qlnx_pci_read_config_byte(void *ecore_dev, uint32_t pci_reg, uint8_t *reg_value) +{ + *reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev, + pci_reg, 1); + return 0; +} + +uint32_t +qlnx_pci_read_config_word(void *ecore_dev, uint32_t pci_reg, + uint16_t *reg_value) +{ + *reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev, + pci_reg, 2); + return 0; +} + +uint32_t +qlnx_pci_read_config_dword(void *ecore_dev, uint32_t pci_reg, + uint32_t *reg_value) +{ + *reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev, + pci_reg, 4); + return 0; +} + +void +qlnx_pci_write_config_byte(void *ecore_dev, uint32_t pci_reg, uint8_t reg_value) +{ + pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev, + pci_reg, reg_value, 1); + return; +} + +void +qlnx_pci_write_config_word(void *ecore_dev, uint32_t pci_reg, + uint16_t reg_value) +{ + pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev, + pci_reg, reg_value, 2); + return; +} + +void +qlnx_pci_write_config_dword(void *ecore_dev, uint32_t pci_reg, + uint32_t reg_value) +{ + pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev, + pci_reg, reg_value, 4); + return; +} + +int +qlnx_pci_find_capability(void *ecore_dev, int cap) +{ + int reg; + qlnx_host_t *ha; + + ha = ecore_dev; + + if (pci_find_cap(ha->pci_dev, PCIY_EXPRESS, ®) == 0) + return reg; + else { + QL_DPRINT1(ha, "failed\n"); + return 0; + } +} + +int +qlnx_pci_find_ext_capability(void *ecore_dev, int ext_cap) +{ + int reg; + qlnx_host_t *ha; + + ha = ecore_dev; + + if (pci_find_extcap(ha->pci_dev, ext_cap, ®) == 0) + return reg; + else { + QL_DPRINT1(ha, "failed\n"); + return 0; + } +} + +uint32_t +qlnx_reg_rd32(void *hwfn, uint32_t reg_addr) +{ + uint32_t data32; + struct ecore_hwfn *p_hwfn; + + p_hwfn = hwfn; + + data32 = bus_read_4(((qlnx_host_t *)p_hwfn->p_dev)->pci_reg, \ + (bus_size_t)(p_hwfn->reg_offset + reg_addr)); + + return (data32); +} + +void +qlnx_reg_wr32(void *hwfn, uint32_t reg_addr, uint32_t value) +{ + struct ecore_hwfn *p_hwfn = hwfn; + + bus_write_4(((qlnx_host_t *)p_hwfn->p_dev)->pci_reg, \ + (bus_size_t)(p_hwfn->reg_offset + reg_addr), value); + + return; +} + +void +qlnx_reg_wr16(void *hwfn, uint32_t reg_addr, uint16_t value) +{ + struct ecore_hwfn *p_hwfn = hwfn; + + bus_write_2(((qlnx_host_t *)p_hwfn->p_dev)->pci_reg, \ + (bus_size_t)(p_hwfn->reg_offset + reg_addr), value); + return; +} + +void +qlnx_dbell_wr32_db(void *hwfn, void *reg_addr, uint32_t value) +{ + struct ecore_dev *cdev; + struct ecore_hwfn *p_hwfn; + uint32_t offset; + + p_hwfn = hwfn; + + cdev = p_hwfn->p_dev; + + offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)(p_hwfn->doorbells)); + bus_write_4(((qlnx_host_t *)cdev)->pci_dbells, offset, value); + + return; +} + +void +qlnx_dbell_wr32(void *hwfn, uint32_t reg_addr, uint32_t value) +{ + struct ecore_hwfn *p_hwfn = hwfn; + + bus_write_4(((qlnx_host_t *)p_hwfn->p_dev)->pci_dbells, \ + (bus_size_t)(p_hwfn->db_offset + reg_addr), value); + + return; +} + +uint32_t +qlnx_direct_reg_rd32(void *p_hwfn, uint32_t *reg_addr) +{ + uint32_t data32; + bus_size_t offset; + struct ecore_dev *cdev; + + cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev; + offset = (bus_size_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview)); + + data32 = bus_read_4(((qlnx_host_t *)cdev)->pci_reg, offset); + + return (data32); +} + +void +qlnx_direct_reg_wr32(void *p_hwfn, void *reg_addr, uint32_t value) +{ + bus_size_t offset; + struct ecore_dev *cdev; + + cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev; + offset = (bus_size_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview)); + + bus_write_4(((qlnx_host_t *)cdev)->pci_reg, offset, value); + + return; +} + +void +qlnx_direct_reg_wr64(void *p_hwfn, void *reg_addr, uint64_t value) +{ + bus_size_t offset; + struct ecore_dev *cdev; + + cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev; + offset = (bus_size_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview)); + + bus_write_8(((qlnx_host_t *)cdev)->pci_reg, offset, value); + return; +} + +void * +qlnx_zalloc(uint32_t size) +{ + caddr_t va; + + va = malloc((unsigned long)size, M_QLNXBUF, M_NOWAIT); + bzero(va, size); + return ((void *)va); +} + +void +qlnx_barrier(void *p_dev) +{ + qlnx_host_t *ha; + + ha = ((struct ecore_dev *) p_dev)->ha; + bus_barrier(ha->pci_reg, 0, 0, BUS_SPACE_BARRIER_WRITE); +} + +void +qlnx_link_update(void *p_hwfn) +{ + qlnx_host_t *ha; + int prev_link_state; + + ha = (qlnx_host_t *)((struct ecore_hwfn *)p_hwfn)->p_dev; + + qlnx_fill_link(ha, p_hwfn, &ha->if_link); + + prev_link_state = ha->link_up; + ha->link_up = ha->if_link.link_up; + + if (prev_link_state != ha->link_up) { + if (ha->link_up) { + if_link_state_change(ha->ifp, LINK_STATE_UP); + } else { + if_link_state_change(ha->ifp, LINK_STATE_DOWN); + } + } +#ifndef QLNX_VF +#ifdef CONFIG_ECORE_SRIOV + + if (qlnx_vf_device(ha) != 0) { + if (ha->sriov_initialized) + qlnx_inform_vf_link_state(p_hwfn, ha); + } + +#endif /* #ifdef CONFIG_ECORE_SRIOV */ +#endif /* #ifdef QLNX_VF */ + + return; +} + +static void +__qlnx_osal_vf_fill_acquire_resc_req(struct ecore_hwfn *p_hwfn, + struct ecore_vf_acquire_sw_info *p_sw_info) +{ + p_sw_info->driver_version = (QLNX_VERSION_MAJOR << 24) | + (QLNX_VERSION_MINOR << 16) | + QLNX_VERSION_BUILD; + p_sw_info->os_type = VFPF_ACQUIRE_OS_FREEBSD; + + return; +} + +void +qlnx_osal_vf_fill_acquire_resc_req(void *p_hwfn, void *p_resc_req, + void *p_sw_info) +{ + __qlnx_osal_vf_fill_acquire_resc_req(p_hwfn, p_sw_info); + + return; +} + +void +qlnx_fill_link(qlnx_host_t *ha, struct ecore_hwfn *hwfn, + struct qlnx_link_output *if_link) +{ + struct ecore_mcp_link_params link_params; + struct ecore_mcp_link_state link_state; + uint8_t p_change; + struct ecore_ptt *p_ptt = NULL; + + memset(if_link, 0, sizeof(*if_link)); + memset(&link_params, 0, sizeof(struct ecore_mcp_link_params)); + memset(&link_state, 0, sizeof(struct ecore_mcp_link_state)); + + ha = (qlnx_host_t *)hwfn->p_dev; + + /* Prepare source inputs */ + /* we only deal with physical functions */ + if (qlnx_vf_device(ha) != 0) { + p_ptt = ecore_ptt_acquire(hwfn); + + if (p_ptt == NULL) { + QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); + return; + } + + ecore_mcp_get_media_type(hwfn, p_ptt, &if_link->media_type); + ecore_ptt_release(hwfn, p_ptt); + + memcpy(&link_params, ecore_mcp_get_link_params(hwfn), + sizeof(link_params)); + memcpy(&link_state, ecore_mcp_get_link_state(hwfn), + sizeof(link_state)); + } else { + ecore_mcp_get_media_type(hwfn, NULL, &if_link->media_type); + ecore_vf_read_bulletin(hwfn, &p_change); + ecore_vf_get_link_params(hwfn, &link_params); + ecore_vf_get_link_state(hwfn, &link_state); + } + + /* Set the link parameters to pass to protocol driver */ + if (link_state.link_up) { + if_link->link_up = true; + if_link->speed = link_state.speed; + } + + if_link->supported_caps = QLNX_LINK_CAP_FIBRE; + + if (link_params.speed.autoneg) + if_link->supported_caps |= QLNX_LINK_CAP_Autoneg; + + if (link_params.pause.autoneg || + (link_params.pause.forced_rx && link_params.pause.forced_tx)) + if_link->supported_caps |= QLNX_LINK_CAP_Asym_Pause; + + if (link_params.pause.autoneg || link_params.pause.forced_rx || + link_params.pause.forced_tx) + if_link->supported_caps |= QLNX_LINK_CAP_Pause; + + if (link_params.speed.advertised_speeds & + NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G) + if_link->supported_caps |= QLNX_LINK_CAP_1000baseT_Half | + QLNX_LINK_CAP_1000baseT_Full; + + if (link_params.speed.advertised_speeds & + NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G) + if_link->supported_caps |= QLNX_LINK_CAP_10000baseKR_Full; + + if (link_params.speed.advertised_speeds & + NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G) + if_link->supported_caps |= QLNX_LINK_CAP_25000baseKR_Full; + + if (link_params.speed.advertised_speeds & + NVM_CFG1_PORT_DRV_LINK_SPEED_40G) + if_link->supported_caps |= QLNX_LINK_CAP_40000baseLR4_Full; + + if (link_params.speed.advertised_speeds & + NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G) + if_link->supported_caps |= QLNX_LINK_CAP_50000baseKR2_Full; + + if (link_params.speed.advertised_speeds & + NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G) + if_link->supported_caps |= QLNX_LINK_CAP_100000baseKR4_Full; + + if_link->advertised_caps = if_link->supported_caps; + + if_link->autoneg = link_params.speed.autoneg; + if_link->duplex = QLNX_LINK_DUPLEX; + + /* Link partner capabilities */ + + if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_1G_HD) + if_link->link_partner_caps |= QLNX_LINK_CAP_1000baseT_Half; + + if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_1G_FD) + if_link->link_partner_caps |= QLNX_LINK_CAP_1000baseT_Full; + + if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_10G) + if_link->link_partner_caps |= QLNX_LINK_CAP_10000baseKR_Full; + + if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_25G) + if_link->link_partner_caps |= QLNX_LINK_CAP_25000baseKR_Full; + + if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_40G) + if_link->link_partner_caps |= QLNX_LINK_CAP_40000baseLR4_Full; + + if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_50G) + if_link->link_partner_caps |= QLNX_LINK_CAP_50000baseKR2_Full; + + if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_100G) + if_link->link_partner_caps |= QLNX_LINK_CAP_100000baseKR4_Full; + + if (link_state.an_complete) + if_link->link_partner_caps |= QLNX_LINK_CAP_Autoneg; + + if (link_state.partner_adv_pause) + if_link->link_partner_caps |= QLNX_LINK_CAP_Pause; + + if ((link_state.partner_adv_pause == + ECORE_LINK_PARTNER_ASYMMETRIC_PAUSE) || + (link_state.partner_adv_pause == + ECORE_LINK_PARTNER_BOTH_PAUSE)) + if_link->link_partner_caps |= QLNX_LINK_CAP_Asym_Pause; + + return; +} + +void +qlnx_schedule_recovery(void *p_hwfn) +{ + qlnx_host_t *ha; + + ha = (qlnx_host_t *)((struct ecore_hwfn *)p_hwfn)->p_dev; + + if (qlnx_vf_device(ha) != 0) { + taskqueue_enqueue(ha->err_taskqueue, &ha->err_task); + } + + return; +} + +static int +qlnx_nic_setup(struct ecore_dev *cdev, struct ecore_pf_params *func_params) +{ + int rc, i; + + for (i = 0; i < cdev->num_hwfns; i++) { + struct ecore_hwfn *p_hwfn = &cdev->hwfns[i]; + p_hwfn->pf_params = *func_params; + +#ifdef QLNX_ENABLE_IWARP + if (qlnx_vf_device((qlnx_host_t *)cdev) != 0) { + p_hwfn->using_ll2 = true; + } +#endif /* #ifdef QLNX_ENABLE_IWARP */ + } + + rc = ecore_resc_alloc(cdev); + if (rc) + goto qlnx_nic_setup_exit; + + ecore_resc_setup(cdev); + +qlnx_nic_setup_exit: + + return rc; +} + +static int +qlnx_nic_start(struct ecore_dev *cdev) +{ + int rc; + struct ecore_hw_init_params params; + + bzero(¶ms, sizeof (struct ecore_hw_init_params)); + + params.p_tunn = NULL; + params.b_hw_start = true; + params.int_mode = cdev->int_mode; + params.allow_npar_tx_switch = true; + params.bin_fw_data = NULL; + + rc = ecore_hw_init(cdev, ¶ms); + if (rc) { + ecore_resc_free(cdev); + return rc; + } + + return 0; +} + +static int +qlnx_slowpath_start(qlnx_host_t *ha) +{ + struct ecore_dev *cdev; + struct ecore_pf_params pf_params; + int rc; + + memset(&pf_params, 0, sizeof(struct ecore_pf_params)); + pf_params.eth_pf_params.num_cons = + (ha->num_rss) * (ha->num_tc + 1); + +#ifdef QLNX_ENABLE_IWARP + if (qlnx_vf_device(ha) != 0) { + if(ha->personality == ECORE_PCI_ETH_IWARP) { + device_printf(ha->pci_dev, "setting parameters required by iWARP dev\n"); + pf_params.rdma_pf_params.num_qps = 1024; + pf_params.rdma_pf_params.num_srqs = 1024; + pf_params.rdma_pf_params.gl_pi = ECORE_ROCE_PROTOCOL_INDEX; + pf_params.rdma_pf_params.rdma_protocol = ECORE_RDMA_PROTOCOL_IWARP; + } else if(ha->personality == ECORE_PCI_ETH_ROCE) { + device_printf(ha->pci_dev, "setting parameters required by RoCE dev\n"); + pf_params.rdma_pf_params.num_qps = 8192; + pf_params.rdma_pf_params.num_srqs = 8192; + //pf_params.rdma_pf_params.min_dpis = 0; + pf_params.rdma_pf_params.min_dpis = 8; + pf_params.rdma_pf_params.roce_edpm_mode = 0; + pf_params.rdma_pf_params.gl_pi = ECORE_ROCE_PROTOCOL_INDEX; + pf_params.rdma_pf_params.rdma_protocol = ECORE_RDMA_PROTOCOL_ROCE; + } + } +#endif /* #ifdef QLNX_ENABLE_IWARP */ + + cdev = &ha->cdev; + + rc = qlnx_nic_setup(cdev, &pf_params); + if (rc) + goto qlnx_slowpath_start_exit; + + cdev->int_mode = ECORE_INT_MODE_MSIX; + cdev->int_coalescing_mode = ECORE_COAL_MODE_ENABLE; + +#ifdef QLNX_MAX_COALESCE + cdev->rx_coalesce_usecs = 255; + cdev->tx_coalesce_usecs = 255; +#endif + + rc = qlnx_nic_start(cdev); + + ha->rx_coalesce_usecs = cdev->rx_coalesce_usecs; + ha->tx_coalesce_usecs = cdev->tx_coalesce_usecs; + +#ifdef QLNX_USER_LLDP + (void)qlnx_set_lldp_tlvx(ha, NULL); +#endif /* #ifdef QLNX_USER_LLDP */ + +qlnx_slowpath_start_exit: + + return (rc); +} + +static int +qlnx_slowpath_stop(qlnx_host_t *ha) +{ + struct ecore_dev *cdev; + device_t dev = ha->pci_dev; + int i; + + cdev = &ha->cdev; + + ecore_hw_stop(cdev); + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + if (ha->sp_handle[i]) + (void)bus_teardown_intr(dev, ha->sp_irq[i], + ha->sp_handle[i]); + + ha->sp_handle[i] = NULL; + + if (ha->sp_irq[i]) + (void) bus_release_resource(dev, SYS_RES_IRQ, + ha->sp_irq_rid[i], ha->sp_irq[i]); + ha->sp_irq[i] = NULL; + } + + ecore_resc_free(cdev); + + return 0; +} + +static void +qlnx_set_id(struct ecore_dev *cdev, char name[NAME_SIZE], + char ver_str[VER_SIZE]) +{ + int i; + + memcpy(cdev->name, name, NAME_SIZE); + + for_each_hwfn(cdev, i) { + snprintf(cdev->hwfns[i].name, NAME_SIZE, "%s-%d", name, i); + } + + cdev->drv_type = DRV_ID_DRV_TYPE_FREEBSD; + + return ; +} + +void +qlnx_get_protocol_stats(void *cdev, int proto_type, void *proto_stats) +{ + enum ecore_mcp_protocol_type type; + union ecore_mcp_protocol_stats *stats; + struct ecore_eth_stats eth_stats; + qlnx_host_t *ha; + + ha = cdev; + stats = proto_stats; + type = proto_type; + + switch (type) { + case ECORE_MCP_LAN_STATS: + ecore_get_vport_stats((struct ecore_dev *)cdev, ð_stats); + stats->lan_stats.ucast_rx_pkts = eth_stats.common.rx_ucast_pkts; + stats->lan_stats.ucast_tx_pkts = eth_stats.common.tx_ucast_pkts; + stats->lan_stats.fcs_err = -1; + break; + + default: + ha->err_get_proto_invalid_type++; + + QL_DPRINT1(ha, "invalid protocol type 0x%x\n", type); + break; + } + return; +} + +static int +qlnx_get_mfw_version(qlnx_host_t *ha, uint32_t *mfw_ver) +{ + struct ecore_hwfn *p_hwfn; + struct ecore_ptt *p_ptt; + + p_hwfn = &ha->cdev.hwfns[0]; + p_ptt = ecore_ptt_acquire(p_hwfn); + + if (p_ptt == NULL) { + QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); + return (-1); + } + ecore_mcp_get_mfw_ver(p_hwfn, p_ptt, mfw_ver, NULL); + + ecore_ptt_release(p_hwfn, p_ptt); + + return (0); +} + +static int +qlnx_get_flash_size(qlnx_host_t *ha, uint32_t *flash_size) +{ + struct ecore_hwfn *p_hwfn; + struct ecore_ptt *p_ptt; + + p_hwfn = &ha->cdev.hwfns[0]; + p_ptt = ecore_ptt_acquire(p_hwfn); + + if (p_ptt == NULL) { + QL_DPRINT1(ha,"ecore_ptt_acquire failed\n"); + return (-1); + } + ecore_mcp_get_flash_size(p_hwfn, p_ptt, flash_size); + + ecore_ptt_release(p_hwfn, p_ptt); + + return (0); +} + +static int +qlnx_alloc_mem_arrays(qlnx_host_t *ha) +{ + bzero(&ha->txq_array[0], (sizeof(struct qlnx_tx_queue) * QLNX_MAX_RSS)); + bzero(&ha->rxq_array[0], (sizeof(struct qlnx_rx_queue) * QLNX_MAX_RSS)); + bzero(&ha->sb_array[0], (sizeof(struct ecore_sb_info) * QLNX_MAX_RSS)); + + return 0; +} + +static void +qlnx_init_fp(qlnx_host_t *ha) +{ + int rss_id, txq_array_index, tc; + + for (rss_id = 0; rss_id < ha->num_rss; rss_id++) { + struct qlnx_fastpath *fp = &ha->fp_array[rss_id]; + + fp->rss_id = rss_id; + fp->edev = ha; + fp->sb_info = &ha->sb_array[rss_id]; + fp->rxq = &ha->rxq_array[rss_id]; + fp->rxq->rxq_id = rss_id; + + for (tc = 0; tc < ha->num_tc; tc++) { + txq_array_index = tc * ha->num_rss + rss_id; + fp->txq[tc] = &ha->txq_array[txq_array_index]; + fp->txq[tc]->index = txq_array_index; + } + + snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", qlnx_name_str, + rss_id); + + fp->tx_ring_full = 0; + + /* reset all the statistics counters */ + + fp->tx_pkts_processed = 0; + fp->tx_pkts_freed = 0; + fp->tx_pkts_transmitted = 0; + fp->tx_pkts_completed = 0; + +#ifdef QLNX_TRACE_PERF_DATA + fp->tx_pkts_trans_ctx = 0; + fp->tx_pkts_compl_ctx = 0; + fp->tx_pkts_trans_fp = 0; + fp->tx_pkts_compl_fp = 0; + fp->tx_pkts_compl_intr = 0; +#endif + fp->tx_lso_wnd_min_len = 0; + fp->tx_defrag = 0; + fp->tx_nsegs_gt_elem_left = 0; + fp->tx_tso_max_nsegs = 0; + fp->tx_tso_min_nsegs = 0; + fp->err_tx_nsegs_gt_elem_left = 0; + fp->err_tx_dmamap_create = 0; + fp->err_tx_defrag_dmamap_load = 0; + fp->err_tx_non_tso_max_seg = 0; + fp->err_tx_dmamap_load = 0; + fp->err_tx_defrag = 0; + fp->err_tx_free_pkt_null = 0; + fp->err_tx_cons_idx_conflict = 0; + + fp->rx_pkts = 0; + fp->err_m_getcl = 0; + fp->err_m_getjcl = 0; + } + return; +} + +void +qlnx_free_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info) +{ + struct ecore_dev *cdev; + + cdev = &ha->cdev; + + if (sb_info->sb_virt) { + OSAL_DMA_FREE_COHERENT(cdev, ((void *)sb_info->sb_virt), + (sb_info->sb_phys), (sizeof(*sb_info->sb_virt))); + sb_info->sb_virt = NULL; + } +} + +static int +qlnx_sb_init(struct ecore_dev *cdev, struct ecore_sb_info *sb_info, + void *sb_virt_addr, bus_addr_t sb_phy_addr, u16 sb_id) +{ + struct ecore_hwfn *p_hwfn; + int hwfn_index, rc; + u16 rel_sb_id; + + hwfn_index = sb_id % cdev->num_hwfns; + p_hwfn = &cdev->hwfns[hwfn_index]; + rel_sb_id = sb_id / cdev->num_hwfns; + + QL_DPRINT2(((qlnx_host_t *)cdev), + "hwfn_index = %d p_hwfn = %p sb_id = 0x%x rel_sb_id = 0x%x \ + sb_info = %p sb_virt_addr = %p sb_phy_addr = %p\n", + hwfn_index, p_hwfn, sb_id, rel_sb_id, sb_info, + sb_virt_addr, (void *)sb_phy_addr); + + rc = ecore_int_sb_init(p_hwfn, p_hwfn->p_main_ptt, sb_info, + sb_virt_addr, sb_phy_addr, rel_sb_id); + + return rc; +} + +/* This function allocates fast-path status block memory */ +int +qlnx_alloc_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info, u16 sb_id) +{ + struct status_block_e4 *sb_virt; + bus_addr_t sb_phys; + int rc; + uint32_t size; + struct ecore_dev *cdev; + + cdev = &ha->cdev; + + size = sizeof(*sb_virt); + sb_virt = OSAL_DMA_ALLOC_COHERENT(cdev, (&sb_phys), size); + + if (!sb_virt) { + QL_DPRINT1(ha, "Status block allocation failed\n"); + return -ENOMEM; + } + + rc = qlnx_sb_init(cdev, sb_info, sb_virt, sb_phys, sb_id); + if (rc) { + OSAL_DMA_FREE_COHERENT(cdev, sb_virt, sb_phys, size); + } + + return rc; +} + +static void +qlnx_free_rx_buffers(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) +{ + int i; + struct sw_rx_data *rx_buf; + + for (i = 0; i < rxq->num_rx_buffers; i++) { + rx_buf = &rxq->sw_rx_ring[i]; + + if (rx_buf->data != NULL) { + if (rx_buf->map != NULL) { + bus_dmamap_unload(ha->rx_tag, rx_buf->map); + bus_dmamap_destroy(ha->rx_tag, rx_buf->map); + rx_buf->map = NULL; + } + m_freem(rx_buf->data); + rx_buf->data = NULL; + } + } + return; +} + +static void +qlnx_free_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) +{ + struct ecore_dev *cdev; + int i; + + cdev = &ha->cdev; + + qlnx_free_rx_buffers(ha, rxq); + + for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) { + qlnx_free_tpa_mbuf(ha, &rxq->tpa_info[i]); + if (rxq->tpa_info[i].mpf != NULL) + m_freem(rxq->tpa_info[i].mpf); + } + + bzero((void *)&rxq->sw_rx_ring[0], + (sizeof (struct sw_rx_data) * RX_RING_SIZE)); + + /* Free the real RQ ring used by FW */ + if (rxq->rx_bd_ring.p_virt_addr) { + ecore_chain_free(cdev, &rxq->rx_bd_ring); + rxq->rx_bd_ring.p_virt_addr = NULL; + } + + /* Free the real completion ring used by FW */ + if (rxq->rx_comp_ring.p_virt_addr && + rxq->rx_comp_ring.pbl_sp.p_virt_table) { + ecore_chain_free(cdev, &rxq->rx_comp_ring); + rxq->rx_comp_ring.p_virt_addr = NULL; + rxq->rx_comp_ring.pbl_sp.p_virt_table = NULL; + } + +#ifdef QLNX_SOFT_LRO + { + struct lro_ctrl *lro; + + lro = &rxq->lro; + tcp_lro_free(lro); + } +#endif /* #ifdef QLNX_SOFT_LRO */ + + return; +} + +static int +qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) +{ + register struct mbuf *mp; + uint16_t rx_buf_size; + struct sw_rx_data *sw_rx_data; + struct eth_rx_bd *rx_bd; + dma_addr_t dma_addr; + bus_dmamap_t map; + bus_dma_segment_t segs[1]; + int nsegs; + int ret; + + rx_buf_size = rxq->rx_buf_size; + + mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size); + + if (mp == NULL) { + QL_DPRINT1(ha, "Failed to allocate Rx data\n"); + return -ENOMEM; + } + + mp->m_len = mp->m_pkthdr.len = rx_buf_size; + + map = (bus_dmamap_t)0; + + ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, map, mp, segs, &nsegs, + BUS_DMA_NOWAIT); + dma_addr = segs[0].ds_addr; + + if (ret || !dma_addr || (nsegs != 1)) { + m_freem(mp); + QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n", + ret, (long long unsigned int)dma_addr, nsegs); + return -ENOMEM; + } + + sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod]; + sw_rx_data->data = mp; + sw_rx_data->dma_addr = dma_addr; + sw_rx_data->map = map; + + /* Advance PROD and get BD pointer */ + rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring); + rx_bd->addr.hi = htole32(U64_HI(dma_addr)); + rx_bd->addr.lo = htole32(U64_LO(dma_addr)); + bus_dmamap_sync(ha->rx_tag, map, BUS_DMASYNC_PREREAD); + + rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1); + + return 0; +} + +static int +qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size, + struct qlnx_agg_info *tpa) +{ + struct mbuf *mp; + dma_addr_t dma_addr; + bus_dmamap_t map; + bus_dma_segment_t segs[1]; + int nsegs; + int ret; + struct sw_rx_data *rx_buf; + + mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size); + + if (mp == NULL) { + QL_DPRINT1(ha, "Failed to allocate Rx data\n"); + return -ENOMEM; + } + + mp->m_len = mp->m_pkthdr.len = rx_buf_size; + + map = (bus_dmamap_t)0; + + ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, map, mp, segs, &nsegs, + BUS_DMA_NOWAIT); + dma_addr = segs[0].ds_addr; + + if (ret || !dma_addr || (nsegs != 1)) { + m_freem(mp); + QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n", + ret, (long long unsigned int)dma_addr, nsegs); + return -ENOMEM; + } + + rx_buf = &tpa->rx_buf; + + memset(rx_buf, 0, sizeof (struct sw_rx_data)); + + rx_buf->data = mp; + rx_buf->dma_addr = dma_addr; + rx_buf->map = map; + + bus_dmamap_sync(ha->rx_tag, map, BUS_DMASYNC_PREREAD); + + return (0); +} + +static void +qlnx_free_tpa_mbuf(qlnx_host_t *ha, struct qlnx_agg_info *tpa) +{ + struct sw_rx_data *rx_buf; + + rx_buf = &tpa->rx_buf; + + if (rx_buf->data != NULL) { + if (rx_buf->map != NULL) { + bus_dmamap_unload(ha->rx_tag, rx_buf->map); + bus_dmamap_destroy(ha->rx_tag, rx_buf->map); + rx_buf->map = NULL; + } + m_freem(rx_buf->data); + rx_buf->data = NULL; + } + return; +} + +/* This function allocates all memory needed per Rx queue */ +static int +qlnx_alloc_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) +{ + int i, rc, num_allocated; + struct ecore_dev *cdev; + + cdev = &ha->cdev; + + rxq->num_rx_buffers = RX_RING_SIZE; + + rxq->rx_buf_size = ha->rx_buf_size; + + /* Allocate the parallel driver ring for Rx buffers */ + bzero((void *)&rxq->sw_rx_ring[0], + (sizeof (struct sw_rx_data) * RX_RING_SIZE)); + + /* Allocate FW Rx ring */ + + rc = ecore_chain_alloc(cdev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_NEXT_PTR, + ECORE_CHAIN_CNT_TYPE_U16, + RX_RING_SIZE, + sizeof(struct eth_rx_bd), + &rxq->rx_bd_ring, NULL); + + if (rc) + goto err; + + /* Allocate FW completion ring */ + rc = ecore_chain_alloc(cdev, + ECORE_CHAIN_USE_TO_CONSUME, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U16, + RX_RING_SIZE, + sizeof(union eth_rx_cqe), + &rxq->rx_comp_ring, NULL); + + if (rc) + goto err; + + /* Allocate buffers for the Rx ring */ + + for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) { + rc = qlnx_alloc_tpa_mbuf(ha, rxq->rx_buf_size, + &rxq->tpa_info[i]); + if (rc) + break; + } + + for (i = 0; i < rxq->num_rx_buffers; i++) { + rc = qlnx_alloc_rx_buffer(ha, rxq); + if (rc) + break; + } + num_allocated = i; + if (!num_allocated) { + QL_DPRINT1(ha, "Rx buffers allocation failed\n"); + goto err; + } else if (num_allocated < rxq->num_rx_buffers) { + QL_DPRINT1(ha, "Allocated less buffers than" + " desired (%d allocated)\n", num_allocated); + } + +#ifdef QLNX_SOFT_LRO + + { + struct lro_ctrl *lro; + + lro = &rxq->lro; + + if (tcp_lro_init_args(lro, ifp, 0, rxq->num_rx_buffers)) { + QL_DPRINT1(ha, "tcp_lro_init[%d] failed\n", + rxq->rxq_id); + goto err; + } + + lro->ifp = ha->ifp; + } +#endif /* #ifdef QLNX_SOFT_LRO */ + return 0; + +err: + qlnx_free_mem_rxq(ha, rxq); + return -ENOMEM; +} + +static void +qlnx_free_mem_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq) +{ + struct ecore_dev *cdev; + + cdev = &ha->cdev; + + bzero((void *)&txq->sw_tx_ring[0], + (sizeof (struct sw_tx_bd) * TX_RING_SIZE)); + + /* Free the real RQ ring used by FW */ + if (txq->tx_pbl.p_virt_addr) { + ecore_chain_free(cdev, &txq->tx_pbl); + txq->tx_pbl.p_virt_addr = NULL; + } + return; +} + +/* This function allocates all memory needed per Tx queue */ +static int +qlnx_alloc_mem_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq) +{ + int ret = ECORE_SUCCESS; + union eth_tx_bd_types *p_virt; + struct ecore_dev *cdev; + + cdev = &ha->cdev; + + bzero((void *)&txq->sw_tx_ring[0], + (sizeof (struct sw_tx_bd) * TX_RING_SIZE)); + + /* Allocate the real Tx ring to be used by FW */ + ret = ecore_chain_alloc(cdev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U16, + TX_RING_SIZE, + sizeof(*p_virt), + &txq->tx_pbl, NULL); + + if (ret != ECORE_SUCCESS) { + goto err; + } + + txq->num_tx_buffers = TX_RING_SIZE; + + return 0; + +err: + qlnx_free_mem_txq(ha, fp, txq); + return -ENOMEM; +} + +static void +qlnx_free_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp) +{ + struct mbuf *mp; + if_t ifp = ha->ifp; + + if (mtx_initialized(&fp->tx_mtx)) { + if (fp->tx_br != NULL) { + mtx_lock(&fp->tx_mtx); + + while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) { + fp->tx_pkts_freed++; + m_freem(mp); + } + + mtx_unlock(&fp->tx_mtx); + + buf_ring_free(fp->tx_br, M_DEVBUF); + fp->tx_br = NULL; + } + mtx_destroy(&fp->tx_mtx); + } + return; +} + +static void +qlnx_free_mem_fp(qlnx_host_t *ha, struct qlnx_fastpath *fp) +{ + int tc; + + qlnx_free_mem_sb(ha, fp->sb_info); + + qlnx_free_mem_rxq(ha, fp->rxq); + + for (tc = 0; tc < ha->num_tc; tc++) + qlnx_free_mem_txq(ha, fp, fp->txq[tc]); + + return; +} + +static int +qlnx_alloc_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp) +{ + snprintf(fp->tx_mtx_name, sizeof(fp->tx_mtx_name), + "qlnx%d_fp%d_tx_mq_lock", ha->dev_unit, fp->rss_id); + + mtx_init(&fp->tx_mtx, fp->tx_mtx_name, NULL, MTX_DEF); + + fp->tx_br = buf_ring_alloc(TX_RING_SIZE, M_DEVBUF, + M_NOWAIT, &fp->tx_mtx); + if (fp->tx_br == NULL) { + QL_DPRINT1(ha, "buf_ring_alloc failed for fp[%d, %d]\n", + ha->dev_unit, fp->rss_id); + return -ENOMEM; + } + return 0; +} + +static int +qlnx_alloc_mem_fp(qlnx_host_t *ha, struct qlnx_fastpath *fp) +{ + int rc, tc; + + rc = qlnx_alloc_mem_sb(ha, fp->sb_info, fp->rss_id); + if (rc) + goto err; + + if (ha->rx_jumbo_buf_eq_mtu) { + if (ha->max_frame_size <= MCLBYTES) + ha->rx_buf_size = MCLBYTES; + else if (ha->max_frame_size <= MJUMPAGESIZE) + ha->rx_buf_size = MJUMPAGESIZE; + else if (ha->max_frame_size <= MJUM9BYTES) + ha->rx_buf_size = MJUM9BYTES; + else if (ha->max_frame_size <= MJUM16BYTES) + ha->rx_buf_size = MJUM16BYTES; + } else { + if (ha->max_frame_size <= MCLBYTES) + ha->rx_buf_size = MCLBYTES; + else + ha->rx_buf_size = MJUMPAGESIZE; + } + + rc = qlnx_alloc_mem_rxq(ha, fp->rxq); + if (rc) + goto err; + + for (tc = 0; tc < ha->num_tc; tc++) { + rc = qlnx_alloc_mem_txq(ha, fp, fp->txq[tc]); + if (rc) + goto err; + } + + return 0; + +err: + qlnx_free_mem_fp(ha, fp); + return -ENOMEM; +} + +static void +qlnx_free_mem_load(qlnx_host_t *ha) +{ + int i; + + for (i = 0; i < ha->num_rss; i++) { + struct qlnx_fastpath *fp = &ha->fp_array[i]; + + qlnx_free_mem_fp(ha, fp); + } + return; +} + +static int +qlnx_alloc_mem_load(qlnx_host_t *ha) +{ + int rc = 0, rss_id; + + for (rss_id = 0; rss_id < ha->num_rss; rss_id++) { + struct qlnx_fastpath *fp = &ha->fp_array[rss_id]; + + rc = qlnx_alloc_mem_fp(ha, fp); + if (rc) + break; + } + return (rc); +} + +static int +qlnx_start_vport(struct ecore_dev *cdev, + u8 vport_id, + u16 mtu, + u8 drop_ttl0_flg, + u8 inner_vlan_removal_en_flg, + u8 tx_switching, + u8 hw_lro_enable) +{ + int rc, i; + struct ecore_sp_vport_start_params vport_start_params = { 0 }; + qlnx_host_t *ha __unused; + + ha = (qlnx_host_t *)cdev; + + vport_start_params.remove_inner_vlan = inner_vlan_removal_en_flg; + vport_start_params.tx_switching = 0; + vport_start_params.handle_ptp_pkts = 0; + vport_start_params.only_untagged = 0; + vport_start_params.drop_ttl0 = drop_ttl0_flg; + + vport_start_params.tpa_mode = + (hw_lro_enable ? ECORE_TPA_MODE_RSC : ECORE_TPA_MODE_NONE); + vport_start_params.max_buffers_per_cqe = QLNX_TPA_MAX_AGG_BUFFERS; + + vport_start_params.vport_id = vport_id; + vport_start_params.mtu = mtu; + + QL_DPRINT2(ha, "Setting mtu to %d and VPORT ID = %d\n", mtu, vport_id); + + for_each_hwfn(cdev, i) { + struct ecore_hwfn *p_hwfn = &cdev->hwfns[i]; + + vport_start_params.concrete_fid = p_hwfn->hw_info.concrete_fid; + vport_start_params.opaque_fid = p_hwfn->hw_info.opaque_fid; + + rc = ecore_sp_vport_start(p_hwfn, &vport_start_params); + + if (rc) { + QL_DPRINT1(ha, "Failed to start VPORT V-PORT %d" + " with MTU %d\n" , vport_id, mtu); + return -ENOMEM; + } + + ecore_hw_start_fastpath(p_hwfn); + + QL_DPRINT2(ha, "Started V-PORT %d with MTU %d\n", + vport_id, mtu); + } + return 0; +} + +static int +qlnx_update_vport(struct ecore_dev *cdev, + struct qlnx_update_vport_params *params) +{ + struct ecore_sp_vport_update_params sp_params; + int rc, i, j, fp_index; + struct ecore_hwfn *p_hwfn; + struct ecore_rss_params *rss; + qlnx_host_t *ha = (qlnx_host_t *)cdev; + struct qlnx_fastpath *fp; + + memset(&sp_params, 0, sizeof(sp_params)); + /* Translate protocol params into sp params */ + sp_params.vport_id = params->vport_id; + + sp_params.update_vport_active_rx_flg = + params->update_vport_active_rx_flg; + sp_params.vport_active_rx_flg = params->vport_active_rx_flg; + + sp_params.update_vport_active_tx_flg = + params->update_vport_active_tx_flg; + sp_params.vport_active_tx_flg = params->vport_active_tx_flg; + + sp_params.update_inner_vlan_removal_flg = + params->update_inner_vlan_removal_flg; + sp_params.inner_vlan_removal_flg = params->inner_vlan_removal_flg; + + sp_params.sge_tpa_params = params->sge_tpa_params; + + /* RSS - is a bit tricky, since upper-layer isn't familiar with hwfns. + * We need to re-fix the rss values per engine for CMT. + */ + if (params->rss_params->update_rss_config) + sp_params.rss_params = params->rss_params; + else + sp_params.rss_params = NULL; + + for_each_hwfn(cdev, i) { + p_hwfn = &cdev->hwfns[i]; + + if ((cdev->num_hwfns > 1) && + params->rss_params->update_rss_config && + params->rss_params->rss_enable) { + rss = params->rss_params; + + for (j = 0; j < ECORE_RSS_IND_TABLE_SIZE; j++) { + fp_index = ((cdev->num_hwfns * j) + i) % + ha->num_rss; + + fp = &ha->fp_array[fp_index]; + rss->rss_ind_table[j] = fp->rxq->handle; + } + + for (j = 0; j < ECORE_RSS_IND_TABLE_SIZE;) { + QL_DPRINT3(ha, "%p %p %p %p %p %p %p %p \n", + rss->rss_ind_table[j], + rss->rss_ind_table[j+1], + rss->rss_ind_table[j+2], + rss->rss_ind_table[j+3], + rss->rss_ind_table[j+4], + rss->rss_ind_table[j+5], + rss->rss_ind_table[j+6], + rss->rss_ind_table[j+7]); + j += 8; + } + } + + sp_params.opaque_fid = p_hwfn->hw_info.opaque_fid; + + QL_DPRINT1(ha, "Update sp vport ID=%d\n", params->vport_id); + + rc = ecore_sp_vport_update(p_hwfn, &sp_params, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc) { + QL_DPRINT1(ha, "Failed to update VPORT\n"); + return rc; + } + + QL_DPRINT2(ha, "Updated V-PORT %d: tx_active_flag %d, \ + rx_active_flag %d [tx_update %d], [rx_update %d]\n", + params->vport_id, params->vport_active_tx_flg, + params->vport_active_rx_flg, + params->update_vport_active_tx_flg, + params->update_vport_active_rx_flg); + } + + return 0; +} + +static void +qlnx_reuse_rx_data(struct qlnx_rx_queue *rxq) +{ + struct eth_rx_bd *rx_bd_cons = + ecore_chain_consume(&rxq->rx_bd_ring); + struct eth_rx_bd *rx_bd_prod = + ecore_chain_produce(&rxq->rx_bd_ring); + struct sw_rx_data *sw_rx_data_cons = + &rxq->sw_rx_ring[rxq->sw_rx_cons]; + struct sw_rx_data *sw_rx_data_prod = + &rxq->sw_rx_ring[rxq->sw_rx_prod]; + + sw_rx_data_prod->data = sw_rx_data_cons->data; + memcpy(rx_bd_prod, rx_bd_cons, sizeof(struct eth_rx_bd)); + + rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); + rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1); + + return; +} + +static void +qlnx_update_rx_prod(struct ecore_hwfn *p_hwfn, struct qlnx_rx_queue *rxq) +{ + + uint16_t bd_prod; + uint16_t cqe_prod; + union { + struct eth_rx_prod_data rx_prod_data; + uint32_t data32; + } rx_prods; + + bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring); + cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring); + + /* Update producers */ + rx_prods.rx_prod_data.bd_prod = htole16(bd_prod); + rx_prods.rx_prod_data.cqe_prod = htole16(cqe_prod); + + /* Make sure that the BD and SGE data is updated before updating the + * producers since FW might read the BD/SGE right after the producer + * is updated. + */ + wmb(); + +#ifdef ECORE_CONFIG_DIRECT_HWFN + internal_ram_wr(p_hwfn, rxq->hw_rxq_prod_addr, + sizeof(rx_prods), &rx_prods.data32); +#else + internal_ram_wr(rxq->hw_rxq_prod_addr, + sizeof(rx_prods), &rx_prods.data32); +#endif + + /* mmiowb is needed to synchronize doorbell writes from more than one + * processor. It guarantees that the write arrives to the device before + * the napi lock is released and another qlnx_poll is called (possibly + * on another CPU). Without this barrier, the next doorbell can bypass + * this doorbell. This is applicable to IA64/Altix systems. + */ + wmb(); + + return; +} + +static uint32_t qlnx_hash_key[] = { + ((0x6d << 24)|(0x5a << 16)|(0x56 << 8)|0xda), + ((0x25 << 24)|(0x5b << 16)|(0x0e << 8)|0xc2), + ((0x41 << 24)|(0x67 << 16)|(0x25 << 8)|0x3d), + ((0x43 << 24)|(0xa3 << 16)|(0x8f << 8)|0xb0), + ((0xd0 << 24)|(0xca << 16)|(0x2b << 8)|0xcb), + ((0xae << 24)|(0x7b << 16)|(0x30 << 8)|0xb4), + ((0x77 << 24)|(0xcb << 16)|(0x2d << 8)|0xa3), + ((0x80 << 24)|(0x30 << 16)|(0xf2 << 8)|0x0c), + ((0x6a << 24)|(0x42 << 16)|(0xb7 << 8)|0x3b), + ((0xbe << 24)|(0xac << 16)|(0x01 << 8)|0xfa)}; + +static int +qlnx_start_queues(qlnx_host_t *ha) +{ + int rc, tc, i, vport_id = 0, + drop_ttl0_flg = 1, vlan_removal_en = 1, + tx_switching = 0, hw_lro_enable = 0; + struct ecore_dev *cdev = &ha->cdev; + struct ecore_rss_params *rss_params = &ha->rss_params; + struct qlnx_update_vport_params vport_update_params; + if_t ifp; + struct ecore_hwfn *p_hwfn; + struct ecore_sge_tpa_params tpa_params; + struct ecore_queue_start_common_params qparams; + struct qlnx_fastpath *fp; + + ifp = ha->ifp; + + QL_DPRINT1(ha, "Num RSS = %d\n", ha->num_rss); + + if (!ha->num_rss) { + QL_DPRINT1(ha, "Cannot update V-VPORT as active as there" + " are no Rx queues\n"); + return -EINVAL; + } + +#ifndef QLNX_SOFT_LRO + hw_lro_enable = if_getcapenable(ifp) & IFCAP_LRO; +#endif /* #ifndef QLNX_SOFT_LRO */ + + rc = qlnx_start_vport(cdev, vport_id, if_getmtu(ifp), drop_ttl0_flg, + vlan_removal_en, tx_switching, hw_lro_enable); + + if (rc) { + QL_DPRINT1(ha, "Start V-PORT failed %d\n", rc); + return rc; + } + + QL_DPRINT2(ha, "Start vport ramrod passed, " + "vport_id = %d, MTU = %d, vlan_removal_en = %d\n", + vport_id, (int)(if_getmtu(ifp) + 0xe), vlan_removal_en); + + for_each_rss(i) { + struct ecore_rxq_start_ret_params rx_ret_params; + struct ecore_txq_start_ret_params tx_ret_params; + + fp = &ha->fp_array[i]; + p_hwfn = &cdev->hwfns[(fp->rss_id % cdev->num_hwfns)]; + + bzero(&qparams, sizeof(struct ecore_queue_start_common_params)); + bzero(&rx_ret_params, + sizeof (struct ecore_rxq_start_ret_params)); + + qparams.queue_id = i ; + qparams.vport_id = vport_id; + qparams.stats_id = vport_id; + qparams.p_sb = fp->sb_info; + qparams.sb_idx = RX_PI; + + + rc = ecore_eth_rx_queue_start(p_hwfn, + p_hwfn->hw_info.opaque_fid, + &qparams, + fp->rxq->rx_buf_size, /* bd_max_bytes */ + /* bd_chain_phys_addr */ + fp->rxq->rx_bd_ring.p_phys_addr, + /* cqe_pbl_addr */ + ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring), + /* cqe_pbl_size */ + ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring), + &rx_ret_params); + + if (rc) { + QL_DPRINT1(ha, "Start RXQ #%d failed %d\n", i, rc); + return rc; + } + + fp->rxq->hw_rxq_prod_addr = rx_ret_params.p_prod; + fp->rxq->handle = rx_ret_params.p_handle; + fp->rxq->hw_cons_ptr = + &fp->sb_info->sb_virt->pi_array[RX_PI]; + + qlnx_update_rx_prod(p_hwfn, fp->rxq); + + for (tc = 0; tc < ha->num_tc; tc++) { + struct qlnx_tx_queue *txq = fp->txq[tc]; + + bzero(&qparams, + sizeof(struct ecore_queue_start_common_params)); + bzero(&tx_ret_params, + sizeof (struct ecore_txq_start_ret_params)); + + qparams.queue_id = txq->index / cdev->num_hwfns ; + qparams.vport_id = vport_id; + qparams.stats_id = vport_id; + qparams.p_sb = fp->sb_info; + qparams.sb_idx = TX_PI(tc); + + rc = ecore_eth_tx_queue_start(p_hwfn, + p_hwfn->hw_info.opaque_fid, + &qparams, tc, + /* bd_chain_phys_addr */ + ecore_chain_get_pbl_phys(&txq->tx_pbl), + ecore_chain_get_page_cnt(&txq->tx_pbl), + &tx_ret_params); + + if (rc) { + QL_DPRINT1(ha, "Start TXQ #%d failed %d\n", + txq->index, rc); + return rc; + } + + txq->doorbell_addr = tx_ret_params.p_doorbell; + txq->handle = tx_ret_params.p_handle; + + txq->hw_cons_ptr = + &fp->sb_info->sb_virt->pi_array[TX_PI(tc)]; + SET_FIELD(txq->tx_db.data.params, + ETH_DB_DATA_DEST, DB_DEST_XCM); + SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, + DB_AGG_CMD_SET); + SET_FIELD(txq->tx_db.data.params, + ETH_DB_DATA_AGG_VAL_SEL, + DQ_XCM_ETH_TX_BD_PROD_CMD); + + txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD; + } + } + + /* Fill struct with RSS params */ + if (ha->num_rss > 1) { + rss_params->update_rss_config = 1; + rss_params->rss_enable = 1; + rss_params->update_rss_capabilities = 1; + rss_params->update_rss_ind_table = 1; + rss_params->update_rss_key = 1; + rss_params->rss_caps = ECORE_RSS_IPV4 | ECORE_RSS_IPV6 | + ECORE_RSS_IPV4_TCP | ECORE_RSS_IPV6_TCP; + rss_params->rss_table_size_log = 7; /* 2^7 = 128 */ + + for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) { + fp = &ha->fp_array[(i % ha->num_rss)]; + rss_params->rss_ind_table[i] = fp->rxq->handle; + } + + for (i = 0; i < ECORE_RSS_KEY_SIZE; i++) + rss_params->rss_key[i] = (__le32)qlnx_hash_key[i]; + + } else { + memset(rss_params, 0, sizeof(*rss_params)); + } + + /* Prepare and send the vport enable */ + memset(&vport_update_params, 0, sizeof(vport_update_params)); + vport_update_params.vport_id = vport_id; + vport_update_params.update_vport_active_tx_flg = 1; + vport_update_params.vport_active_tx_flg = 1; + vport_update_params.update_vport_active_rx_flg = 1; + vport_update_params.vport_active_rx_flg = 1; + vport_update_params.rss_params = rss_params; + vport_update_params.update_inner_vlan_removal_flg = 1; + vport_update_params.inner_vlan_removal_flg = 1; + + if (hw_lro_enable) { + memset(&tpa_params, 0, sizeof (struct ecore_sge_tpa_params)); + + tpa_params.max_buffers_per_cqe = QLNX_TPA_MAX_AGG_BUFFERS; + + tpa_params.update_tpa_en_flg = 1; + tpa_params.tpa_ipv4_en_flg = 1; + tpa_params.tpa_ipv6_en_flg = 1; + + tpa_params.update_tpa_param_flg = 1; + tpa_params.tpa_pkt_split_flg = 0; + tpa_params.tpa_hdr_data_split_flg = 0; + tpa_params.tpa_gro_consistent_flg = 0; + tpa_params.tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM; + tpa_params.tpa_max_size = (uint16_t)(-1); + tpa_params.tpa_min_size_to_start = if_getmtu(ifp) / 2; + tpa_params.tpa_min_size_to_cont = if_getmtu(ifp) / 2; + + vport_update_params.sge_tpa_params = &tpa_params; + } + + rc = qlnx_update_vport(cdev, &vport_update_params); + if (rc) { + QL_DPRINT1(ha, "Update V-PORT failed %d\n", rc); + return rc; + } + + return 0; +} + +static int +qlnx_drain_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp, + struct qlnx_tx_queue *txq) +{ + uint16_t hw_bd_cons; + uint16_t ecore_cons_idx; + + QL_DPRINT2(ha, "enter\n"); + + hw_bd_cons = le16toh(*txq->hw_cons_ptr); + + while (hw_bd_cons != + (ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl))) { + mtx_lock(&fp->tx_mtx); + + (void)qlnx_tx_int(ha, fp, txq); + + mtx_unlock(&fp->tx_mtx); + + qlnx_mdelay(__func__, 2); + + hw_bd_cons = le16toh(*txq->hw_cons_ptr); + } + + QL_DPRINT2(ha, "[%d, %d]: done\n", fp->rss_id, txq->index); + + return 0; +} + +static int +qlnx_stop_queues(qlnx_host_t *ha) +{ + struct qlnx_update_vport_params vport_update_params; + struct ecore_dev *cdev; + struct qlnx_fastpath *fp; + int rc, tc, i; + + cdev = &ha->cdev; + + /* Disable the vport */ + + memset(&vport_update_params, 0, sizeof(vport_update_params)); + + vport_update_params.vport_id = 0; + vport_update_params.update_vport_active_tx_flg = 1; + vport_update_params.vport_active_tx_flg = 0; + vport_update_params.update_vport_active_rx_flg = 1; + vport_update_params.vport_active_rx_flg = 0; + vport_update_params.rss_params = &ha->rss_params; + vport_update_params.rss_params->update_rss_config = 0; + vport_update_params.rss_params->rss_enable = 0; + vport_update_params.update_inner_vlan_removal_flg = 0; + vport_update_params.inner_vlan_removal_flg = 0; + + QL_DPRINT1(ha, "Update vport ID= %d\n", vport_update_params.vport_id); + + rc = qlnx_update_vport(cdev, &vport_update_params); + if (rc) { + QL_DPRINT1(ha, "Failed to update vport\n"); + return rc; + } + + /* Flush Tx queues. If needed, request drain from MCP */ + for_each_rss(i) { + fp = &ha->fp_array[i]; + + for (tc = 0; tc < ha->num_tc; tc++) { + struct qlnx_tx_queue *txq = fp->txq[tc]; + + rc = qlnx_drain_txq(ha, fp, txq); + if (rc) + return rc; + } + } + + /* Stop all Queues in reverse order*/ + for (i = ha->num_rss - 1; i >= 0; i--) { + struct ecore_hwfn *p_hwfn = &cdev->hwfns[(i % cdev->num_hwfns)]; + + fp = &ha->fp_array[i]; + + /* Stop the Tx Queue(s)*/ + for (tc = 0; tc < ha->num_tc; tc++) { + int tx_queue_id __unused; + + tx_queue_id = tc * ha->num_rss + i; + rc = ecore_eth_tx_queue_stop(p_hwfn, + fp->txq[tc]->handle); + + if (rc) { + QL_DPRINT1(ha, "Failed to stop TXQ #%d\n", + tx_queue_id); + return rc; + } + } + + /* Stop the Rx Queue*/ + rc = ecore_eth_rx_queue_stop(p_hwfn, fp->rxq->handle, false, + false); + if (rc) { + QL_DPRINT1(ha, "Failed to stop RXQ #%d\n", i); + return rc; + } + } + + /* Stop the vport */ + for_each_hwfn(cdev, i) { + struct ecore_hwfn *p_hwfn = &cdev->hwfns[i]; + + rc = ecore_sp_vport_stop(p_hwfn, p_hwfn->hw_info.opaque_fid, 0); + + if (rc) { + QL_DPRINT1(ha, "Failed to stop VPORT\n"); + return rc; + } + } + + return rc; +} + +static int +qlnx_set_ucast_rx_mac(qlnx_host_t *ha, + enum ecore_filter_opcode opcode, + unsigned char mac[ETH_ALEN]) +{ + struct ecore_filter_ucast ucast; + struct ecore_dev *cdev; + int rc; + + cdev = &ha->cdev; + + bzero(&ucast, sizeof(struct ecore_filter_ucast)); + + ucast.opcode = opcode; + ucast.type = ECORE_FILTER_MAC; + ucast.is_rx_filter = 1; + ucast.vport_to_add_to = 0; + memcpy(&ucast.mac[0], mac, ETH_ALEN); + + rc = ecore_filter_ucast_cmd(cdev, &ucast, ECORE_SPQ_MODE_CB, NULL); + + return (rc); +} + +static int +qlnx_remove_all_ucast_mac(qlnx_host_t *ha) +{ + struct ecore_filter_ucast ucast; + struct ecore_dev *cdev; + int rc; + + bzero(&ucast, sizeof(struct ecore_filter_ucast)); + + ucast.opcode = ECORE_FILTER_REPLACE; + ucast.type = ECORE_FILTER_MAC; + ucast.is_rx_filter = 1; + + cdev = &ha->cdev; + + rc = ecore_filter_ucast_cmd(cdev, &ucast, ECORE_SPQ_MODE_CB, NULL); + + return (rc); +} + +static int +qlnx_remove_all_mcast_mac(qlnx_host_t *ha) +{ + struct ecore_filter_mcast *mcast; + struct ecore_dev *cdev; + int rc, i; + + cdev = &ha->cdev; + + mcast = &ha->ecore_mcast; + bzero(mcast, sizeof(struct ecore_filter_mcast)); + + mcast->opcode = ECORE_FILTER_REMOVE; + + for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { + if (ha->mcast[i].addr[0] || ha->mcast[i].addr[1] || + ha->mcast[i].addr[2] || ha->mcast[i].addr[3] || + ha->mcast[i].addr[4] || ha->mcast[i].addr[5]) { + memcpy(&mcast->mac[i][0], &ha->mcast[i].addr[0], ETH_ALEN); + mcast->num_mc_addrs++; + } + } + mcast = &ha->ecore_mcast; + + rc = ecore_filter_mcast_cmd(cdev, mcast, ECORE_SPQ_MODE_CB, NULL); + + bzero(ha->mcast, (sizeof(qlnx_mcast_t) * QLNX_MAX_NUM_MULTICAST_ADDRS)); + ha->nmcast = 0; + + return (rc); +} + +static int +qlnx_clean_filters(qlnx_host_t *ha) +{ + int rc = 0; + + /* Remove all unicast macs */ + rc = qlnx_remove_all_ucast_mac(ha); + if (rc) + return rc; + + /* Remove all multicast macs */ + rc = qlnx_remove_all_mcast_mac(ha); + if (rc) + return rc; + + rc = qlnx_set_ucast_rx_mac(ha, ECORE_FILTER_FLUSH, ha->primary_mac); + + return (rc); +} + +static int +qlnx_set_rx_accept_filter(qlnx_host_t *ha, uint8_t filter) +{ + struct ecore_filter_accept_flags accept; + int rc = 0; + struct ecore_dev *cdev; + + cdev = &ha->cdev; + + bzero(&accept, sizeof(struct ecore_filter_accept_flags)); + + accept.update_rx_mode_config = 1; + accept.rx_accept_filter = filter; + + accept.update_tx_mode_config = 1; + accept.tx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | + ECORE_ACCEPT_MCAST_MATCHED | ECORE_ACCEPT_BCAST; + + rc = ecore_filter_accept_cmd(cdev, 0, accept, false, false, + ECORE_SPQ_MODE_CB, NULL); + + return (rc); +} + +static int +qlnx_set_rx_mode(qlnx_host_t *ha) +{ + int rc = 0; + uint8_t filter; + const if_t ifp = ha->ifp; + const struct ifaddr *ifa; + struct sockaddr_dl *sdl; + + ifa = if_getifaddr(ifp); + if (if_gettype(ifp) == IFT_ETHER && ifa != NULL && + ifa->ifa_addr != NULL) { + sdl = (struct sockaddr_dl *) ifa->ifa_addr; + + rc = qlnx_set_ucast_rx_mac(ha, ECORE_FILTER_REPLACE, LLADDR(sdl)); + } else { + rc = qlnx_set_ucast_rx_mac(ha, ECORE_FILTER_REPLACE, ha->primary_mac); + } + if (rc) + return rc; + + rc = qlnx_remove_all_mcast_mac(ha); + if (rc) + return rc; + + filter = ECORE_ACCEPT_UCAST_MATCHED | + ECORE_ACCEPT_MCAST_MATCHED | + ECORE_ACCEPT_BCAST; + + if (qlnx_vf_device(ha) == 0 || (if_getflags(ha->ifp) & IFF_PROMISC)) { + filter |= ECORE_ACCEPT_UCAST_UNMATCHED; + filter |= ECORE_ACCEPT_MCAST_UNMATCHED; + } else if (if_getflags(ha->ifp) & IFF_ALLMULTI) { + filter |= ECORE_ACCEPT_MCAST_UNMATCHED; + } + ha->filter = filter; + + rc = qlnx_set_rx_accept_filter(ha, filter); + + return (rc); +} + +static int +qlnx_set_link(qlnx_host_t *ha, bool link_up) +{ + int i, rc = 0; + struct ecore_dev *cdev; + struct ecore_hwfn *hwfn; + struct ecore_ptt *ptt; + + if (qlnx_vf_device(ha) == 0) + return (0); + + cdev = &ha->cdev; + + for_each_hwfn(cdev, i) { + hwfn = &cdev->hwfns[i]; + + ptt = ecore_ptt_acquire(hwfn); + if (!ptt) + return -EBUSY; + + rc = ecore_mcp_set_link(hwfn, ptt, link_up); + + ecore_ptt_release(hwfn, ptt); + + if (rc) + return rc; + } + return (rc); +} + +static uint64_t +qlnx_get_counter(if_t ifp, ift_counter cnt) +{ + qlnx_host_t *ha; + uint64_t count; + + ha = (qlnx_host_t *)if_getsoftc(ifp); + + switch (cnt) { + case IFCOUNTER_IPACKETS: + count = ha->hw_stats.common.rx_ucast_pkts + + ha->hw_stats.common.rx_mcast_pkts + + ha->hw_stats.common.rx_bcast_pkts; + break; + + case IFCOUNTER_IERRORS: + count = ha->hw_stats.common.rx_crc_errors + + ha->hw_stats.common.rx_align_errors + + ha->hw_stats.common.rx_oversize_packets + + ha->hw_stats.common.rx_undersize_packets; + break; + + case IFCOUNTER_OPACKETS: + count = ha->hw_stats.common.tx_ucast_pkts + + ha->hw_stats.common.tx_mcast_pkts + + ha->hw_stats.common.tx_bcast_pkts; + break; + + case IFCOUNTER_OERRORS: + count = ha->hw_stats.common.tx_err_drop_pkts; + break; + + case IFCOUNTER_COLLISIONS: + return (0); + + case IFCOUNTER_IBYTES: + count = ha->hw_stats.common.rx_ucast_bytes + + ha->hw_stats.common.rx_mcast_bytes + + ha->hw_stats.common.rx_bcast_bytes; + break; + + case IFCOUNTER_OBYTES: + count = ha->hw_stats.common.tx_ucast_bytes + + ha->hw_stats.common.tx_mcast_bytes + + ha->hw_stats.common.tx_bcast_bytes; + break; + + case IFCOUNTER_IMCASTS: + count = ha->hw_stats.common.rx_mcast_bytes; + break; + + case IFCOUNTER_OMCASTS: + count = ha->hw_stats.common.tx_mcast_bytes; + break; + + case IFCOUNTER_IQDROPS: + case IFCOUNTER_OQDROPS: + case IFCOUNTER_NOPROTO: + + default: + return (if_get_counter_default(ifp, cnt)); + } + return (count); +} + +static void +qlnx_timer(void *arg) +{ + qlnx_host_t *ha; + + ha = (qlnx_host_t *)arg; + + if (ha->error_recovery) { + ha->error_recovery = 0; + taskqueue_enqueue(ha->err_taskqueue, &ha->err_task); + return; + } + + ecore_get_vport_stats(&ha->cdev, &ha->hw_stats); + + if (ha->storm_stats_gather) + qlnx_sample_storm_stats(ha); + + callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha); + + return; +} + +static int +qlnx_load(qlnx_host_t *ha) +{ + int i; + int rc = 0; + device_t dev; + + dev = ha->pci_dev; + + QL_DPRINT2(ha, "enter\n"); + + rc = qlnx_alloc_mem_arrays(ha); + if (rc) + goto qlnx_load_exit0; + + qlnx_init_fp(ha); + + rc = qlnx_alloc_mem_load(ha); + if (rc) + goto qlnx_load_exit1; + + QL_DPRINT2(ha, "Allocated %d RSS queues on %d TC/s\n", + ha->num_rss, ha->num_tc); + + for (i = 0; i < ha->num_rss; i++) { + if ((rc = bus_setup_intr(dev, ha->irq_vec[i].irq, + (INTR_TYPE_NET | INTR_MPSAFE), + NULL, qlnx_fp_isr, &ha->irq_vec[i], + &ha->irq_vec[i].handle))) { + QL_DPRINT1(ha, "could not setup interrupt\n"); + goto qlnx_load_exit2; + } + + QL_DPRINT2(ha, "rss_id = %d irq_rid %d \ + irq %p handle %p\n", i, + ha->irq_vec[i].irq_rid, + ha->irq_vec[i].irq, ha->irq_vec[i].handle); + + bus_bind_intr(dev, ha->irq_vec[i].irq, (i % mp_ncpus)); + } + + rc = qlnx_start_queues(ha); + if (rc) + goto qlnx_load_exit2; + + QL_DPRINT2(ha, "Start VPORT, RXQ and TXQ succeeded\n"); + + /* Add primary mac and set Rx filters */ + rc = qlnx_set_rx_mode(ha); + if (rc) + goto qlnx_load_exit2; + + /* Ask for link-up using current configuration */ + qlnx_set_link(ha, true); + + if (qlnx_vf_device(ha) == 0) + qlnx_link_update(&ha->cdev.hwfns[0]); + + ha->state = QLNX_STATE_OPEN; + + bzero(&ha->hw_stats, sizeof(struct ecore_eth_stats)); + + if (ha->flags.callout_init) + callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha); + + goto qlnx_load_exit0; + +qlnx_load_exit2: + qlnx_free_mem_load(ha); + +qlnx_load_exit1: + ha->num_rss = 0; + +qlnx_load_exit0: + QL_DPRINT2(ha, "exit [%d]\n", rc); + return rc; +} + +static void +qlnx_drain_soft_lro(qlnx_host_t *ha) +{ +#ifdef QLNX_SOFT_LRO + + if_t ifp; + int i; + + ifp = ha->ifp; + + if (if_getcapenable(ifp) & IFCAP_LRO) { + for (i = 0; i < ha->num_rss; i++) { + struct qlnx_fastpath *fp = &ha->fp_array[i]; + struct lro_ctrl *lro; + + lro = &fp->rxq->lro; + + tcp_lro_flush_all(lro); + } + } + +#endif /* #ifdef QLNX_SOFT_LRO */ + + return; +} + +static void +qlnx_unload(qlnx_host_t *ha) +{ + struct ecore_dev *cdev; + device_t dev; + int i; + + cdev = &ha->cdev; + dev = ha->pci_dev; + + QL_DPRINT2(ha, "enter\n"); + QL_DPRINT1(ha, " QLNX STATE = %d\n",ha->state); + + if (ha->state == QLNX_STATE_OPEN) { + qlnx_set_link(ha, false); + qlnx_clean_filters(ha); + qlnx_stop_queues(ha); + ecore_hw_stop_fastpath(cdev); + + for (i = 0; i < ha->num_rss; i++) { + if (ha->irq_vec[i].handle) { + (void)bus_teardown_intr(dev, + ha->irq_vec[i].irq, + ha->irq_vec[i].handle); + ha->irq_vec[i].handle = NULL; + } + } + + qlnx_drain_fp_taskqueues(ha); + qlnx_drain_soft_lro(ha); + qlnx_free_mem_load(ha); + } + + if (ha->flags.callout_init) + callout_drain(&ha->qlnx_callout); + + qlnx_mdelay(__func__, 1000); + + ha->state = QLNX_STATE_CLOSED; + + QL_DPRINT2(ha, "exit\n"); + return; +} + +static int +qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index) +{ + int rval = -1; + struct ecore_hwfn *p_hwfn; + struct ecore_ptt *p_ptt; + + ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver()); + + p_hwfn = &ha->cdev.hwfns[hwfn_index]; + p_ptt = ecore_ptt_acquire(p_hwfn); + + if (!p_ptt) { + QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); + return (rval); + } + + rval = ecore_dbg_grc_get_dump_buf_size(p_hwfn, p_ptt, num_dwords); + + if (rval == DBG_STATUS_OK) + rval = 0; + else { + QL_DPRINT1(ha, "ecore_dbg_grc_get_dump_buf_size failed" + "[0x%x]\n", rval); + } + + ecore_ptt_release(p_hwfn, p_ptt); + + return (rval); +} + +static int +qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index) +{ + int rval = -1; + struct ecore_hwfn *p_hwfn; + struct ecore_ptt *p_ptt; + + ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver()); + + p_hwfn = &ha->cdev.hwfns[hwfn_index]; + p_ptt = ecore_ptt_acquire(p_hwfn); + + if (!p_ptt) { + QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); + return (rval); + } + + rval = ecore_dbg_idle_chk_get_dump_buf_size(p_hwfn, p_ptt, num_dwords); + + if (rval == DBG_STATUS_OK) + rval = 0; + else { + QL_DPRINT1(ha, "ecore_dbg_idle_chk_get_dump_buf_size failed" + " [0x%x]\n", rval); + } + + ecore_ptt_release(p_hwfn, p_ptt); + + return (rval); +} + +static void +qlnx_sample_storm_stats(qlnx_host_t *ha) +{ + int i, index; + struct ecore_dev *cdev; + qlnx_storm_stats_t *s_stats; + uint32_t reg; + struct ecore_ptt *p_ptt; + struct ecore_hwfn *hwfn; + + if (ha->storm_stats_index >= QLNX_STORM_STATS_SAMPLES_PER_HWFN) { + ha->storm_stats_gather = 0; + return; + } + + cdev = &ha->cdev; + + for_each_hwfn(cdev, i) { + hwfn = &cdev->hwfns[i]; + + p_ptt = ecore_ptt_acquire(hwfn); + if (!p_ptt) + return; + + index = ha->storm_stats_index + + (i * QLNX_STORM_STATS_SAMPLES_PER_HWFN); + + s_stats = &ha->storm_stats[index]; + + /* XSTORM */ + reg = XSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; + s_stats->xstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = XSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; + s_stats->xstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = XSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; + s_stats->xstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = XSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; + s_stats->xstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); + + /* YSTORM */ + reg = YSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; + s_stats->ystorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = YSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; + s_stats->ystorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = YSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; + s_stats->ystorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = YSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; + s_stats->ystorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); + + /* PSTORM */ + reg = PSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; + s_stats->pstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = PSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; + s_stats->pstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = PSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; + s_stats->pstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = PSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; + s_stats->pstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); + + /* TSTORM */ + reg = TSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; + s_stats->tstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = TSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; + s_stats->tstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = TSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; + s_stats->tstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = TSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; + s_stats->tstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); + + /* MSTORM */ + reg = MSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; + s_stats->mstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = MSEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; + s_stats->mstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = MSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; + s_stats->mstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = MSEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; + s_stats->mstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); + + /* USTORM */ + reg = USEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; + s_stats->ustorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = USEM_REG_FAST_MEMORY + + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; + s_stats->ustorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = USEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; + s_stats->ustorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); + + reg = USEM_REG_FAST_MEMORY + + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; + s_stats->ustorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); + + ecore_ptt_release(hwfn, p_ptt); + } + + ha->storm_stats_index++; + + return; +} + +/* + * Name: qlnx_dump_buf8 + * Function: dumps a buffer as bytes + */ +static void +qlnx_dump_buf8(qlnx_host_t *ha, const char *msg, void *dbuf, uint32_t len) +{ + device_t dev; + uint32_t i = 0; + uint8_t *buf; + + dev = ha->pci_dev; + buf = dbuf; + + device_printf(dev, "%s: %s 0x%x dump start\n", __func__, msg, len); + + while (len >= 16) { + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x %02x %02x %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15]); + i += 16; + len -= 16; + buf += 16; + } + switch (len) { + case 1: + device_printf(dev,"0x%08x: %02x\n", i, buf[0]); + break; + case 2: + device_printf(dev,"0x%08x: %02x %02x\n", i, buf[0], buf[1]); + break; + case 3: + device_printf(dev,"0x%08x: %02x %02x %02x\n", + i, buf[0], buf[1], buf[2]); + break; + case 4: + device_printf(dev,"0x%08x: %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3]); + break; + case 5: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4]); + break; + case 6: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]); + break; + case 7: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6]); + break; + case 8: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7]); + break; + case 9: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7], buf[8]); + break; + case 10: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7], buf[8], buf[9]); + break; + case 11: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7], buf[8], buf[9], buf[10]); + break; + case 12: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7], buf[8], buf[9], buf[10], buf[11]); + break; + case 13: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7], buf[8], buf[9], buf[10], buf[11], buf[12]); + break; + case 14: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], + buf[13]); + break; + case 15: + device_printf(dev,"0x%08x:" + " %02x %02x %02x %02x %02x %02x %02x %02x" + " %02x %02x %02x %02x %02x %02x %02x\n", i, + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], + buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], + buf[13], buf[14]); + break; + default: + break; + } + + device_printf(dev, "%s: %s dump end\n", __func__, msg); + + return; +} + +#ifdef CONFIG_ECORE_SRIOV + +static void +__qlnx_osal_iov_vf_cleanup(struct ecore_hwfn *p_hwfn, uint8_t rel_vf_id) +{ + struct ecore_public_vf_info *vf_info; + + vf_info = ecore_iov_get_public_vf_info(p_hwfn, rel_vf_id, false); + + if (!vf_info) + return; + + /* Clear the VF mac */ + memset(vf_info->forced_mac, 0, ETH_ALEN); + + vf_info->forced_vlan = 0; + + return; +} + +void +qlnx_osal_iov_vf_cleanup(void *p_hwfn, uint8_t relative_vf_id) +{ + __qlnx_osal_iov_vf_cleanup(p_hwfn, relative_vf_id); + return; +} + +static int +__qlnx_iov_chk_ucast(struct ecore_hwfn *p_hwfn, int vfid, + struct ecore_filter_ucast *params) +{ + struct ecore_public_vf_info *vf; + + if (!ecore_iov_vf_has_vport_instance(p_hwfn, vfid)) { + QL_DPRINT1(((qlnx_host_t *)p_hwfn->p_dev), + "VF[%d] vport not initialized\n", vfid); + return ECORE_INVAL; + } + + vf = ecore_iov_get_public_vf_info(p_hwfn, vfid, true); + if (!vf) + return -EINVAL; + + /* No real decision to make; Store the configured MAC */ + if (params->type == ECORE_FILTER_MAC || + params->type == ECORE_FILTER_MAC_VLAN) + memcpy(params->mac, vf->forced_mac, ETH_ALEN); + + return 0; +} + +int +qlnx_iov_chk_ucast(void *p_hwfn, int vfid, void *params) +{ + return (__qlnx_iov_chk_ucast(p_hwfn, vfid, params)); +} + +static int +__qlnx_iov_update_vport(struct ecore_hwfn *hwfn, uint8_t vfid, + struct ecore_sp_vport_update_params *params, uint16_t * tlvs) +{ + if (!ecore_iov_vf_has_vport_instance(hwfn, vfid)) { + QL_DPRINT1(((qlnx_host_t *)hwfn->p_dev), + "VF[%d] vport not initialized\n", vfid); + return ECORE_INVAL; + } + + /* Untrusted VFs can't even be trusted to know that fact. + * Simply indicate everything is configured fine, and trace + * configuration 'behind their back'. + */ + if (!(*tlvs & BIT(ECORE_IOV_VP_UPDATE_ACCEPT_PARAM))) + return 0; + + return 0; + +} +int +qlnx_iov_update_vport(void *hwfn, uint8_t vfid, void *params, uint16_t *tlvs) +{ + return(__qlnx_iov_update_vport(hwfn, vfid, params, tlvs)); +} + +static int +qlnx_find_hwfn_index(struct ecore_hwfn *p_hwfn) +{ + int i; + struct ecore_dev *cdev; + + cdev = p_hwfn->p_dev; + + for (i = 0; i < cdev->num_hwfns; i++) { + if (&cdev->hwfns[i] == p_hwfn) + break; + } + + if (i >= cdev->num_hwfns) + return (-1); + + return (i); +} + +static int +__qlnx_pf_vf_msg(struct ecore_hwfn *p_hwfn, uint16_t rel_vf_id) +{ + qlnx_host_t *ha = (qlnx_host_t *)p_hwfn->p_dev; + int i; + + QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p rel_vf_id = %d\n", + ha, p_hwfn->p_dev, p_hwfn, rel_vf_id); + + if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) + return (-1); + + if (ha->sriov_task[i].pf_taskqueue != NULL) { + atomic_testandset_32(&ha->sriov_task[i].flags, + QLNX_SRIOV_TASK_FLAGS_VF_PF_MSG); + + taskqueue_enqueue(ha->sriov_task[i].pf_taskqueue, + &ha->sriov_task[i].pf_task); + } + + return (ECORE_SUCCESS); +} + +int +qlnx_pf_vf_msg(void *p_hwfn, uint16_t relative_vf_id) +{ + return (__qlnx_pf_vf_msg(p_hwfn, relative_vf_id)); +} + +static void +__qlnx_vf_flr_update(struct ecore_hwfn *p_hwfn) +{ + qlnx_host_t *ha = (qlnx_host_t *)p_hwfn->p_dev; + int i; + + if (!ha->sriov_initialized) + return; + + QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p \n", + ha, p_hwfn->p_dev, p_hwfn); + + if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) + return; + + if (ha->sriov_task[i].pf_taskqueue != NULL) { + atomic_testandset_32(&ha->sriov_task[i].flags, + QLNX_SRIOV_TASK_FLAGS_VF_FLR_UPDATE); + + taskqueue_enqueue(ha->sriov_task[i].pf_taskqueue, + &ha->sriov_task[i].pf_task); + } + + return; +} + +void +qlnx_vf_flr_update(void *p_hwfn) +{ + __qlnx_vf_flr_update(p_hwfn); + + return; +} + +#ifndef QLNX_VF + +static void +qlnx_vf_bulleting_update(struct ecore_hwfn *p_hwfn) +{ + qlnx_host_t *ha = (qlnx_host_t *)p_hwfn->p_dev; + int i; + + QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p \n", + ha, p_hwfn->p_dev, p_hwfn); + + if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) + return; + + QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p i = %d\n", + ha, p_hwfn->p_dev, p_hwfn, i); + + if (ha->sriov_task[i].pf_taskqueue != NULL) { + atomic_testandset_32(&ha->sriov_task[i].flags, + QLNX_SRIOV_TASK_FLAGS_BULLETIN_UPDATE); + + taskqueue_enqueue(ha->sriov_task[i].pf_taskqueue, + &ha->sriov_task[i].pf_task); + } +} + +static void +qlnx_initialize_sriov(qlnx_host_t *ha) +{ + device_t dev; + nvlist_t *pf_schema, *vf_schema; + int iov_error; + + dev = ha->pci_dev; + + pf_schema = pci_iov_schema_alloc_node(); + vf_schema = pci_iov_schema_alloc_node(); + + pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); + pci_iov_schema_add_bool(vf_schema, "allow-set-mac", + IOV_SCHEMA_HASDEFAULT, FALSE); + pci_iov_schema_add_bool(vf_schema, "allow-promisc", + IOV_SCHEMA_HASDEFAULT, FALSE); + pci_iov_schema_add_uint16(vf_schema, "num-queues", + IOV_SCHEMA_HASDEFAULT, 1); + + iov_error = pci_iov_attach(dev, pf_schema, vf_schema); + + if (iov_error != 0) { + ha->sriov_initialized = 0; + } else { + device_printf(dev, "SRIOV initialized\n"); + ha->sriov_initialized = 1; + } + + return; +} + +static void +qlnx_sriov_disable(qlnx_host_t *ha) +{ + struct ecore_dev *cdev; + int i, j; + + cdev = &ha->cdev; + + ecore_iov_set_vfs_to_disable(cdev, true); + + for_each_hwfn(cdev, i) { + struct ecore_hwfn *hwfn = &cdev->hwfns[i]; + struct ecore_ptt *ptt = ecore_ptt_acquire(hwfn); + + if (!ptt) { + QL_DPRINT1(ha, "Failed to acquire ptt\n"); + return; + } + /* Clean WFQ db and configure equal weight for all vports */ + ecore_clean_wfq_db(hwfn, ptt); + + ecore_for_each_vf(hwfn, j) { + int k = 0; + + if (!ecore_iov_is_valid_vfid(hwfn, j, true, false)) + continue; + + if (ecore_iov_is_vf_started(hwfn, j)) { + /* Wait until VF is disabled before releasing */ + + for (k = 0; k < 100; k++) { + if (!ecore_iov_is_vf_stopped(hwfn, j)) { + qlnx_mdelay(__func__, 10); + } else + break; + } + } + + if (k < 100) + ecore_iov_release_hw_for_vf(&cdev->hwfns[i], + ptt, j); + else { + QL_DPRINT1(ha, + "Timeout waiting for VF's FLR to end\n"); + } + } + ecore_ptt_release(hwfn, ptt); + } + + ecore_iov_set_vfs_to_disable(cdev, false); + + return; +} + +static void +qlnx_sriov_enable_qid_config(struct ecore_hwfn *hwfn, u16 vfid, + struct ecore_iov_vf_init_params *params) +{ + u16 base, i; + + /* Since we have an equal resource distribution per-VF, and we assume + * PF has acquired the ECORE_PF_L2_QUE first queues, we start setting + * sequentially from there. + */ + base = FEAT_NUM(hwfn, ECORE_PF_L2_QUE) + vfid * params->num_queues; + + params->rel_vf_id = vfid; + + for (i = 0; i < params->num_queues; i++) { + params->req_rx_queue[i] = base + i; + params->req_tx_queue[i] = base + i; + } + + /* PF uses indices 0 for itself; Set vport/RSS afterwards */ + params->vport_id = vfid + 1; + params->rss_eng_id = vfid + 1; + + return; +} + +static int +qlnx_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *nvlist_params) +{ + qlnx_host_t *ha; + struct ecore_dev *cdev; + struct ecore_iov_vf_init_params params; + int ret, j, i; + uint32_t max_vfs; + + if ((ha = device_get_softc(dev)) == NULL) { + device_printf(dev, "%s: cannot get softc\n", __func__); + return (-1); + } + + if (qlnx_create_pf_taskqueues(ha) != 0) + goto qlnx_iov_init_err0; + + cdev = &ha->cdev; + + max_vfs = RESC_NUM(&cdev->hwfns[0], ECORE_VPORT); + + QL_DPRINT2(ha," dev = %p enter num_vfs = %d max_vfs = %d\n", + dev, num_vfs, max_vfs); + + if (num_vfs >= max_vfs) { + QL_DPRINT1(ha, "Can start at most %d VFs\n", + (RESC_NUM(&cdev->hwfns[0], ECORE_VPORT) - 1)); + goto qlnx_iov_init_err0; + } + + ha->vf_attr = malloc(((sizeof (qlnx_vf_attr_t) * num_vfs)), M_QLNXBUF, + M_NOWAIT); + + if (ha->vf_attr == NULL) + goto qlnx_iov_init_err0; + + memset(¶ms, 0, sizeof(params)); + + /* Initialize HW for VF access */ + for_each_hwfn(cdev, j) { + struct ecore_hwfn *hwfn = &cdev->hwfns[j]; + struct ecore_ptt *ptt = ecore_ptt_acquire(hwfn); + + /* Make sure not to use more than 16 queues per VF */ + params.num_queues = min_t(int, + (FEAT_NUM(hwfn, ECORE_VF_L2_QUE) / num_vfs), + 16); + + if (!ptt) { + QL_DPRINT1(ha, "Failed to acquire ptt\n"); + goto qlnx_iov_init_err1; + } + + for (i = 0; i < num_vfs; i++) { + if (!ecore_iov_is_valid_vfid(hwfn, i, false, true)) + continue; + + qlnx_sriov_enable_qid_config(hwfn, i, ¶ms); + + ret = ecore_iov_init_hw_for_vf(hwfn, ptt, ¶ms); + + if (ret) { + QL_DPRINT1(ha, "Failed to enable VF[%d]\n", i); + ecore_ptt_release(hwfn, ptt); + goto qlnx_iov_init_err1; + } + } + + ecore_ptt_release(hwfn, ptt); + } + + ha->num_vfs = num_vfs; + qlnx_inform_vf_link_state(&cdev->hwfns[0], ha); + + QL_DPRINT2(ha," dev = %p exit num_vfs = %d\n", dev, num_vfs); + + return (0); + +qlnx_iov_init_err1: + qlnx_sriov_disable(ha); + +qlnx_iov_init_err0: + qlnx_destroy_pf_taskqueues(ha); + ha->num_vfs = 0; + + return (-1); +} + +static void +qlnx_iov_uninit(device_t dev) +{ + qlnx_host_t *ha; + + if ((ha = device_get_softc(dev)) == NULL) { + device_printf(dev, "%s: cannot get softc\n", __func__); + return; + } + + QL_DPRINT2(ha," dev = %p enter\n", dev); + + qlnx_sriov_disable(ha); + qlnx_destroy_pf_taskqueues(ha); + + free(ha->vf_attr, M_QLNXBUF); + ha->vf_attr = NULL; + + ha->num_vfs = 0; + + QL_DPRINT2(ha," dev = %p exit\n", dev); + return; +} + +static int +qlnx_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) +{ + qlnx_host_t *ha; + qlnx_vf_attr_t *vf_attr; + unsigned const char *mac; + size_t size; + struct ecore_hwfn *p_hwfn; + + if ((ha = device_get_softc(dev)) == NULL) { + device_printf(dev, "%s: cannot get softc\n", __func__); + return (-1); + } + + QL_DPRINT2(ha," dev = %p enter vfnum = %d\n", dev, vfnum); + + if (vfnum > (ha->num_vfs - 1)) { + QL_DPRINT1(ha, " VF[%d] is greater than max allowed [%d]\n", + vfnum, (ha->num_vfs - 1)); + } + + vf_attr = &ha->vf_attr[vfnum]; + + if (nvlist_exists_binary(params, "mac-addr")) { + mac = nvlist_get_binary(params, "mac-addr", &size); + bcopy(mac, vf_attr->mac_addr, ETHER_ADDR_LEN); + device_printf(dev, + "%s: mac_addr = %02x:%02x:%02x:%02x:%02x:%02x\n", + __func__, vf_attr->mac_addr[0], + vf_attr->mac_addr[1], vf_attr->mac_addr[2], + vf_attr->mac_addr[3], vf_attr->mac_addr[4], + vf_attr->mac_addr[5]); + p_hwfn = &ha->cdev.hwfns[0]; + ecore_iov_bulletin_set_mac(p_hwfn, vf_attr->mac_addr, + vfnum); + } + + QL_DPRINT2(ha," dev = %p exit vfnum = %d\n", dev, vfnum); + return (0); +} + +static void +qlnx_handle_vf_msg(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn) +{ + uint64_t events[ECORE_VF_ARRAY_LENGTH]; + struct ecore_ptt *ptt; + int i; + + ptt = ecore_ptt_acquire(p_hwfn); + if (!ptt) { + QL_DPRINT1(ha, "Can't acquire PTT; re-scheduling\n"); + __qlnx_pf_vf_msg(p_hwfn, 0); + return; + } + + ecore_iov_pf_get_pending_events(p_hwfn, events); + + QL_DPRINT2(ha, "Event mask of VF events:" + "0x%" PRIu64 "0x%" PRIu64 " 0x%" PRIu64 "\n", + events[0], events[1], events[2]); + + ecore_for_each_vf(p_hwfn, i) { + /* Skip VFs with no pending messages */ + if (!(events[i / 64] & (1ULL << (i % 64)))) + continue; + + QL_DPRINT2(ha, + "Handling VF message from VF 0x%02x [Abs 0x%02x]\n", + i, p_hwfn->p_dev->p_iov_info->first_vf_in_pf + i); + + /* Copy VF's message to PF's request buffer for that VF */ + if (ecore_iov_copy_vf_msg(p_hwfn, ptt, i)) + continue; + + ecore_iov_process_mbx_req(p_hwfn, ptt, i); + } + + ecore_ptt_release(p_hwfn, ptt); + + return; +} + +static void +qlnx_handle_vf_flr_update(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn) +{ + struct ecore_ptt *ptt; + int ret; + + ptt = ecore_ptt_acquire(p_hwfn); + + if (!ptt) { + QL_DPRINT1(ha, "Can't acquire PTT; re-scheduling\n"); + __qlnx_vf_flr_update(p_hwfn); + return; + } + + ret = ecore_iov_vf_flr_cleanup(p_hwfn, ptt); + + if (ret) { + QL_DPRINT1(ha, "ecore_iov_vf_flr_cleanup failed; re-scheduling\n"); + } + + ecore_ptt_release(p_hwfn, ptt); + + return; +} + +static void +qlnx_handle_bulletin_update(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn) +{ + struct ecore_ptt *ptt; + int i; + + ptt = ecore_ptt_acquire(p_hwfn); + + if (!ptt) { + QL_DPRINT1(ha, "Can't acquire PTT; re-scheduling\n"); + qlnx_vf_bulleting_update(p_hwfn); + return; + } + + ecore_for_each_vf(p_hwfn, i) { + QL_DPRINT1(ha, "ecore_iov_post_vf_bulletin[%p, %d]\n", + p_hwfn, i); + ecore_iov_post_vf_bulletin(p_hwfn, i, ptt); + } + + ecore_ptt_release(p_hwfn, ptt); + + return; +} + +static void +qlnx_pf_taskqueue(void *context, int pending) +{ + struct ecore_hwfn *p_hwfn; + qlnx_host_t *ha; + int i; + + p_hwfn = context; + + if (p_hwfn == NULL) + return; + + ha = (qlnx_host_t *)(p_hwfn->p_dev); + + if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) + return; + + if (atomic_testandclear_32(&ha->sriov_task[i].flags, + QLNX_SRIOV_TASK_FLAGS_VF_PF_MSG)) + qlnx_handle_vf_msg(ha, p_hwfn); + + if (atomic_testandclear_32(&ha->sriov_task[i].flags, + QLNX_SRIOV_TASK_FLAGS_VF_FLR_UPDATE)) + qlnx_handle_vf_flr_update(ha, p_hwfn); + + if (atomic_testandclear_32(&ha->sriov_task[i].flags, + QLNX_SRIOV_TASK_FLAGS_BULLETIN_UPDATE)) + qlnx_handle_bulletin_update(ha, p_hwfn); + + return; +} + +static int +qlnx_create_pf_taskqueues(qlnx_host_t *ha) +{ + int i; + uint8_t tq_name[32]; + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i]; + + bzero(tq_name, sizeof (tq_name)); + snprintf(tq_name, sizeof (tq_name), "ql_pf_tq_%d", i); + + TASK_INIT(&ha->sriov_task[i].pf_task, 0, qlnx_pf_taskqueue, p_hwfn); + + ha->sriov_task[i].pf_taskqueue = taskqueue_create(tq_name, M_NOWAIT, + taskqueue_thread_enqueue, + &ha->sriov_task[i].pf_taskqueue); + + if (ha->sriov_task[i].pf_taskqueue == NULL) + return (-1); + + taskqueue_start_threads(&ha->sriov_task[i].pf_taskqueue, 1, + PI_NET, "%s", tq_name); + + QL_DPRINT1(ha, "%p\n", ha->sriov_task[i].pf_taskqueue); + } + + return (0); +} + +static void +qlnx_destroy_pf_taskqueues(qlnx_host_t *ha) +{ + int i; + + for (i = 0; i < ha->cdev.num_hwfns; i++) { + if (ha->sriov_task[i].pf_taskqueue != NULL) { + taskqueue_drain(ha->sriov_task[i].pf_taskqueue, + &ha->sriov_task[i].pf_task); + taskqueue_free(ha->sriov_task[i].pf_taskqueue); + ha->sriov_task[i].pf_taskqueue = NULL; + } + } + return; +} + +static void +qlnx_inform_vf_link_state(struct ecore_hwfn *p_hwfn, qlnx_host_t *ha) +{ + struct ecore_mcp_link_capabilities caps; + struct ecore_mcp_link_params params; + struct ecore_mcp_link_state link; + int i; + + if (!p_hwfn->pf_iov_info) + return; + + memset(¶ms, 0, sizeof(struct ecore_mcp_link_params)); + memset(&link, 0, sizeof(struct ecore_mcp_link_state)); + memset(&caps, 0, sizeof(struct ecore_mcp_link_capabilities)); + + memcpy(&caps, ecore_mcp_get_link_capabilities(p_hwfn), sizeof(caps)); + memcpy(&link, ecore_mcp_get_link_state(p_hwfn), sizeof(link)); + memcpy(¶ms, ecore_mcp_get_link_params(p_hwfn), sizeof(params)); + + QL_DPRINT2(ha, "called\n"); + + /* Update bulletin of all future possible VFs with link configuration */ + for (i = 0; i < p_hwfn->p_dev->p_iov_info->total_vfs; i++) { + /* Modify link according to the VF's configured link state */ + + link.link_up = false; + + if (ha->link_up) { + link.link_up = true; + /* Set speed according to maximum supported by HW. + * that is 40G for regular devices and 100G for CMT + * mode devices. + */ + link.speed = (p_hwfn->p_dev->num_hwfns > 1) ? + 100000 : link.speed; + } + QL_DPRINT2(ha, "link [%d] = %d\n", i, link.link_up); + ecore_iov_set_link(p_hwfn, i, ¶ms, &link, &caps); + } + + qlnx_vf_bulleting_update(p_hwfn); + + return; +} +#endif /* #ifndef QLNX_VF */ +#endif /* #ifdef CONFIG_ECORE_SRIOV */ |