summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--share/man/man4/ena.44
-rw-r--r--sys/contrib/ena-com/ena_com.c471
-rw-r--r--sys/contrib/ena-com/ena_com.h37
-rw-r--r--sys/contrib/ena-com/ena_defs/ena_admin_defs.h152
-rw-r--r--sys/contrib/ena-com/ena_defs/ena_common_defs.h2
-rw-r--r--sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h2
-rw-r--r--sys/contrib/ena-com/ena_defs/ena_gen_info.h6
-rw-r--r--sys/contrib/ena-com/ena_defs/ena_regs_defs.h2
-rw-r--r--sys/contrib/ena-com/ena_eth_com.c129
-rw-r--r--sys/contrib/ena-com/ena_eth_com.h19
-rw-r--r--sys/contrib/ena-com/ena_plat.h77
-rw-r--r--sys/dev/ena/ena.c179
-rw-r--r--sys/dev/ena/ena.h32
-rw-r--r--sys/dev/ena/ena_datapath.c90
-rw-r--r--sys/dev/ena/ena_datapath.h2
-rw-r--r--sys/dev/ena/ena_netmap.c61
-rw-r--r--sys/dev/ena/ena_netmap.h2
-rw-r--r--sys/dev/ena/ena_sysctl.c99
-rw-r--r--sys/dev/ena/ena_sysctl.h2
-rw-r--r--sys/modules/ena/Makefile4
20 files changed, 851 insertions, 521 deletions
diff --git a/share/man/man4/ena.4 b/share/man/man4/ena.4
index 358c2b786bbb..d73413f9a7b0 100644
--- a/share/man/man4/ena.4
+++ b/share/man/man4/ena.4
@@ -1,4 +1,6 @@
-.\" Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
+.\" SPDX-License-Identifier: BSD-2-Clause
+.\"
+.\" Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/ena-com/ena_com.c b/sys/contrib/ena-com/ena_com.c
index dde0c3357f63..8c63c1a03f76 100644
--- a/sys/contrib/ena-com/ena_com.c
+++ b/sys/contrib/ena-com/ena_com.c
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -70,9 +70,9 @@
#define ENA_REGS_ADMIN_INTR_MASK 1
-#define ENA_MIN_POLL_US 100
+#define ENA_MIN_ADMIN_POLL_US 100
-#define ENA_MAX_POLL_US 5000
+#define ENA_MAX_ADMIN_POLL_US 5000
/*****************************************************************************/
/*****************************************************************************/
@@ -106,7 +106,7 @@ static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
dma_addr_t addr)
{
if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
- ena_trc_err("dma address has more bits that the device supports\n");
+ ena_trc_err(ena_dev, "DMA address has more bits that the device supports\n");
return ENA_COM_INVAL;
}
@@ -116,16 +116,17 @@ static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
return 0;
}
-static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
+static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)
{
- struct ena_com_admin_sq *sq = &queue->sq;
- u16 size = ADMIN_SQ_SIZE(queue->q_depth);
+ struct ena_com_dev *ena_dev = admin_queue->ena_dev;
+ struct ena_com_admin_sq *sq = &admin_queue->sq;
+ u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);
- ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, size, sq->entries, sq->dma_addr,
+ ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, sq->entries, sq->dma_addr,
sq->mem_handle);
if (!sq->entries) {
- ena_trc_err("memory allocation failed\n");
+ ena_trc_err(ena_dev, "Memory allocation failed\n");
return ENA_COM_NO_MEM;
}
@@ -138,16 +139,17 @@ static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
return 0;
}
-static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
+static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)
{
- struct ena_com_admin_cq *cq = &queue->cq;
- u16 size = ADMIN_CQ_SIZE(queue->q_depth);
+ struct ena_com_dev *ena_dev = admin_queue->ena_dev;
+ struct ena_com_admin_cq *cq = &admin_queue->cq;
+ u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);
- ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, size, cq->entries, cq->dma_addr,
+ ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, cq->entries, cq->dma_addr,
cq->mem_handle);
if (!cq->entries) {
- ena_trc_err("memory allocation failed\n");
+ ena_trc_err(ena_dev, "Memory allocation failed\n");
return ENA_COM_NO_MEM;
}
@@ -157,22 +159,22 @@ static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
return 0;
}
-static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
+static int ena_com_admin_init_aenq(struct ena_com_dev *ena_dev,
struct ena_aenq_handlers *aenq_handlers)
{
- struct ena_com_aenq *aenq = &dev->aenq;
+ struct ena_com_aenq *aenq = &ena_dev->aenq;
u32 addr_low, addr_high, aenq_caps;
u16 size;
- dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
+ ena_dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
- ENA_MEM_ALLOC_COHERENT(dev->dmadev, size,
+ ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, size,
aenq->entries,
aenq->dma_addr,
aenq->mem_handle);
if (!aenq->entries) {
- ena_trc_err("memory allocation failed\n");
+ ena_trc_err(ena_dev, "Memory allocation failed\n");
return ENA_COM_NO_MEM;
}
@@ -182,18 +184,18 @@ static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
- ENA_REG_WRITE32(dev->bus, addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
- ENA_REG_WRITE32(dev->bus, addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
+ ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
+ ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
aenq_caps = 0;
- aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
+ aenq_caps |= ena_dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
aenq_caps |= (sizeof(struct ena_admin_aenq_entry) <<
ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
- ENA_REG_WRITE32(dev->bus, aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
+ ENA_REG_WRITE32(ena_dev->bus, aenq_caps, ena_dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
if (unlikely(!aenq_handlers)) {
- ena_trc_err("aenq handlers pointer is NULL\n");
+ ena_trc_err(ena_dev, "AENQ handlers pointer is NULL\n");
return ENA_COM_INVAL;
}
@@ -209,31 +211,34 @@ static void comp_ctxt_release(struct ena_com_admin_queue *queue,
ATOMIC32_DEC(&queue->outstanding_cmds);
}
-static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
+static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,
u16 command_id, bool capture)
{
- if (unlikely(command_id >= queue->q_depth)) {
- ena_trc_err("command id is larger than the queue size. cmd_id: %u queue size %d\n",
- command_id, queue->q_depth);
+ if (unlikely(command_id >= admin_queue->q_depth)) {
+ ena_trc_err(admin_queue->ena_dev,
+ "Command id is larger than the queue size. cmd_id: %u queue size %d\n",
+ command_id, admin_queue->q_depth);
return NULL;
}
- if (unlikely(!queue->comp_ctx)) {
- ena_trc_err("Completion context is NULL\n");
+ if (unlikely(!admin_queue->comp_ctx)) {
+ ena_trc_err(admin_queue->ena_dev,
+ "Completion context is NULL\n");
return NULL;
}
- if (unlikely(queue->comp_ctx[command_id].occupied && capture)) {
- ena_trc_err("Completion context is occupied\n");
+ if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {
+ ena_trc_err(admin_queue->ena_dev,
+ "Completion context is occupied\n");
return NULL;
}
if (capture) {
- ATOMIC32_INC(&queue->outstanding_cmds);
- queue->comp_ctx[command_id].occupied = true;
+ ATOMIC32_INC(&admin_queue->outstanding_cmds);
+ admin_queue->comp_ctx[command_id].occupied = true;
}
- return &queue->comp_ctx[command_id];
+ return &admin_queue->comp_ctx[command_id];
}
static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
@@ -254,7 +259,7 @@ static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queu
/* In case of queue FULL */
cnt = (u16)ATOMIC32_READ(&admin_queue->outstanding_cmds);
if (cnt >= admin_queue->q_depth) {
- ena_trc_dbg("admin queue is full.\n");
+ ena_trc_dbg(admin_queue->ena_dev, "Admin queue is full.\n");
admin_queue->stats.out_of_space++;
return ERR_PTR(ENA_COM_NO_SPACE);
}
@@ -296,20 +301,21 @@ static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queu
return comp_ctx;
}
-static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
+static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)
{
- size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
+ struct ena_com_dev *ena_dev = admin_queue->ena_dev;
+ size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);
struct ena_comp_ctx *comp_ctx;
u16 i;
- queue->comp_ctx = ENA_MEM_ALLOC(queue->q_dmadev, size);
- if (unlikely(!queue->comp_ctx)) {
- ena_trc_err("memory allocation failed\n");
+ admin_queue->comp_ctx = ENA_MEM_ALLOC(admin_queue->q_dmadev, size);
+ if (unlikely(!admin_queue->comp_ctx)) {
+ ena_trc_err(ena_dev, "Memory allocation failed\n");
return ENA_COM_NO_MEM;
}
- for (i = 0; i < queue->q_depth; i++) {
- comp_ctx = get_comp_ctxt(queue, i, false);
+ for (i = 0; i < admin_queue->q_depth; i++) {
+ comp_ctx = get_comp_ctxt(admin_queue, i, false);
if (comp_ctx)
ENA_WAIT_EVENT_INIT(comp_ctx->wait_event);
}
@@ -377,7 +383,7 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
}
if (!io_sq->desc_addr.virt_addr) {
- ena_trc_err("memory allocation failed\n");
+ ena_trc_err(ena_dev, "Memory allocation failed\n");
return ENA_COM_NO_MEM;
}
}
@@ -402,7 +408,7 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
io_sq->bounce_buf_ctrl.base_buffer = ENA_MEM_ALLOC(ena_dev->dmadev, size);
if (!io_sq->bounce_buf_ctrl.base_buffer) {
- ena_trc_err("bounce buffer memory allocation failed\n");
+ ena_trc_err(ena_dev, "Bounce buffer memory allocation failed\n");
return ENA_COM_NO_MEM;
}
@@ -449,23 +455,25 @@ static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
io_cq->bus = ena_dev->bus;
- ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev,
- size,
- io_cq->cdesc_addr.virt_addr,
- io_cq->cdesc_addr.phys_addr,
- io_cq->cdesc_addr.mem_handle,
- ctx->numa_node,
- prev_node);
+ ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(ena_dev->dmadev,
+ size,
+ io_cq->cdesc_addr.virt_addr,
+ io_cq->cdesc_addr.phys_addr,
+ io_cq->cdesc_addr.mem_handle,
+ ctx->numa_node,
+ prev_node,
+ ENA_CDESC_RING_SIZE_ALIGNMENT);
if (!io_cq->cdesc_addr.virt_addr) {
- ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
- size,
- io_cq->cdesc_addr.virt_addr,
- io_cq->cdesc_addr.phys_addr,
- io_cq->cdesc_addr.mem_handle);
+ ENA_MEM_ALLOC_COHERENT_ALIGNED(ena_dev->dmadev,
+ size,
+ io_cq->cdesc_addr.virt_addr,
+ io_cq->cdesc_addr.phys_addr,
+ io_cq->cdesc_addr.mem_handle,
+ ENA_CDESC_RING_SIZE_ALIGNMENT);
}
if (!io_cq->cdesc_addr.virt_addr) {
- ena_trc_err("memory allocation failed\n");
+ ena_trc_err(ena_dev, "Memory allocation failed\n");
return ENA_COM_NO_MEM;
}
@@ -486,7 +494,8 @@ static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *a
comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
if (unlikely(!comp_ctx)) {
- ena_trc_err("comp_ctx is NULL. Changing the admin queue running state\n");
+ ena_trc_err(admin_queue->ena_dev,
+ "comp_ctx is NULL. Changing the admin queue running state\n");
admin_queue->running_state = false;
return;
}
@@ -538,10 +547,12 @@ static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_qu
admin_queue->stats.completed_cmd += comp_num;
}
-static int ena_com_comp_status_to_errno(u8 comp_status)
+static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,
+ u8 comp_status)
{
if (unlikely(comp_status != 0))
- ena_trc_err("admin command failed[%u]\n", comp_status);
+ ena_trc_err(admin_queue->ena_dev,
+ "Admin command failed[%u]\n", comp_status);
switch (comp_status) {
case ENA_ADMIN_SUCCESS:
@@ -555,15 +566,17 @@ static int ena_com_comp_status_to_errno(u8 comp_status)
case ENA_ADMIN_ILLEGAL_PARAMETER:
case ENA_ADMIN_UNKNOWN_ERROR:
return ENA_COM_INVAL;
+ case ENA_ADMIN_RESOURCE_BUSY:
+ return ENA_COM_TRY_AGAIN;
}
return ENA_COM_INVAL;
}
-static inline void ena_delay_exponential_backoff_us(u32 exp, u32 delay_us)
+static void ena_delay_exponential_backoff_us(u32 exp, u32 delay_us)
{
- delay_us = ENA_MAX32(ENA_MIN_POLL_US, delay_us);
- delay_us = ENA_MIN32(delay_us * (1 << exp), ENA_MAX_POLL_US);
+ delay_us = ENA_MAX32(ENA_MIN_ADMIN_POLL_US, delay_us);
+ delay_us = ENA_MIN32(delay_us * (1U << exp), ENA_MAX_ADMIN_POLL_US);
ENA_USLEEP(delay_us);
}
@@ -586,7 +599,8 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c
break;
if (ENA_TIME_EXPIRE(timeout)) {
- ena_trc_err("Wait for completion (polling) timeout\n");
+ ena_trc_err(admin_queue->ena_dev,
+ "Wait for completion (polling) timeout\n");
/* ENA didn't have any completion */
ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
admin_queue->stats.no_completion++;
@@ -597,11 +611,12 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c
goto err;
}
- ena_delay_exponential_backoff_us(exp++, admin_queue->ena_dev->ena_min_poll_delay_us);
+ ena_delay_exponential_backoff_us(exp++,
+ admin_queue->ena_dev->ena_min_poll_delay_us);
}
if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
- ena_trc_err("Command was aborted\n");
+ ena_trc_err(admin_queue->ena_dev, "Command was aborted\n");
ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
admin_queue->stats.aborted_cmd++;
ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
@@ -610,15 +625,16 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c
}
ENA_WARN(comp_ctx->status != ENA_CMD_COMPLETED,
- "Invalid comp status %d\n", comp_ctx->status);
+ admin_queue->ena_dev, "Invalid comp status %d\n",
+ comp_ctx->status);
- ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
+ ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);
err:
comp_ctxt_release(admin_queue, comp_ctx);
return ret;
}
-/**
+/*
* Set the LLQ configurations of the firmware
*
* The driver provides only the enabled feature values to the device,
@@ -643,13 +659,9 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev)
cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header;
cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl;
- if (llq_info->disable_meta_caching)
- cmd.u.llq.accel_mode.u.set.enabled_flags |=
- BIT(ENA_ADMIN_DISABLE_META_CACHING);
-
- if (llq_info->max_entries_in_tx_burst)
- cmd.u.llq.accel_mode.u.set.enabled_flags |=
- BIT(ENA_ADMIN_LIMIT_TX_BURST);
+ cmd.u.llq.accel_mode.u.set.enabled_flags =
+ BIT(ENA_ADMIN_DISABLE_META_CACHING) |
+ BIT(ENA_ADMIN_LIMIT_TX_BURST);
ret = ena_com_execute_admin_command(admin_queue,
(struct ena_admin_aq_entry *)&cmd,
@@ -658,7 +670,7 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev)
sizeof(resp));
if (unlikely(ret))
- ena_trc_err("Failed to set LLQ configurations: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to set LLQ configurations: %d\n", ret);
return ret;
}
@@ -668,6 +680,7 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
struct ena_llq_configurations *llq_default_cfg)
{
struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
+ struct ena_admin_accel_mode_get llq_accel_mode_get;
u16 supported_feat;
int rc;
@@ -679,7 +692,7 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
llq_info->header_location_ctrl =
llq_default_cfg->llq_header_location;
} else {
- ena_trc_err("Invalid header location control, supported: 0x%x\n",
+ ena_trc_err(ena_dev, "Invalid header location control, supported: 0x%x\n",
supported_feat);
return -EINVAL;
}
@@ -694,12 +707,12 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
} else if (supported_feat & ENA_ADMIN_SINGLE_DESC_PER_ENTRY) {
llq_info->desc_stride_ctrl = ENA_ADMIN_SINGLE_DESC_PER_ENTRY;
} else {
- ena_trc_err("Invalid desc_stride_ctrl, supported: 0x%x\n",
+ ena_trc_err(ena_dev, "Invalid desc_stride_ctrl, supported: 0x%x\n",
supported_feat);
return -EINVAL;
}
- ena_trc_err("Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
+ ena_trc_err(ena_dev, "Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_stride_ctrl,
supported_feat,
llq_info->desc_stride_ctrl);
@@ -723,11 +736,12 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
llq_info->desc_list_entry_size = 256;
} else {
- ena_trc_err("Invalid entry_size_ctrl, supported: 0x%x\n", supported_feat);
+ ena_trc_err(ena_dev, "Invalid entry_size_ctrl, supported: 0x%x\n",
+ supported_feat);
return -EINVAL;
}
- ena_trc_err("Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
+ ena_trc_err(ena_dev, "Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_ring_entry_size,
supported_feat,
llq_info->desc_list_entry_size);
@@ -736,7 +750,7 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
/* The desc list entry size should be whole multiply of 8
* This requirement comes from __iowrite64_copy()
*/
- ena_trc_err("illegal entry size %d\n",
+ ena_trc_err(ena_dev, "Illegal entry size %d\n",
llq_info->desc_list_entry_size);
return -EINVAL;
}
@@ -760,29 +774,31 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
} else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8) {
llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8;
} else {
- ena_trc_err("Invalid descs_num_before_header, supported: 0x%x\n",
+ ena_trc_err(ena_dev, "Invalid descs_num_before_header, supported: 0x%x\n",
supported_feat);
return -EINVAL;
}
- ena_trc_err("Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
+ ena_trc_err(ena_dev, "Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_num_decs_before_header,
supported_feat,
llq_info->descs_num_before_header);
}
/* Check for accelerated queue supported */
+ llq_accel_mode_get = llq_features->accel_mode.u.get;
+
llq_info->disable_meta_caching =
- llq_features->accel_mode.u.get.supported_flags &
- BIT(ENA_ADMIN_DISABLE_META_CACHING);
+ !!(llq_accel_mode_get.supported_flags &
+ BIT(ENA_ADMIN_DISABLE_META_CACHING));
- if (llq_features->accel_mode.u.get.supported_flags & BIT(ENA_ADMIN_LIMIT_TX_BURST))
+ if (llq_accel_mode_get.supported_flags & BIT(ENA_ADMIN_LIMIT_TX_BURST))
llq_info->max_entries_in_tx_burst =
- llq_features->accel_mode.u.get.max_tx_burst_size /
+ llq_accel_mode_get.max_tx_burst_size /
llq_default_cfg->llq_ring_entry_size_value;
rc = ena_com_set_llq(ena_dev);
if (rc)
- ena_trc_err("Cannot set LLQ configuration: %d\n", rc);
+ ena_trc_err(ena_dev, "Cannot set LLQ configuration: %d\n", rc);
return rc;
}
@@ -808,13 +824,15 @@ static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *com
ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
if (comp_ctx->status == ENA_CMD_COMPLETED) {
- ena_trc_err("The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n",
+ ena_trc_err(admin_queue->ena_dev,
+ "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n",
comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF");
/* Check if fallback to polling is enabled */
if (admin_queue->auto_polling)
admin_queue->polling = true;
} else {
- ena_trc_err("The ena device didn't send a completion for the admin cmd %d status %d\n",
+ ena_trc_err(admin_queue->ena_dev,
+ "The ena device didn't send a completion for the admin cmd %d status %d\n",
comp_ctx->cmd_opcode, comp_ctx->status);
}
/* Check if shifted to polling mode.
@@ -828,7 +846,7 @@ static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *com
}
}
- ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
+ ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);
err:
comp_ctxt_release(admin_queue, comp_ctx);
return ret;
@@ -876,7 +894,7 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
}
if (unlikely(i == timeout)) {
- ena_trc_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
+ ena_trc_err(ena_dev, "Reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
mmio_read->seq_num,
offset,
read_resp->req_id,
@@ -886,7 +904,7 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
}
if (read_resp->reg_off != offset) {
- ena_trc_err("Read failure: wrong offset provided\n");
+ ena_trc_err(ena_dev, "Read failure: wrong offset provided\n");
ret = ENA_MMIO_READ_TIMEOUT;
} else {
ret = read_resp->reg_val;
@@ -945,7 +963,7 @@ static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
sizeof(destroy_resp));
if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
- ena_trc_err("failed to destroy io sq error: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to destroy io sq error: %d\n", ret);
return ret;
}
@@ -1001,7 +1019,7 @@ static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
- ena_trc_err("Reg read timeout occurred\n");
+ ena_trc_err(ena_dev, "Reg read timeout occurred\n");
return ENA_COM_TIMER_EXPIRED;
}
@@ -1041,7 +1059,7 @@ static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
int ret;
if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
- ena_trc_dbg("Feature %d isn't supported\n", feature_id);
+ ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", feature_id);
return ENA_COM_UNSUPPORTED;
}
@@ -1060,7 +1078,7 @@ static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
&get_cmd.control_buffer.address,
control_buf_dma_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
@@ -1077,7 +1095,7 @@ static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
sizeof(*get_resp));
if (unlikely(ret))
- ena_trc_err("Failed to submit get_feature command %d error: %d\n",
+ ena_trc_err(ena_dev, "Failed to submit get_feature command %d error: %d\n",
feature_id, ret);
return ret;
@@ -1108,13 +1126,9 @@ static void ena_com_hash_key_fill_default_key(struct ena_com_dev *ena_dev)
ENA_RSS_FILL_KEY(&hash_key->key, sizeof(hash_key->key));
/* The key buffer is stored in the device in an array of
- * uint32 elements. Therefore the number of elements can be derived
- * by dividing the buffer length by the size of each array element.
- * In current implementation each element is sized at uint32_t
- * so it's actually a division by 4 but if the element size changes,
- * there is no need to rewrite this code.
+ * uint32 elements.
*/
- hash_key->keys_num = sizeof(hash_key->key) / sizeof(hash_key->key[0]);
+ hash_key->key_parts = ENA_ADMIN_RSS_KEY_PARTS;
}
static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
@@ -1187,13 +1201,13 @@ static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
int ret;
ret = ena_com_get_feature(ena_dev, &get_resp,
- ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG, 0);
+ ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG, 0);
if (unlikely(ret))
return ret;
if ((get_resp.u.ind_table.min_size > log_size) ||
(get_resp.u.ind_table.max_size < log_size)) {
- ena_trc_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
+ ena_trc_err(ena_dev, "Indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
1 << log_size,
1 << get_resp.u.ind_table.min_size,
1 << get_resp.u.ind_table.max_size);
@@ -1297,7 +1311,7 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
&create_cmd.sq_ba,
io_sq->desc_addr.phys_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
}
@@ -1308,7 +1322,7 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
(struct ena_admin_acq_entry *)&cmd_completion,
sizeof(cmd_completion));
if (unlikely(ret)) {
- ena_trc_err("Failed to create IO SQ. error: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to create IO SQ. error: %d\n", ret);
return ret;
}
@@ -1326,7 +1340,7 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
cmd_completion.llq_descriptors_offset);
}
- ena_trc_dbg("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
+ ena_trc_dbg(ena_dev, "Created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
return ret;
}
@@ -1360,7 +1374,7 @@ static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
u16 prev_intr_delay_resolution = ena_dev->intr_delay_resolution;
if (unlikely(!intr_delay_resolution)) {
- ena_trc_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
+ ena_trc_err(ena_dev, "Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
}
@@ -1396,23 +1410,25 @@ int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
comp, comp_size);
if (IS_ERR(comp_ctx)) {
if (comp_ctx == ERR_PTR(ENA_COM_NO_DEVICE))
- ena_trc_dbg("Failed to submit command [%ld]\n",
+ ena_trc_dbg(admin_queue->ena_dev,
+ "Failed to submit command [%ld]\n",
PTR_ERR(comp_ctx));
else
- ena_trc_err("Failed to submit command [%ld]\n",
+ ena_trc_err(admin_queue->ena_dev,
+ "Failed to submit command [%ld]\n",
PTR_ERR(comp_ctx));
- return PTR_ERR(comp_ctx);
+ return (int)PTR_ERR(comp_ctx);
}
ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
if (unlikely(ret)) {
if (admin_queue->running_state)
- ena_trc_err("Failed to process command. ret = %d\n",
- ret);
+ ena_trc_err(admin_queue->ena_dev,
+ "Failed to process command. ret = %d\n", ret);
else
- ena_trc_dbg("Failed to process command. ret = %d\n",
- ret);
+ ena_trc_dbg(admin_queue->ena_dev,
+ "Failed to process command. ret = %d\n", ret);
}
return ret;
}
@@ -1441,7 +1457,7 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
&create_cmd.cq_ba,
io_cq->cdesc_addr.phys_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
@@ -1451,7 +1467,7 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
(struct ena_admin_acq_entry *)&cmd_completion,
sizeof(cmd_completion));
if (unlikely(ret)) {
- ena_trc_err("Failed to create IO CQ. error: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to create IO CQ. error: %d\n", ret);
return ret;
}
@@ -1470,7 +1486,7 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
(u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
cmd_completion.numa_node_register_offset);
- ena_trc_dbg("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
+ ena_trc_dbg(ena_dev, "Created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
return ret;
}
@@ -1480,7 +1496,7 @@ int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
struct ena_com_io_cq **io_cq)
{
if (qid >= ENA_TOTAL_NUM_QUEUES) {
- ena_trc_err("Invalid queue number %d but the max is %d\n",
+ ena_trc_err(ena_dev, "Invalid queue number %d but the max is %d\n",
qid, ENA_TOTAL_NUM_QUEUES);
return ENA_COM_INVAL;
}
@@ -1546,7 +1562,7 @@ int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
sizeof(destroy_resp));
if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
- ena_trc_err("Failed to destroy IO CQ. error: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to destroy IO CQ. error: %d\n", ret);
return ret;
}
@@ -1570,7 +1586,7 @@ void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
{
u16 depth = ena_dev->aenq.q_depth;
- ENA_WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
+ ENA_WARN(ena_dev->aenq.head != depth, ena_dev, "Invalid AENQ state\n");
/* Init head_db to mark that all entries in the queue
* are initially available
@@ -1588,12 +1604,12 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG, 0);
if (ret) {
- ena_trc_info("Can't get aenq configuration\n");
+ ena_trc_info(ena_dev, "Can't get aenq configuration\n");
return ret;
}
if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
- ena_trc_warn("Trying to set unsupported aenq events. supported flag: 0x%x asked flag: 0x%x\n",
+ ena_trc_warn(ena_dev, "Trying to set unsupported aenq events. supported flag: 0x%x asked flag: 0x%x\n",
get_resp.u.aenq.supported_groups,
groups_flag);
return ENA_COM_UNSUPPORTED;
@@ -1614,7 +1630,7 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
sizeof(resp));
if (unlikely(ret))
- ena_trc_err("Failed to config AENQ ret: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to config AENQ ret: %d\n", ret);
return ret;
}
@@ -1622,20 +1638,20 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
{
u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
- int width;
+ u32 width;
if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
- ena_trc_err("Reg read timeout occurred\n");
+ ena_trc_err(ena_dev, "Reg read timeout occurred\n");
return ENA_COM_TIMER_EXPIRED;
}
width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
- ena_trc_dbg("ENA dma width: %d\n", width);
+ ena_trc_dbg(ena_dev, "ENA dma width: %d\n", width);
if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
- ena_trc_err("DMA width illegal value: %d\n", width);
+ ena_trc_err(ena_dev, "DMA width illegal value: %d\n", width);
return ENA_COM_INVAL;
}
@@ -1659,16 +1675,16 @@ int ena_com_validate_version(struct ena_com_dev *ena_dev)
if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
(ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
- ena_trc_err("Reg read timeout occurred\n");
+ ena_trc_err(ena_dev, "Reg read timeout occurred\n");
return ENA_COM_TIMER_EXPIRED;
}
- ena_trc_info("ena device version: %d.%d\n",
+ ena_trc_info(ena_dev, "ENA device version: %d.%d\n",
(ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
- ena_trc_info("ena controller version: %d.%d.%d implementation version %d\n",
+ ena_trc_info(ena_dev, "ENA controller version: %d.%d.%d implementation version %d\n",
(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK)
>> ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK)
@@ -1684,13 +1700,29 @@ int ena_com_validate_version(struct ena_com_dev *ena_dev)
/* Validate the ctrl version without the implementation ID */
if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
- ena_trc_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
+ ena_trc_err(ena_dev, "ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
return -1;
}
return 0;
}
+static void
+ena_com_free_ena_admin_queue_comp_ctx(struct ena_com_dev *ena_dev,
+ struct ena_com_admin_queue *admin_queue)
+
+{
+ if (!admin_queue->comp_ctx)
+ return;
+
+ ENA_WAIT_EVENTS_DESTROY(admin_queue);
+ ENA_MEM_FREE(ena_dev->dmadev,
+ admin_queue->comp_ctx,
+ (admin_queue->q_depth * sizeof(struct ena_comp_ctx)));
+
+ admin_queue->comp_ctx = NULL;
+}
+
void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
{
struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
@@ -1699,12 +1731,8 @@ void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
struct ena_com_aenq *aenq = &ena_dev->aenq;
u16 size;
- ENA_WAIT_EVENT_DESTROY(admin_queue->comp_ctx->wait_event);
- if (admin_queue->comp_ctx)
- ENA_MEM_FREE(ena_dev->dmadev,
- admin_queue->comp_ctx,
- (admin_queue->q_depth * sizeof(struct ena_comp_ctx)));
- admin_queue->comp_ctx = NULL;
+ ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);
+
size = ADMIN_SQ_SIZE(admin_queue->q_depth);
if (sq->entries)
ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, sq->entries,
@@ -1820,12 +1848,12 @@ int ena_com_admin_init(struct ena_com_dev *ena_dev,
dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
- ena_trc_err("Reg read timeout occurred\n");
+ ena_trc_err(ena_dev, "Reg read timeout occurred\n");
return ENA_COM_TIMER_EXPIRED;
}
if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
- ena_trc_err("Device isn't ready, abort com init\n");
+ ena_trc_err(ena_dev, "Device isn't ready, abort com init\n");
return ENA_COM_NO_DEVICE;
}
@@ -1903,7 +1931,7 @@ int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
int ret;
if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
- ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n",
+ ena_trc_err(ena_dev, "Qid (%d) is bigger than max num of queues (%d)\n",
ctx->qid, ENA_TOTAL_NUM_QUEUES);
return ENA_COM_INVAL;
}
@@ -1962,7 +1990,7 @@ void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
struct ena_com_io_cq *io_cq;
if (qid >= ENA_TOTAL_NUM_QUEUES) {
- ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n",
+ ena_trc_err(ena_dev, "Qid (%d) is bigger than max num of queues (%d)\n",
qid, ENA_TOTAL_NUM_QUEUES);
return;
}
@@ -1995,6 +2023,7 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
sizeof(get_resp.u.dev_attr));
+
ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
@@ -2061,17 +2090,6 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
else
return rc;
- rc = ena_com_get_feature(ena_dev, &get_resp,
- ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG, 0);
- if (!rc)
- memcpy(&get_feat_ctx->ind_table, &get_resp.u.ind_table,
- sizeof(get_resp.u.ind_table));
- else if (rc == ENA_COM_UNSUPPORTED)
- memset(&get_feat_ctx->ind_table, 0x0,
- sizeof(get_feat_ctx->ind_table));
- else
- return rc;
-
return 0;
}
@@ -2083,10 +2101,10 @@ void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
/* ena_handle_specific_aenq_event:
* return the handler that is relevant to the specific event group
*/
-static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
+static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *ena_dev,
u16 group)
{
- struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers;
+ struct ena_aenq_handlers *aenq_handlers = ena_dev->aenq.aenq_handlers;
if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
return aenq_handlers->handlers[group];
@@ -2098,11 +2116,11 @@ static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
* handles the aenq incoming events.
* pop events from the queue and apply the specific handler
*/
-void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
+void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data)
{
struct ena_admin_aenq_entry *aenq_e;
struct ena_admin_aenq_common_desc *aenq_common;
- struct ena_com_aenq *aenq = &dev->aenq;
+ struct ena_com_aenq *aenq = &ena_dev->aenq;
u64 timestamp;
ena_aenq_handler handler_cb;
u16 masked_head, processed = 0;
@@ -2123,13 +2141,14 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
timestamp = (u64)aenq_common->timestamp_low |
((u64)aenq_common->timestamp_high << 32);
- ena_trc_dbg("AENQ! Group[%x] Syndrom[%x] timestamp: [%" ENA_PRIu64 "s]\n",
+
+ ena_trc_dbg(ena_dev, "AENQ! Group[%x] Syndrome[%x] timestamp: [%" ENA_PRIu64 "s]\n",
aenq_common->group,
- aenq_common->syndrom,
+ aenq_common->syndrome,
timestamp);
/* Handle specific event*/
- handler_cb = ena_com_get_specific_aenq_cb(dev,
+ handler_cb = ena_com_get_specific_aenq_cb(ena_dev,
aenq_common->group);
handler_cb(data, aenq_e); /* call the actual event handler*/
@@ -2154,8 +2173,8 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
/* write the aenq doorbell after all AENQ descriptors were read */
mb();
- ENA_REG_WRITE32_RELAXED(dev->bus, (u32)aenq->head,
- dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
+ ENA_REG_WRITE32_RELAXED(ena_dev->bus, (u32)aenq->head,
+ ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
mmiowb();
}
#ifdef ENA_EXTENDED_STATS
@@ -2191,19 +2210,19 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev,
if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
(cap == ENA_MMIO_READ_TIMEOUT))) {
- ena_trc_err("Reg read32 timeout occurred\n");
+ ena_trc_err(ena_dev, "Reg read32 timeout occurred\n");
return ENA_COM_TIMER_EXPIRED;
}
if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
- ena_trc_err("Device isn't ready, can't reset device\n");
+ ena_trc_err(ena_dev, "Device isn't ready, can't reset device\n");
return ENA_COM_INVAL;
}
timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
if (timeout == 0) {
- ena_trc_err("Invalid timeout value\n");
+ ena_trc_err(ena_dev, "Invalid timeout value\n");
return ENA_COM_INVAL;
}
@@ -2219,7 +2238,7 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev,
rc = wait_for_reset_state(ena_dev, timeout,
ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
if (rc != 0) {
- ena_trc_err("Reset indication didn't turn on\n");
+ ena_trc_err(ena_dev, "Reset indication didn't turn on\n");
return rc;
}
@@ -2227,7 +2246,7 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev,
ENA_REG_WRITE32(ena_dev->bus, 0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
rc = wait_for_reset_state(ena_dev, timeout, 0);
if (rc != 0) {
- ena_trc_err("Reset indication didn't turn off\n");
+ ena_trc_err(ena_dev, "Reset indication didn't turn off\n");
return rc;
}
@@ -2264,7 +2283,22 @@ static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
sizeof(*get_resp));
if (unlikely(ret))
- ena_trc_err("Failed to get stats. error: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to get stats. error: %d\n", ret);
+
+ return ret;
+}
+
+int ena_com_get_eni_stats(struct ena_com_dev *ena_dev,
+ struct ena_admin_eni_stats *stats)
+{
+ struct ena_com_stats_ctx ctx;
+ int ret;
+
+ memset(&ctx, 0x0, sizeof(ctx));
+ ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENI);
+ if (likely(ret == 0))
+ memcpy(stats, &ctx.get_resp.u.eni_stats,
+ sizeof(ctx.get_resp.u.eni_stats));
return ret;
}
@@ -2278,8 +2312,8 @@ int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
memset(&ctx, 0x0, sizeof(ctx));
ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
if (likely(ret == 0))
- memcpy(stats, &ctx.get_resp.basic_stats,
- sizeof(ctx.get_resp.basic_stats));
+ memcpy(stats, &ctx.get_resp.u.basic_stats,
+ sizeof(ctx.get_resp.u.basic_stats));
return ret;
}
@@ -2306,7 +2340,7 @@ int ena_com_get_dev_extended_stats(struct ena_com_dev *ena_dev, char *buff,
&get_cmd->u.control_buffer.address,
phys_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
goto free_ext_stats_mem;
}
get_cmd->u.control_buffer.length = len;
@@ -2337,7 +2371,7 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
int ret;
if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
- ena_trc_dbg("Feature %d isn't supported\n", ENA_ADMIN_MTU);
+ ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", ENA_ADMIN_MTU);
return ENA_COM_UNSUPPORTED;
}
@@ -2347,7 +2381,7 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
cmd.aq_common_descriptor.flags = 0;
cmd.feat_common.feature_id = ENA_ADMIN_MTU;
- cmd.u.mtu.mtu = mtu;
+ cmd.u.mtu.mtu = (u32)mtu;
ret = ena_com_execute_admin_command(admin_queue,
(struct ena_admin_aq_entry *)&cmd,
@@ -2356,7 +2390,7 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
sizeof(resp));
if (unlikely(ret))
- ena_trc_err("Failed to set mtu %d. error: %d\n", mtu, ret);
+ ena_trc_err(ena_dev, "Failed to set mtu %d. error: %d\n", mtu, ret);
return ret;
}
@@ -2370,7 +2404,7 @@ int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
ret = ena_com_get_feature(ena_dev, &resp,
ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0);
if (unlikely(ret)) {
- ena_trc_err("Failed to get offload capabilities %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to get offload capabilities %d\n", ret);
return ret;
}
@@ -2390,7 +2424,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
if (!ena_com_check_supported_feature_id(ena_dev,
ENA_ADMIN_RSS_HASH_FUNCTION)) {
- ena_trc_dbg("Feature %d isn't supported\n",
+ ena_trc_dbg(ena_dev, "Feature %d isn't supported\n",
ENA_ADMIN_RSS_HASH_FUNCTION);
return ENA_COM_UNSUPPORTED;
}
@@ -2402,7 +2436,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
return ret;
if (!(get_resp.u.flow_hash_func.supported_func & BIT(rss->hash_func))) {
- ena_trc_err("Func hash %d isn't supported by device, abort\n",
+ ena_trc_err(ena_dev, "Func hash %d isn't supported by device, abort\n",
rss->hash_func);
return ENA_COM_UNSUPPORTED;
}
@@ -2420,7 +2454,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
&cmd.control_buffer.address,
rss->hash_key_dma_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
@@ -2432,7 +2466,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
(struct ena_admin_acq_entry *)&resp,
sizeof(resp));
if (unlikely(ret)) {
- ena_trc_err("Failed to set hash function %d. error: %d\n",
+ ena_trc_err(ena_dev, "Failed to set hash function %d. error: %d\n",
rss->hash_func, ret);
return ENA_COM_INVAL;
}
@@ -2464,7 +2498,7 @@ int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
return rc;
if (!(BIT(func) & get_resp.u.flow_hash_func.supported_func)) {
- ena_trc_err("Flow hash function %d isn't supported\n", func);
+ ena_trc_err(ena_dev, "Flow hash function %d isn't supported\n", func);
return ENA_COM_UNSUPPORTED;
}
@@ -2472,20 +2506,20 @@ int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
case ENA_ADMIN_TOEPLITZ:
if (key) {
if (key_len != sizeof(hash_key->key)) {
- ena_trc_err("key len (%hu) doesn't equal the supported size (%zu)\n",
+ ena_trc_err(ena_dev, "key len (%hu) doesn't equal the supported size (%zu)\n",
key_len, sizeof(hash_key->key));
return ENA_COM_INVAL;
}
memcpy(hash_key->key, key, key_len);
rss->hash_init_val = init_val;
- hash_key->keys_num = key_len / sizeof(hash_key->key[0]);
+ hash_key->key_parts = key_len / sizeof(hash_key->key[0]);
}
break;
case ENA_ADMIN_CRC32:
rss->hash_init_val = init_val;
break;
default:
- ena_trc_err("Invalid hash function (%d)\n", func);
+ ena_trc_err(ena_dev, "Invalid hash function (%d)\n", func);
return ENA_COM_INVAL;
}
@@ -2533,7 +2567,8 @@ int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key)
ena_dev->rss.hash_key;
if (key)
- memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2);
+ memcpy(key, hash_key->key,
+ (size_t)(hash_key->key_parts) * sizeof(hash_key->key[0]));
return 0;
}
@@ -2570,7 +2605,7 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
if (!ena_com_check_supported_feature_id(ena_dev,
ENA_ADMIN_RSS_HASH_INPUT)) {
- ena_trc_dbg("Feature %d isn't supported\n",
+ ena_trc_dbg(ena_dev, "Feature %d isn't supported\n",
ENA_ADMIN_RSS_HASH_INPUT);
return ENA_COM_UNSUPPORTED;
}
@@ -2589,7 +2624,7 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
&cmd.control_buffer.address,
rss->hash_ctrl_dma_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
cmd.control_buffer.length = sizeof(*hash_ctrl);
@@ -2600,7 +2635,7 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
(struct ena_admin_acq_entry *)&resp,
sizeof(resp));
if (unlikely(ret))
- ena_trc_err("Failed to set hash input. error: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to set hash input. error: %d\n", ret);
return ret;
}
@@ -2650,7 +2685,7 @@ int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
available_fields = hash_ctrl->selected_fields[i].fields &
hash_ctrl->supported_fields[i].fields;
if (available_fields != hash_ctrl->selected_fields[i].fields) {
- ena_trc_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
+ ena_trc_err(ena_dev, "Hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
i, hash_ctrl->supported_fields[i].fields,
hash_ctrl->selected_fields[i].fields);
return ENA_COM_UNSUPPORTED;
@@ -2676,7 +2711,7 @@ int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
int rc;
if (proto >= ENA_ADMIN_RSS_PROTO_NUM) {
- ena_trc_err("Invalid proto num (%u)\n", proto);
+ ena_trc_err(ena_dev, "Invalid proto num (%u)\n", proto);
return ENA_COM_INVAL;
}
@@ -2688,7 +2723,7 @@ int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
/* Make sure all the fields are supported */
supported_fields = hash_ctrl->supported_fields[proto].fields;
if ((hash_fields & supported_fields) != hash_fields) {
- ena_trc_err("proto %d doesn't support the required fields %x. supports only: %x\n",
+ ena_trc_err(ena_dev, "Proto %d doesn't support the required fields %x. supports only: %x\n",
proto, hash_fields, supported_fields);
}
@@ -2728,15 +2763,15 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
int ret;
if (!ena_com_check_supported_feature_id(ena_dev,
- ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
- ena_trc_dbg("Feature %d isn't supported\n",
- ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
+ ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG)) {
+ ena_trc_dbg(ena_dev, "Feature %d isn't supported\n",
+ ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG);
return ENA_COM_UNSUPPORTED;
}
ret = ena_com_ind_tbl_convert_to_device(ena_dev);
if (ret) {
- ena_trc_err("Failed to convert host indirection table to device table\n");
+ ena_trc_err(ena_dev, "Failed to convert host indirection table to device table\n");
return ret;
}
@@ -2745,7 +2780,7 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
cmd.aq_common_descriptor.flags =
ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
- cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG;
+ cmd.feat_common.feature_id = ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG;
cmd.u.ind_table.size = rss->tbl_log_size;
cmd.u.ind_table.inline_index = 0xFFFFFFFF;
@@ -2753,11 +2788,11 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
&cmd.control_buffer.address,
rss->rss_ind_tbl_dma_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
- cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
+ cmd.control_buffer.length = (u32)(1ULL << rss->tbl_log_size) *
sizeof(struct ena_admin_rss_ind_table_entry);
ret = ena_com_execute_admin_command(admin_queue,
@@ -2767,7 +2802,7 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
sizeof(resp));
if (unlikely(ret))
- ena_trc_err("Failed to set indirect table. error: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to set indirect table. error: %d\n", ret);
return ret;
}
@@ -2779,11 +2814,11 @@ int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
u32 tbl_size;
int i, rc;
- tbl_size = (1ULL << rss->tbl_log_size) *
+ tbl_size = (u32)(1ULL << rss->tbl_log_size) *
sizeof(struct ena_admin_rss_ind_table_entry);
rc = ena_com_get_feature_ex(ena_dev, &get_resp,
- ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG,
+ ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG,
rss->rss_ind_tbl_dma_addr,
tbl_size, 0);
if (unlikely(rc))
@@ -2932,7 +2967,7 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
&cmd.u.host_attr.debug_ba,
host_attr->debug_area_dma_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
@@ -2940,7 +2975,7 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
&cmd.u.host_attr.os_info_ba,
host_attr->host_info_dma_addr);
if (unlikely(ret)) {
- ena_trc_err("memory address set failed\n");
+ ena_trc_err(ena_dev, "Memory address set failed\n");
return ret;
}
@@ -2953,7 +2988,7 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
sizeof(resp));
if (unlikely(ret))
- ena_trc_err("Failed to set host attributes: %d\n", ret);
+ ena_trc_err(ena_dev, "Failed to set host attributes: %d\n", ret);
return ret;
}
@@ -2965,12 +3000,13 @@ bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
ENA_ADMIN_INTERRUPT_MODERATION);
}
-static int ena_com_update_nonadaptive_moderation_interval(u32 coalesce_usecs,
+static int ena_com_update_nonadaptive_moderation_interval(struct ena_com_dev *ena_dev,
+ u32 coalesce_usecs,
u32 intr_delay_resolution,
u32 *intr_moder_interval)
{
if (!intr_delay_resolution) {
- ena_trc_err("Illegal interrupt delay granularity value\n");
+ ena_trc_err(ena_dev, "Illegal interrupt delay granularity value\n");
return ENA_COM_FAULT;
}
@@ -2979,11 +3015,11 @@ static int ena_com_update_nonadaptive_moderation_interval(u32 coalesce_usecs,
return 0;
}
-
int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
u32 tx_coalesce_usecs)
{
- return ena_com_update_nonadaptive_moderation_interval(tx_coalesce_usecs,
+ return ena_com_update_nonadaptive_moderation_interval(ena_dev,
+ tx_coalesce_usecs,
ena_dev->intr_delay_resolution,
&ena_dev->intr_moder_tx_interval);
}
@@ -2991,7 +3027,8 @@ int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_de
int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
u32 rx_coalesce_usecs)
{
- return ena_com_update_nonadaptive_moderation_interval(rx_coalesce_usecs,
+ return ena_com_update_nonadaptive_moderation_interval(ena_dev,
+ rx_coalesce_usecs,
ena_dev->intr_delay_resolution,
&ena_dev->intr_moder_rx_interval);
}
@@ -3007,12 +3044,12 @@ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
if (rc) {
if (rc == ENA_COM_UNSUPPORTED) {
- ena_trc_dbg("Feature %d isn't supported\n",
+ ena_trc_dbg(ena_dev, "Feature %d isn't supported\n",
ENA_ADMIN_INTERRUPT_MODERATION);
rc = 0;
} else {
- ena_trc_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
- rc);
+ ena_trc_err(ena_dev,
+ "Failed to get interrupt moderation admin cmd. rc: %d\n", rc);
}
/* no moderation supported, disable adaptive support */
@@ -3060,7 +3097,7 @@ int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
(llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc));
if (unlikely(ena_dev->tx_max_header_size == 0)) {
- ena_trc_err("the size of the LLQ entry is smaller than needed\n");
+ ena_trc_err(ena_dev, "The size of the LLQ entry is smaller than needed\n");
return -EINVAL;
}
diff --git a/sys/contrib/ena-com/ena_com.h b/sys/contrib/ena-com/ena_com.h
index c1b9540edd0b..414301bdaf91 100644
--- a/sys/contrib/ena-com/ena_com.h
+++ b/sys/contrib/ena-com/ena_com.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -51,6 +51,8 @@
#define ADMIN_CQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_acq_entry))
#define ADMIN_AENQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aenq_entry))
+#define ENA_CDESC_RING_SIZE_ALIGNMENT (1 << 12) /* 4K */
+
/*****************************************************************************/
/*****************************************************************************/
/* ENA adaptive interrupt moderation settings */
@@ -326,6 +328,7 @@ struct ena_com_dev {
void __iomem *mem_bar;
void *dmadev;
void *bus;
+ ena_netdev *net_device;
enum ena_admin_placement_policy_type tx_mem_queue_type;
u32 tx_max_header_size;
@@ -363,7 +366,6 @@ struct ena_com_dev_get_features_ctx {
struct ena_admin_feature_offload_desc offload;
struct ena_admin_ena_hw_hints hw_hints;
struct ena_admin_feature_llq_desc llq;
- struct ena_admin_feature_rss_ind_table ind_table;
};
struct ena_com_create_io_ctx {
@@ -548,7 +550,7 @@ void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev);
* This method goes over the async event notification queue and calls the proper
* aenq handler.
*/
-void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data);
+void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data);
/* ena_com_abort_admin_commands - Abort all the outstanding admin commands.
* @ena_dev: ENA communication layer struct
@@ -628,6 +630,15 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
struct ena_admin_basic_stats *stats);
+/* ena_com_get_eni_stats - Get extended network interface statistics
+ * @ena_dev: ENA communication layer struct
+ * @stats: stats return value
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_get_eni_stats(struct ena_com_dev *ena_dev,
+ struct ena_admin_eni_stats *stats);
+
/* ena_com_set_dev_mtu - Configure the device mtu.
* @ena_dev: ENA communication layer struct
* @mtu: mtu value
@@ -961,6 +972,26 @@ int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
struct ena_admin_feature_llq_desc *llq_features,
struct ena_llq_configurations *llq_default_config);
+/* ena_com_io_sq_to_ena_dev - Extract ena_com_dev using contained field io_sq.
+ * @io_sq: IO submit queue struct
+ *
+ * @return - ena_com_dev struct extracted from io_sq
+ */
+static inline struct ena_com_dev *ena_com_io_sq_to_ena_dev(struct ena_com_io_sq *io_sq)
+{
+ return container_of(io_sq, struct ena_com_dev, io_sq_queues[io_sq->qid]);
+}
+
+/* ena_com_io_cq_to_ena_dev - Extract ena_com_dev using contained field io_cq.
+ * @io_sq: IO submit queue struct
+ *
+ * @return - ena_com_dev struct extracted from io_sq
+ */
+static inline struct ena_com_dev *ena_com_io_cq_to_ena_dev(struct ena_com_io_cq *io_cq)
+{
+ return container_of(io_cq, struct ena_com_dev, io_cq_queues[io_cq->qid]);
+}
+
static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev)
{
return ena_dev->adaptive_coalescing;
diff --git a/sys/contrib/ena-com/ena_defs/ena_admin_defs.h b/sys/contrib/ena-com/ena_defs/ena_admin_defs.h
index 52cdb9e5e394..edfdad3473d7 100644
--- a/sys/contrib/ena-com/ena_defs/ena_admin_defs.h
+++ b/sys/contrib/ena-com/ena_defs/ena_admin_defs.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -36,6 +36,8 @@
#define ENA_ADMIN_EXTRA_PROPERTIES_STRING_LEN 32
#define ENA_ADMIN_EXTRA_PROPERTIES_COUNT 32
+#define ENA_ADMIN_RSS_KEY_PARTS 10
+
enum ena_admin_aq_opcode {
ENA_ADMIN_CREATE_SQ = 1,
ENA_ADMIN_DESTROY_SQ = 2,
@@ -58,6 +60,7 @@ enum ena_admin_aq_completion_status {
ENA_ADMIN_RESOURCE_BUSY = 7,
};
+/* subcommands for the set/get feature admin commands */
enum ena_admin_aq_feature_id {
ENA_ADMIN_DEVICE_ATTRIBUTES = 1,
ENA_ADMIN_MAX_QUEUES_NUM = 2,
@@ -68,7 +71,7 @@ enum ena_admin_aq_feature_id {
ENA_ADMIN_MAX_QUEUES_EXT = 7,
ENA_ADMIN_RSS_HASH_FUNCTION = 10,
ENA_ADMIN_STATELESS_OFFLOAD_CONFIG = 11,
- ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG = 12,
+ ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG = 12,
ENA_ADMIN_MTU = 14,
ENA_ADMIN_RSS_HASH_INPUT = 18,
ENA_ADMIN_INTERRUPT_MODERATION = 20,
@@ -122,6 +125,8 @@ enum ena_admin_completion_policy_type {
enum ena_admin_get_stats_type {
ENA_ADMIN_GET_STATS_TYPE_BASIC = 0,
ENA_ADMIN_GET_STATS_TYPE_EXTENDED = 1,
+ /* extra HW stats for specific network interface */
+ ENA_ADMIN_GET_STATS_TYPE_ENI = 2,
};
enum ena_admin_get_stats_scope {
@@ -198,7 +203,7 @@ struct ena_admin_acq_common_desc {
uint16_t extended_status;
/* indicates to the driver which AQ entry has been consumed by the
- * device and could be reused
+ * device and could be reused
*/
uint16_t sq_head_indx;
};
@@ -243,8 +248,8 @@ struct ena_admin_aq_create_sq_cmd {
*/
uint8_t sq_caps_3;
- /* associated completion queue id. This CQ must be created prior to
- * SQ creation
+ /* associated completion queue id. This CQ must be created prior to SQ
+ * creation
*/
uint16_t cq_idx;
@@ -383,7 +388,7 @@ struct ena_admin_aq_get_stats_cmd {
uint16_t queue_idx;
/* device id, value 0xFFFF means mine. only privileged device can get
- * stats of other device
+ * stats of other device
*/
uint16_t device_id;
};
@@ -415,10 +420,43 @@ struct ena_admin_basic_stats {
uint32_t tx_drops_high;
};
+/* ENI Statistics Command. */
+struct ena_admin_eni_stats {
+ /* The number of packets shaped due to inbound aggregate BW
+ * allowance being exceeded
+ */
+ uint64_t bw_in_allowance_exceeded;
+
+ /* The number of packets shaped due to outbound aggregate BW
+ * allowance being exceeded
+ */
+ uint64_t bw_out_allowance_exceeded;
+
+ /* The number of packets shaped due to PPS allowance being exceeded */
+ uint64_t pps_allowance_exceeded;
+
+ /* The number of packets shaped due to connection tracking
+ * allowance being exceeded and leading to failure in establishment
+ * of new connections
+ */
+ uint64_t conntrack_allowance_exceeded;
+
+ /* The number of packets shaped due to linklocal packet rate
+ * allowance being exceeded
+ */
+ uint64_t linklocal_allowance_exceeded;
+};
+
struct ena_admin_acq_get_stats_resp {
struct ena_admin_acq_common_desc acq_common_desc;
- struct ena_admin_basic_stats basic_stats;
+ union {
+ uint64_t raw[7];
+
+ struct ena_admin_basic_stats basic_stats;
+
+ struct ena_admin_eni_stats eni_stats;
+ } u;
};
struct ena_admin_get_set_feature_common_desc {
@@ -432,8 +470,8 @@ struct ena_admin_get_set_feature_common_desc {
uint8_t feature_id;
/* The driver specifies the max feature version it supports and the
- * device responds with the currently supported feature version. The
- * field is zero based
+ * device responds with the currently supported feature version. The
+ * field is zero based
*/
uint8_t feature_version;
@@ -445,7 +483,9 @@ struct ena_admin_device_attr_feature_desc {
uint32_t device_version;
- /* bitmap of ena_admin_aq_feature_id */
+ /* bitmap of ena_admin_aq_feature_id, which represents supported
+ * subcommands for the set/get feature admin commands.
+ */
uint32_t supported_features;
uint32_t reserved3;
@@ -531,32 +571,30 @@ struct ena_admin_feature_llq_desc {
uint32_t max_llq_depth;
- /* specify the header locations the device supports. bitfield of
- * enum ena_admin_llq_header_location.
+ /* specify the header locations the device supports. bitfield of enum
+ * ena_admin_llq_header_location.
*/
uint16_t header_location_ctrl_supported;
/* the header location the driver selected to use. */
uint16_t header_location_ctrl_enabled;
- /* if inline header is specified - this is the size of descriptor
- * list entry. If header in a separate ring is specified - this is
- * the size of header ring entry. bitfield of enum
- * ena_admin_llq_ring_entry_size. specify the entry sizes the device
- * supports
+ /* if inline header is specified - this is the size of descriptor list
+ * entry. If header in a separate ring is specified - this is the size
+ * of header ring entry. bitfield of enum ena_admin_llq_ring_entry_size.
+ * specify the entry sizes the device supports
*/
uint16_t entry_size_ctrl_supported;
/* the entry size the driver selected to use. */
uint16_t entry_size_ctrl_enabled;
- /* valid only if inline header is specified. First entry associated
- * with the packet includes descriptors and header. Rest of the
- * entries occupied by descriptors. This parameter defines the max
- * number of descriptors precedding the header in the first entry.
- * The field is bitfield of enum
- * ena_admin_llq_num_descs_before_header and specify the values the
- * device supports
+ /* valid only if inline header is specified. First entry associated with
+ * the packet includes descriptors and header. Rest of the entries
+ * occupied by descriptors. This parameter defines the max number of
+ * descriptors precedding the header in the first entry. The field is
+ * bitfield of enum ena_admin_llq_num_descs_before_header and specify
+ * the values the device supports
*/
uint16_t desc_num_before_header_supported;
@@ -564,7 +602,7 @@ struct ena_admin_feature_llq_desc {
uint16_t desc_num_before_header_enabled;
/* valid only if inline was chosen. bitfield of enum
- * ena_admin_llq_stride_ctrl
+ * ena_admin_llq_stride_ctrl
*/
uint16_t descriptors_stride_ctrl_supported;
@@ -574,8 +612,8 @@ struct ena_admin_feature_llq_desc {
/* reserved */
uint32_t reserved1;
- /* accelerated low latency queues requirment. driver needs to
- * support those requirments in order to use accelerated llq
+ /* accelerated low latency queues requirement. driver needs to
+ * support those requirements in order to use accelerated llq
*/
struct ena_admin_accel_mode_req accel_mode;
};
@@ -599,8 +637,8 @@ struct ena_admin_queue_ext_feature_fields {
uint32_t max_tx_header_size;
- /* Maximum Descriptors number, including meta descriptor, allowed for
- * a single Tx packet
+ /* Maximum Descriptors number, including meta descriptor, allowed for a
+ * single Tx packet
*/
uint16_t max_per_packet_tx_descs;
@@ -623,8 +661,8 @@ struct ena_admin_queue_feature_desc {
uint32_t max_header_size;
- /* Maximum Descriptors number, including meta descriptor, allowed for
- * a single Tx packet
+ /* Maximum Descriptors number, including meta descriptor, allowed for a
+ * single Tx packet
*/
uint16_t max_packet_tx_descs;
@@ -720,11 +758,11 @@ enum ena_admin_hash_functions {
};
struct ena_admin_feature_rss_flow_hash_control {
- uint32_t keys_num;
+ uint32_t key_parts;
uint32_t reserved;
- uint32_t key[10];
+ uint32_t key[ENA_ADMIN_RSS_KEY_PARTS];
};
struct ena_admin_feature_rss_flow_hash_function {
@@ -859,11 +897,12 @@ struct ena_admin_host_info {
uint16_t reserved;
- /* 0 : mutable_rss_table_size
+ /* 0 : reserved
* 1 : rx_offset
* 2 : interrupt_moderation
- * 3 : map_rx_buf_bidirectional
- * 31:4 : reserved
+ * 3 : rx_buf_mirroring
+ * 4 : rss_configurable_function_key
+ * 31:5 : reserved
*/
uint32_t driver_supported_features;
};
@@ -945,7 +984,7 @@ struct ena_admin_queue_ext_feature_desc {
struct ena_admin_queue_ext_feature_fields max_queue_ext;
uint32_t raw[10];
- } ;
+ };
};
struct ena_admin_get_feat_resp {
@@ -1028,7 +1067,7 @@ struct ena_admin_set_feat_resp {
struct ena_admin_aenq_common_desc {
uint16_t group;
- uint16_t syndrom;
+ uint16_t syndrome;
/* 0 : phase
* 7:1 : reserved - MBZ
@@ -1052,7 +1091,7 @@ enum ena_admin_aenq_group {
ENA_ADMIN_AENQ_GROUPS_NUM = 5,
};
-enum ena_admin_aenq_notification_syndrom {
+enum ena_admin_aenq_notification_syndrome {
ENA_ADMIN_SUSPEND = 0,
ENA_ADMIN_RESUME = 1,
ENA_ADMIN_UPDATE_HINTS = 2,
@@ -1181,13 +1220,14 @@ struct ena_admin_ena_mmio_req_read_less_resp {
#define ENA_ADMIN_HOST_INFO_DEVICE_MASK GENMASK(7, 3)
#define ENA_ADMIN_HOST_INFO_BUS_SHIFT 8
#define ENA_ADMIN_HOST_INFO_BUS_MASK GENMASK(15, 8)
-#define ENA_ADMIN_HOST_INFO_MUTABLE_RSS_TABLE_SIZE_MASK BIT(0)
#define ENA_ADMIN_HOST_INFO_RX_OFFSET_SHIFT 1
#define ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK BIT(1)
#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_SHIFT 2
#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK BIT(2)
-#define ENA_ADMIN_HOST_INFO_MAP_RX_BUF_BIDIRECTIONAL_SHIFT 3
-#define ENA_ADMIN_HOST_INFO_MAP_RX_BUF_BIDIRECTIONAL_MASK BIT(3)
+#define ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_SHIFT 3
+#define ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK BIT(3)
+#define ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT 4
+#define ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK BIT(4)
/* feature_rss_ind_table */
#define ENA_ADMIN_FEATURE_RSS_IND_TABLE_ONE_ENTRY_UPDATE_MASK BIT(0)
@@ -1609,16 +1649,6 @@ static inline void set_ena_admin_host_info_bus(struct ena_admin_host_info *p, ui
p->bdf |= (val << ENA_ADMIN_HOST_INFO_BUS_SHIFT) & ENA_ADMIN_HOST_INFO_BUS_MASK;
}
-static inline uint32_t get_ena_admin_host_info_mutable_rss_table_size(const struct ena_admin_host_info *p)
-{
- return p->driver_supported_features & ENA_ADMIN_HOST_INFO_MUTABLE_RSS_TABLE_SIZE_MASK;
-}
-
-static inline void set_ena_admin_host_info_mutable_rss_table_size(struct ena_admin_host_info *p, uint32_t val)
-{
- p->driver_supported_features |= val & ENA_ADMIN_HOST_INFO_MUTABLE_RSS_TABLE_SIZE_MASK;
-}
-
static inline uint32_t get_ena_admin_host_info_rx_offset(const struct ena_admin_host_info *p)
{
return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK) >> ENA_ADMIN_HOST_INFO_RX_OFFSET_SHIFT;
@@ -1639,14 +1669,24 @@ static inline void set_ena_admin_host_info_interrupt_moderation(struct ena_admin
p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_SHIFT) & ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK;
}
-static inline uint32_t get_ena_admin_host_info_map_rx_buf_bidirectional(const struct ena_admin_host_info *p)
+static inline uint32_t get_ena_admin_host_info_rx_buf_mirroring(const struct ena_admin_host_info *p)
+{
+ return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK) >> ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_SHIFT;
+}
+
+static inline void set_ena_admin_host_info_rx_buf_mirroring(struct ena_admin_host_info *p, uint32_t val)
+{
+ p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_SHIFT) & ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK;
+}
+
+static inline uint32_t get_ena_admin_host_info_rss_configurable_function_key(const struct ena_admin_host_info *p)
{
- return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_MAP_RX_BUF_BIDIRECTIONAL_MASK) >> ENA_ADMIN_HOST_INFO_MAP_RX_BUF_BIDIRECTIONAL_SHIFT;
+ return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK) >> ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT;
}
-static inline void set_ena_admin_host_info_map_rx_buf_bidirectional(struct ena_admin_host_info *p, uint32_t val)
+static inline void set_ena_admin_host_info_rss_configurable_function_key(struct ena_admin_host_info *p, uint32_t val)
{
- p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_MAP_RX_BUF_BIDIRECTIONAL_SHIFT) & ENA_ADMIN_HOST_INFO_MAP_RX_BUF_BIDIRECTIONAL_MASK;
+ p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT) & ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
}
static inline uint8_t get_ena_admin_feature_rss_ind_table_one_entry_update(const struct ena_admin_feature_rss_ind_table *p)
diff --git a/sys/contrib/ena-com/ena_defs/ena_common_defs.h b/sys/contrib/ena-com/ena_defs/ena_common_defs.h
index 88b90d44a79a..ee49ff6e2776 100644
--- a/sys/contrib/ena-com/ena_defs/ena_common_defs.h
+++ b/sys/contrib/ena-com/ena_defs/ena_common_defs.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
diff --git a/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h b/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h
index 14f44d0d9a86..817375a947c5 100644
--- a/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h
+++ b/sys/contrib/ena-com/ena_defs/ena_eth_io_defs.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
diff --git a/sys/contrib/ena-com/ena_defs/ena_gen_info.h b/sys/contrib/ena-com/ena_defs/ena_gen_info.h
index 83ed024ae4cc..726750a67d4e 100644
--- a/sys/contrib/ena-com/ena_defs/ena_gen_info.h
+++ b/sys/contrib/ena-com/ena_defs/ena_gen_info.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -30,5 +30,5 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-#define ENA_GEN_DATE "Mon Apr 20 15:41:59 DST 2020"
-#define ENA_GEN_COMMIT "daa45ac"
+#define ENA_GEN_DATE "Fri Sep 18 17:09:00 IDT 2020"
+#define ENA_GEN_COMMIT "0f80d82"
diff --git a/sys/contrib/ena-com/ena_defs/ena_regs_defs.h b/sys/contrib/ena-com/ena_defs/ena_regs_defs.h
index 53ac662b6189..bdd91ef2e026 100644
--- a/sys/contrib/ena-com/ena_defs/ena_regs_defs.h
+++ b/sys/contrib/ena-com/ena_defs/ena_regs_defs.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
diff --git a/sys/contrib/ena-com/ena_eth_com.c b/sys/contrib/ena-com/ena_eth_com.c
index 58ddb82246fd..47ca4e4afdb6 100644
--- a/sys/contrib/ena-com/ena_eth_com.c
+++ b/sys/contrib/ena-com/ena_eth_com.c
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -85,12 +85,14 @@ static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq,
if (is_llq_max_tx_burst_exists(io_sq)) {
if (unlikely(!io_sq->entries_in_tx_burst_left)) {
- ena_trc_err("Error: trying to send more packets than tx burst allows\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Error: trying to send more packets than tx burst allows\n");
return ENA_COM_NO_SPACE;
}
io_sq->entries_in_tx_burst_left--;
- ena_trc_dbg("decreasing entries_in_tx_burst_left of queue %d to %d\n",
+ ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
+ "Decreasing entries_in_tx_burst_left of queue %d to %d\n",
io_sq->qid, io_sq->entries_in_tx_burst_left);
}
@@ -129,12 +131,14 @@ static int ena_com_write_header_to_bounce(struct ena_com_io_sq *io_sq,
llq_info->descs_num_before_header * io_sq->desc_entry_size;
if (unlikely((header_offset + header_len) > llq_info->desc_list_entry_size)) {
- ena_trc_err("trying to write header larger than llq entry can accommodate\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Trying to write header larger than llq entry can accommodate\n");
return ENA_COM_FAULT;
}
if (unlikely(!bounce_buffer)) {
- ena_trc_err("bounce buffer is NULL\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Bounce buffer is NULL\n");
return ENA_COM_FAULT;
}
@@ -152,7 +156,8 @@ static void *get_sq_desc_llq(struct ena_com_io_sq *io_sq)
bounce_buffer = pkt_ctrl->curr_bounce_buf;
if (unlikely(!bounce_buffer)) {
- ena_trc_err("bounce buffer is NULL\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Bounce buffer is NULL\n");
return NULL;
}
@@ -177,7 +182,8 @@ static int ena_com_close_bounce_buffer(struct ena_com_io_sq *io_sq)
rc = ena_com_write_bounce_buffer_to_dev(io_sq,
pkt_ctrl->curr_bounce_buf);
if (unlikely(rc)) {
- ena_trc_err("failed to write bounce buffer to device\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Failed to write bounce buffer to device\n");
return rc;
}
@@ -210,7 +216,8 @@ static int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq)
rc = ena_com_write_bounce_buffer_to_dev(io_sq,
pkt_ctrl->curr_bounce_buf);
if (unlikely(rc)) {
- ena_trc_err("failed to write bounce buffer to device\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Failed to write bounce buffer to device\n");
return rc;
}
@@ -280,7 +287,8 @@ static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
io_cq->cur_rx_pkt_cdesc_count = 0;
io_cq->cur_rx_pkt_cdesc_start_idx = head_masked;
- ena_trc_dbg("ena q_id: %d packets were completed. first desc idx %u descs# %d\n",
+ ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq),
+ "ENA q_id: %d packets were completed. first desc idx %u descs# %d\n",
io_cq->qid, *first_cdesc_idx, count);
} else {
io_cq->cur_rx_pkt_cdesc_count += count;
@@ -296,6 +304,9 @@ static int ena_com_create_meta(struct ena_com_io_sq *io_sq,
struct ena_eth_io_tx_meta_desc *meta_desc = NULL;
meta_desc = get_sq_desc(io_sq);
+ if (unlikely(!meta_desc))
+ return ENA_COM_FAULT;
+
memset(meta_desc, 0x0, sizeof(struct ena_eth_io_tx_meta_desc));
meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_DESC_MASK;
@@ -303,7 +314,7 @@ static int ena_com_create_meta(struct ena_com_io_sq *io_sq,
meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK;
/* bits 0-9 of the mss */
- meta_desc->word2 |= (ena_meta->mss <<
+ meta_desc->word2 |= ((u32)ena_meta->mss <<
ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) &
ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK;
/* bits 10-13 of the mss */
@@ -313,7 +324,7 @@ static int ena_com_create_meta(struct ena_com_io_sq *io_sq,
/* Extended meta desc */
meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
- meta_desc->len_ctrl |= (io_sq->phase <<
+ meta_desc->len_ctrl |= ((u32)io_sq->phase <<
ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) &
ENA_ETH_IO_TX_META_DESC_PHASE_MASK;
@@ -326,7 +337,7 @@ static int ena_com_create_meta(struct ena_com_io_sq *io_sq,
ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) &
ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK;
- meta_desc->word2 |= (ena_meta->l4_hdr_len <<
+ meta_desc->word2 |= ((u32)ena_meta->l4_hdr_len <<
ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) &
ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK;
@@ -348,20 +359,23 @@ static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
*have_meta = true;
return ena_com_create_meta(io_sq, ena_meta);
- } else if (ena_com_meta_desc_changed(io_sq, ena_tx_ctx)) {
+ }
+
+ if (ena_com_meta_desc_changed(io_sq, ena_tx_ctx)) {
*have_meta = true;
/* Cache the meta desc */
memcpy(&io_sq->cached_tx_meta, ena_meta,
sizeof(struct ena_com_tx_meta));
return ena_com_create_meta(io_sq, ena_meta);
- } else {
- *have_meta = false;
- return ENA_COM_OK;
}
+
+ *have_meta = false;
+ return ENA_COM_OK;
}
-static void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
- struct ena_eth_io_rx_cdesc_base *cdesc)
+static void ena_com_rx_set_flags(struct ena_com_io_cq *io_cq,
+ struct ena_com_rx_ctx *ena_rx_ctx,
+ struct ena_eth_io_rx_cdesc_base *cdesc)
{
ena_rx_ctx->l3_proto = cdesc->status &
ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK;
@@ -382,7 +396,8 @@ static void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
(cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) >>
ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT;
- ena_trc_dbg("ena_rx_ctx->l3_proto %d ena_rx_ctx->l4_proto %d\nena_rx_ctx->l3_csum_err %d ena_rx_ctx->l4_csum_err %d\nhash frag %d frag: %d cdesc_status: %x\n",
+ ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq),
+ "l3_proto %d l4_proto %d l3_csum_err %d l4_csum_err %d hash %d frag %d cdesc_status %x\n",
ena_rx_ctx->l3_proto,
ena_rx_ctx->l4_proto,
ena_rx_ctx->l3_csum_err,
@@ -411,23 +426,26 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
u64 addr_hi;
ENA_WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_TX,
- "wrong Q type");
+ ena_com_io_sq_to_ena_dev(io_sq), "wrong Q type");
/* num_bufs +1 for potential meta desc */
if (unlikely(!ena_com_sq_have_enough_space(io_sq, num_bufs + 1))) {
- ena_trc_dbg("Not enough space in the tx queue\n");
+ ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
+ "Not enough space in the tx queue\n");
return ENA_COM_NO_MEM;
}
if (unlikely(header_len > io_sq->tx_max_header_size)) {
- ena_trc_err("header size is too large %d max header: %d\n",
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Header size is too large %d max header: %d\n",
header_len, io_sq->tx_max_header_size);
return ENA_COM_INVAL;
}
if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV
&& !buffer_to_push)) {
- ena_trc_err("push header wasn't provided on LLQ mode\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Push header wasn't provided on LLQ mode\n");
return ENA_COM_INVAL;
}
@@ -437,7 +455,8 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
rc = ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx, &have_meta);
if (unlikely(rc)) {
- ena_trc_err("failed to create and store tx meta desc\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Failed to create and store tx meta desc\n");
return rc;
}
@@ -445,7 +464,8 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
if (unlikely(!num_bufs && !header_len)) {
rc = ena_com_close_bounce_buffer(io_sq);
if (rc)
- ena_trc_err("failed to write buffers to LLQ\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Failed to write buffers to LLQ\n");
*nb_hw_desc = io_sq->tail - start_tail;
return rc;
}
@@ -459,16 +479,16 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
if (!have_meta)
desc->len_ctrl |= ENA_ETH_IO_TX_DESC_FIRST_MASK;
- desc->buff_addr_hi_hdr_sz |= (header_len <<
+ desc->buff_addr_hi_hdr_sz |= ((u32)header_len <<
ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) &
ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK;
- desc->len_ctrl |= (io_sq->phase << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) &
+ desc->len_ctrl |= ((u32)io_sq->phase << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) &
ENA_ETH_IO_TX_DESC_PHASE_MASK;
desc->len_ctrl |= ENA_ETH_IO_TX_DESC_COMP_REQ_MASK;
/* Bits 0-9 */
- desc->meta_ctrl |= (ena_tx_ctx->req_id <<
+ desc->meta_ctrl |= ((u32)ena_tx_ctx->req_id <<
ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) &
ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK;
@@ -506,7 +526,8 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
if (likely(i != 0)) {
rc = ena_com_sq_update_tail(io_sq);
if (unlikely(rc)) {
- ena_trc_err("failed to update sq tail\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Failed to update sq tail\n");
return rc;
}
@@ -516,7 +537,7 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
memset(desc, 0x0, sizeof(struct ena_eth_io_tx_desc));
- desc->len_ctrl |= (io_sq->phase <<
+ desc->len_ctrl |= ((u32)io_sq->phase <<
ENA_ETH_IO_TX_DESC_PHASE_SHIFT) &
ENA_ETH_IO_TX_DESC_PHASE_MASK;
}
@@ -538,13 +559,15 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
rc = ena_com_sq_update_tail(io_sq);
if (unlikely(rc)) {
- ena_trc_err("failed to update sq tail of the last descriptor\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Failed to update sq tail of the last descriptor\n");
return rc;
}
rc = ena_com_close_bounce_buffer(io_sq);
if (rc)
- ena_trc_err("failed when closing bounce buffer\n");
+ ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq),
+ "Failed when closing bounce buffer\n");
*nb_hw_desc = io_sq->tail - start_tail;
return rc;
@@ -556,12 +579,13 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
{
struct ena_com_rx_buf_info *ena_buf = &ena_rx_ctx->ena_bufs[0];
struct ena_eth_io_rx_cdesc_base *cdesc = NULL;
+ u16 q_depth = io_cq->q_depth;
u16 cdesc_idx = 0;
u16 nb_hw_desc;
u16 i = 0;
ENA_WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX,
- "wrong Q type");
+ ena_com_io_cq_to_ena_dev(io_cq), "wrong Q type");
nb_hw_desc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx);
if (nb_hw_desc == 0) {
@@ -569,11 +593,13 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
return 0;
}
- ena_trc_dbg("fetch rx packet: queue %d completed desc: %d\n",
+ ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq),
+ "Fetch rx packet: queue %d completed desc: %d\n",
io_cq->qid, nb_hw_desc);
if (unlikely(nb_hw_desc > ena_rx_ctx->max_bufs)) {
- ena_trc_err("Too many RX cdescs (%d) > MAX(%d)\n",
+ ena_trc_err(ena_com_io_cq_to_ena_dev(io_cq),
+ "Too many RX cdescs (%d) > MAX(%d)\n",
nb_hw_desc, ena_rx_ctx->max_bufs);
return ENA_COM_NO_SPACE;
}
@@ -582,21 +608,30 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
ena_rx_ctx->pkt_offset = cdesc->offset;
do {
- ena_buf->len = cdesc->length;
- ena_buf->req_id = cdesc->req_id;
- ena_buf++;
- } while ((++i < nb_hw_desc) && (cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i)));
+ ena_buf[i].len = cdesc->length;
+ ena_buf[i].req_id = cdesc->req_id;
+ if (unlikely(ena_buf[i].req_id >= q_depth))
+ return ENA_COM_EIO;
+
+ if (++i >= nb_hw_desc)
+ break;
+
+ cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i);
+
+ } while (1);
/* Update SQ head ptr */
io_sq->next_to_comp += nb_hw_desc;
- ena_trc_dbg("[%s][QID#%d] Updating SQ head to: %d\n", __func__,
+ ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq),
+ "[%s][QID#%d] Updating SQ head to: %d\n", __func__,
io_sq->qid, io_sq->next_to_comp);
/* Get rx flags from the last pkt */
- ena_com_rx_set_flags(ena_rx_ctx, cdesc);
+ ena_com_rx_set_flags(io_cq, ena_rx_ctx, cdesc);
ena_rx_ctx->descs = nb_hw_desc;
+
return 0;
}
@@ -607,7 +642,7 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
struct ena_eth_io_rx_desc *desc;
ENA_WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX,
- "wrong Q type");
+ ena_com_io_sq_to_ena_dev(io_sq), "wrong Q type");
if (unlikely(!ena_com_sq_have_enough_space(io_sq, 1)))
return ENA_COM_NO_SPACE;
@@ -621,12 +656,16 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
desc->length = ena_buf->len;
desc->ctrl = ENA_ETH_IO_RX_DESC_FIRST_MASK |
- ENA_ETH_IO_RX_DESC_LAST_MASK |
- (io_sq->phase & ENA_ETH_IO_RX_DESC_PHASE_MASK) |
- ENA_ETH_IO_RX_DESC_COMP_REQ_MASK;
+ ENA_ETH_IO_RX_DESC_LAST_MASK |
+ ENA_ETH_IO_RX_DESC_COMP_REQ_MASK |
+ (io_sq->phase & ENA_ETH_IO_RX_DESC_PHASE_MASK);
desc->req_id = req_id;
+ ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
+ "[%s] Adding single RX desc, Queue: %u, req_id: %u\n",
+ __func__, io_sq->qid, req_id);
+
desc->buff_addr_lo = (u32)ena_buf->paddr;
desc->buff_addr_hi =
((ena_buf->paddr & GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32);
diff --git a/sys/contrib/ena-com/ena_eth_com.h b/sys/contrib/ena-com/ena_eth_com.h
index 4b91221ea093..85675bb004b9 100644
--- a/sys/contrib/ena-com/ena_eth_com.h
+++ b/sys/contrib/ena-com/ena_eth_com.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -171,7 +171,8 @@ static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
llq_info->descs_per_entry);
}
- ena_trc_dbg("queue: %d num_descs: %d num_entries_needed: %d\n",
+ ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
+ "Queue: %d num_descs: %d num_entries_needed: %d\n",
io_sq->qid, num_descs, num_entries_needed);
return num_entries_needed > io_sq->entries_in_tx_burst_left;
@@ -182,14 +183,16 @@ static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst;
u16 tail = io_sq->tail;
- ena_trc_dbg("write submission queue doorbell for queue: %d tail: %d\n",
+ ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
+ "Write submission queue doorbell for queue: %d tail: %d\n",
io_sq->qid, tail);
ENA_REG_WRITE32(io_sq->bus, tail, io_sq->db_addr);
if (is_llq_max_tx_burst_exists(io_sq)) {
- ena_trc_dbg("reset available entries in tx burst for queue %d to %d\n",
- io_sq->qid, max_entries_in_tx_burst);
+ ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq),
+ "Reset available entries in tx burst for queue %d to %d\n",
+ io_sq->qid, max_entries_in_tx_burst);
io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst;
}
@@ -207,7 +210,8 @@ static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq)
need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH);
if (unlikely(need_update)) {
- ena_trc_dbg("Write completion queue doorbell for queue %d: head: %d\n",
+ ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq),
+ "Write completion queue doorbell for queue %d: head: %d\n",
io_cq->qid, head);
ENA_REG_WRITE32(io_cq->bus, head, io_cq->cq_head_db_reg);
io_cq->last_head_update = head;
@@ -271,7 +275,8 @@ static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq,
*req_id = READ_ONCE16(cdesc->req_id);
if (unlikely(*req_id >= io_cq->q_depth)) {
- ena_trc_err("Invalid req id %d\n", cdesc->req_id);
+ ena_trc_err(ena_com_io_cq_to_ena_dev(io_cq),
+ "Invalid req id %d\n", cdesc->req_id);
return ENA_COM_INVAL;
}
diff --git a/sys/contrib/ena-com/ena_plat.h b/sys/contrib/ena-com/ena_plat.h
index e3536cdf3573..8fe1ec9aa731 100644
--- a/sys/contrib/ena-com/ena_plat.h
+++ b/sys/contrib/ena-com/ena_plat.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -106,24 +106,37 @@ extern struct ena_bus_space ebs;
#define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */
#define ENA_NETMAP (1 << 9) /* Detailed info about netmap. */
+#define DEFAULT_ALLOC_ALIGNMENT 8
+
extern int ena_log_level;
-#define ena_trace_raw(level, fmt, args...) \
+#define container_of(ptr, type, member) \
+ ({ \
+ const __typeof(((type *)0)->member) *__p = (ptr); \
+ (type *)((uintptr_t)__p - offsetof(type, member)); \
+ })
+
+#define ena_trace_raw(ctx, level, fmt, args...) \
do { \
+ ((void)(ctx)); \
if (((level) & ena_log_level) != (level)) \
break; \
printf(fmt, ##args); \
} while (0)
-#define ena_trace(level, fmt, args...) \
- ena_trace_raw(level, "%s() [TID:%d]: " \
+#define ena_trace(ctx, level, fmt, args...) \
+ ena_trace_raw(ctx, level, "%s() [TID:%d]: " \
fmt, __func__, curthread->td_tid, ##args)
-#define ena_trc_dbg(format, arg...) ena_trace(ENA_DBG, format, ##arg)
-#define ena_trc_info(format, arg...) ena_trace(ENA_INFO, format, ##arg)
-#define ena_trc_warn(format, arg...) ena_trace(ENA_WARNING, format, ##arg)
-#define ena_trc_err(format, arg...) ena_trace(ENA_ALERT, format, ##arg)
+#define ena_trc_dbg(ctx, format, arg...) \
+ ena_trace(ctx, ENA_DBG, format, ##arg)
+#define ena_trc_info(ctx, format, arg...) \
+ ena_trace(ctx, ENA_INFO, format, ##arg)
+#define ena_trc_warn(ctx, format, arg...) \
+ ena_trace(ctx, ENA_WARNING, format, ##arg)
+#define ena_trc_err(ctx, format, arg...) \
+ ena_trace(ctx, ENA_ALERT, format, ##arg)
#define unlikely(x) __predict_false(!!(x))
#define likely(x) __predict_true(!!(x))
@@ -134,19 +147,10 @@ extern int ena_log_level;
#define MAX_ERRNO 4095
#define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
-#define ENA_ASSERT(cond, format, arg...) \
- do { \
- if (unlikely(!(cond))) { \
- ena_trc_err( \
- "Assert failed on %s:%s:%d:" format, \
- __FILE__, __func__, __LINE__, ##arg); \
- } \
- } while (0)
-
-#define ENA_WARN(cond, format, arg...) \
+#define ENA_WARN(cond, ctx, format, arg...) \
do { \
if (unlikely((cond))) { \
- ena_trc_warn(format, ##arg); \
+ ena_trc_warn(ctx, format, ##arg); \
} \
} while (0)
@@ -185,6 +189,7 @@ static inline long PTR_ERR(const void *ptr)
#define ENA_COM_NO_DEVICE ENODEV
#define ENA_COM_PERMISSION EPERM
#define ENA_COM_TIMER_EXPIRED ETIMEDOUT
+#define ENA_COM_EIO EIO
#define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
#define ENA_USLEEP(x) pause_sbt("ena", SBT_1US * (x), SBT_1US, 0)
@@ -233,10 +238,17 @@ static inline long PTR_ERR(const void *ptr)
cv_init(&((waitqueue).wq), "cv"); \
mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \
} while (0)
-#define ENA_WAIT_EVENT_DESTROY(waitqueue) \
+#define ENA_WAIT_EVENTS_DESTROY(admin_queue) \
do { \
- cv_destroy(&((waitqueue).wq)); \
- mtx_destroy(&((waitqueue).mtx)); \
+ struct ena_comp_ctx *comp_ctx; \
+ int i; \
+ for (i = 0; i < admin_queue->q_depth; i++) { \
+ comp_ctx = get_comp_ctxt(admin_queue, i, false); \
+ if (comp_ctx != NULL) { \
+ cv_destroy(&((comp_ctx->wait_event).wq)); \
+ mtx_destroy(&((comp_ctx->wait_event).mtx)); \
+ } \
+ } \
} while (0)
#define ENA_WAIT_EVENT_CLEAR(waitqueue) \
cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description)
@@ -281,11 +293,12 @@ typedef uint32_t ena_atomic32_t;
#define ENA_PRIu64 PRIu64
typedef uint64_t ena_time_t;
+typedef struct ifnet ena_netdev;
void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
int error);
int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
- int mapflags);
+ int mapflags, bus_size_t alignment);
static inline uint32_t
ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
@@ -313,20 +326,30 @@ ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
(void)(size); \
free(ptr, M_DEVBUF); \
} while (0)
-#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, node, \
- dev_node) \
+#define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \
+ handle, node, dev_node, alignment) \
do { \
((virt) = NULL); \
(void)(dev_node); \
} while (0)
-#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \
+#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, \
+ node, dev_node) \
+ ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, \
+ phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT)
+
+#define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma, \
+ alignment) \
do { \
- ena_dma_alloc((dmadev), (size), &(dma), 0); \
+ ena_dma_alloc((dmadev), (size), &(dma), 0, alignment); \
(virt) = (void *)(dma).vaddr; \
(phys) = (dma).paddr; \
} while (0)
+#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \
+ ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, \
+ phys, dma, DEFAULT_ALLOC_ALIGNMENT)
+
#define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \
do { \
(void)size; \
diff --git a/sys/dev/ena/ena.c b/sys/dev/ena/ena.c
index 1bd25c231967..70989b7a36b9 100644
--- a/sys/dev/ena/ena.c
+++ b/sys/dev/ena/ena.c
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -167,15 +167,16 @@ static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *);
static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *);
static void unimplemented_aenq_handler(void *,
struct ena_admin_aenq_entry *);
+static int ena_copy_eni_metrics(struct ena_adapter *);
static void ena_timer_service(void *);
static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION;
static ena_vendor_info_t ena_vendor_info_array[] = {
{ PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0},
- { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_PF, 0},
+ { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF_RSERV0, 0},
{ PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0},
- { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_VF, 0},
+ { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF_RSERV0, 0},
/* Last entry */
{ 0, 0, 0 }
};
@@ -195,7 +196,7 @@ ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
int
ena_dma_alloc(device_t dmadev, bus_size_t size,
- ena_mem_handle_t *dma , int mapflags)
+ ena_mem_handle_t *dma, int mapflags, bus_size_t alignment)
{
struct ena_adapter* adapter = device_get_softc(dmadev);
uint32_t maxsize;
@@ -209,7 +210,7 @@ ena_dma_alloc(device_t dmadev, bus_size_t size,
dma_space_addr = BUS_SPACE_MAXADDR;
error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */
- 8, 0, /* alignment, bounds */
+ alignment, 0, /* alignment, bounds */
dma_space_addr, /* lowaddr of exclusion window */
BUS_SPACE_MAXADDR,/* highaddr of exclusion window */
NULL, NULL, /* filter, filterarg */
@@ -221,14 +222,14 @@ ena_dma_alloc(device_t dmadev, bus_size_t size,
NULL, /* lockarg */
&dma->tag);
if (unlikely(error != 0)) {
- ena_trace(ENA_ALERT, "bus_dma_tag_create failed: %d\n", error);
+ ena_trace(NULL, ENA_ALERT, "bus_dma_tag_create failed: %d\n", error);
goto fail_tag;
}
error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr,
BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
if (unlikely(error != 0)) {
- ena_trace(ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n",
+ ena_trace(NULL, ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n",
(uintmax_t)size, error);
goto fail_map_create;
}
@@ -237,7 +238,7 @@ ena_dma_alloc(device_t dmadev, bus_size_t size,
error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr,
size, ena_dmamap_callback, &dma->paddr, mapflags);
if (unlikely((error != 0) || (dma->paddr == 0))) {
- ena_trace(ENA_ALERT, ": bus_dmamap_load failed: %d\n", error);
+ ena_trace(NULL, ENA_ALERT, ": bus_dmamap_load failed: %d\n", error);
goto fail_map_load;
}
@@ -310,7 +311,7 @@ ena_probe(device_t dev)
while (ent->vendor_id != 0) {
if ((pci_vendor_id == ent->vendor_id) &&
(pci_device_id == ent->device_id)) {
- ena_trace(ENA_DBG, "vendor=%x device=%x\n",
+ ena_trace(NULL, ENA_DBG, "vendor=%x device=%x\n",
pci_vendor_id, pci_device_id);
sprintf(adapter_name, DEVICE_DESC);
@@ -340,7 +341,7 @@ ena_change_mtu(if_t ifp, int new_mtu)
rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
if (likely(rc == 0)) {
- ena_trace(ENA_DBG, "set MTU to %d\n", new_mtu);
+ ena_trace(NULL, ENA_DBG, "set MTU to %d\n", new_mtu);
if_setmtu(ifp, new_mtu);
} else {
device_printf(adapter->pdev, "Failed to set MTU to %d\n",
@@ -661,7 +662,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
err = bus_dmamap_create(adapter->tx_buf_tag, 0,
&tx_ring->tx_buffer_info[i].dmamap);
if (unlikely(err != 0)) {
- ena_trace(ENA_ALERT,
+ ena_trace(NULL, ENA_ALERT,
"Unable to create Tx DMA map for buffer %d\n",
i);
goto err_map_release;
@@ -674,7 +675,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
err = bus_dmamap_create(adapter->tx_buf_tag, 0,
&map[j]);
if (unlikely(err != 0)) {
- ena_trace(ENA_ALERT, "Unable to create "
+ ena_trace(NULL, ENA_ALERT, "Unable to create "
"Tx DMA for buffer %d %d\n", i, j);
goto err_map_release;
}
@@ -688,7 +689,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT,
taskqueue_thread_enqueue, &tx_ring->enqueue_tq);
if (unlikely(tx_ring->enqueue_tq == NULL)) {
- ena_trace(ENA_ALERT,
+ ena_trace(NULL, ENA_ALERT,
"Unable to create taskqueue for enqueue task\n");
i = tx_ring->ring_size;
goto err_map_release;
@@ -873,7 +874,7 @@ ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
err = bus_dmamap_create(adapter->rx_buf_tag, 0,
&(rx_ring->rx_buffer_info[i].map));
if (err != 0) {
- ena_trace(ENA_ALERT,
+ ena_trace(NULL, ENA_ALERT,
"Unable to create Rx DMA map for buffer %d\n", i);
goto err_buf_info_unmap;
}
@@ -886,7 +887,7 @@ ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
device_printf(adapter->pdev,
"LRO[%d] Initialization failed!\n", qid);
} else {
- ena_trace(ENA_INFO,
+ ena_trace(NULL, ENA_INFO,
"RX Soft LRO[%d] Initialized\n", qid);
rx_ring->lro.ifp = adapter->ifp;
}
@@ -1017,13 +1018,13 @@ ena_alloc_rx_mbuf(struct ena_adapter *adapter,
rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen;
/* Map packets for DMA */
- ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
+ ena_trace(NULL, ENA_DBG | ENA_RSC | ENA_RXPTH,
"Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n",
adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len);
error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map,
rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
if (unlikely((error != 0) || (nsegs != 1))) {
- ena_trace(ENA_WARNING, "failed to map mbuf, error: %d, "
+ ena_trace(NULL, ENA_WARNING, "failed to map mbuf, error: %d, "
"nsegs: %d\n", error, nsegs);
counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1);
goto exit;
@@ -1036,7 +1037,7 @@ ena_alloc_rx_mbuf(struct ena_adapter *adapter,
ena_buf->paddr = segs[0].ds_addr;
ena_buf->len = mlen;
- ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
+ ena_trace(NULL, ENA_DBG | ENA_RSC | ENA_RXPTH,
"ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr);
@@ -1054,7 +1055,7 @@ ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
{
if (rx_info->mbuf == NULL) {
- ena_trace(ENA_WARNING, "Trying to free unallocated buffer\n");
+ ena_trace(NULL, ENA_WARNING, "Trying to free unallocated buffer\n");
return;
}
@@ -1079,7 +1080,7 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
uint32_t i;
int rc;
- ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d\n",
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d\n",
rx_ring->qid);
next_to_use = rx_ring->next_to_use;
@@ -1087,7 +1088,7 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
for (i = 0; i < num; i++) {
struct ena_rx_buffer *rx_info;
- ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC,
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH | ENA_RSC,
"RX buffer - next to use: %d\n", next_to_use);
req_id = rx_ring->free_rx_ids[next_to_use];
@@ -1099,7 +1100,7 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
#endif /* DEV_NETMAP */
rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
if (unlikely(rc != 0)) {
- ena_trace(ENA_WARNING,
+ ena_trace(NULL, ENA_WARNING,
"failed to alloc buffer for rx queue %d\n",
rx_ring->qid);
break;
@@ -1107,7 +1108,7 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
&rx_info->ena_buf, req_id);
if (unlikely(rc != 0)) {
- ena_trace(ENA_WARNING,
+ ena_trace(NULL, ENA_WARNING,
"failed to add buffer for rx queue %d\n",
rx_ring->qid);
break;
@@ -1118,7 +1119,7 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
if (unlikely(i < num)) {
counter_u64_add(rx_ring->rx_stats.refil_partial, 1);
- ena_trace(ENA_WARNING,
+ ena_trace(NULL, ENA_WARNING,
"refilled rx qid %d with only %d mbufs (from %d)\n",
rx_ring->qid, i, num);
}
@@ -1327,7 +1328,7 @@ ena_refill_all_rx_bufs(struct ena_adapter *adapter)
bufs_num = rx_ring->ring_size - 1;
rc = ena_refill_rx_bufs(rx_ring, bufs_num);
if (unlikely(rc != bufs_num))
- ena_trace(ENA_WARNING, "refilling Queue %d failed. "
+ ena_trace(NULL, ENA_WARNING, "refilling Queue %d failed. "
"Allocated %d buffers from: %d\n", i, rc, bufs_num);
#ifdef DEV_NETMAP
rx_ring->initialized = true;
@@ -1368,7 +1369,7 @@ ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid)
qid, i);
print_once = false;
} else {
- ena_trace(ENA_DBG,
+ ena_trace(NULL, ENA_DBG,
"free uncompleted tx mbuf qid %d idx 0x%x\n",
qid, i);
}
@@ -1584,7 +1585,7 @@ ena_enable_msix(struct ena_adapter *adapter)
adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry),
M_DEVBUF, M_WAITOK | M_ZERO);
- ena_trace(ENA_DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs);
+ ena_trace(NULL, ENA_DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs);
for (i = 0; i < msix_vecs; i++) {
adapter->msix_entries[i].entry = i;
@@ -1662,7 +1663,7 @@ ena_setup_io_intr(struct ena_adapter *adapter)
adapter->irq_tbl[irq_idx].data = &adapter->que[i];
adapter->irq_tbl[irq_idx].vector =
adapter->msix_entries[irq_idx].vector;
- ena_trace(ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n",
+ ena_trace(NULL, ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n",
adapter->msix_entries[irq_idx].vector);
/*
@@ -1712,7 +1713,7 @@ ena_request_mgmnt_irq(struct ena_adapter *adapter)
return (rc);
err_res_free:
- ena_trace(ENA_INFO | ENA_ADMQ, "releasing resource for irq %d\n",
+ ena_trace(NULL, ENA_INFO | ENA_ADMQ, "releasing resource for irq %d\n",
irq->vector);
rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
irq->vector, irq->res);
@@ -1765,7 +1766,7 @@ ena_request_io_irq(struct ena_adapter *adapter)
}
irq->requested = true;
- ena_trace(ENA_INFO, "queue %d - cpu %d\n",
+ ena_trace(NULL, ENA_INFO, "queue %d - cpu %d\n",
i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
}
@@ -1812,7 +1813,7 @@ ena_free_mgmnt_irq(struct ena_adapter *adapter)
irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
if (irq->requested) {
- ena_trace(ENA_INFO | ENA_ADMQ, "tear down irq: %d\n",
+ ena_trace(NULL, ENA_INFO | ENA_ADMQ, "tear down irq: %d\n",
irq->vector);
rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
if (unlikely(rc != 0))
@@ -1822,7 +1823,7 @@ ena_free_mgmnt_irq(struct ena_adapter *adapter)
}
if (irq->res != NULL) {
- ena_trace(ENA_INFO | ENA_ADMQ, "release resource irq: %d\n",
+ ena_trace(NULL, ENA_INFO | ENA_ADMQ, "release resource irq: %d\n",
irq->vector);
rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
irq->vector, irq->res);
@@ -1842,7 +1843,7 @@ ena_free_io_irq(struct ena_adapter *adapter)
for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
irq = &adapter->irq_tbl[i];
if (irq->requested) {
- ena_trace(ENA_INFO | ENA_IOQ, "tear down irq: %d\n",
+ ena_trace(NULL, ENA_INFO | ENA_IOQ, "tear down irq: %d\n",
irq->vector);
rc = bus_teardown_intr(adapter->pdev, irq->res,
irq->cookie);
@@ -1854,7 +1855,7 @@ ena_free_io_irq(struct ena_adapter *adapter)
}
if (irq->res != NULL) {
- ena_trace(ENA_INFO | ENA_IOQ, "release resource irq: %d\n",
+ ena_trace(NULL, ENA_INFO | ENA_IOQ, "release resource irq: %d\n",
irq->vector);
rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
irq->vector, irq->res);
@@ -2001,21 +2002,21 @@ create_queues_with_size_backoff(struct ena_adapter *adapter)
/* Allocate transmit descriptors */
rc = ena_setup_all_tx_resources(adapter);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "err_setup_tx\n");
+ ena_trace(NULL, ENA_ALERT, "err_setup_tx\n");
goto err_setup_tx;
}
/* Allocate receive descriptors */
rc = ena_setup_all_rx_resources(adapter);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "err_setup_rx\n");
+ ena_trace(NULL, ENA_ALERT, "err_setup_rx\n");
goto err_setup_rx;
}
/* Create IO queues for Rx & Tx */
rc = ena_create_io_queues(adapter);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT,
+ ena_trace(NULL, ENA_ALERT,
"create IO queues failed\n");
goto err_io_que;
}
@@ -2032,7 +2033,7 @@ err_setup_tx:
* error straightaway.
*/
if (unlikely(rc != ENOMEM)) {
- ena_trace(ENA_ALERT,
+ ena_trace(NULL, ENA_ALERT,
"Queue creation failed with error code: %d\n", rc);
return (rc);
}
@@ -2087,12 +2088,12 @@ ena_up(struct ena_adapter *adapter)
/* setup interrupts for IO queues */
rc = ena_setup_io_intr(adapter);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "error setting up IO interrupt\n");
+ ena_trace(NULL, ENA_ALERT, "error setting up IO interrupt\n");
goto error;
}
rc = ena_request_io_irq(adapter);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "err_req_irq\n");
+ ena_trace(NULL, ENA_ALERT, "err_req_irq\n");
goto error;
}
@@ -2107,7 +2108,7 @@ ena_up(struct ena_adapter *adapter)
rc = create_queues_with_size_backoff(adapter);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT,
+ ena_trace(NULL, ENA_ALERT,
"error creating queues with size backoff\n");
goto err_create_queues_with_backoff;
}
@@ -2189,7 +2190,7 @@ static void
ena_media_status(if_t ifp, struct ifmediareq *ifmr)
{
struct ena_adapter *adapter = if_getsoftc(ifp);
- ena_trace(ENA_DBG, "enter\n");
+ ena_trace(NULL, ENA_DBG, "enter\n");
ENA_LOCK_LOCK(adapter);
@@ -2198,7 +2199,7 @@ ena_media_status(if_t ifp, struct ifmediareq *ifmr)
if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) {
ENA_LOCK_UNLOCK(adapter);
- ena_trace(ENA_INFO, "Link is down\n");
+ ena_trace(NULL, ENA_INFO, "Link is down\n");
return;
}
@@ -2392,7 +2393,7 @@ ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter,
ifp = adapter->ifp = if_gethandle(IFT_ETHER);
if (unlikely(ifp == NULL)) {
- ena_trace(ENA_ALERT, "can not allocate ifnet structure\n");
+ ena_trace(NULL, ENA_ALERT, "can not allocate ifnet structure\n");
return (ENXIO);
}
if_initname(ifp, device_get_name(pdev), device_get_unit(pdev));
@@ -2528,7 +2529,7 @@ ena_enable_wc(struct resource *res)
/* Enable write combining */
rc = pmap_change_attr(va, len, PAT_WRITE_COMBINING);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "pmap_change_attr failed, %d\n", rc);
+ ena_trace(NULL, ENA_ALERT, "pmap_change_attr failed, %d\n", rc);
return (rc);
}
@@ -2729,7 +2730,7 @@ ena_rss_init_default_deferred(void *arg)
dc = devclass_find("ena");
if (unlikely(dc == NULL)) {
- ena_trace(ENA_ALERT, "No devclass ena\n");
+ ena_trace(NULL, ENA_ALERT, "No devclass ena\n");
return;
}
@@ -2760,7 +2761,7 @@ ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev)
/* Allocate only the host info */
rc = ena_com_allocate_host_info(ena_dev);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "Cannot allocate host info\n");
+ ena_trace(NULL, ENA_ALERT, "Cannot allocate host info\n");
return;
}
@@ -2781,13 +2782,15 @@ ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev)
(DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
(DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
host_info->num_cpus = mp_ncpus;
+ host_info->driver_supported_features =
+ ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK;
rc = ena_com_set_host_attributes(ena_dev);
if (unlikely(rc != 0)) {
if (rc == EOPNOTSUPP)
- ena_trace(ENA_WARNING, "Cannot set host attributes\n");
+ ena_trace(NULL, ENA_WARNING, "Cannot set host attributes\n");
else
- ena_trace(ENA_ALERT, "Cannot set host attributes\n");
+ ena_trace(NULL, ENA_ALERT, "Cannot set host attributes\n");
goto err;
}
@@ -3041,7 +3044,7 @@ check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
if (unlikely(time_offset > adapter->missing_tx_timeout)) {
if (!tx_buf->print_once)
- ena_trace(ENA_WARNING, "Found a Tx that wasn't "
+ ena_trace(NULL, ENA_WARNING, "Found a Tx that wasn't "
"completed on time, qid %d, index %d.\n",
tx_ring->qid, i);
@@ -3197,6 +3200,44 @@ static void ena_update_hints(struct ena_adapter *adapter,
}
}
+/**
+ * ena_copy_eni_metrics - Get and copy ENI metrics from the HW.
+ * @adapter: ENA device adapter
+ *
+ * Returns 0 on success, EOPNOTSUPP if current HW doesn't support those metrics
+ * and other error codes on failure.
+ *
+ * This function can possibly cause a race with other calls to the admin queue.
+ * Because of that, the caller should either lock this function or make sure
+ * that there is no race in the current context.
+ */
+static int
+ena_copy_eni_metrics(struct ena_adapter *adapter)
+{
+ static bool print_once = true;
+ int rc;
+
+ rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics);
+
+ if (rc != 0) {
+ if (rc == ENA_COM_UNSUPPORTED) {
+ if (print_once) {
+ device_printf(adapter->pdev,
+ "Retrieving ENI metrics is not supported.\n");
+ print_once = false;
+ } else {
+ ena_trace(NULL, ENA_DBG,
+ "Retrieving ENI metrics is not supported.\n");
+ }
+ } else {
+ device_printf(adapter->pdev,
+ "Failed to get ENI metrics: %d\n", rc);
+ }
+ }
+
+ return (rc);
+}
+
static void
ena_timer_service(void *data)
{
@@ -3212,6 +3253,38 @@ ena_timer_service(void *data)
check_for_empty_rx_ring(adapter);
+ /*
+ * User controller update of the ENI metrics.
+ * If the delay was set to 0, then the stats shouldn't be updated at
+ * all.
+ * Otherwise, wait 'eni_metrics_sample_interval' seconds, before
+ * updating stats.
+ * As timer service is executed every second, it's enough to increment
+ * appropriate counter each time the timer service is executed.
+ */
+ if ((adapter->eni_metrics_sample_interval != 0) &&
+ (++adapter->eni_metrics_sample_interval_cnt >=
+ adapter->eni_metrics_sample_interval)) {
+ /*
+ * There is no race with other admin queue calls, as:
+ * - Timer service runs after interface is up, so all
+ * configuration calls to the admin queue are finished.
+ * - After interface is up, the driver doesn't use (at least
+ * for now) other functions writing to the admin queue.
+ *
+ * It may change in the future, so in that situation, the lock
+ * will be needed. ENA_LOCK_*() cannot be used for that purpose,
+ * as callout ena_timer_service is protected by them. It could
+ * lead to the deadlock if callout_drain() would hold the lock
+ * before ena_copy_eni_metrics() was executed. It's advised to
+ * use separate lock in that situation which will be used only
+ * for the admin queue.
+ */
+ (void)ena_copy_eni_metrics(adapter);
+ adapter->eni_metrics_sample_interval_cnt = 0;
+ }
+
+
if (host_info != NULL)
ena_update_host_info(host_info, adapter->ifp);
@@ -3778,11 +3851,11 @@ static void ena_notification(void *adapter_data,
struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
struct ena_admin_ena_hw_hints *hints;
- ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
+ ENA_WARN(NULL, aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
"Invalid group(%x) expected %x\n", aenq_e->aenq_common_desc.group,
ENA_ADMIN_NOTIFICATION);
- switch (aenq_e->aenq_common_desc.syndrom) {
+ switch (aenq_e->aenq_common_desc.syndrome) {
case ENA_ADMIN_UPDATE_HINTS:
hints =
(struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4);
@@ -3791,7 +3864,7 @@ static void ena_notification(void *adapter_data,
default:
device_printf(adapter->pdev,
"Invalid aenq notification link state %d\n",
- aenq_e->aenq_common_desc.syndrom);
+ aenq_e->aenq_common_desc.syndrome);
}
}
diff --git a/sys/dev/ena/ena.h b/sys/dev/ena/ena.h
index ee57f53f365b..28da047e2e1b 100644
--- a/sys/dev/ena/ena.h
+++ b/sys/dev/ena/ena.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -40,7 +40,7 @@
#include "ena-com/ena_eth_com.h"
#define DRV_MODULE_VER_MAJOR 2
-#define DRV_MODULE_VER_MINOR 2
+#define DRV_MODULE_VER_MINOR 3
#define DRV_MODULE_VER_SUBMINOR 0
#define DRV_MODULE_NAME "ena"
@@ -150,10 +150,10 @@
*/
#define PCI_VENDOR_ID_AMAZON 0x1d0f
-#define PCI_DEV_ID_ENA_PF 0x0ec2
-#define PCI_DEV_ID_ENA_LLQ_PF 0x1ec2
-#define PCI_DEV_ID_ENA_VF 0xec20
-#define PCI_DEV_ID_ENA_LLQ_VF 0xec21
+#define PCI_DEV_ID_ENA_PF 0x0ec2
+#define PCI_DEV_ID_ENA_PF_RSERV0 0x1ec2
+#define PCI_DEV_ID_ENA_VF 0xec20
+#define PCI_DEV_ID_ENA_VF_RSERV0 0xec21
/*
* Flags indicating current ENA driver state
@@ -463,9 +463,13 @@ struct ena_adapter {
uint32_t missing_tx_threshold;
bool disable_meta_caching;
+ uint16_t eni_metrics_sample_interval;
+ uint16_t eni_metrics_sample_interval_cnt;
+
/* Statistics */
struct ena_stats_dev dev_stats;
struct ena_hw_stats hw_stats;
+ struct ena_admin_eni_stats eni_metrics;
enum ena_regs_reset_reason_types reset_reason;
};
@@ -514,20 +518,4 @@ ena_trigger_reset(struct ena_adapter *adapter,
}
}
-static inline int
-validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
-{
- if (likely(req_id < rx_ring->ring_size))
- return (0);
-
- device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n",
- req_id);
- counter_u64_add(rx_ring->rx_stats.bad_req_id, 1);
-
- /* Trigger device reset */
- ena_trigger_reset(rx_ring->adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
-
- return (EFAULT);
-}
-
#endif /* !(ENA_H) */
diff --git a/sys/dev/ena/ena_datapath.c b/sys/dev/ena/ena_datapath.c
index 51beabc283f7..252a665b7938 100644
--- a/sys/dev/ena/ena_datapath.c
+++ b/sys/dev/ena/ena_datapath.c
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -75,7 +75,7 @@ ena_cleanup(void *arg, int pending)
if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
return;
- ena_trace(ENA_DBG, "MSI-X TX/RX routine\n");
+ ena_trace(NULL, ENA_DBG, "MSI-X TX/RX routine\n");
tx_ring = que->tx_ring;
rx_ring = que->rx_ring;
@@ -266,7 +266,7 @@ ena_tx_cleanup(struct ena_ring *tx_ring)
bus_dmamap_unload(adapter->tx_buf_tag,
tx_info->dmamap);
- ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed\n",
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed\n",
tx_ring->qid, mbuf);
m_freem(mbuf);
@@ -291,7 +291,7 @@ ena_tx_cleanup(struct ena_ring *tx_ring)
work_done = TX_BUDGET - budget;
- ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d\n",
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d\n",
tx_ring->qid, work_done);
/* If there is still something to commit update ring state */
@@ -395,7 +395,6 @@ ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
struct ena_rx_buffer *rx_info;
struct ena_adapter *adapter;
unsigned int descs = ena_rx_ctx->descs;
- int rc;
uint16_t ntc, len, req_id, buf = 0;
ntc = *next_to_clean;
@@ -403,17 +402,13 @@ ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
len = ena_bufs[buf].len;
req_id = ena_bufs[buf].req_id;
- rc = validate_rx_req_id(rx_ring, req_id);
- if (unlikely(rc != 0))
- return (NULL);
-
rx_info = &rx_ring->rx_buffer_info[req_id];
if (unlikely(rx_info->mbuf == NULL)) {
device_printf(adapter->pdev, "NULL mbuf in rx_info");
return (NULL);
}
- ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx\n",
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx\n",
rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
@@ -422,12 +417,16 @@ ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
mbuf->m_flags |= M_PKTHDR;
mbuf->m_pkthdr.len = len;
mbuf->m_len = len;
+ // Only for the first segment the data starts at specific offset
+ mbuf->m_data = mtodo(mbuf, ena_rx_ctx->pkt_offset);
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH,
+ "Mbuf data offset=%u\n", ena_rx_ctx->pkt_offset);
mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
/* Fill mbuf with hash key and it's interpretation for optimization */
ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
- ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d\n",
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d\n",
mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
/* DMA address is not needed anymore, unmap it */
@@ -445,16 +444,6 @@ ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
++buf;
len = ena_bufs[buf].len;
req_id = ena_bufs[buf].req_id;
- rc = validate_rx_req_id(rx_ring, req_id);
- if (unlikely(rc != 0)) {
- /*
- * If the req_id is invalid, then the device will be
- * reset. In that case we must free all mbufs that
- * were already gathered.
- */
- m_freem(mbuf);
- return (NULL);
- }
rx_info = &rx_ring->rx_buffer_info[req_id];
if (unlikely(rx_info->mbuf == NULL)) {
@@ -477,11 +466,11 @@ ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
BUS_DMASYNC_POSTREAD);
if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
- ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p\n",
+ ena_trace(NULL, ENA_WARNING, "Failed to append Rx mbuf %p\n",
mbuf);
}
- ena_trace(ENA_DBG | ENA_RXPTH,
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH,
"rx mbuf updated. len %d\n", mbuf->m_pkthdr.len);
/* Free already appended mbuf, it won't be useful anymore */
@@ -512,7 +501,7 @@ ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
/* ipv4 checksum error */
mbuf->m_pkthdr.csum_flags = 0;
counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
- ena_trace(ENA_DBG, "RX IPv4 header checksum error\n");
+ ena_trace(NULL, ENA_DBG, "RX IPv4 header checksum error\n");
return;
}
@@ -523,7 +512,7 @@ ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
/* TCP/UDP checksum error */
mbuf->m_pkthdr.csum_flags = 0;
counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
- ena_trace(ENA_DBG, "RX L4 checksum error\n");
+ ena_trace(NULL, ENA_DBG, "RX L4 checksum error\n");
} else {
mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
@@ -543,6 +532,7 @@ ena_rx_cleanup(struct ena_ring *rx_ring)
struct ena_com_rx_ctx ena_rx_ctx;
struct ena_com_io_cq* io_cq;
struct ena_com_io_sq* io_sq;
+ enum ena_regs_reset_reason_types reset_reason;
if_t ifp;
uint16_t ena_qid;
uint16_t next_to_clean;
@@ -569,23 +559,35 @@ ena_rx_cleanup(struct ena_ring *rx_ring)
return (0);
#endif /* DEV_NETMAP */
- ena_trace(ENA_DBG, "rx: qid %d\n", qid);
+ ena_trace(NULL, ENA_DBG, "rx: qid %d\n", qid);
do {
ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
ena_rx_ctx.descs = 0;
+ ena_rx_ctx.pkt_offset = 0;
+
bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
-
- if (unlikely(rc != 0))
- goto error;
+ if (unlikely(rc != 0)) {
+ if (rc == ENA_COM_NO_SPACE) {
+ counter_u64_add(rx_ring->rx_stats.bad_desc_num,
+ 1);
+ reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
+ } else {
+ counter_u64_add(rx_ring->rx_stats.bad_req_id,
+ 1);
+ reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
+ }
+ ena_trigger_reset(adapter, reset_reason);
+ return (0);
+ }
if (unlikely(ena_rx_ctx.descs == 0))
break;
- ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. "
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. "
"descs #: %d l3 proto %d l4 proto %d hash: %x\n",
rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
@@ -638,7 +640,7 @@ ena_rx_cleanup(struct ena_ring *rx_ring)
do_if_input = 0;
}
if (do_if_input != 0) {
- ena_trace(ENA_DBG | ENA_RXPTH,
+ ena_trace(NULL, ENA_DBG | ENA_RXPTH,
"calling if_input() with mbuf %p\n", mbuf);
(*ifp->if_input)(ifp, mbuf);
}
@@ -664,14 +666,6 @@ ena_rx_cleanup(struct ena_ring *rx_ring)
tcp_lro_flush_all(&rx_ring->lro);
return (RX_BUDGET - budget);
-
-error:
- counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1);
-
- /* Too many desc from the device. Trigger reset */
- ena_trigger_reset(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS);
-
- return (0);
}
static void
@@ -827,7 +821,7 @@ ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info,
rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->dmamap, mbuf,
segs, &nsegs, BUS_DMA_NOWAIT);
if (unlikely((rc != 0) || (nsegs == 0))) {
- ena_trace(ENA_WARNING,
+ ena_trace(NULL, ENA_WARNING,
"dmamap load failed! err: %d nsegs: %d\n", rc, nsegs);
goto dma_error;
}
@@ -859,7 +853,7 @@ ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info,
counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
}
- ena_trace(ENA_DBG | ENA_TXPTH,
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH,
"mbuf: %p header_buf->vaddr: %p push_len: %d\n",
mbuf, *push_hdr, *header_len);
@@ -937,12 +931,12 @@ ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);
if (unlikely(rc != 0)) {
- ena_trace(ENA_WARNING,
+ ena_trace(NULL, ENA_WARNING,
"Failed to collapse mbuf! err: %d\n", rc);
return (rc);
}
- ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len);
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len);
next_to_use = tx_ring->next_to_use;
req_id = tx_ring->free_tx_ids[next_to_use];
@@ -951,7 +945,7 @@ ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len);
if (unlikely(rc != 0)) {
- ena_trace(ENA_WARNING, "Failed to map TX mbuf\n");
+ ena_trace(NULL, ENA_WARNING, "Failed to map TX mbuf\n");
return (rc);
}
memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
@@ -966,7 +960,7 @@ ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
if (tx_ring->acum_pkts == DB_THRESHOLD ||
ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) {
- ena_trace(ENA_DBG | ENA_TXPTH,
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH,
"llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
tx_ring->que->id);
ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
@@ -978,7 +972,7 @@ ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc);
if (unlikely(rc != 0)) {
if (likely(rc == ENA_COM_NO_MEM)) {
- ena_trace(ENA_DBG | ENA_TXPTH,
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH,
"tx ring[%d] if out of space\n", tx_ring->que->id);
} else {
device_printf(adapter->pdev,
@@ -1011,7 +1005,7 @@ ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
*/
if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
adapter->max_tx_sgl_size + 2))) {
- ena_trace(ENA_DBG | ENA_TXPTH, "Stop queue %d\n",
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH, "Stop queue %d\n",
tx_ring->que->id);
tx_ring->running = false;
@@ -1064,7 +1058,7 @@ ena_start_xmit(struct ena_ring *tx_ring)
io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {
- ena_trace(ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and"
+ ena_trace(NULL, ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and"
" header csum flags %#jx\n",
mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags);
diff --git a/sys/dev/ena/ena_datapath.h b/sys/dev/ena/ena_datapath.h
index 86d875cbe015..4886ff1e6391 100644
--- a/sys/dev/ena/ena_datapath.h
+++ b/sys/dev/ena/ena_datapath.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
diff --git a/sys/dev/ena/ena_netmap.c b/sys/dev/ena/ena_netmap.c
index 20a341173c8c..2f645a7fbd19 100644
--- a/sys/dev/ena/ena_netmap.c
+++ b/sys/dev/ena/ena_netmap.c
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -88,7 +88,7 @@ ena_netmap_attach(struct ena_adapter *adapter)
{
struct netmap_adapter na;
- ena_trace(ENA_NETMAP, "netmap attach\n");
+ ena_trace(NULL, ENA_NETMAP, "netmap attach\n");
bzero(&na, sizeof(na));
na.na_flags = NAF_MOREFRAG;
@@ -126,12 +126,12 @@ ena_netmap_alloc_rx_slot(struct ena_adapter *adapter,
nm_i = kring->nr_hwcur;
head = kring->rhead;
- ena_trace(ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, "
+ ena_trace(NULL, ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, "
"rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur,
kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail);
if ((nm_i == head) && rx_ring->initialized) {
- ena_trace(ENA_NETMAP, "No free slots in netmap ring\n");
+ ena_trace(NULL, ENA_NETMAP, "No free slots in netmap ring\n");
return (ENOMEM);
}
@@ -150,7 +150,7 @@ ena_netmap_alloc_rx_slot(struct ena_adapter *adapter,
rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr);
if (rc != 0) {
- ena_trace(ENA_WARNING, "DMA mapping error\n");
+ ena_trace(NULL, ENA_WARNING, "DMA mapping error\n");
return (rc);
}
bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
@@ -210,7 +210,7 @@ ena_netmap_free_rx_slot(struct ena_adapter *adapter,
slot = &kring->ring->slot[nm_i];
- ENA_ASSERT(slot->buf_idx == 0, "Overwrite slot buf\n");
+ ENA_WARN(slot->buf_idx != 0, NULL, "Overwrite slot buf\n");
slot->buf_idx = rx_info->netmap_buf_idx;
slot->flags = NS_BUF_CHANGED;
@@ -252,7 +252,7 @@ ena_netmap_reset_ring(struct ena_adapter *adapter, int qid, enum txrx x)
return;
netmap_reset(NA(adapter->ifp), x, qid, 0);
- ena_trace(ENA_NETMAP, "%s ring %d is in netmap mode\n",
+ ena_trace(NULL, ENA_NETMAP, "%s ring %d is in netmap mode\n",
(x == NR_TX) ? "Tx" : "Rx", qid);
}
@@ -282,7 +282,7 @@ ena_netmap_reg(struct netmap_adapter *na, int onoff)
ena_down(adapter);
if (onoff) {
- ena_trace(ENA_NETMAP, "netmap on\n");
+ ena_trace(NULL, ENA_NETMAP, "netmap on\n");
for_rx_tx(t) {
for (i = 0; i <= nma_get_nrings(na, t); i++) {
kring = NMR(na, t)[i];
@@ -293,7 +293,7 @@ ena_netmap_reg(struct netmap_adapter *na, int onoff)
}
nm_set_native_flags(na);
} else {
- ena_trace(ENA_NETMAP, "netmap off\n");
+ ena_trace(NULL, ENA_NETMAP, "netmap off\n");
nm_clear_native_flags(na);
for_rx_tx(t) {
for (i = 0; i <= nma_get_nrings(na, t); i++) {
@@ -307,7 +307,7 @@ ena_netmap_reg(struct netmap_adapter *na, int onoff)
rc = ena_up(adapter);
if (rc != 0) {
- ena_trace(ENA_WARNING, "ena_up failed with rc=%d\n", rc);
+ ena_trace(NULL, ENA_WARNING, "ena_up failed with rc=%d\n", rc);
adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE;
nm_clear_native_flags(na);
ena_destroy_device(adapter, false);
@@ -401,7 +401,7 @@ ena_netmap_tx_frame(struct ena_netmap_ctx *ctx)
adapter = ctx->adapter;
if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) {
- ena_trace(ENA_WARNING, "Too many slots per packet\n");
+ ena_trace(NULL, ENA_WARNING, "Too many slots per packet\n");
return (EINVAL);
}
@@ -438,7 +438,7 @@ ena_netmap_tx_frame(struct ena_netmap_ctx *ctx)
rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc);
if (unlikely(rc != 0)) {
if (likely(rc == ENA_COM_NO_MEM)) {
- ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
+ ena_trace(NULL, ENA_NETMAP | ENA_DBG | ENA_TXPTH,
"Tx ring[%d] is out of space\n", tx_ring->que->id);
} else {
device_printf(adapter->pdev,
@@ -532,13 +532,13 @@ ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot,
*vaddr = PNMB(na, slot, paddr);
if (unlikely(vaddr == NULL)) {
- ena_trace(ENA_ALERT, "Slot address is NULL\n");
+ ena_trace(NULL, ENA_ALERT, "Slot address is NULL\n");
return (EINVAL);
}
rc = netmap_load_map(na, dmatag, dmamap, *vaddr);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "Failed to map slot %d for DMA\n",
+ ena_trace(NULL, ENA_ALERT, "Failed to map slot %d for DMA\n",
slot->buf_idx);
return (EINVAL);
}
@@ -626,7 +626,7 @@ ena_netmap_tx_map_slots(struct ena_netmap_ctx *ctx,
delta = push_len - slot_head_len;
}
- ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
+ ena_trace(NULL, ENA_NETMAP | ENA_DBG | ENA_TXPTH,
"slot: %d header_buf->vaddr: %p push_len: %d\n",
slot->buf_idx, *push_hdr, push_len);
@@ -860,7 +860,7 @@ ena_netmap_tx_clean_one(struct ena_netmap_ctx *ctx, uint16_t req_id)
/* Next, retain the sockets back to the userspace */
for (n = 0; n < nm_info->sockets_used; n++) {
ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
- ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0,
+ ENA_WARN(ctx->slots[ctx->nm_i].buf_idx != 0, NULL,
"Tx idx is not 0.\n");
ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
@@ -882,7 +882,7 @@ validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
if (likely(req_id < tx_ring->ring_size))
return (0);
- ena_trace(ENA_WARNING, "Invalid req_id: %hu\n", req_id);
+ ena_trace(NULL, ENA_WARNING, "Invalid req_id: %hu\n", req_id);
counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
@@ -949,6 +949,7 @@ static inline int
ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
{
struct ena_com_rx_ctx ena_rx_ctx;
+ enum ena_regs_reset_reason_types reset_reason;
int rc, len = 0;
uint16_t buf, nm;
@@ -959,16 +960,22 @@ ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx);
if (unlikely(rc != 0)) {
- ena_trace(ENA_ALERT, "Too many desc from the device.\n");
- counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
- ena_trigger_reset(ctx->adapter,
- ENA_REGS_RESET_TOO_MANY_RX_DESCS);
+ ena_trace(NULL, ENA_ALERT,
+ "Failed to read pkt from the device with error: %d\n", rc);
+ if (rc == ENA_COM_NO_SPACE) {
+ counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
+ reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
+ } else {
+ counter_u64_add(ctx->ring->rx_stats.bad_req_id, 1);
+ reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
+ }
+ ena_trigger_reset(ctx->adapter, reset_reason);
return (rc);
}
if (unlikely(ena_rx_ctx.descs == 0))
return (ENA_NETMAP_NO_MORE_FRAMES);
- ena_trace(ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:"
+ ena_trace(NULL, ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:"
" %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid,
ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto,
ena_rx_ctx.hash);
@@ -1017,19 +1024,15 @@ ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len)
{
struct ena_rx_buffer *rx_info;
uint16_t req_id;
- int rc;
req_id = ctx->ring->ena_bufs[buf].req_id;
- rc = validate_rx_req_id(ctx->ring, req_id);
- if (unlikely(rc != 0))
- return (rc);
-
rx_info = &ctx->ring->rx_buffer_info[req_id];
bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map,
BUS_DMASYNC_POSTREAD);
netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map);
- ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, "Rx idx is not 0.\n");
+ ENA_WARN(ctx->slots[ctx->nm_i].buf_idx != 0, NULL,
+ "Rx idx is not 0.\n");
ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx;
rx_info->netmap_buf_idx = 0;
@@ -1041,7 +1044,7 @@ ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len)
ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len;
*len += ctx->slots[ctx->nm_i].len;
ctx->ring->free_rx_ids[ctx->nt] = req_id;
- ena_trace(ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n",
+ ena_trace(NULL, ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n",
rx_info, ctx->slots[ctx->nm_i].buf_idx,
(uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i);
diff --git a/sys/dev/ena/ena_netmap.h b/sys/dev/ena/ena_netmap.h
index 7f13da21e2b4..a418e0b6317e 100644
--- a/sys/dev/ena/ena_netmap.h
+++ b/sys/dev/ena/ena_netmap.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
diff --git a/sys/dev/ena/ena_sysctl.c b/sys/dev/ena/ena_sysctl.c
index 55d9c96da4c8..0f1a93faba84 100644
--- a/sys/dev/ena/ena_sysctl.c
+++ b/sys/dev/ena/ena_sysctl.c
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
@@ -34,10 +34,15 @@ __FBSDID("$FreeBSD$");
static void ena_sysctl_add_wd(struct ena_adapter *);
static void ena_sysctl_add_stats(struct ena_adapter *);
+static void ena_sysctl_add_eni_metrics(struct ena_adapter *);
static void ena_sysctl_add_tuneables(struct ena_adapter *);
static int ena_sysctl_buf_ring_size(SYSCTL_HANDLER_ARGS);
static int ena_sysctl_rx_queue_size(SYSCTL_HANDLER_ARGS);
static int ena_sysctl_io_queues_nb(SYSCTL_HANDLER_ARGS);
+static int ena_sysctl_eni_metrics_interval(SYSCTL_HANDLER_ARGS);
+
+/* Limit max ENI sample rate to be an hour. */
+#define ENI_METRICS_MAX_SAMPLE_INTERVAL 3600
static SYSCTL_NODE(_hw, OID_AUTO, ena, CTLFLAG_RD, 0, "ENA driver parameters");
@@ -68,6 +73,7 @@ ena_sysctl_add_nodes(struct ena_adapter *adapter)
{
ena_sysctl_add_wd(adapter);
ena_sysctl_add_stats(adapter);
+ ena_sysctl_add_eni_metrics(adapter);
ena_sysctl_add_tuneables(adapter);
}
@@ -291,6 +297,58 @@ ena_sysctl_add_stats(struct ena_adapter *adapter)
}
static void
+ena_sysctl_add_eni_metrics(struct ena_adapter *adapter)
+{
+ device_t dev;
+ struct ena_admin_eni_stats *eni_metrics;
+
+ struct sysctl_ctx_list *ctx;
+ struct sysctl_oid *tree;
+ struct sysctl_oid_list *child;
+
+ struct sysctl_oid *eni_node;
+ struct sysctl_oid_list *eni_list;
+
+ dev = adapter->pdev;
+
+ ctx = device_get_sysctl_ctx(dev);
+ tree = device_get_sysctl_tree(dev);
+ child = SYSCTL_CHILDREN(tree);
+
+ eni_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "eni_metrics",
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "ENA's ENI metrics");
+ eni_list = SYSCTL_CHILDREN(eni_node);
+
+ eni_metrics = &adapter->eni_metrics;
+
+ SYSCTL_ADD_U64(ctx, eni_list, OID_AUTO, "bw_in_allowance_exceeded",
+ CTLFLAG_RD, &eni_metrics->bw_in_allowance_exceeded, 0,
+ "Inbound BW allowance exceeded");
+ SYSCTL_ADD_U64(ctx, eni_list, OID_AUTO, "bw_out_allowance_exceeded",
+ CTLFLAG_RD, &eni_metrics->bw_out_allowance_exceeded, 0,
+ "Outbound BW allowance exceeded");
+ SYSCTL_ADD_U64(ctx, eni_list, OID_AUTO, "pps_allowance_exceeded",
+ CTLFLAG_RD, &eni_metrics->pps_allowance_exceeded, 0,
+ "PPS allowance exceeded");
+ SYSCTL_ADD_U64(ctx, eni_list, OID_AUTO, "conntrack_allowance_exceeded",
+ CTLFLAG_RD, &eni_metrics->conntrack_allowance_exceeded, 0,
+ "Connection tracking allowance exceeded");
+ SYSCTL_ADD_U64(ctx, eni_list, OID_AUTO, "linklocal_allowance_exceeded",
+ CTLFLAG_RD, &eni_metrics->linklocal_allowance_exceeded, 0,
+ "Linklocal packet rate allowance exceeded");
+
+ /*
+ * Tuneable, which determines how often ENI metrics will be read.
+ * 0 means it's turned off. Maximum allowed value is limited by:
+ * ENI_METRICS_MAX_SAMPLE_INTERVAL.
+ */
+ SYSCTL_ADD_PROC(ctx, eni_list, OID_AUTO, "sample_interval",
+ CTLTYPE_U16 | CTLFLAG_RW | CTLFLAG_MPSAFE, adapter, 0,
+ ena_sysctl_eni_metrics_interval, "SU",
+ "Interval in seconds for updating ENI emetrics. 0 turns off the update.");
+}
+
+static void
ena_sysctl_add_tuneables(struct ena_adapter *adapter)
{
device_t dev;
@@ -335,7 +393,7 @@ ena_sysctl_buf_ring_size(SYSCTL_HANDLER_ARGS)
error = sysctl_wire_old_buffer(req, sizeof(val));
if (error == 0) {
val = adapter->buf_ring_size;
- error = sysctl_handle_int(oidp, &val, 0, req);
+ error = sysctl_handle_32(oidp, &val, 0, req);
}
if (error != 0 || req->newptr == NULL)
return (error);
@@ -460,3 +518,40 @@ ena_sysctl_io_queues_nb(SYSCTL_HANDLER_ARGS)
return (error);
}
+
+static int
+ena_sysctl_eni_metrics_interval(SYSCTL_HANDLER_ARGS)
+{
+ struct ena_adapter *adapter = arg1;
+ uint16_t interval;
+ int error;
+
+ error = sysctl_wire_old_buffer(req, sizeof(interval));
+ if (error == 0) {
+ interval = adapter->eni_metrics_sample_interval;
+ error = sysctl_handle_16(oidp, &interval, 0, req);
+ }
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+
+ if (interval > ENI_METRICS_MAX_SAMPLE_INTERVAL) {
+ device_printf(adapter->pdev,
+ "ENI metrics update interval is out of range - maximum allowed value: %d seconds\n",
+ ENI_METRICS_MAX_SAMPLE_INTERVAL);
+ return (EINVAL);
+ }
+
+ if (interval == 0) {
+ device_printf(adapter->pdev,
+ "ENI metrics update is now turned off\n");
+ bzero(&adapter->eni_metrics, sizeof(adapter->eni_metrics));
+ } else {
+ device_printf(adapter->pdev,
+ "ENI metrics update interval is set to: %"PRIu16" seconds\n",
+ interval);
+ }
+
+ adapter->eni_metrics_sample_interval = interval;
+
+ return (0);
+}
diff --git a/sys/dev/ena/ena_sysctl.h b/sys/dev/ena/ena_sysctl.h
index 6041969e57e0..5aa0b795fbb5 100644
--- a/sys/dev/ena/ena_sysctl.h
+++ b/sys/dev/ena/ena_sysctl.h
@@ -1,5 +1,5 @@
/*-
- * BSD LICENSE
+ * SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
diff --git a/sys/modules/ena/Makefile b/sys/modules/ena/Makefile
index 72d0028367e8..497bd835efff 100644
--- a/sys/modules/ena/Makefile
+++ b/sys/modules/ena/Makefile
@@ -1,7 +1,7 @@
#
-# BSD LICENSE
+# SPDX-License-Identifier: BSD-2-Clause
#
-# Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
+# Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-16 13:26:58 +0000