summaryrefslogtreecommitdiff
path: root/ena_eth_com.c
diff options
context:
space:
mode:
Diffstat (limited to 'ena_eth_com.c')
-rw-r--r--ena_eth_com.c112
1 files changed, 73 insertions, 39 deletions
diff --git a/ena_eth_com.c b/ena_eth_com.c
index bbed92d22c47..58ddb82246fd 100644
--- a/ena_eth_com.c
+++ b/ena_eth_com.c
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
+ * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -33,7 +33,7 @@
#include "ena_eth_com.h"
-static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(
+static struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(
struct ena_com_io_cq *io_cq)
{
struct ena_eth_io_rx_cdesc_base *cdesc;
@@ -60,7 +60,7 @@ static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(
return cdesc;
}
-static inline void *get_sq_desc_regular_queue(struct ena_com_io_sq *io_sq)
+static void *get_sq_desc_regular_queue(struct ena_com_io_sq *io_sq)
{
u16 tail_masked;
u32 offset;
@@ -72,7 +72,7 @@ static inline void *get_sq_desc_regular_queue(struct ena_com_io_sq *io_sq)
return (void *)((uintptr_t)io_sq->desc_addr.virt_addr + offset);
}
-static inline int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq,
+static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq,
u8 *bounce_buffer)
{
struct ena_com_llq_info *llq_info = &io_sq->llq_info;
@@ -113,7 +113,7 @@ static inline int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq
return ENA_COM_OK;
}
-static inline int ena_com_write_header_to_bounce(struct ena_com_io_sq *io_sq,
+static int ena_com_write_header_to_bounce(struct ena_com_io_sq *io_sq,
u8 *header_src,
u16 header_len)
{
@@ -143,7 +143,7 @@ static inline int ena_com_write_header_to_bounce(struct ena_com_io_sq *io_sq,
return 0;
}
-static inline void *get_sq_desc_llq(struct ena_com_io_sq *io_sq)
+static void *get_sq_desc_llq(struct ena_com_io_sq *io_sq)
{
struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
u8 *bounce_buffer;
@@ -163,7 +163,7 @@ static inline void *get_sq_desc_llq(struct ena_com_io_sq *io_sq)
return sq_desc;
}
-static inline int ena_com_close_bounce_buffer(struct ena_com_io_sq *io_sq)
+static int ena_com_close_bounce_buffer(struct ena_com_io_sq *io_sq)
{
struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
struct ena_com_llq_info *llq_info = &io_sq->llq_info;
@@ -176,8 +176,10 @@ static inline int ena_com_close_bounce_buffer(struct ena_com_io_sq *io_sq)
if (pkt_ctrl->idx) {
rc = ena_com_write_bounce_buffer_to_dev(io_sq,
pkt_ctrl->curr_bounce_buf);
- if (unlikely(rc))
+ if (unlikely(rc)) {
+ ena_trc_err("failed to write bounce buffer to device\n");
return rc;
+ }
pkt_ctrl->curr_bounce_buf =
ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
@@ -190,7 +192,7 @@ static inline int ena_com_close_bounce_buffer(struct ena_com_io_sq *io_sq)
return ENA_COM_OK;
}
-static inline void *get_sq_desc(struct ena_com_io_sq *io_sq)
+static void *get_sq_desc(struct ena_com_io_sq *io_sq)
{
if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
return get_sq_desc_llq(io_sq);
@@ -198,7 +200,7 @@ static inline void *get_sq_desc(struct ena_com_io_sq *io_sq)
return get_sq_desc_regular_queue(io_sq);
}
-static inline int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq)
+static int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq)
{
struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl;
struct ena_com_llq_info *llq_info = &io_sq->llq_info;
@@ -207,13 +209,15 @@ static inline int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq)
if (!pkt_ctrl->descs_left_in_line) {
rc = ena_com_write_bounce_buffer_to_dev(io_sq,
pkt_ctrl->curr_bounce_buf);
- if (unlikely(rc))
+ if (unlikely(rc)) {
+ ena_trc_err("failed to write bounce buffer to device\n");
return rc;
+ }
pkt_ctrl->curr_bounce_buf =
ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
- memset(io_sq->llq_buf_ctrl.curr_bounce_buf,
- 0x0, llq_info->desc_list_entry_size);
+ memset(io_sq->llq_buf_ctrl.curr_bounce_buf,
+ 0x0, llq_info->desc_list_entry_size);
pkt_ctrl->idx = 0;
if (unlikely(llq_info->desc_stride_ctrl == ENA_ADMIN_SINGLE_DESC_PER_ENTRY))
@@ -226,7 +230,7 @@ static inline int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq)
return ENA_COM_OK;
}
-static inline int ena_com_sq_update_tail(struct ena_com_io_sq *io_sq)
+static int ena_com_sq_update_tail(struct ena_com_io_sq *io_sq)
{
if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
return ena_com_sq_update_llq_tail(io_sq);
@@ -240,7 +244,7 @@ static inline int ena_com_sq_update_tail(struct ena_com_io_sq *io_sq)
return ENA_COM_OK;
}
-static inline struct ena_eth_io_rx_cdesc_base *
+static struct ena_eth_io_rx_cdesc_base *
ena_com_rx_cdesc_idx_to_ptr(struct ena_com_io_cq *io_cq, u16 idx)
{
idx &= (io_cq->q_depth - 1);
@@ -249,7 +253,7 @@ static inline struct ena_eth_io_rx_cdesc_base *
idx * io_cq->cdesc_entry_size_in_bytes);
}
-static inline u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
+static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
u16 *first_cdesc_idx)
{
struct ena_eth_io_rx_cdesc_base *cdesc;
@@ -286,11 +290,10 @@ static inline u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
return count;
}
-static inline int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
- struct ena_com_tx_ctx *ena_tx_ctx)
+static int ena_com_create_meta(struct ena_com_io_sq *io_sq,
+ struct ena_com_tx_meta *ena_meta)
{
struct ena_eth_io_tx_meta_desc *meta_desc = NULL;
- struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
meta_desc = get_sq_desc(io_sq);
memset(meta_desc, 0x0, sizeof(struct ena_eth_io_tx_meta_desc));
@@ -310,12 +313,13 @@ static inline int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io
/* Extended meta desc */
meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
- meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
meta_desc->len_ctrl |= (io_sq->phase <<
ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) &
ENA_ETH_IO_TX_META_DESC_PHASE_MASK;
meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_FIRST_MASK;
+ meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
+
meta_desc->word2 |= ena_meta->l3_hdr_len &
ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK;
meta_desc->word2 |= (ena_meta->l3_hdr_offset <<
@@ -326,16 +330,37 @@ static inline int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io
ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) &
ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK;
- meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
+ return ena_com_sq_update_tail(io_sq);
+}
- /* Cached the meta desc */
- memcpy(&io_sq->cached_tx_meta, ena_meta,
- sizeof(struct ena_com_tx_meta));
+static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
+ struct ena_com_tx_ctx *ena_tx_ctx,
+ bool *have_meta)
+{
+ struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
- return ena_com_sq_update_tail(io_sq);
+ /* When disable meta caching is set, don't bother to save the meta and
+ * compare it to the stored version, just create the meta
+ */
+ if (io_sq->disable_meta_caching) {
+ if (unlikely(!ena_tx_ctx->meta_valid))
+ return ENA_COM_INVAL;
+
+ *have_meta = true;
+ return ena_com_create_meta(io_sq, ena_meta);
+ } else if (ena_com_meta_desc_changed(io_sq, ena_tx_ctx)) {
+ *have_meta = true;
+ /* Cache the meta desc */
+ memcpy(&io_sq->cached_tx_meta, ena_meta,
+ sizeof(struct ena_com_tx_meta));
+ return ena_com_create_meta(io_sq, ena_meta);
+ } else {
+ *have_meta = false;
+ return ENA_COM_OK;
+ }
}
-static inline void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
+static void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
struct ena_eth_io_rx_cdesc_base *cdesc)
{
ena_rx_ctx->l3_proto = cdesc->status &
@@ -401,24 +426,26 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
}
if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV
- && !buffer_to_push))
+ && !buffer_to_push)) {
+ ena_trc_err("push header wasn't provided on LLQ mode\n");
return ENA_COM_INVAL;
+ }
rc = ena_com_write_header_to_bounce(io_sq, buffer_to_push, header_len);
if (unlikely(rc))
return rc;
- have_meta = ena_tx_ctx->meta_valid && ena_com_meta_desc_changed(io_sq,
- ena_tx_ctx);
- if (have_meta) {
- rc = ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx);
- if (unlikely(rc))
- return rc;
+ rc = ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx, &have_meta);
+ if (unlikely(rc)) {
+ ena_trc_err("failed to create and store tx meta desc\n");
+ return rc;
}
/* If the caller doesn't want to send packets */
if (unlikely(!num_bufs && !header_len)) {
rc = ena_com_close_bounce_buffer(io_sq);
+ if (rc)
+ ena_trc_err("failed to write buffers to LLQ\n");
*nb_hw_desc = io_sq->tail - start_tail;
return rc;
}
@@ -478,8 +505,10 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
/* The first desc share the same desc as the header */
if (likely(i != 0)) {
rc = ena_com_sq_update_tail(io_sq);
- if (unlikely(rc))
+ if (unlikely(rc)) {
+ ena_trc_err("failed to update sq tail\n");
return rc;
+ }
desc = get_sq_desc(io_sq);
if (unlikely(!desc))
@@ -508,10 +537,14 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
desc->len_ctrl |= ENA_ETH_IO_TX_DESC_LAST_MASK;
rc = ena_com_sq_update_tail(io_sq);
- if (unlikely(rc))
+ if (unlikely(rc)) {
+ ena_trc_err("failed to update sq tail of the last descriptor\n");
return rc;
+ }
rc = ena_com_close_bounce_buffer(io_sq);
+ if (rc)
+ ena_trc_err("failed when closing bounce buffer\n");
*nb_hw_desc = io_sq->tail - start_tail;
return rc;
@@ -525,7 +558,7 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
struct ena_eth_io_rx_cdesc_base *cdesc = NULL;
u16 cdesc_idx = 0;
u16 nb_hw_desc;
- u16 i;
+ u16 i = 0;
ENA_WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX,
"wrong Q type");
@@ -545,13 +578,14 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
return ENA_COM_NO_SPACE;
}
- for (i = 0; i < nb_hw_desc; i++) {
- cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i);
+ cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx);
+ ena_rx_ctx->pkt_offset = cdesc->offset;
+ do {
ena_buf->len = cdesc->length;
ena_buf->req_id = cdesc->req_id;
ena_buf++;
- }
+ } while ((++i < nb_hw_desc) && (cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i)));
/* Update SQ head ptr */
io_sq->next_to_comp += nb_hw_desc;
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-22 10:17:02 +0000