aboutsummaryrefslogtreecommitdiff
path: root/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev/mlx5/mlx5_en/mlx5_en_rx.c')
-rw-r--r--sys/dev/mlx5/mlx5_en/mlx5_en_rx.c102
1 files changed, 80 insertions, 22 deletions
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index b52dee102a3d..6b53db6fea23 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -27,7 +27,9 @@
#include "opt_ratelimit.h"
#include <dev/mlx5/mlx5_en/en.h>
+#include <netinet/ip_var.h>
#include <machine/in_cksum.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
static inline int
mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
@@ -43,25 +45,21 @@ mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
if (rq->mbuf[ix].mbuf != NULL)
return (0);
- mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR,
- MLX5E_MAX_RX_BYTES);
+ mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz);
if (unlikely(mb == NULL))
return (-ENOMEM);
- mb->m_len = MLX5E_MAX_RX_BYTES;
- mb->m_pkthdr.len = MLX5E_MAX_RX_BYTES;
+ mb->m_len = rq->wqe_sz;
+ mb->m_pkthdr.len = rq->wqe_sz;
for (i = 1; i < rq->nsegs; i++) {
- if (mb_head->m_pkthdr.len >= rq->wqe_sz)
- break;
- mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0,
- MLX5E_MAX_RX_BYTES);
+ mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, rq->wqe_sz);
if (unlikely(mb == NULL)) {
m_freem(mb_head);
return (-ENOMEM);
}
- mb->m_len = MLX5E_MAX_RX_BYTES;
- mb_head->m_pkthdr.len += MLX5E_MAX_RX_BYTES;
+ mb->m_len = rq->wqe_sz;
+ mb_head->m_pkthdr.len += rq->wqe_sz;
}
/* rewind to first mbuf in chain */
mb = mb_head;
@@ -69,6 +67,9 @@ mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
/* get IP header aligned */
m_adj(mb, MLX5E_NET_IP_ALIGN);
+ err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, &rq->mbuf[ix]);
+ if (err)
+ goto err_free_mbuf;
err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
mb, segs, &nsegs, BUS_DMA_NOWAIT);
if (err != 0)
@@ -124,6 +125,27 @@ mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
mlx5_wq_ll_update_db_record(&rq->wq);
}
+static uint32_t
+csum_reduce(uint32_t val)
+{
+ while (val > 0xffff)
+ val = (val >> 16) + (val & 0xffff);
+ return (val);
+}
+
+static u_short
+csum_buf(uint32_t val, void *buf, int len)
+{
+ u_short x;
+
+ MPASS(len % 2 == 0);
+ for (int i = 0; i < len; i += 2) {
+ bcopy((char *)buf + i, &x, 2);
+ val = csum_reduce(val + x);
+ }
+ return (val);
+}
+
static void
mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
{
@@ -135,6 +157,7 @@ mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
struct ip *ip4 = NULL;
struct tcphdr *th;
uint32_t *ts_ptr;
+ uint32_t tcp_csum;
uint8_t l4_hdr_type;
int tcp_ack;
@@ -164,10 +187,10 @@ mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
ts_ptr = (uint32_t *)(th + 1);
if (get_cqe_lro_tcppsh(cqe))
- th->th_flags |= TH_PUSH;
+ tcp_set_flags(th, tcp_get_flags(th) | TH_PUSH);
if (tcp_ack) {
- th->th_flags |= TH_ACK;
+ tcp_set_flags(th, tcp_get_flags(th) | TH_ACK);
th->th_ack = cqe->lro_ack_seq_num;
th->th_win = cqe->lro_tcp_win;
@@ -183,29 +206,55 @@ mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
* +--------+--------+--------+--------+
*/
if (get_cqe_lro_timestamp_valid(cqe) &&
- (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 |
+ (__predict_true(*ts_ptr == ntohl(TCPOPT_NOP << 24 |
TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 |
- TCPOLEN_TIMESTAMP))) {
+ TCPOLEN_TIMESTAMP)))) {
/*
* cqe->timestamp is 64bit long.
* [0-31] - timestamp.
* [32-64] - timestamp echo replay.
*/
- ts_ptr[1] = *(uint32_t *)&cqe->timestamp;
ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1);
}
}
if (ip4) {
+ struct ipovly io;
+
ip4->ip_ttl = cqe->lro_min_ttl;
ip4->ip_len = cpu_to_be16(tot_len);
ip4->ip_sum = 0;
- ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2);
+ ip4->ip_sum = in_cksum_skip(mb, (ip4->ip_hl << 2) +
+ ETHER_HDR_LEN, ETHER_HDR_LEN);
+
+ /* TCP checksum: data */
+ tcp_csum = cqe->check_sum;
+
+ /* TCP checksum: IP pseudoheader */
+ bzero(io.ih_x1, sizeof(io.ih_x1));
+ io.ih_pr = IPPROTO_TCP;
+ io.ih_len = htons(ntohs(ip4->ip_len) - sizeof(*ip4));
+ io.ih_src = ip4->ip_src;
+ io.ih_dst = ip4->ip_dst;
+ tcp_csum = csum_buf(tcp_csum, &io, sizeof(io));
+
+ /* TCP checksum: TCP header */
+ th->th_sum = 0;
+ tcp_csum = csum_buf(tcp_csum, th, th->th_off * 4);
+ th->th_sum = ~tcp_csum & 0xffff;
} else {
ip6->ip6_hlim = cqe->lro_min_ttl;
ip6->ip6_plen = cpu_to_be16(tot_len -
sizeof(struct ip6_hdr));
+
+ /* TCP checksum */
+ th->th_sum = 0;
+ tcp_csum = ~in6_cksum_partial_l2(mb, IPPROTO_TCP,
+ sizeof(struct ether_header),
+ sizeof(struct ether_header) + sizeof(struct ip6_hdr),
+ tot_len - sizeof(struct ip6_hdr), th->th_off * 4) & 0xffff;
+ tcp_csum = csum_reduce(tcp_csum + cqe->check_sum);
+ th->th_sum = ~tcp_csum & 0xffff;
}
- /* TODO: handle tcp checksum */
}
static uint64_t
@@ -273,9 +322,8 @@ mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp)
}
static inline void
-mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
- struct mlx5e_rq *rq, struct mbuf *mb,
- u32 cqe_bcnt)
+mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq,
+ struct mbuf *mb, struct mlx5e_rq_mbuf *mr, u32 cqe_bcnt)
{
if_t ifp = rq->ifp;
struct mlx5e_channel *c;
@@ -418,6 +466,8 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
default:
break;
}
+
+ mlx5e_accel_ipsec_handle_rx(mb, cqe, mr);
}
static inline void
@@ -563,7 +613,9 @@ mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
("Filter returned %d!\n", rv));
}
}
- if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt &&
+ if (!mlx5e_accel_ipsec_flow(cqe) /* tag is already assigned
+ to rq->mbuf */ &&
+ MHLEN - MLX5E_NET_IP_ALIGN >= byte_cnt &&
(mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
/* set maximum mbuf length */
mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN;
@@ -580,7 +632,8 @@ mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
rq->mbuf[wqe_counter].dma_map);
}
rx_common:
- mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt);
+ mlx5e_build_rx_mbuf(cqe, rq, mb, &rq->mbuf[wqe_counter],
+ byte_cnt);
rq->stats.bytes += byte_cnt;
rq->stats.packets++;
#ifdef NUMA
@@ -644,6 +697,9 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
mtx_unlock(&c->iq.lock);
mtx_lock(&rq->mtx);
+ if (rq->enabled == 0)
+ goto out;
+ rq->processing++;
/*
* Polling the entire CQ without posting new WQEs results in
@@ -664,6 +720,8 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes);
mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock));
tcp_lro_flush_all(&rq->lro);
+ rq->processing--;
+out:
mtx_unlock(&rq->mtx);
for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-06 03:17:29 +0000