4 files changed, 32 insertions, 20 deletions

diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 773f3f56026a..0265683e090f 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -797,6 +797,7 @@ struct mlx5e_sq {
 	struct	mtx comp_lock;
 	struct	mlx5e_sq_stats stats;
 	struct	callout cev_callout;
+	int	db_inhibit;
 
 	/* data path */
 #define	mlx5e_sq_zero_start dma_tag
@@ -1154,8 +1155,13 @@ int	mlx5e_add_all_vxlan_rules(struct mlx5e_priv *priv);
 void	mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv);
 
 static inline void
-mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe)
+mlx5e_tx_notify_hw(struct mlx5e_sq *sq, bool force)
 {
+	if (unlikely((force == false && sq->db_inhibit != 0) || sq->doorbell.d64 == 0)) {
+		/* skip writing the doorbell record */
+		return;
+	}
+
 	/* ensure wqe is visible to device before updating doorbell record */
 	wmb();
 
@@ -1167,8 +1173,10 @@ mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe)
 	 */
 	wmb();
 
-	mlx5_write64(wqe, sq->uar_map,
+	mlx5_write64(sq->doorbell.d32, sq->uar_map,
 	    MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock));
+
+	sq->doorbell.d64 = 0;
 }
 
 static inline void
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index 64d8752abad6..4209e3fde72f 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -1799,11 +1799,7 @@ mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
 		atomic_thread_fence_rel();
 	}
 done:
-	/* Check if we need to write the doorbell */
-	if (likely(sq->doorbell.d64 != 0)) {
-		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-		sq->doorbell.d64 = 0;
-	}
+	mlx5e_tx_notify_hw(sq, false);
 }
 
 void
@@ -3859,8 +3855,7 @@ mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
 
 	sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
 	sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
-	mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-	sq->doorbell.d64 = 0;
+	mlx5e_tx_notify_hw(sq, true);
 }
 
 void
@@ -4854,10 +4849,8 @@ mlx5_en_debugnet_transmit(struct ifnet *dev, struct mbuf *m)
 		err = 0;
 	}
 
-	if (likely(sq->doorbell.d64 != 0)) {
-		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-		sq->doorbell.d64 = 0;
-	}
+	mlx5e_tx_notify_hw(sq, true);
+
 	return (err);
 }
 
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index e943a65709cf..ec6b027b324e 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -566,6 +566,7 @@ wq_ll_pop:
 void
 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
 {
+	struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq);
 	struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq);
 	int i = 0;
 
@@ -584,6 +585,11 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
 		rq->ifp->if_input(rq->ifp, mb);
 	}
 #endif
+	for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
+		mtx_lock(&c->sq[j].lock);
+		c->sq[j].db_inhibit++;
+		mtx_unlock(&c->sq[j].lock);
+	}
 
 	mtx_lock(&rq->mtx);
 
@@ -607,4 +613,12 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
 	mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock));
 	tcp_lro_flush_all(&rq->lro);
 	mtx_unlock(&rq->mtx);
+
+	for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
+		mtx_lock(&c->sq[j].lock);
+		c->sq[j].db_inhibit--;
+		/* Update the doorbell record, if any. */
+		mlx5e_tx_notify_hw(c->sq + j, true);
+		mtx_unlock(&c->sq[j].lock);
+	}
 }
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
index e2051896a003..35ab3d016392 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015-2019 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -1122,11 +1122,8 @@ mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb)
 		err = ENOBUFS;
 	}
 
-	/* Check if we need to write the doorbell */
-	if (likely(sq->doorbell.d64 != 0)) {
-		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
-		sq->doorbell.d64 = 0;
-	}
+	/* Write the doorbell record, if any. */
+	mlx5e_tx_notify_hw(sq, false);
 
 	/*
 	 * Check if we need to start the event timer which flushes the