13 files changed, 512 insertions, 58 deletions

diff --git a/sys/dev/qat/include/adf_cfg_device.h b/sys/dev/qat/include/adf_cfg_device.h
index 40fb91119f03..9def937cc9db 100644
--- a/sys/dev/qat/include/adf_cfg_device.h
+++ b/sys/dev/qat/include/adf_cfg_device.h
@@ -28,7 +28,7 @@
 #define ADF_CFG_STATIC_CONF_NUM_DC_ACCEL_UNITS 2
 #define ADF_CFG_STATIC_CONF_NUM_INLINE_ACCEL_UNITS 0
 #define ADF_CFG_STATIC_CONF_INST_NUM_DC 2
-#define ADF_CFG_STATIC_CONF_INST_NUM_CY_POLL 2
+#define ADF_CFG_STATIC_CONF_INST_NUM_CY_POLL 6
 #define ADF_CFG_STATIC_CONF_INST_NUM_CY_IRQ 2
 
 #define ADF_CFG_FW_STRING_TO_ID(str, acc, id)                                  \
diff --git a/sys/dev/qat/include/common/adf_accel_devices.h b/sys/dev/qat/include/common/adf_accel_devices.h
index ad0e74335259..9503069ac2a2 100644
--- a/sys/dev/qat/include/common/adf_accel_devices.h
+++ b/sys/dev/qat/include/common/adf_accel_devices.h
@@ -19,6 +19,7 @@
 #define ADF_200XXVF_DEVICE_NAME "200xxvf"
 #define ADF_C4XXX_DEVICE_NAME "c4xxx"
 #define ADF_C4XXXVF_DEVICE_NAME "c4xxxvf"
+#define ADF_4XXX_DEVICE_NAME "4xxx"
 #define ADF_DH895XCC_PCI_DEVICE_ID 0x435
 #define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443
 #define ADF_C62X_PCI_DEVICE_ID 0x37c8
@@ -31,8 +32,17 @@
 #define ADF_D15XXIOV_PCI_DEVICE_ID 0x6f55
 #define ADF_C4XXX_PCI_DEVICE_ID 0x18a0
 #define ADF_C4XXXIOV_PCI_DEVICE_ID 0x18a1
+#define ADF_4XXX_PCI_DEVICE_ID 0x4940
+#define ADF_401XX_PCI_DEVICE_ID 0x4942
 
 #define IS_QAT_GEN3(ID) ({ (ID == ADF_C4XXX_PCI_DEVICE_ID); })
+static inline bool
+IS_QAT_GEN4(const unsigned int id)
+{
+	return (id == ADF_4XXX_PCI_DEVICE_ID || id == ADF_401XX_PCI_DEVICE_ID);
+}
+
+#define IS_QAT_GEN3_OR_GEN4(ID) (IS_QAT_GEN3(ID) || IS_QAT_GEN4(ID))
 #define ADF_VF2PF_SET_SIZE 32
 #define ADF_MAX_VF2PF_SET 4
 #define ADF_VF2PF_SET_OFFSET(set_nr) ((set_nr)*ADF_VF2PF_SET_SIZE)
@@ -50,7 +60,7 @@
 #define ADF_PCI_MAX_BARS 3
 #define ADF_DEVICE_NAME_LENGTH 32
 #define ADF_ETR_MAX_RINGS_PER_BANK 16
-#define ADF_MAX_MSIX_VECTOR_NAME 16
+#define ADF_MAX_MSIX_VECTOR_NAME 32
 #define ADF_DEVICE_NAME_PREFIX "qat_"
 #define ADF_STOP_RETRY 50
 #define ADF_NUM_THREADS_PER_AE (8)
@@ -58,7 +68,6 @@
 #define ADF_NUM_PKE_STRAND (2)
 #define ADF_AE_STRAND0_THREAD (8)
 #define ADF_AE_STRAND1_THREAD (9)
-#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE + ADF_NUM_PKE_STRAND)
 #define ADF_CFG_NUM_SERVICES 4
 #define ADF_SRV_TYPE_BIT_LEN 3
 #define ADF_SRV_TYPE_MASK 0x7
@@ -75,6 +84,8 @@
 #define GET_SRV_TYPE(ena_srv_mask, srv)                                        \
 	(((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK)
 
+#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.csr_ops)
+
 #define ADF_DEFAULT_RING_TO_SRV_MAP                                            \
 	(CRYPTO | CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                   \
 	 NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                                \
@@ -156,7 +167,9 @@ enum adf_accel_unit_services {
 	ADF_ACCEL_SERVICE_NULL = 0,
 	ADF_ACCEL_INLINE_CRYPTO = 1,
 	ADF_ACCEL_CRYPTO = 2,
-	ADF_ACCEL_COMPRESSION = 4
+	ADF_ACCEL_COMPRESSION = 4,
+	ADF_ACCEL_ASYM = 8,
+	ADF_ACCEL_ADMIN = 16
 };
 
 struct adf_ae_info {
@@ -182,6 +195,7 @@ struct adf_accel_unit_info {
 	u32 dc_ae_msk;
 	u8 num_cy_au;
 	u8 num_dc_au;
+	u8 num_asym_au;
 	u8 num_inline_au;
 	struct adf_accel_unit *au;
 	const struct adf_ae_info *ae_info;
@@ -231,6 +245,60 @@ struct admin_info {
 	u32 mailbox_offset;
 } __packed;
 
+struct adf_hw_csr_ops {
+	u64 (*build_csr_ring_base_addr)(bus_addr_t addr, u32 size);
+	u32 (*read_csr_ring_head)(struct resource *csr_base_addr,
+				  u32 bank,
+				  u32 ring);
+	void (*write_csr_ring_head)(struct resource *csr_base_addr,
+				    u32 bank,
+				    u32 ring,
+				    u32 value);
+	u32 (*read_csr_ring_tail)(struct resource *csr_base_addr,
+				  u32 bank,
+				  u32 ring);
+	void (*write_csr_ring_tail)(struct resource *csr_base_addr,
+				    u32 bank,
+				    u32 ring,
+				    u32 value);
+	u32 (*read_csr_e_stat)(struct resource *csr_base_addr, u32 bank);
+	void (*write_csr_ring_config)(struct resource *csr_base_addr,
+				      u32 bank,
+				      u32 ring,
+				      u32 value);
+	void (*write_csr_ring_base)(struct resource *csr_base_addr,
+				    u32 bank,
+				    u32 ring,
+				    bus_addr_t addr);
+	void (*write_csr_int_flag)(struct resource *csr_base_addr,
+				   u32 bank,
+				   u32 value);
+	void (*write_csr_int_srcsel)(struct resource *csr_base_addr, u32 bank);
+	void (*write_csr_int_col_en)(struct resource *csr_base_addr,
+				     u32 bank,
+				     u32 value);
+	void (*write_csr_int_col_ctl)(struct resource *csr_base_addr,
+				      u32 bank,
+				      u32 value);
+	void (*write_csr_int_flag_and_col)(struct resource *csr_base_addr,
+					   u32 bank,
+					   u32 value);
+	u32 (*read_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
+					u32 bank);
+	void (*write_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
+					  u32 bank,
+					  u32 value);
+};
+
+struct adf_hw_csr_info {
+	struct adf_hw_csr_ops csr_ops;
+	u32 csr_addr_offset;
+	u32 ring_bundle_size;
+	u32 bank_int_flag_clear_mask;
+	u32 num_rings_per_int_srcsel;
+	u32 arb_enable_mask;
+};
+
 struct adf_cfg_device_data;
 struct adf_accel_dev;
 struct adf_etr_data;
@@ -282,8 +350,10 @@ struct adf_hw_device_data {
 	void (*exit_arb)(struct adf_accel_dev *accel_dev);
 	void (*get_arb_mapping)(struct adf_accel_dev *accel_dev,
 				const uint32_t **cfg);
+	int (*init_device)(struct adf_accel_dev *accel_dev);
 	int (*get_heartbeat_status)(struct adf_accel_dev *accel_dev);
 	uint32_t (*get_ae_clock)(struct adf_hw_device_data *self);
+	uint32_t (*get_hb_clock)(struct adf_hw_device_data *self);
 	void (*disable_iov)(struct adf_accel_dev *accel_dev);
 	void (*configure_iov_threads)(struct adf_accel_dev *accel_dev,
 				      bool enable);
@@ -298,6 +368,8 @@ struct adf_hw_device_data {
 	void (*restore_device)(struct adf_accel_dev *accel_dev);
 	uint32_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev,
 					enum adf_accel_unit_services services);
+	enum adf_accel_unit_services (
+	    *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num);
 	int (*add_pke_stats)(struct adf_accel_dev *accel_dev);
 	void (*remove_pke_stats)(struct adf_accel_dev *accel_dev);
 	int (*add_misc_error)(struct adf_accel_dev *accel_dev);
@@ -311,6 +383,14 @@ struct adf_hw_device_data {
 				    enum adf_accel_unit_services services);
 	void (*pre_reset)(struct adf_accel_dev *accel_dev);
 	void (*post_reset)(struct adf_accel_dev *accel_dev);
+	void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
+	void (*get_ring_svc_map_data)(int ring_pair_index,
+				      u16 ring_to_svc_map,
+				      u8 *serv_type,
+				      int *ring_index,
+				      int *num_rings_per_srv,
+				      int bundle_num);
+	struct adf_hw_csr_info csr_info;
 	const char *fw_name;
 	const char *fw_mmp_name;
 	bool reset_ack;
@@ -320,7 +400,10 @@ struct adf_hw_device_data {
 	uint16_t accel_mask;
 	u32 aerucm_mask;
 	u32 ae_mask;
+	u32 admin_ae_mask;
 	u32 service_mask;
+	u32 service_to_load_mask;
+	u32 heartbeat_ctr_num;
 	uint16_t tx_rings_mask;
 	uint8_t tx_rx_gap;
 	uint8_t num_banks;
diff --git a/sys/dev/qat/include/common/adf_cfg.h b/sys/dev/qat/include/common/adf_cfg.h
index edc4813cb69e..58502c8605b8 100644
--- a/sys/dev/qat/include/common/adf_cfg.h
+++ b/sys/dev/qat/include/common/adf_cfg.h
@@ -76,4 +76,14 @@ int adf_cfg_get_services_enabled(struct adf_accel_dev *accel_dev,
 int adf_cfg_restore_section(struct adf_accel_dev *accel_dev,
 			    struct adf_cfg_section *section);
 void adf_cfg_keyval_del_all(struct list_head *head);
+
+static inline int
+adf_cy_inst_cross_banks(struct adf_accel_dev *accel_dev)
+{
+	if (accel_dev->hw_device->num_rings_per_bank == 2)
+		return 1;
+	else
+		return 0;
+}
+
 #endif
diff --git a/sys/dev/qat/include/common/adf_cfg_common.h b/sys/dev/qat/include/common/adf_cfg_common.h
index 68fb5e8a98b3..65fc60fc8c3d 100644
--- a/sys/dev/qat/include/common/adf_cfg_common.h
+++ b/sys/dev/qat/include/common/adf_cfg_common.h
@@ -27,7 +27,7 @@
 #define ADF_MAX_ACCELENGINES 12
 #define ADF_CFG_STORAGE_ENABLED 1
 #define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES)
-#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000
+#define ADF_GEN2_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000
 #define ADF_WDT_TIMER_SYM_COMP_MS 3
 #define ADF_MIN_HB_TIMER_MS 100
 #define ADF_CFG_MAX_NUM_OF_SECTIONS 16
@@ -87,7 +87,8 @@ enum adf_device_type {
 	DEV_200XX,
 	DEV_200XXVF,
 	DEV_C4XXX,
-	DEV_C4XXXVF
+	DEV_C4XXXVF,
+	DEV_4XXX
 };
 
 enum adf_cfg_fw_image_type {
@@ -158,6 +159,7 @@ struct adf_cfg_bundle {
 	/* contains all the info about rings */
 	struct adf_cfg_ring **rings;
 	u16 in_use;
+	u16 max_cfg_svc_num;
 };
 
 struct adf_cfg_instance {
diff --git a/sys/dev/qat/include/common/adf_cfg_strings.h b/sys/dev/qat/include/common/adf_cfg_strings.h
index 2f05dadadc45..933ffe0ba6ad 100644
--- a/sys/dev/qat/include/common/adf_cfg_strings.h
+++ b/sys/dev/qat/include/common/adf_cfg_strings.h
@@ -22,6 +22,8 @@
 #define ADF_RING_DC_RX "RingRx"
 #define ADF_ETRMGR_BANK "Bank"
 #define ADF_RING_BANK_NUM "BankNumber"
+#define ADF_RING_BANK_NUM_ASYM "BankNumberAsym"
+#define ADF_RING_BANK_NUM_SYM "BankNumberSym"
 #define ADF_CY "Cy"
 #define ADF_DC "Dc"
 #define ADF_DC_EXTENDED_FEATURES "Device_DcExtendedFeatures"
@@ -112,6 +114,8 @@
 #define ADF_CY_CORE_AFFINITY_FORMAT ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY
 #define ADF_DC_CORE_AFFINITY_FORMAT ADF_DC "%d" ADF_ETRMGR_CORE_AFFINITY
 #define ADF_CY_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM
+#define ADF_CY_ASYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_ASYM
+#define ADF_CY_SYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_SYM
 #define ADF_DC_BANK_NUM_FORMAT ADF_DC "%d" ADF_RING_BANK_NUM
 #define ADF_CY_ASYM_TX_FORMAT ADF_CY "%d" ADF_RING_ASYM_TX
 #define ADF_CY_SYM_TX_FORMAT ADF_CY "%d" ADF_RING_SYM_TX
diff --git a/sys/dev/qat/include/common/adf_common_drv.h b/sys/dev/qat/include/common/adf_common_drv.h
index 3bb35ed55da3..7ec380540336 100644
--- a/sys/dev/qat/include/common/adf_common_drv.h
+++ b/sys/dev/qat/include/common/adf_common_drv.h
@@ -203,10 +203,14 @@ int adf_init_gen2_arb(struct adf_accel_dev *accel_dev);
 void adf_exit_arb(struct adf_accel_dev *accel_dev);
 void adf_disable_arb(struct adf_accel_dev *accel_dev);
 void adf_update_ring_arb(struct adf_etr_ring_data *ring);
-void
-adf_enable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask);
-void
-adf_disable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask);
+void adf_enable_ring_arb(struct adf_accel_dev *accel_dev,
+			 void *csr_addr,
+			 unsigned int bank_nr,
+			 unsigned int mask);
+void adf_disable_ring_arb(struct adf_accel_dev *accel_dev,
+			  void *csr_addr,
+			  unsigned int bank_nr,
+			  unsigned int mask);
 int adf_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
 struct adf_accel_dev *adf_devmgr_get_dev_by_bdf(struct adf_pci_address *addr);
 struct adf_accel_dev *adf_devmgr_get_dev_by_pci_bus(u8 bus);
@@ -238,9 +242,7 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev);
 
 int qat_hal_init(struct adf_accel_dev *accel_dev);
 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle);
-void qat_hal_start(struct icp_qat_fw_loader_handle *handle,
-		   unsigned char ae,
-		   unsigned int ctx_mask);
+int qat_hal_start(struct icp_qat_fw_loader_handle *handle);
 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle,
 		  unsigned char ae,
 		  unsigned int ctx_mask);
diff --git a/sys/dev/qat/include/common/adf_gen2_hw_data.h b/sys/dev/qat/include/common/adf_gen2_hw_data.h
new file mode 100644
index 000000000000..395abec81b9f
--- /dev/null
+++ b/sys/dev/qat/include/common/adf_gen2_hw_data.h
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: BSD-3-Clause  */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#ifndef ADF_GEN2_HW_DATA_H_
+#define ADF_GEN2_HW_DATA_H_
+
+#include "adf_accel_devices.h"
+#include "adf_cfg_common.h"
+
+/* Transport access */
+#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL
+#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL
+#define ADF_RING_CSR_RING_CONFIG 0x000
+#define ADF_RING_CSR_RING_LBASE 0x040
+#define ADF_RING_CSR_RING_UBASE 0x080
+#define ADF_RING_CSR_RING_HEAD 0x0C0
+#define ADF_RING_CSR_RING_TAIL 0x100
+#define ADF_RING_CSR_E_STAT 0x14C
+#define ADF_RING_CSR_INT_FLAG 0x170
+#define ADF_RING_CSR_INT_SRCSEL 0x174
+#define ADF_RING_CSR_INT_SRCSEL_2 0x178
+#define ADF_RING_CSR_INT_COL_EN 0x17C
+#define ADF_RING_CSR_INT_COL_CTL 0x180
+#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184
+#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000
+#define ADF_RING_BUNDLE_SIZE 0x1000
+#define ADF_GEN2_RX_RINGS_OFFSET 8
+#define ADF_GEN2_TX_RINGS_MASK 0xFF
+
+#define BUILD_RING_BASE_ADDR(addr, size)                                       \
+	(((addr) >> 6) & (GENMASK_ULL(63, 0) << (size)))
+#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring)                          \
+	ADF_CSR_RD(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD +  \
+		       ((ring) << 2))
+#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring)                          \
+	ADF_CSR_RD(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL +  \
+		       ((ring) << 2))
+#define READ_CSR_E_STAT(csr_base_addr, bank)                                   \
+	ADF_CSR_RD(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_E_STAT)
+#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value)                \
+	ADF_CSR_WR(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) +                           \
+		       ADF_RING_CSR_RING_CONFIG + ((ring) << 2),               \
+		   value)
+#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value)                  \
+	do {                                                                   \
+		u32 l_base = 0, u_base = 0;                                    \
+		l_base = (u32)((value)&0xFFFFFFFF);                            \
+		u_base = (u32)(((value)&0xFFFFFFFF00000000ULL) >> 32);         \
+		ADF_CSR_WR(csr_base_addr,                                      \
+			   (ADF_RING_BUNDLE_SIZE * (bank)) +                   \
+			       ADF_RING_CSR_RING_LBASE + ((ring) << 2),        \
+			   l_base);                                            \
+		ADF_CSR_WR(csr_base_addr,                                      \
+			   (ADF_RING_BUNDLE_SIZE * (bank)) +                   \
+			       ADF_RING_CSR_RING_UBASE + ((ring) << 2),        \
+			   u_base);                                            \
+	} while (0)
+
+#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value)                  \
+	ADF_CSR_WR(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD +  \
+		       ((ring) << 2),                                          \
+		   value)
+#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value)                  \
+	ADF_CSR_WR(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL +  \
+		       ((ring) << 2),                                          \
+		   value)
+#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value)                         \
+	ADF_CSR_WR(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_FLAG,    \
+		   value)
+#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank)                              \
+	do {                                                                   \
+		ADF_CSR_WR(csr_base_addr,                                      \
+			   (ADF_RING_BUNDLE_SIZE * (bank)) +                   \
+			       ADF_RING_CSR_INT_SRCSEL,                        \
+			   ADF_BANK_INT_SRC_SEL_MASK_0);                       \
+		ADF_CSR_WR(csr_base_addr,                                      \
+			   (ADF_RING_BUNDLE_SIZE * (bank)) +                   \
+			       ADF_RING_CSR_INT_SRCSEL_2,                      \
+			   ADF_BANK_INT_SRC_SEL_MASK_X);                       \
+	} while (0)
+#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value)                       \
+	ADF_CSR_WR(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_EN,  \
+		   value)
+#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value)                      \
+	ADF_CSR_WR(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_CTL, \
+		   ADF_RING_CSR_INT_COL_CTL_ENABLE | (value))
+#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value)                 \
+	ADF_CSR_WR(csr_base_addr,                                              \
+		   (ADF_RING_BUNDLE_SIZE * (bank)) +                           \
+		       ADF_RING_CSR_INT_FLAG_AND_COL,                          \
+		   value)
+
+/* AE to function map */
+#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190)
+#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310)
+#define AE2FUNCTION_MAP_REG_SIZE 4
+#define AE2FUNCTION_MAP_VALID BIT(7)
+
+#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index)                      \
+	ADF_CSR_RD(pmisc_bar_addr,                                             \
+		   AE2FUNCTION_MAP_A_OFFSET +                                  \
+		       AE2FUNCTION_MAP_REG_SIZE * (index))
+#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value)              \
+	ADF_CSR_WR(pmisc_bar_addr,                                             \
+		   AE2FUNCTION_MAP_A_OFFSET +                                  \
+		       AE2FUNCTION_MAP_REG_SIZE * (index),                     \
+		   value)
+#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index)                      \
+	ADF_CSR_RD(pmisc_bar_addr,                                             \
+		   AE2FUNCTION_MAP_B_OFFSET +                                  \
+		       AE2FUNCTION_MAP_REG_SIZE * (index))
+#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value)              \
+	ADF_CSR_WR(pmisc_bar_addr,                                             \
+		   AE2FUNCTION_MAP_B_OFFSET +                                  \
+		       AE2FUNCTION_MAP_REG_SIZE * (index),                     \
+		   value)
+
+/* Admin Interface Offsets */
+#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574)
+#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578)
+#define ADF_MAILBOX_BASE_OFFSET 0x20970
+
+/* Arbiter configuration */
+#define ADF_ARB_OFFSET 0x30000
+#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180
+#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0))
+#define ADF_ARB_REG_SLOT 0x1000
+#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C
+
+#define READ_CSR_RING_SRV_ARB_EN(csr_addr, index)                              \
+	ADF_CSR_RD(csr_addr,                                                   \
+		   ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)))
+
+#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value)                      \
+	ADF_CSR_WR(csr_addr,                                                   \
+		   ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)), \
+		   value)
+
+/* Power gating */
+#define ADF_POWERGATE_DC BIT(23)
+#define ADF_POWERGATE_PKE BIT(24)
+
+/* Default ring mapping */
+#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP                                       \
+	(CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT |                            \
+	 CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                            \
+	 UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                            \
+	 COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+/* Error detection and correction */
+#define ADF_GEN2_AE_CTX_ENABLES(i) ((i)*0x1000 + 0x20818)
+#define ADF_GEN2_AE_MISC_CONTROL(i) ((i)*0x1000 + 0x20960)
+#define ADF_GEN2_ENABLE_AE_ECC_ERR BIT(28)
+#define ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12))
+#define ADF_GEN2_UERRSSMSH(i) ((i)*0x4000 + 0x18)
+#define ADF_GEN2_CERRSSMSH(i) ((i)*0x4000 + 0x10)
+#define ADF_GEN2_ERRSSMSH_EN BIT(3)
+
+#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE + ADF_NUM_PKE_STRAND)
+
+void adf_gen2_init_hw_csr_info(struct adf_hw_csr_info *csr_info);
+
+#endif
diff --git a/sys/dev/qat/include/common/adf_gen4_hw_data.h b/sys/dev/qat/include/common/adf_gen4_hw_data.h
new file mode 100644
index 000000000000..c0ef0c92772e
--- /dev/null
+++ b/sys/dev/qat/include/common/adf_gen4_hw_data.h
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: BSD-3-Clause  */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#ifndef ADF_GEN4_HW_CSR_DATA_H_
+#define ADF_GEN4_HW_CSR_DATA_H_
+
+#include "adf_accel_devices.h"
+
+/* Transport access */
+#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL
+#define ADF_RING_CSR_RING_CONFIG 0x1000
+#define ADF_RING_CSR_RING_LBASE 0x1040
+#define ADF_RING_CSR_RING_UBASE 0x1080
+#define ADF_RING_CSR_RING_HEAD 0x0C0
+#define ADF_RING_CSR_RING_TAIL 0x100
+#define ADF_RING_CSR_E_STAT 0x14C
+#define ADF_RING_CSR_INT_FLAG 0x170
+#define ADF_RING_CSR_INT_SRCSEL 0x174
+#define ADF_RING_CSR_INT_COL_CTL 0x180
+#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184
+#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000
+#define ADF_RING_CSR_INT_COL_EN 0x17C
+#define ADF_RING_CSR_ADDR_OFFSET 0x100000
+#define ADF_RING_BUNDLE_SIZE 0x2000
+
+#define BUILD_RING_BASE_ADDR(addr, size)                                       \
+	((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6)
+#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring)                          \
+	ADF_CSR_RD((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_RING_HEAD + ((ring) << 2))
+#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring)                          \
+	ADF_CSR_RD((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_RING_TAIL + ((ring) << 2))
+#define READ_CSR_E_STAT(csr_base_addr, bank)                                   \
+	ADF_CSR_RD((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_E_STAT)
+#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value)                \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_RING_CONFIG + ((ring) << 2),               \
+		   value)
+#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value)                  \
+	do {                                                                   \
+		struct resource *_csr_base_addr = csr_base_addr;               \
+		u32 _bank = bank;                                              \
+		u32 _ring = ring;                                              \
+		dma_addr_t _value = value;                                     \
+		u32 l_base = 0, u_base = 0;                                    \
+		l_base = lower_32_bits(_value);                                \
+		u_base = upper_32_bits(_value);                                \
+		ADF_CSR_WR((_csr_base_addr),                                   \
+			   ADF_RING_CSR_ADDR_OFFSET +                          \
+			       ADF_RING_BUNDLE_SIZE * (_bank) +                \
+			       ADF_RING_CSR_RING_LBASE + ((_ring) << 2),       \
+			   l_base);                                            \
+		ADF_CSR_WR((_csr_base_addr),                                   \
+			   ADF_RING_CSR_ADDR_OFFSET +                          \
+			       ADF_RING_BUNDLE_SIZE * (_bank) +                \
+			       ADF_RING_CSR_RING_UBASE + ((_ring) << 2),       \
+			   u_base);                                            \
+	} while (0)
+
+#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value)                  \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_RING_HEAD + ((ring) << 2),                 \
+		   value)
+#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value)                  \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_RING_TAIL + ((ring) << 2),                 \
+		   value)
+#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value)                         \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_INT_FLAG,                                  \
+		   (value))
+#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank)                              \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_INT_SRCSEL,                                \
+		   ADF_BANK_INT_SRC_SEL_MASK)
+#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value)                       \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_INT_COL_EN,                                \
+		   (value))
+#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value)                      \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_INT_COL_CTL,                               \
+		   ADF_RING_CSR_INT_COL_CTL_ENABLE | (value))
+#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value)                 \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_INT_FLAG_AND_COL,                          \
+		   (value))
+
+/* Arbiter configuration */
+#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C
+
+#define READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank)                          \
+	ADF_CSR_RD((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_RING_SRV_ARB_EN)
+
+#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value)                  \
+	ADF_CSR_WR((csr_base_addr),                                            \
+		   ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) +  \
+		       ADF_RING_CSR_RING_SRV_ARB_EN,                           \
+		   (value))
+
+/* WDT timers
+ *
+ * Timeout is in cycles. Clock speed may vary across products but this
+ * value should be a few milli-seconds.
+ */
+#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL
+#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000
+#define ADF_SSMWDTL_OFFSET 0x54
+#define ADF_SSMWDTH_OFFSET 0x5C
+#define ADF_SSMWDTPKEL_OFFSET 0x58
+#define ADF_SSMWDTPKEH_OFFSET 0x60
+
+#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE)
+
+int adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
+void adf_gen4_init_hw_csr_info(struct adf_hw_csr_info *csr_info);
+#endif
diff --git a/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h b/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
index a8afb5a4b377..7f6e80eeb431 100644
--- a/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
+++ b/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
@@ -16,6 +16,7 @@ struct icp_qat_fw_loader_ae_data {
 struct icp_qat_fw_loader_hal_handle {
 	struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE];
 	unsigned int ae_mask;
+	unsigned int admin_ae_mask;
 	unsigned int slice_mask;
 	unsigned int revision_id;
 	unsigned int ae_max_num;
diff --git a/sys/dev/qat/include/common/icp_qat_hal.h b/sys/dev/qat/include/common/icp_qat_hal.h
index 3a7475f25333..68e12826a7e8 100644
--- a/sys/dev/qat/include/common/icp_qat_hal.h
+++ b/sys/dev/qat/include/common/icp_qat_hal.h
@@ -52,23 +52,32 @@ enum hal_ae_csr {
 };
 
 enum fcu_csr {
-	FCU_CONTROL = 0x0,
-	FCU_STATUS = 0x4,
-	FCU_DRAM_ADDR_LO = 0xc,
+	FCU_CONTROL = 0x00,
+	FCU_STATUS = 0x04,
+	FCU_DRAM_ADDR_LO = 0x0c,
 	FCU_DRAM_ADDR_HI = 0x10,
 	FCU_RAMBASE_ADDR_HI = 0x14,
 	FCU_RAMBASE_ADDR_LO = 0x18
 };
 
 enum fcu_csr_c4xxx {
-	FCU_CONTROL_C4XXX = 0x0,
-	FCU_STATUS_C4XXX = 0x4,
-	FCU_STATUS1_C4XXX = 0xc,
+	FCU_CONTROL_C4XXX = 0x00,
+	FCU_STATUS_C4XXX = 0x04,
+	FCU_STATUS1_C4XXX = 0x0c,
 	FCU_AE_LOADED_C4XXX = 0x10,
 	FCU_DRAM_ADDR_LO_C4XXX = 0x14,
 	FCU_DRAM_ADDR_HI_C4XXX = 0x18,
 };
 
+enum fcu_csr_4xxx {
+	FCU_CONTROL_4XXX = 0x00,
+	FCU_STATUS_4XXX = 0x04,
+	FCU_ME_BROADCAST_MASK_TYPE = 0x08,
+	FCU_AE_LOADED_4XXX = 0x10,
+	FCU_DRAM_ADDR_LO_4XXX = 0x14,
+	FCU_DRAM_ADDR_HI_4XXX = 0x18,
+};
+
 enum fcu_cmd {
 	FCU_CTRL_CMD_NOOP = 0,
 	FCU_CTRL_CMD_AUTH = 1,
@@ -104,6 +113,7 @@ enum fcu_sts {
 #define LCS_STATUS (0x1)
 #define MMC_SHARE_CS_BITPOS 2
 #define GLOBAL_CSR 0xA00
+#define FCU_CTRL_BROADCAST_POS 0x4
 #define FCU_CTRL_AE_POS 0x8
 #define FCU_AUTH_STS_MASK 0x7
 #define FCU_STS_DONE_POS 0x9
@@ -111,20 +121,26 @@ enum fcu_sts {
 #define FCU_LOADED_AE_POS 0x16
 #define FW_AUTH_WAIT_PERIOD 10
 #define FW_AUTH_MAX_RETRY 300
+#define FW_BROADCAST_MAX_RETRY 300
 #define FCU_OFFSET 0x8c0
 #define FCU_OFFSET_C4XXX 0x1000
+#define FCU_OFFSET_4XXX 0x1000
 #define MAX_CPP_NUM 2
 #define AE_CPP_NUM 2
 #define AES_PER_CPP 16
 #define SLICES_PER_CPP 6
 #define ICP_QAT_AE_OFFSET 0x20000
 #define ICP_QAT_AE_OFFSET_C4XXX 0x40000
+#define ICP_QAT_AE_OFFSET_4XXX 0x600000
 #define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000)
 #define ICP_QAT_CAP_OFFSET_C4XXX 0x70000
+#define ICP_QAT_CAP_OFFSET_4XXX 0x640000
 #define LOCAL_TO_XFER_REG_OFFSET 0x800
 #define ICP_QAT_EP_OFFSET 0x3a000
 #define ICP_QAT_EP_OFFSET_C4XXX 0x60000
+#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */
 #define MEM_CFG_ERR_BIT 0x20
+#define AE_TG_NUM_CPM2X 4
 
 #define CAP_CSR_ADDR(csr) (csr + handle->hal_cap_g_ctl_csr_addr_v)
 #define SET_CAP_CSR(handle, csr, val)                                          \
@@ -133,20 +149,17 @@ enum fcu_sts {
 	ADF_CSR_RD(handle->hal_misc_addr_v, CAP_CSR_ADDR(csr))
 #define SET_GLB_CSR(handle, csr, val)                                          \
 	({                                                                     \
-		typeof(handle) handle_ = (handle);                             \
-		typeof(csr) csr_ = (csr);                                      \
-		typeof(val) val_ = (val);                                      \
-		(IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ?   \
-		    SET_CAP_CSR(handle_, (csr_), (val_)) :                     \
-		    SET_CAP_CSR(handle_, csr_ + GLOBAL_CSR, val_);             \
+		u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev));     \
+		(IS_QAT_GEN3_OR_GEN4(dev_id)) ?                                \
+		    SET_CAP_CSR((handle), (csr), (val)) :                      \
+		    SET_CAP_CSR((handle), (csr) + GLOBAL_CSR, val);            \
 	})
 #define GET_GLB_CSR(handle, csr)                                               \
 	({                                                                     \
-		typeof(handle) handle_ = (handle);                             \
-		typeof(csr) csr_ = (csr);                                      \
-		(IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ?   \
-		    (GET_CAP_CSR(handle_, (csr_))) :                           \
-		    (GET_CAP_CSR(handle_, (GLOBAL_CSR + (csr_))));             \
+		u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev));     \
+		(IS_QAT_GEN3_OR_GEN4(dev_id)) ?                                \
+		    GET_CAP_CSR((handle), (csr)) :                             \
+		    GET_CAP_CSR((handle), (csr) + GLOBAL_CSR);                 \
 	})
 #define SET_FCU_CSR(handle, csr, val)                                          \
 	({                                                                     \
@@ -157,7 +170,12 @@ enum fcu_sts {
 		    SET_CAP_CSR(handle_,                                       \
 				((csr_) + FCU_OFFSET_C4XXX),                   \
 				(val_)) :                                      \
-		    SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_));       \
+		    ((IS_QAT_GEN4(                                             \
+			 pci_get_device(GET_DEV(handle_->accel_dev)))) ?       \
+			 SET_CAP_CSR(handle_,                                  \
+				     ((csr_) + FCU_OFFSET_4XXX),               \
+				     (val_)) :                                 \
+			 SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_))); \
 	})
 #define GET_FCU_CSR(handle, csr)                                               \
 	({                                                                     \
@@ -165,7 +183,10 @@ enum fcu_sts {
 		typeof(csr) csr_ = (csr);                                      \
 		(IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ?   \
 		    GET_CAP_CSR(handle_, (FCU_OFFSET_C4XXX + (csr_))) :        \
-		    GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_)));               \
+		    ((IS_QAT_GEN4(                                             \
+			 pci_get_device(GET_DEV(handle_->accel_dev)))) ?       \
+			 GET_CAP_CSR(handle_, (FCU_OFFSET_4XXX + (csr_))) :    \
+			 GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_))));         \
 	})
 #define AE_CSR(handle, ae)                                                     \
 	((handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12))
@@ -184,13 +205,19 @@ enum fcu_sts {
 	ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val)
 #define GET_CSR_OFFSET(device_id, cap_offset_, ae_offset_, ep_offset_)         \
 	({                                                                     \
-		int gen3 = IS_QAT_GEN3(device_id);                             \
-		cap_offset_ =                                                  \
-		    (gen3 ? ICP_QAT_CAP_OFFSET_C4XXX : ICP_QAT_CAP_OFFSET);    \
-		ae_offset_ =                                                   \
-		    (gen3 ? ICP_QAT_AE_OFFSET_C4XXX : ICP_QAT_AE_OFFSET);      \
-		ep_offset_ =                                                   \
-		    (gen3 ? ICP_QAT_EP_OFFSET_C4XXX : ICP_QAT_EP_OFFSET);      \
+		if (IS_QAT_GEN3(device_id)) {                                  \
+			cap_offset_ = ICP_QAT_CAP_OFFSET_C4XXX;                \
+			ae_offset_ = ICP_QAT_AE_OFFSET_C4XXX;                  \
+			ep_offset_ = ICP_QAT_EP_OFFSET_C4XXX;                  \
+		} else if (IS_QAT_GEN4(device_id)) {                           \
+			cap_offset_ = ICP_QAT_CAP_OFFSET_4XXX;                 \
+			ae_offset_ = ICP_QAT_AE_OFFSET_4XXX;                   \
+			ep_offset_ = ICP_QAT_EP_OFFSET_4XXX;                   \
+		} else {                                                       \
+			cap_offset_ = ICP_QAT_CAP_OFFSET;                      \
+			ae_offset_ = ICP_QAT_AE_OFFSET;                        \
+			ep_offset_ = ICP_QAT_EP_OFFSET;                        \
+		}                                                              \
 	})
 
 #endif
diff --git a/sys/dev/qat/include/common/icp_qat_uclo.h b/sys/dev/qat/include/common/icp_qat_uclo.h
index 21a1c2fc8ace..1bdddce1d85e 100644
--- a/sys/dev/qat/include/common/icp_qat_uclo.h
+++ b/sys/dev/qat/include/common/icp_qat_uclo.h
@@ -9,6 +9,7 @@
 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000
 #define ICP_QAT_AC_200XX_DEV_TYPE 0x02000000
 #define ICP_QAT_AC_C4XXX_DEV_TYPE 0x04000000
+#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000
 #define ICP_QAT_UCLO_MAX_AE 32
 #define ICP_QAT_UCLO_MAX_CTX 8
 #define ICP_QAT_UCLO_MAX_CPPNUM 2
@@ -17,6 +18,7 @@
 #define ICP_QAT_UCLO_MAX_XFER_REG 128
 #define ICP_QAT_UCLO_MAX_GPR_REG 128
 #define ICP_QAT_UCLO_MAX_LMEM_REG 1024
+#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280
 #define ICP_QAT_UCLO_AE_ALL_CTX 0xff
 #define ICP_QAT_UOF_OBJID_LEN 8
 #define ICP_QAT_UOF_FID 0xc6c2
@@ -46,22 +48,42 @@
 #define ICP_QAT_SUOF_IMAG "SUF_IMAG"
 #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long))
 #define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long))
-#define ICP_QAT_CSS_FWSK_MODULUS_LEN 256
-#define ICP_QAT_CSS_FWSK_EXPONENT_LEN 4
-#define ICP_QAT_CSS_FWSK_PAD_LEN 252
-#define ICP_QAT_CSS_FWSK_PUB_LEN                                               \
-	(ICP_QAT_CSS_FWSK_MODULUS_LEN + ICP_QAT_CSS_FWSK_EXPONENT_LEN +        \
-	 ICP_QAT_CSS_FWSK_PAD_LEN)
-#define ICP_QAT_CSS_SIGNATURE_LEN 256
+
+#define DSS_FWSK_MODULUS_LEN 384 // RSA3K
+#define DSS_FWSK_EXPONENT_LEN 4
+#define DSS_FWSK_PADDING_LEN 380
+#define DSS_SIGNATURE_LEN 384 // RSA3K
+
+#define CSS_FWSK_MODULUS_LEN 256 // RSA2K
+#define CSS_FWSK_EXPONENT_LEN 4
+#define CSS_FWSK_PADDING_LEN 252
+#define CSS_SIGNATURE_LEN 256 // RSA2K
+
+#define ICP_QAT_CSS_FWSK_MODULUS_LEN(ID)                                       \
+	(IS_QAT_GEN4(ID) ? DSS_FWSK_MODULUS_LEN : CSS_FWSK_MODULUS_LEN)
+
+#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID)                                      \
+	(IS_QAT_GEN4(ID) ? DSS_FWSK_EXPONENT_LEN : CSS_FWSK_EXPONENT_LEN)
+
+#define ICP_QAT_CSS_FWSK_PAD_LEN(ID)                                           \
+	(IS_QAT_GEN4(ID) ? DSS_FWSK_PADDING_LEN : CSS_FWSK_PADDING_LEN)
+
+#define ICP_QAT_CSS_FWSK_PUB_LEN(ID)                                           \
+	(ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) +                                    \
+	 ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) + ICP_QAT_CSS_FWSK_PAD_LEN(ID))
+
+#define ICP_QAT_CSS_SIGNATURE_LEN(ID)                                          \
+	(IS_QAT_GEN4(ID) ? DSS_SIGNATURE_LEN : CSS_SIGNATURE_LEN)
+
 #define ICP_QAT_CSS_AE_IMG_LEN                                                 \
 	(sizeof(struct icp_qat_simg_ae_mode) + ICP_QAT_SIMG_AE_INIT_SEQ_LEN +  \
 	 ICP_QAT_SIMG_AE_INSTS_LEN)
-#define ICP_QAT_CSS_AE_SIMG_LEN                                                \
-	(sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN +           \
-	 ICP_QAT_CSS_SIGNATURE_LEN + ICP_QAT_CSS_AE_IMG_LEN)
-#define ICP_QAT_AE_IMG_OFFSET                                                  \
-	(sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN +       \
-	 ICP_QAT_CSS_FWSK_EXPONENT_LEN + ICP_QAT_CSS_SIGNATURE_LEN)
+#define ICP_QAT_CSS_AE_SIMG_LEN(ID)                                            \
+	(sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN(ID) +       \
+	 ICP_QAT_CSS_SIGNATURE_LEN(ID) + ICP_QAT_CSS_AE_IMG_LEN)
+#define ICP_QAT_AE_IMG_OFFSET(ID)                                              \
+	(sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) +   \
+	 ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) + ICP_QAT_CSS_SIGNATURE_LEN(ID))
 #define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000
 
 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode)&0xf)
diff --git a/sys/dev/qat/include/icp_qat_fw_init_admin.h b/sys/dev/qat/include/icp_qat_fw_init_admin.h
index 6f88de144770..9c1482158443 100644
--- a/sys/dev/qat/include/icp_qat_fw_init_admin.h
+++ b/sys/dev/qat/include/icp_qat_fw_init_admin.h
@@ -200,10 +200,6 @@ struct icp_qat_fw_init_admin_hb_cnt {
 	u16 req_heartbeat_cnt;
 };
 
-struct icp_qat_fw_init_admin_hb_stats {
-	struct icp_qat_fw_init_admin_hb_cnt stats[ADF_NUM_HB_CNT_PER_AE];
-};
-
 #define ICP_QAT_FW_COMN_HEARTBEAT_OK 0
 #define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1
 #define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS 0
diff --git a/sys/dev/qat/include/qat_ocf_utils.h b/sys/dev/qat/include/qat_ocf_utils.h
index 0cacd8f0a84f..30a7e9b7f8ec 100644
--- a/sys/dev/qat/include/qat_ocf_utils.h
+++ b/sys/dev/qat/include/qat_ocf_utils.h
@@ -45,8 +45,11 @@ static inline CpaBoolean
 is_use_sep_digest(const struct crypto_session_params *csp)
 {
 	/* Use separated digest for all digest/hash operations,
-	 * including GMAC */
-	if (CSP_MODE_DIGEST == csp->csp_mode || CSP_MODE_ETA == csp->csp_mode)
+	 * including GMAC. ETA and AEAD use separated digest
+	 * due to FW limitation to specify offset to digest
+	 * appended to pay-load buffer. */
+	if (CSP_MODE_DIGEST == csp->csp_mode || CSP_MODE_ETA == csp->csp_mode ||
+	    CSP_MODE_AEAD == csp->csp_mode)
 		return CPA_TRUE;
 
 	return CPA_FALSE;