1 files changed, 87 insertions, 21 deletions

diff --git a/module/zfs/qat_crypt.c b/module/zfs/qat_crypt.c
index 5a5113e68a5e..34c19b5823a8 100644
--- a/module/zfs/qat_crypt.c
+++ b/module/zfs/qat_crypt.c
@@ -31,6 +31,7 @@
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <linux/completion.h>
+#include <linux/mod_compat.h>
 #include <sys/zfs_context.h>
 #include <sys/zio_crypt.h>
 #include "lac/cpa_cy_im.h"
@@ -50,7 +51,7 @@
 static Cpa32U inst_num = 0;
 static Cpa16U num_inst = 0;
 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
-static boolean_t qat_crypt_init_done = B_FALSE;
+static boolean_t qat_cy_init_done = B_FALSE;
 int zfs_qat_encrypt_disable = 0;
 int zfs_qat_checksum_disable = 0;
 
@@ -76,7 +77,7 @@ boolean_t
 qat_crypt_use_accel(size_t s_len)
 {
 	return (!zfs_qat_encrypt_disable &&
-	    qat_crypt_init_done &&
+	    qat_cy_init_done &&
 	    s_len >= QAT_MIN_BUF_SIZE &&
 	    s_len <= QAT_MAX_BUF_SIZE);
 }
@@ -85,26 +86,29 @@ boolean_t
 qat_checksum_use_accel(size_t s_len)
 {
 	return (!zfs_qat_checksum_disable &&
-	    qat_crypt_init_done &&
+	    qat_cy_init_done &&
 	    s_len >= QAT_MIN_BUF_SIZE &&
 	    s_len <= QAT_MAX_BUF_SIZE);
 }
 
 void
-qat_crypt_clean(void)
+qat_cy_clean(void)
 {
 	for (Cpa16U i = 0; i < num_inst; i++)
 		cpaCyStopInstance(cy_inst_handles[i]);
 
 	num_inst = 0;
-	qat_crypt_init_done = B_FALSE;
+	qat_cy_init_done = B_FALSE;
 }
 
 int
-qat_crypt_init(void)
+qat_cy_init(void)
 {
 	CpaStatus status = CPA_STATUS_FAIL;
 
+	if (qat_cy_init_done)
+		return (0);
+
 	status = cpaCyGetNumInstances(&num_inst);
 	if (status != CPA_STATUS_SUCCESS)
 		return (-1);
@@ -131,21 +135,21 @@ qat_crypt_init(void)
 			goto error;
 	}
 
-	qat_crypt_init_done = B_TRUE;
+	qat_cy_init_done = B_TRUE;
 	return (0);
 
 error:
-	qat_crypt_clean();
+	qat_cy_clean();
 	return (-1);
 }
 
 void
-qat_crypt_fini(void)
+qat_cy_fini(void)
 {
-	if (!qat_crypt_init_done)
+	if (!qat_cy_init_done)
 		return;
 
-	qat_crypt_clean();
+	qat_cy_clean();
 }
 
 static CpaStatus
@@ -351,6 +355,21 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 	    nr_bufs * sizeof (CpaFlatBuffer));
 	if (status != CPA_STATUS_SUCCESS)
 		goto fail;
+	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
+	    ZIO_DATA_MAC_LEN);
+	if (status != CPA_STATUS_SUCCESS)
+		goto fail;
+	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
+	    ZIO_DATA_IV_LEN);
+	if (status != CPA_STATUS_SUCCESS)
+		goto fail;
+	if (aad_len > 0) {
+		status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
+		    aad_len);
+		if (status != CPA_STATUS_SUCCESS)
+			goto fail;
+		bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
+	}
 
 	bytes_left = enc_len;
 	data = src_buf;
@@ -389,18 +408,13 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 
 	op_data.sessionCtx = cy_session_ctx;
 	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
-	op_data.pIv = NULL; /* set this later as the J0 block */
-	op_data.ivLenInBytes = 0;
 	op_data.cryptoStartSrcOffsetInBytes = 0;
 	op_data.messageLenToCipherInBytes = 0;
 	op_data.hashStartSrcOffsetInBytes = 0;
 	op_data.messageLenToHashInBytes = 0;
-	op_data.pDigestResult = 0;
 	op_data.messageLenToCipherInBytes = enc_len;
 	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
-	op_data.pDigestResult = digest_buf;
-	op_data.pAdditionalAuthData = aad_buf;
-	op_data.pIv = iv_buf;
+	bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);
 
 	cb.verify_result = CPA_FALSE;
 	init_completion(&cb.complete);
@@ -420,6 +434,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 		goto fail;
 	}
 
+	/* save digest result to digest_buf */
+	bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
 	if (dir == QAT_ENCRYPT)
 		QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
 	else
@@ -435,6 +451,10 @@ fail:
 		kunmap(out_pages[i]);
 
 	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
+	if (aad_len > 0)
+		QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
+	QAT_PHYS_CONTIG_FREE(op_data.pIv);
+	QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
 	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
 	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
 	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
@@ -557,10 +577,56 @@ fail:
 	return (status);
 }
 
-module_param(zfs_qat_encrypt_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
+static int
+param_set_qat_encrypt(const char *val, struct kernel_param *kp)
+{
+	int ret;
+	int *pvalue = kp->arg;
+	ret = param_set_int(val, kp);
+	if (ret)
+		return (ret);
+	/*
+	 * zfs_qat_encrypt_disable = 0: enable qat encrypt
+	 * try to initialize qat instance if it has not been done
+	 */
+	if (*pvalue == 0 && !qat_cy_init_done) {
+		ret = qat_cy_init();
+		if (ret != 0) {
+			zfs_qat_encrypt_disable = 1;
+			return (ret);
+		}
+	}
+	return (ret);
+}
+
+static int
+param_set_qat_checksum(const char *val, struct kernel_param *kp)
+{
+	int ret;
+	int *pvalue = kp->arg;
+	ret = param_set_int(val, kp);
+	if (ret)
+		return (ret);
+	/*
+	 * set_checksum_param_ops = 0: enable qat checksum
+	 * try to initialize qat instance if it has not been done
+	 */
+	if (*pvalue == 0 && !qat_cy_init_done) {
+		ret = qat_cy_init();
+		if (ret != 0) {
+			zfs_qat_checksum_disable = 1;
+			return (ret);
+		}
+	}
+	return (ret);
+}
+
+module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
+    param_get_int, &zfs_qat_encrypt_disable, 0644);
+MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
 
-module_param(zfs_qat_checksum_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
+module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
+    param_get_int, &zfs_qat_checksum_disable, 0644);
+MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
 
 #endif