5 files changed, 199 insertions, 182 deletions

diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index d14d1e7fe750c..c585a665e9f59 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -488,41 +488,44 @@ static int
 g_eli_newsession(struct g_eli_worker *wr)
 {
 	struct g_eli_softc *sc;
-	struct cryptoini crie, cria;
+	struct crypto_session_params csp;
 	int error;
+	void *key;
 
 	sc = wr->w_softc;
 
-	bzero(&crie, sizeof(crie));
-	crie.cri_alg = sc->sc_ealgo;
-	crie.cri_klen = sc->sc_ekeylen;
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_CIPHER;
+	csp.csp_cipher_alg = sc->sc_ealgo;
+	csp.csp_ivlen = g_eli_ivlen(sc->sc_ealgo);
+	csp.csp_cipher_klen = sc->sc_ekeylen / 8;
 	if (sc->sc_ealgo == CRYPTO_AES_XTS)
-		crie.cri_klen <<= 1;
+		csp.csp_cipher_klen <<= 1;
 	if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) {
-		crie.cri_key = g_eli_key_hold(sc, 0,
+		key = g_eli_key_hold(sc, 0,
 		    LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize);
+		csp.csp_cipher_key = key;
 	} else {
-		crie.cri_key = sc->sc_ekey;
+		key = NULL;
+		csp.csp_cipher_key = sc->sc_ekey;
 	}
 	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
-		bzero(&cria, sizeof(cria));
-		cria.cri_alg = sc->sc_aalgo;
-		cria.cri_klen = sc->sc_akeylen;
-		cria.cri_key = sc->sc_akey;
-		crie.cri_next = &cria;
+		csp.csp_mode = CSP_MODE_ETA;
+		csp.csp_auth_alg = sc->sc_aalgo;
+		csp.csp_auth_klen = G_ELI_AUTH_SECKEYLEN;
 	}
 
 	switch (sc->sc_crypto) {
 	case G_ELI_CRYPTO_SW:
-		error = crypto_newsession(&wr->w_sid, &crie,
+		error = crypto_newsession(&wr->w_sid, &csp,
 		    CRYPTOCAP_F_SOFTWARE);
 		break;
 	case G_ELI_CRYPTO_HW:
-		error = crypto_newsession(&wr->w_sid, &crie,
+		error = crypto_newsession(&wr->w_sid, &csp,
 		    CRYPTOCAP_F_HARDWARE);
 		break;
 	case G_ELI_CRYPTO_UNKNOWN:
-		error = crypto_newsession(&wr->w_sid, &crie,
+		error = crypto_newsession(&wr->w_sid, &csp,
 		    CRYPTOCAP_F_HARDWARE);
 		if (error == 0) {
 			mtx_lock(&sc->sc_queue_mtx);
@@ -530,7 +533,7 @@ g_eli_newsession(struct g_eli_worker *wr)
 				sc->sc_crypto = G_ELI_CRYPTO_HW;
 			mtx_unlock(&sc->sc_queue_mtx);
 		} else {
-			error = crypto_newsession(&wr->w_sid, &crie,
+			error = crypto_newsession(&wr->w_sid, &csp,
 			    CRYPTOCAP_F_SOFTWARE);
 			mtx_lock(&sc->sc_queue_mtx);
 			if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
@@ -542,8 +545,12 @@ g_eli_newsession(struct g_eli_worker *wr)
 		panic("%s: invalid condition", __func__);
 	}
 
-	if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0)
-		g_eli_key_drop(sc, crie.cri_key);
+	if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) {
+		if (error)
+			g_eli_key_drop(sc, key);
+		else
+			wr->w_first_key = key;
+	}
 
 	return (error);
 }
@@ -551,8 +558,14 @@ g_eli_newsession(struct g_eli_worker *wr)
 static void
 g_eli_freesession(struct g_eli_worker *wr)
 {
+	struct g_eli_softc *sc;
 
 	crypto_freesession(wr->w_sid);
+	if (wr->w_first_key != NULL) {
+		sc = wr->w_softc;
+		g_eli_key_drop(sc, wr->w_first_key);
+		wr->w_first_key = NULL;
+	}
 }
 
 static void
diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h
index e387782d949b7..dab9d13ccff7c 100644
--- a/sys/geom/eli/g_eli.h
+++ b/sys/geom/eli/g_eli.h
@@ -163,6 +163,7 @@ extern u_int g_eli_batch;
 struct g_eli_worker {
 	struct g_eli_softc	*w_softc;
 	struct proc		*w_proc;
+	void			*w_first_key;
 	u_int			 w_number;
 	crypto_session_t	 w_sid;
 	boolean_t		 w_active;
@@ -574,6 +575,25 @@ g_eli_keylen(u_int algo, u_int keylen)
 }
 
 static __inline u_int
+g_eli_ivlen(u_int algo)
+{
+
+	switch (algo) {
+	case CRYPTO_AES_XTS:
+		return (AES_XTS_IV_LEN);
+	case CRYPTO_AES_CBC:
+		return (AES_BLOCK_LEN);
+	case CRYPTO_BLF_CBC:
+		return (BLOWFISH_BLOCK_LEN);
+	case CRYPTO_CAMELLIA_CBC:
+		return (CAMELLIA_BLOCK_LEN);
+	case CRYPTO_3DES_CBC:
+		return (DES3_BLOCK_LEN);
+	}
+	return (0);
+}
+
+static __inline u_int
 g_eli_hashlen(u_int algo)
 {
 
diff --git a/sys/geom/eli/g_eli_crypto.c b/sys/geom/eli/g_eli_crypto.c
index 41918f8898cad..ae88dadcaff70 100644
--- a/sys/geom/eli/g_eli_crypto.c
+++ b/sys/geom/eli/g_eli_crypto.c
@@ -61,50 +61,40 @@ static int
 g_eli_crypto_cipher(u_int algo, int enc, u_char *data, size_t datasize,
     const u_char *key, size_t keysize)
 {
-	struct cryptoini cri;
+	struct crypto_session_params csp;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	crypto_session_t sid;
-	u_char *p;
 	int error;
 
 	KASSERT(algo != CRYPTO_AES_XTS,
 	    ("%s: CRYPTO_AES_XTS unexpected here", __func__));
 
-	bzero(&cri, sizeof(cri));
-	cri.cri_alg = algo;
-	cri.cri_key = __DECONST(void *, key);
-	cri.cri_klen = keysize;
-	error = crypto_newsession(&sid, &cri, CRYPTOCAP_F_SOFTWARE);
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_CIPHER;
+	csp.csp_cipher_alg = algo;
+	csp.csp_ivlen = g_eli_ivlen(algo);
+	csp.csp_cipher_key = key;
+	csp.csp_cipher_klen = keysize / 8;
+	error = crypto_newsession(&sid, &csp, CRYPTOCAP_F_SOFTWARE);
 	if (error != 0)
 		return (error);
-	p = malloc(sizeof(*crp) + sizeof(*crd), M_ELI, M_NOWAIT | M_ZERO);
-	if (p == NULL) {
+	crp = crypto_getreq(sid, M_NOWAIT);
+	if (crp == NULL) {
 		crypto_freesession(sid);
 		return (ENOMEM);
 	}
-	crp = (struct cryptop *)p;	p += sizeof(*crp);
-	crd = (struct cryptodesc *)p;	p += sizeof(*crd);
 
-	crd->crd_skip = 0;
-	crd->crd_len = datasize;
-	crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-	if (enc)
-		crd->crd_flags |= CRD_F_ENCRYPT;
-	crd->crd_alg = algo;
-	crd->crd_key = __DECONST(void *, key);
-	crd->crd_klen = keysize;
-	bzero(crd->crd_iv, sizeof(crd->crd_iv));
-	crd->crd_next = NULL;
+	crp->crp_payload_start = 0;
+	crp->crp_payload_length = datasize;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
+	crp->crp_op = enc ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+	memset(crp->crp_iv, 0, sizeof(crp->crp_iv));
 
-	crp->crp_session = sid;
-	crp->crp_ilen = datasize;
-	crp->crp_olen = datasize;
 	crp->crp_opaque = NULL;
 	crp->crp_callback = g_eli_crypto_done;
+	crp->crp_buf_type = CRYPTO_BUF_CONTIG;
+	crp->crp_ilen = datasize;
 	crp->crp_buf = (void *)data;
-	crp->crp_flags = CRYPTO_F_CBIFSYNC;
-	crp->crp_desc = crd;
 
 	error = crypto_dispatch(crp);
 	if (error == 0) {
@@ -113,7 +103,7 @@ g_eli_crypto_cipher(u_int algo, int enc, u_char *data, size_t datasize,
 		error = crp->crp_etype;
 	}
 
-	free(crp, M_ELI);
+	crypto_freereq(crp);
 	crypto_freesession(sid);
 	return (error);
 }
diff --git a/sys/geom/eli/g_eli_integrity.c b/sys/geom/eli/g_eli_integrity.c
index a58e9f4358f60..7ec1b5662e6ec 100644
--- a/sys/geom/eli/g_eli_integrity.c
+++ b/sys/geom/eli/g_eli_integrity.c
@@ -140,31 +140,51 @@ g_eli_auth_read_done(struct cryptop *crp)
 	}
 	bp = (struct bio *)crp->crp_opaque;
 	bp->bio_inbed++;
+	sc = bp->bio_to->geom->softc;
 	if (crp->crp_etype == 0) {
-		bp->bio_completed += crp->crp_olen;
-		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%jd completed=%jd).",
-		    bp->bio_inbed, bp->bio_children, (intmax_t)crp->crp_olen, (intmax_t)bp->bio_completed);
+		bp->bio_completed += crp->crp_payload_length;
+		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%d completed=%jd).",
+		    bp->bio_inbed, bp->bio_children, crp->crp_payload_length, (intmax_t)bp->bio_completed);
 	} else {
-		G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
+		u_int nsec, decr_secsize, encr_secsize, rel_sec;
+		int *errorp;
+
+		/* Sectorsize of decrypted provider eg. 4096. */
+		decr_secsize = bp->bio_to->sectorsize;
+		/* The real sectorsize of encrypted provider, eg. 512. */
+		encr_secsize =
+		    LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
+		/* Number of sectors from decrypted provider, eg. 2. */
+		nsec = bp->bio_length / decr_secsize;
+		/* Number of sectors from encrypted provider, eg. 18. */
+		nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
+		/* Which relative sector this request decrypted. */
+		rel_sec = ((crp->crp_buf + crp->crp_payload_start) -
+		    (char *)bp->bio_driver2) / encr_secsize;
+
+		errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec +
+		    sizeof(int) * rel_sec);
+		*errorp = crp->crp_etype;
+		G_ELI_DEBUG(1,
+		    "Crypto READ request failed (%d/%d) error=%d.",
 		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
-		if (bp->bio_error == 0)
-			bp->bio_error = crp->crp_etype;
+		if (bp->bio_error == 0 || bp->bio_error == EINTEGRITY)
+			bp->bio_error = crp->crp_etype == EBADMSG ?
+			    EINTEGRITY : crp->crp_etype;
 	}
-	sc = bp->bio_to->geom->softc;
-	g_eli_key_drop(sc, crp->crp_desc->crd_next->crd_key);
+	if (crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * Do we have all sectors already?
 	 */
 	if (bp->bio_inbed < bp->bio_children)
 		return (0);
+
 	if (bp->bio_error == 0) {
 		u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
-		u_char *srcdata, *dstdata, *auth;
-		off_t coroff, corsize;
+		u_char *srcdata, *dstdata;
 
-		/*
-		 * Verify data integrity based on calculated and read HMACs.
-		 */
 		/* Sectorsize of decrypted provider eg. 4096. */
 		decr_secsize = bp->bio_to->sectorsize;
 		/* The real sectorsize of encrypted provider, eg. 512. */
@@ -180,30 +200,54 @@ g_eli_auth_read_done(struct cryptop *crp)
 
 		srcdata = bp->bio_driver2;
 		dstdata = bp->bio_data;
-		auth = srcdata + encr_secsize * nsec;
+
+		for (i = 1; i <= nsec; i++) {
+			data_secsize = sc->sc_data_per_sector;
+			if ((i % lsec) == 0)
+				data_secsize = decr_secsize % data_secsize;
+			bcopy(srcdata + sc->sc_alen, dstdata, data_secsize);
+			srcdata += encr_secsize;
+			dstdata += data_secsize;
+		}
+	} else if (bp->bio_error == EINTEGRITY) {
+		u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
+		int *errorp;
+		off_t coroff, corsize, dstoff;
+
+		/* Sectorsize of decrypted provider eg. 4096. */
+		decr_secsize = bp->bio_to->sectorsize;
+		/* The real sectorsize of encrypted provider, eg. 512. */
+		encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
+		/* Number of data bytes in one encrypted sector, eg. 480. */
+		data_secsize = sc->sc_data_per_sector;
+		/* Number of sectors from decrypted provider, eg. 2. */
+		nsec = bp->bio_length / decr_secsize;
+		/* Number of sectors from encrypted provider, eg. 18. */
+		nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
+		/* Last sector number in every big sector, eg. 9. */
+		lsec = sc->sc_bytes_per_sector / encr_secsize;
+
+		errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec);
 		coroff = -1;
 		corsize = 0;
+		dstoff = bp->bio_offset;
 
 		for (i = 1; i <= nsec; i++) {
 			data_secsize = sc->sc_data_per_sector;
 			if ((i % lsec) == 0)
 				data_secsize = decr_secsize % data_secsize;
-			if (bcmp(srcdata, auth, sc->sc_alen) != 0) {
+			if (errorp[i - 1] == EBADMSG) {
 				/*
-				 * Curruption detected, remember the offset if
+				 * Corruption detected, remember the offset if
 				 * this is the first corrupted sector and
 				 * increase size.
 				 */
-				if (bp->bio_error == 0)
-					bp->bio_error = -1;
-				if (coroff == -1) {
-					coroff = bp->bio_offset +
-					    (dstdata - (u_char *)bp->bio_data);
-				}
+				if (coroff == -1)
+					coroff = dstoff;
 				corsize += data_secsize;
 			} else {
 				/*
-				 * No curruption, good.
+				 * No corruption, good.
 				 * Report previous corruption if there was one.
 				 */
 				if (coroff != -1) {
@@ -214,12 +258,8 @@ g_eli_auth_read_done(struct cryptop *crp)
 					coroff = -1;
 					corsize = 0;
 				}
-				bcopy(srcdata + sc->sc_alen, dstdata,
-				    data_secsize);
 			}
-			srcdata += encr_secsize;
-			dstdata += data_secsize;
-			auth += sc->sc_alen;
+			dstoff += data_secsize;
 		}
 		/* Report previous corruption if there was one. */
 		if (coroff != -1) {
@@ -231,9 +271,7 @@ g_eli_auth_read_done(struct cryptop *crp)
 	free(bp->bio_driver2, M_ELI);
 	bp->bio_driver2 = NULL;
 	if (bp->bio_error != 0) {
-		if (bp->bio_error == -1)
-			bp->bio_error = EINTEGRITY;
-		else {
+		if (bp->bio_error != EINTEGRITY) {
 			G_ELI_LOGREQ(0, bp,
 			    "Crypto READ request failed (error=%d).",
 			    bp->bio_error);
@@ -277,7 +315,9 @@ g_eli_auth_write_done(struct cryptop *crp)
 			bp->bio_error = crp->crp_etype;
 	}
 	sc = bp->bio_to->geom->softc;
-	g_eli_key_drop(sc, crp->crp_desc->crd_key);
+	if (crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * All sectors are already encrypted?
 	 */
@@ -361,14 +401,16 @@ g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp)
 
 	cbp->bio_length = cp->provider->sectorsize * nsec;
 	size = cbp->bio_length;
-	size += sc->sc_alen * nsec;
-	size += sizeof(struct cryptop) * nsec;
-	size += sizeof(struct cryptodesc) * nsec * 2;
+	size += sizeof(int) * nsec;
 	size += G_ELI_AUTH_SECKEYLEN * nsec;
 	cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
 	bp->bio_driver2 = malloc(size, M_ELI, M_WAITOK);
 	cbp->bio_data = bp->bio_driver2;
 
+	/* Clear the error array. */
+	memset((char *)bp->bio_driver2 + cbp->bio_length, 0,
+	    sizeof(int) * nsec);
+
 	/*
 	 * We read more than what is requested, so we have to be ready to read
 	 * more than MAXPHYS.
@@ -408,10 +450,9 @@ g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp)
 {
 	struct g_eli_softc *sc;
 	struct cryptop *crp;
-	struct cryptodesc *crde, *crda;
 	u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
 	off_t dstoff;
-	u_char *p, *data, *auth, *authkey, *plaindata;
+	u_char *p, *data, *authkey, *plaindata;
 	int error;
 
 	G_ELI_LOGREQ(3, bp, "%s", __func__);
@@ -433,19 +474,15 @@ g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp)
 	/* Destination offset, used for IV generation. */
 	dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
 
-	auth = NULL;	/* Silence compiler warning. */
 	plaindata = bp->bio_data;
 	if (bp->bio_cmd == BIO_READ) {
 		data = bp->bio_driver2;
-		auth = data + encr_secsize * nsec;
-		p = auth + sc->sc_alen * nsec;
+		p = data + encr_secsize * nsec;
+		p += sizeof(int) * nsec;
 	} else {
 		size_t size;
 
 		size = encr_secsize * nsec;
-		size += sizeof(*crp) * nsec;
-		size += sizeof(*crde) * nsec;
-		size += sizeof(*crda) * nsec;
 		size += G_ELI_AUTH_SECKEYLEN * nsec;
 		size += sizeof(uintptr_t);	/* Space for alignment. */
 		data = malloc(size, M_ELI, M_WAITOK);
@@ -460,9 +497,7 @@ g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp)
 #endif
 
 	for (i = 1; i <= nsec; i++, dstoff += encr_secsize) {
-		crp = (struct cryptop *)p;	p += sizeof(*crp);
-		crde = (struct cryptodesc *)p;	p += sizeof(*crde);
-		crda = (struct cryptodesc *)p;	p += sizeof(*crda);
+		crp = crypto_getreq(wr->w_sid, M_WAITOK);
 		authkey = (u_char *)p;		p += G_ELI_AUTH_SECKEYLEN;
 
 		data_secsize = sc->sc_data_per_sector;
@@ -477,21 +512,14 @@ g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp)
 				    encr_secsize - sc->sc_alen - data_secsize);
 		}
 
-		if (bp->bio_cmd == BIO_READ) {
-			/* Remember read HMAC. */
-			bcopy(data, auth, sc->sc_alen);
-			auth += sc->sc_alen;
-			/* TODO: bzero(9) can be commented out later. */
-			bzero(data, sc->sc_alen);
-		} else {
+		if (bp->bio_cmd == BIO_WRITE) {
 			bcopy(plaindata, data + sc->sc_alen, data_secsize);
 			plaindata += data_secsize;
 		}
 
-		crp->crp_session = wr->w_sid;
 		crp->crp_ilen = sc->sc_alen + data_secsize;
-		crp->crp_olen = data_secsize;
 		crp->crp_opaque = (void *)bp;
+		crp->crp_buf_type = CRYPTO_BUF_CONTIG;
 		crp->crp_buf = (void *)data;
 		data += encr_secsize;
 		crp->crp_flags = CRYPTO_F_CBIFSYNC;
@@ -499,41 +527,28 @@ g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp)
 			crp->crp_flags |= CRYPTO_F_BATCH;
 		if (bp->bio_cmd == BIO_WRITE) {
 			crp->crp_callback = g_eli_auth_write_done;
-			crp->crp_desc = crde;
-			crde->crd_next = crda;
-			crda->crd_next = NULL;
+			crp->crp_op = CRYPTO_OP_ENCRYPT |
+			    CRYPTO_OP_COMPUTE_DIGEST;
 		} else {
 			crp->crp_callback = g_eli_auth_read_done;
-			crp->crp_desc = crda;
-			crda->crd_next = crde;
-			crde->crd_next = NULL;
+			crp->crp_op = CRYPTO_OP_DECRYPT |
+			    CRYPTO_OP_VERIFY_DIGEST;
 		}
 
-		crde->crd_skip = sc->sc_alen;
-		crde->crd_len = data_secsize;
-		crde->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-		if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0)
-			crde->crd_flags |= CRD_F_KEY_EXPLICIT;
-		if (bp->bio_cmd == BIO_WRITE)
-			crde->crd_flags |= CRD_F_ENCRYPT;
-		crde->crd_alg = sc->sc_ealgo;
-		crde->crd_key = g_eli_key_hold(sc, dstoff, encr_secsize);
-		crde->crd_klen = sc->sc_ekeylen;
-		if (sc->sc_ealgo == CRYPTO_AES_XTS)
-			crde->crd_klen <<= 1;
-		g_eli_crypto_ivgen(sc, dstoff, crde->crd_iv,
-		    sizeof(crde->crd_iv));
+		crp->crp_digest_start = 0;
+		crp->crp_payload_start = sc->sc_alen;
+		crp->crp_payload_length = data_secsize;
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+		if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0) {
+			crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
+			    encr_secsize);
+		}
+		g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
+		    sizeof(crp->crp_iv));
 
-		crda->crd_skip = sc->sc_alen;
-		crda->crd_len = data_secsize;
-		crda->crd_inject = 0;
-		crda->crd_flags = CRD_F_KEY_EXPLICIT;
-		crda->crd_alg = sc->sc_aalgo;
 		g_eli_auth_keygen(sc, dstoff, authkey);
-		crda->crd_key = authkey;
-		crda->crd_klen = G_ELI_AUTH_SECKEYLEN * 8;
+		crp->crp_auth_key = authkey;
 
-		crp->crp_etype = 0;
 		error = crypto_dispatch(crp);
 		KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
 		    error));
diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c
index 0a9e809e8b350..bfa1b800266b4 100644
--- a/sys/geom/eli/g_eli_privacy.c
+++ b/sys/geom/eli/g_eli_privacy.c
@@ -82,7 +82,7 @@ g_eli_crypto_read_done(struct cryptop *crp)
 	if (crp->crp_etype == 0) {
 		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
 		    bp->bio_inbed, bp->bio_children);
-		bp->bio_completed += crp->crp_olen;
+		bp->bio_completed += crp->crp_ilen;
 	} else {
 		G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
 		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
@@ -90,8 +90,9 @@ g_eli_crypto_read_done(struct cryptop *crp)
 			bp->bio_error = crp->crp_etype;
 	}
 	sc = bp->bio_to->geom->softc;
-	if (sc != NULL)
-		g_eli_key_drop(sc, crp->crp_desc->crd_key);
+	if (sc != NULL && crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * Do we have all sectors already?
 	 */
@@ -143,7 +144,9 @@ g_eli_crypto_write_done(struct cryptop *crp)
 	}
 	gp = bp->bio_to->geom;
 	sc = gp->softc;
-	g_eli_key_drop(sc, crp->crp_desc->crd_key);
+	if (crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * All sectors are already encrypted?
 	 */
@@ -233,11 +236,9 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
 {
 	struct g_eli_softc *sc;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	u_int i, nsec, secsize;
 	off_t dstoff;
-	size_t size;
-	u_char *p, *data;
+	u_char *data;
 	int error;
 
 	G_ELI_LOGREQ(3, bp, "%s", __func__);
@@ -247,71 +248,49 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
 	secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
 	nsec = bp->bio_length / secsize;
 
-	/*
-	 * Calculate how much memory do we need.
-	 * We need separate crypto operation for every single sector.
-	 * It is much faster to calculate total amount of needed memory here and
-	 * do the allocation once instead of allocating memory in pieces (many,
-	 * many pieces).
-	 */
-	size = sizeof(*crp) * nsec;
-	size += sizeof(*crd) * nsec;
+	bp->bio_inbed = 0;
+	bp->bio_children = nsec;
+
 	/*
 	 * If we write the data we cannot destroy current bio_data content,
 	 * so we need to allocate more memory for encrypted data.
 	 */
-	if (bp->bio_cmd == BIO_WRITE)
-		size += bp->bio_length;
-	p = malloc(size, M_ELI, M_WAITOK);
-
-	bp->bio_inbed = 0;
-	bp->bio_children = nsec;
-	bp->bio_driver2 = p;
-
-	if (bp->bio_cmd == BIO_READ)
-		data = bp->bio_data;
-	else {
-		data = p;
-		p += bp->bio_length;
+	if (bp->bio_cmd == BIO_WRITE) {
+		data = malloc(bp->bio_length, M_ELI, M_WAITOK);
+		bp->bio_driver2 = data;
 		bcopy(bp->bio_data, data, bp->bio_length);
-	}
+	} else
+		data = bp->bio_data;
 
 	for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) {
-		crp = (struct cryptop *)p;	p += sizeof(*crp);
-		crd = (struct cryptodesc *)p;	p += sizeof(*crd);
+		crp = crypto_getreq(wr->w_sid, M_WAITOK);
 
-		crp->crp_session = wr->w_sid;
 		crp->crp_ilen = secsize;
-		crp->crp_olen = secsize;
 		crp->crp_opaque = (void *)bp;
+		crp->crp_buf_type = CRYPTO_BUF_CONTIG;
 		crp->crp_buf = (void *)data;
 		data += secsize;
-		if (bp->bio_cmd == BIO_WRITE)
+		if (bp->bio_cmd == BIO_WRITE) {
+			crp->crp_op = CRYPTO_OP_ENCRYPT;
 			crp->crp_callback = g_eli_crypto_write_done;
-		else /* if (bp->bio_cmd == BIO_READ) */
+		} else /* if (bp->bio_cmd == BIO_READ) */ {
+			crp->crp_op = CRYPTO_OP_DECRYPT;
 			crp->crp_callback = g_eli_crypto_read_done;
+		}
 		crp->crp_flags = CRYPTO_F_CBIFSYNC;
 		if (g_eli_batch)
 			crp->crp_flags |= CRYPTO_F_BATCH;
-		crp->crp_desc = crd;
 
-		crd->crd_skip = 0;
-		crd->crd_len = secsize;
-		crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-		if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0)
-			crd->crd_flags |= CRD_F_KEY_EXPLICIT;
-		if (bp->bio_cmd == BIO_WRITE)
-			crd->crd_flags |= CRD_F_ENCRYPT;
-		crd->crd_alg = sc->sc_ealgo;
-		crd->crd_key = g_eli_key_hold(sc, dstoff, secsize);
-		crd->crd_klen = sc->sc_ekeylen;
-		if (sc->sc_ealgo == CRYPTO_AES_XTS)
-			crd->crd_klen <<= 1;
-		g_eli_crypto_ivgen(sc, dstoff, crd->crd_iv,
-		    sizeof(crd->crd_iv));
-		crd->crd_next = NULL;
+		crp->crp_payload_start = 0;
+		crp->crp_payload_length = secsize;
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+		if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0) {
+			crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
+			    secsize);
+		}
+		g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
+		    sizeof(crp->crp_iv));
 
-		crp->crp_etype = 0;
 		error = crypto_dispatch(crp);
 		KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
 		    error));