6 files changed, 706 insertions, 337 deletions

diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c
index e097a22713b3..2f0d9bfdb3c9 100644
--- a/sys/opencrypto/criov.c
+++ b/sys/opencrypto/criov.c
@@ -60,7 +60,7 @@ __FBSDID("$FreeBSD$");
 	}								\
 } while (0)
 
-void
+static void
 cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
 {
 	struct iovec *iov = uio->uio_iov;
@@ -80,7 +80,7 @@ cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
 	}
 }
 
-void
+static void
 cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp)
 {
 	struct iovec *iov = uio->uio_iov;
@@ -103,7 +103,7 @@ cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp)
 /*
  * Return the index and offset of location in iovec list.
  */
-int
+static int
 cuio_getptr(struct uio *uio, int loc, int *off)
 {
 	int ind, len;
@@ -128,11 +128,263 @@ cuio_getptr(struct uio *uio, int loc, int *off)
 	return (-1);
 }
 
+void
+crypto_cursor_init(struct crypto_buffer_cursor *cc,
+    const struct crypto_buffer *cb)
+{
+	memset(cc, 0, sizeof(*cc));
+	cc->cc_type = cb->cb_type;
+	switch (cc->cc_type) {
+	case CRYPTO_BUF_CONTIG:
+		cc->cc_buf = cb->cb_buf;
+		cc->cc_buf_len = cb->cb_buf_len;
+		break;
+	case CRYPTO_BUF_MBUF:
+		cc->cc_mbuf = cb->cb_mbuf;
+		break;
+	case CRYPTO_BUF_UIO:
+		cc->cc_iov = cb->cb_uio->uio_iov;
+		break;
+	default:
+#ifdef INVARIANTS
+		panic("%s: invalid buffer type %d", __func__, cb->cb_type);
+#endif
+		break;
+	}
+}
+
+void
+crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
+{
+	size_t remain;
+
+	switch (cc->cc_type) {
+	case CRYPTO_BUF_CONTIG:
+		MPASS(cc->cc_buf_len >= amount);
+		cc->cc_buf += amount;
+		cc->cc_buf_len -= amount;
+		break;
+	case CRYPTO_BUF_MBUF:
+		for (;;) {
+			remain = cc->cc_mbuf->m_len - cc->cc_offset;
+			if (amount < remain) {
+				cc->cc_offset += amount;
+				break;
+			}
+			amount -= remain;
+			cc->cc_mbuf = cc->cc_mbuf->m_next;
+			cc->cc_offset = 0;
+			if (amount == 0)
+				break;
+		}
+		break;
+	case CRYPTO_BUF_UIO:
+		for (;;) {
+			remain = cc->cc_iov->iov_len - cc->cc_offset;
+			if (amount < remain) {
+				cc->cc_offset += amount;
+				break;
+			}
+			amount -= remain;
+			cc->cc_iov++;
+			cc->cc_offset = 0;
+			if (amount == 0)
+				break;
+		}
+		break;
+	default:
+#ifdef INVARIANTS
+		panic("%s: invalid buffer type %d", __func__, cc->cc_type);
+#endif
+		break;
+	}
+}
+
+void *
+crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
+{
+	switch (cc->cc_type) {
+	case CRYPTO_BUF_CONTIG:
+		return (cc->cc_buf);
+	case CRYPTO_BUF_MBUF:
+		if (cc->cc_mbuf == NULL)
+			return (NULL);
+		KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
+		    ("%s: not supported for unmapped mbufs", __func__));
+		return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
+	case CRYPTO_BUF_UIO:
+		return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
+	default:
+#ifdef INVARIANTS
+		panic("%s: invalid buffer type %d", __func__, cc->cc_type);
+#endif
+		return (NULL);
+	}
+}
+
+size_t
+crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
+{
+	switch (cc->cc_type) {
+	case CRYPTO_BUF_CONTIG:
+		return (cc->cc_buf_len);
+	case CRYPTO_BUF_MBUF:
+		if (cc->cc_mbuf == NULL)
+			return (0);
+		return (cc->cc_mbuf->m_len - cc->cc_offset);
+	case CRYPTO_BUF_UIO:
+		return (cc->cc_iov->iov_len - cc->cc_offset);
+	default:
+#ifdef INVARIANTS
+		panic("%s: invalid buffer type %d", __func__, cc->cc_type);
+#endif
+		return (0);
+	}
+}
+
+void
+crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
+    const void *vsrc)
+{
+	size_t remain, todo;
+	const char *src;
+	char *dst;
+
+	src = vsrc;
+	switch (cc->cc_type) {
+	case CRYPTO_BUF_CONTIG:
+		MPASS(cc->cc_buf_len >= size);
+		memcpy(cc->cc_buf, src, size);
+		cc->cc_buf += size;
+		cc->cc_buf_len -= size;
+		break;
+	case CRYPTO_BUF_MBUF:
+		for (;;) {
+			KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
+			    ("%s: not supported for unmapped mbufs", __func__));
+			dst = mtod(cc->cc_mbuf, char *) + cc->cc_offset;
+			remain = cc->cc_mbuf->m_len - cc->cc_offset;
+			todo = MIN(remain, size);
+			memcpy(dst, src, todo);
+			dst += todo;
+			if (todo < remain) {
+				cc->cc_offset += todo;
+				break;
+			}
+			size -= todo;	
+			cc->cc_mbuf = cc->cc_mbuf->m_next;
+			cc->cc_offset = 0;
+			if (size == 0)
+				break;
+		}
+		break;
+	case CRYPTO_BUF_UIO:
+		for (;;) {
+			dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
+			remain = cc->cc_iov->iov_len - cc->cc_offset;
+			todo = MIN(remain, size);
+			memcpy(dst, src, todo);
+			dst += todo;
+			if (todo < remain) {
+				cc->cc_offset += todo;
+				break;
+			}
+			size -= todo;	
+			cc->cc_iov++;
+			cc->cc_offset = 0;
+			if (size == 0)
+				break;
+		}
+		break;
+	default:
+#ifdef INVARIANTS
+		panic("%s: invalid buffer type %d", __func__, cc->cc_type);
+#endif
+		break;
+	}
+}
+
+void
+crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
+{
+	size_t remain, todo;
+	const char *src;
+	char *dst;
+
+	dst = vdst;
+	switch (cc->cc_type) {
+	case CRYPTO_BUF_CONTIG:
+		MPASS(cc->cc_buf_len >= size);
+		memcpy(dst, cc->cc_buf, size);
+		cc->cc_buf += size;
+		cc->cc_buf_len -= size;
+		break;
+	case CRYPTO_BUF_MBUF:
+		for (;;) {
+			KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
+			    ("%s: not supported for unmapped mbufs", __func__));
+			src = mtod(cc->cc_mbuf, const char *) + cc->cc_offset;
+			remain = cc->cc_mbuf->m_len - cc->cc_offset;
+			todo = MIN(remain, size);
+			memcpy(dst, src, todo);
+			dst += todo;
+			if (todo < remain) {
+				cc->cc_offset += todo;
+				break;
+			}
+			size -= todo;
+			cc->cc_mbuf = cc->cc_mbuf->m_next;
+			cc->cc_offset = 0;
+			if (size == 0)
+				break;
+		}
+		break;
+	case CRYPTO_BUF_UIO:
+		for (;;) {
+			src = (const char *)cc->cc_iov->iov_base +
+			    cc->cc_offset;
+			remain = cc->cc_iov->iov_len - cc->cc_offset;
+			todo = MIN(remain, size);
+			memcpy(dst, src, todo);
+			dst += todo;
+			if (todo < remain) {
+				cc->cc_offset += todo;
+				break;
+			}
+			size -= todo;
+			cc->cc_iov++;
+			cc->cc_offset = 0;
+			if (size == 0)
+				break;
+		}
+		break;
+	default:
+#ifdef INVARIANTS
+		panic("%s: invalid buffer type %d", __func__, cc->cc_type);
+#endif
+		break;
+	}
+}
+
+/*
+ * To avoid advancing 'cursor', make a local copy that gets advanced
+ * instead.
+ */
+void
+crypto_cursor_copydata_noadv(struct crypto_buffer_cursor *cc, int size,
+    void *vdst)
+{
+	struct crypto_buffer_cursor copy;
+
+	copy = *cc;
+	crypto_cursor_copydata(&copy, size, vdst);
+}
+
 /*
  * Apply function f to the data in an iovec list starting "off" bytes from
  * the beginning, continuing for "len" bytes.
  */
-int
+static int
 cuio_apply(struct uio *uio, int off, int len, int (*f)(void *, void *, u_int),
     void *arg)
 {
@@ -159,19 +411,28 @@ cuio_apply(struct uio *uio, int off, int len, int (*f)(void *, void *, u_int),
 void
 crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
 {
+	struct crypto_buffer *cb;
 
-	switch (crp->crp_buf_type) {
+	if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE)
+		cb = &crp->crp_obuf;
+	else
+		cb = &crp->crp_buf;
+	switch (cb->cb_type) {
 	case CRYPTO_BUF_MBUF:
-		m_copyback(crp->crp_mbuf, off, size, src);
+		m_copyback(cb->cb_mbuf, off, size, src);
 		break;
 	case CRYPTO_BUF_UIO:
-		cuio_copyback(crp->crp_uio, off, size, src);
+		cuio_copyback(cb->cb_uio, off, size, src);
 		break;
 	case CRYPTO_BUF_CONTIG:
-		bcopy(src, crp->crp_buf + off, size);
+		MPASS(off + size <= cb->cb_buf_len);
+		bcopy(src, cb->cb_buf + off, size);
 		break;
 	default:
-		panic("invalid crp buf type %d", crp->crp_buf_type);
+#ifdef INVARIANTS
+		panic("invalid crp buf type %d", cb->cb_type);
+#endif
+		break;
 	}
 }
 
@@ -179,88 +440,57 @@ void
 crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
 {
 
-	switch (crp->crp_buf_type) {
+	switch (crp->crp_buf.cb_type) {
 	case CRYPTO_BUF_MBUF:
-		m_copydata(crp->crp_mbuf, off, size, dst);
+		m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
 		break;
 	case CRYPTO_BUF_UIO:
-		cuio_copydata(crp->crp_uio, off, size, dst);
+		cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
 		break;
 	case CRYPTO_BUF_CONTIG:
-		bcopy(crp->crp_buf + off, dst, size);
+		MPASS(off + size <= crp->crp_buf.cb_buf_len);
+		bcopy(crp->crp_buf.cb_buf + off, dst, size);
 		break;
 	default:
-		panic("invalid crp buf type %d", crp->crp_buf_type);
+#ifdef INVARIANTS
+		panic("invalid crp buf type %d", crp->crp_buf.cb_type);
+#endif
+		break;
 	}
 }
 
 int
-crypto_apply(struct cryptop *crp, int off, int len,
+crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
     int (*f)(void *, void *, u_int), void *arg)
 {
 	int error;
 
-	switch (crp->crp_buf_type) {
+	switch (cb->cb_type) {
 	case CRYPTO_BUF_MBUF:
-		error = m_apply(crp->crp_mbuf, off, len, f, arg);
+		error = m_apply(cb->cb_mbuf, off, len, f, arg);
 		break;
 	case CRYPTO_BUF_UIO:
-		error = cuio_apply(crp->crp_uio, off, len, f, arg);
+		error = cuio_apply(cb->cb_uio, off, len, f, arg);
 		break;
 	case CRYPTO_BUF_CONTIG:
-		error = (*f)(arg, crp->crp_buf + off, len);
+		MPASS(off + len <= cb->cb_buf_len);
+		error = (*f)(arg, cb->cb_buf + off, len);
 		break;
 	default:
-		panic("invalid crp buf type %d", crp->crp_buf_type);
+#ifdef INVARIANTS
+		panic("invalid crypto buf type %d", cb->cb_type);
+#endif
+		error = 0;
+		break;
 	}
 	return (error);
 }
 
 int
-crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr, int *cnt,
-    int *allocated)
+crypto_apply(struct cryptop *crp, int off, int len,
+    int (*f)(void *, void *, u_int), void *arg)
 {
-	struct iovec *iov;
-	struct mbuf *m, *mtmp;
-	int i, j;
-
-	*allocated = 0;
-	iov = *iovptr;
-	if (iov == NULL)
-		*cnt = 0;
-
-	m = mbuf;
-	i = 0;
-	while (m != NULL) {
-		if (i == *cnt) {
-			/* we need to allocate a larger array */
-			j = 1;
-			mtmp = m;
-			while ((mtmp = mtmp->m_next) != NULL)
-				j++;
-			iov = malloc(sizeof *iov * (i + j), M_CRYPTO_DATA,
-			    M_NOWAIT);
-			if (iov == NULL)
-				return ENOMEM;
-			*allocated = 1;
-			*cnt = i + j;
-			memcpy(iov, *iovptr, sizeof *iov * i);
-		}
-
-		iov[i].iov_base = m->m_data;
-		iov[i].iov_len = m->m_len;
-
-		i++;
-		m = m->m_next;
-	}
-
-	if (*allocated)
-		KASSERT(*cnt == i, ("did not allocate correct amount: %d != %d",
-		    *cnt, i));
-
-	*iovptr = iov;
-	*cnt = i;
-	return 0;
+	return (crypto_apply_buf(&crp->crp_buf, off, len, f, arg));
 }
 
 static inline void *
@@ -300,17 +530,28 @@ cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
 }
 
 void *
-crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
+crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
+    size_t len)
 {
 
-	switch (crp->crp_buf_type) {
+	switch (cb->cb_type) {
 	case CRYPTO_BUF_MBUF:
-		return (m_contiguous_subsegment(crp->crp_mbuf, skip, len));
+		return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
 	case CRYPTO_BUF_UIO:
-		return (cuio_contiguous_segment(crp->crp_uio, skip, len));
+		return (cuio_contiguous_segment(cb->cb_uio, skip, len));
 	case CRYPTO_BUF_CONTIG:
-		return (crp->crp_buf + skip);
+		MPASS(skip + len <= cb->cb_buf_len);
+		return (cb->cb_buf + skip);
 	default:
-		panic("invalid crp buf type %d", crp->crp_buf_type);
+#ifdef INVARIANTS
+		panic("invalid crp buf type %d", cb->cb_type);
+#endif
+		return (NULL);
 	}
 }
+
+void *
+crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
+{
+	return (crypto_buffer_contiguous_subsegment(&crp->crp_buf, skip, len));
+}
diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
index b4aafdfb6914..73a38ea399d4 100644
--- a/sys/opencrypto/crypto.c
+++ b/sys/opencrypto/crypto.c
@@ -69,12 +69,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/malloc.h>
+#include <sys/mbuf.h>
 #include <sys/proc.h>
 #include <sys/refcount.h>
 #include <sys/sdt.h>
 #include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+#include <sys/uio.h>
 
 #include <ddb/ddb.h>
 
@@ -753,7 +755,7 @@ check_csp(const struct crypto_session_params *csp)
 	struct auth_hash *axf;
 
 	/* Mode-independent checks. */
-	if (csp->csp_flags != 0)
+	if ((csp->csp_flags & ~CSP_F_SEPARATE_OUTPUT) != 0)
 		return (false);
 	if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
 	    csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
@@ -767,7 +769,7 @@ check_csp(const struct crypto_session_params *csp)
 	case CSP_MODE_COMPRESS:
 		if (!alg_is_compression(csp->csp_cipher_alg))
 			return (false);
-		if (csp->csp_flags != 0)
+		if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT)
 			return (false);
 		if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
 		    csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
@@ -1206,20 +1208,66 @@ crypto_unblock(u_int32_t driverid, int what)
 	return err;
 }
 
+size_t
+crypto_buffer_len(struct crypto_buffer *cb)
+{
+	switch (cb->cb_type) {
+	case CRYPTO_BUF_CONTIG:
+		return (cb->cb_buf_len);
+	case CRYPTO_BUF_MBUF:
+		if (cb->cb_mbuf->m_flags & M_PKTHDR)
+			return (cb->cb_mbuf->m_pkthdr.len);
+		return (m_length(cb->cb_mbuf, NULL));
+	case CRYPTO_BUF_UIO:
+		return (cb->cb_uio->uio_resid);
+	default:
+		return (0);
+	}
+}
+
 #ifdef INVARIANTS
 /* Various sanity checks on crypto requests. */
 static void
+cb_sanity(struct crypto_buffer *cb, const char *name)
+{
+	KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
+	    ("incoming crp with invalid %s buffer type", name));
+	if (cb->cb_type == CRYPTO_BUF_CONTIG)
+		KASSERT(cb->cb_buf_len >= 0,
+		    ("incoming crp with -ve %s buffer length", name));
+}
+
+static void
 crp_sanity(struct cryptop *crp)
 {
 	struct crypto_session_params *csp;
+	struct crypto_buffer *out;
+	size_t ilen, len, olen;
 
 	KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
-	KASSERT(crp->crp_ilen >= 0, ("incoming crp with -ve input length"));
+	KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
+	    crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
+	    ("incoming crp with invalid output buffer type"));
 	KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
 	KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
 	    ("incoming crp already done"));
 
 	csp = &crp->crp_session->csp;
+	cb_sanity(&crp->crp_buf, "input");
+	ilen = crypto_buffer_len(&crp->crp_buf);
+	olen = ilen;
+	out = NULL;
+	if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) {
+		if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
+			cb_sanity(&crp->crp_obuf, "output");
+			out = &crp->crp_obuf;
+			olen = crypto_buffer_len(out);
+		}
+	} else
+		KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
+		    ("incoming crp with separate output buffer "
+		    "but no session support"));
+
 	switch (csp->csp_mode) {
 	case CSP_MODE_COMPRESS:
 		KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
@@ -1257,17 +1305,14 @@ crp_sanity(struct cryptop *crp)
 		    ("invalid ETA op %x", crp->crp_op));
 		break;
 	}
-	KASSERT(crp->crp_buf_type >= CRYPTO_BUF_CONTIG &&
-	    crp->crp_buf_type <= CRYPTO_BUF_MBUF,
-	    ("invalid crp buffer type %d", crp->crp_buf_type));
 	if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
 		KASSERT(crp->crp_aad_start == 0 ||
-		    crp->crp_aad_start < crp->crp_ilen,
+		    crp->crp_aad_start < ilen,
 		    ("invalid AAD start"));
 		KASSERT(crp->crp_aad_length != 0 || crp->crp_aad_start == 0,
 		    ("AAD with zero length and non-zero start"));
 		KASSERT(crp->crp_aad_length == 0 ||
-		    crp->crp_aad_start + crp->crp_aad_length <= crp->crp_ilen,
+		    crp->crp_aad_start + crp->crp_aad_length <= ilen,
 		    ("AAD outside input length"));
 	} else {
 		KASSERT(crp->crp_aad_start == 0 && crp->crp_aad_length == 0,
@@ -1282,25 +1327,39 @@ crp_sanity(struct cryptop *crp)
 		KASSERT(crp->crp_iv_start == 0,
 		    ("IV_SEPARATE used with non-zero IV start"));
 	} else {
-		KASSERT(crp->crp_iv_start < crp->crp_ilen,
+		KASSERT(crp->crp_iv_start < ilen,
 		    ("invalid IV start"));
-		KASSERT(crp->crp_iv_start + csp->csp_ivlen <= crp->crp_ilen,
-		    ("IV outside input length"));
+		KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
+		    ("IV outside buffer length"));
 	}
+	/* XXX: payload_start of 0 should always be < ilen? */
 	KASSERT(crp->crp_payload_start == 0 ||
-	    crp->crp_payload_start < crp->crp_ilen,
+	    crp->crp_payload_start < ilen,
 	    ("invalid payload start"));
 	KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
-	    crp->crp_ilen, ("payload outside input length"));
+	    ilen, ("payload outside input buffer"));
+	if (out == NULL) {
+		KASSERT(crp->crp_payload_output_start == 0,
+		    ("payload output start non-zero without output buffer"));
+	} else {
+		KASSERT(crp->crp_payload_output_start < olen,
+		    ("invalid payload output start"));
+		KASSERT(crp->crp_payload_output_start +
+		    crp->crp_payload_length <= olen,
+		    ("payload outside output buffer"));
+	}
 	if (csp->csp_mode == CSP_MODE_DIGEST ||
 	    csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
+		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
+			len = ilen;
+		else
+			len = olen;
 		KASSERT(crp->crp_digest_start == 0 ||
-		    crp->crp_digest_start < crp->crp_ilen,
+		    crp->crp_digest_start < len,
 		    ("invalid digest start"));
 		/* XXX: For the mlen == 0 case this check isn't perfect. */
-		KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <=
-		    crp->crp_ilen,
-		    ("digest outside input length"));
+		KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
+		    ("digest outside buffer"));
 	} else {
 		KASSERT(crp->crp_digest_start == 0,
 		    ("non-zero digest start for request without a digest"));
@@ -2143,10 +2202,10 @@ DB_SHOW_COMMAND(crypto, db_show_crypto)
 	    "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
 	    "Device", "Callback");
 	TAILQ_FOREACH(crp, &crp_q, crp_next) {
-		db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
+		db_printf("%4u %08x %4u %4u %04x %8p %8p\n"
 		    , crp->crp_session->cap->cc_hid
 		    , (int) crypto_ses2caps(crp->crp_session)
-		    , crp->crp_ilen, crp->crp_olen
+		    , crp->crp_olen
 		    , crp->crp_etype
 		    , crp->crp_flags
 		    , device_get_nameunit(crp->crp_session->cap->cc_dev)
diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c
index 653c951e2da1..4046d1b3561a 100644
--- a/sys/opencrypto/cryptodev.c
+++ b/sys/opencrypto/cryptodev.c
@@ -948,10 +948,8 @@ cryptodev_op(
 		goto bail;
 	}
 
-	crp->crp_ilen = cop->len + cse->hashsize;
 	crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
-	crp->crp_buf = cod->buf;
-	crp->crp_buf_type = CRYPTO_BUF_CONTIG;
+	crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize);
 	crp->crp_callback = cryptodev_cb;
 	crp->crp_opaque = cod;
 
@@ -1129,10 +1127,9 @@ cryptodev_aead(
 		goto bail;
 	}
 
-	crp->crp_ilen = caead->aadlen + caead->len + cse->hashsize;
 	crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
-	crp->crp_buf = cod->buf;
-	crp->crp_buf_type = CRYPTO_BUF_CONTIG;
+	crypto_use_buf(crp, cod->buf, caead->aadlen + caead->len +
+	    cse->hashsize);
 	crp->crp_callback = cryptodev_cb;
 	crp->crp_opaque = cod;
 
diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h
index 76553184bc6a..4483fc122f28 100644
--- a/sys/opencrypto/cryptodev.h
+++ b/sys/opencrypto/cryptodev.h
@@ -383,7 +383,9 @@ struct crypto_session_params {
 
 	int		csp_flags;
 
-	int		csp_ivlen;	 /* IV length in bytes. */
+#define	CSP_F_SEPARATE_OUTPUT	0x0001	/* Requests can use separate output */
+
+	int		csp_ivlen;	/* IV length in bytes. */
 
 	int		csp_cipher_alg;
 	int		csp_cipher_klen; /* Key length in bytes. */
@@ -396,6 +398,47 @@ struct crypto_session_params {
 					   0 means all. */
 };
 
+enum crypto_buffer_type {
+	CRYPTO_BUF_NONE = 0,
+	CRYPTO_BUF_CONTIG,
+	CRYPTO_BUF_UIO,
+	CRYPTO_BUF_MBUF,
+	CRYPTO_BUF_LAST = CRYPTO_BUF_MBUF
+};
+
+/*
+ * Description of a data buffer for a request.  Requests can either
+ * have a single buffer that is modified in place or separate input
+ * and output buffers.
+ */
+struct crypto_buffer {
+	union {
+		struct {
+			char	*cb_buf;
+			int	cb_buf_len;
+		};
+		struct mbuf *cb_mbuf;
+		struct uio *cb_uio;
+	};
+	enum crypto_buffer_type cb_type;
+};
+
+/*
+ * A cursor is used to iterate through a crypto request data buffer.
+ */
+struct crypto_buffer_cursor {
+	union {
+		char *cc_buf;
+		struct mbuf *cc_mbuf;
+		struct iovec *cc_iov;
+	};
+	union {
+		int cc_buf_len;
+		size_t cc_offset;
+	};
+	enum crypto_buffer_type cc_type;
+};
+
 /* Structure describing complete operation */
 struct cryptop {
 	TAILQ_ENTRY(cryptop) crp_next;
@@ -403,7 +446,6 @@ struct cryptop {
 	struct task	crp_task;
 
 	crypto_session_t crp_session;	/* Session */
-	int		crp_ilen;	/* Input data total length */
 	int		crp_olen;	/* Result total length */
 
 	int		crp_etype;	/*
@@ -434,12 +476,8 @@ struct cryptop {
 
 	int		crp_op;
 
-	union {
-		caddr_t		crp_buf;	/* Data to be processed */
-		struct mbuf	*crp_mbuf;
-		struct uio	*crp_uio;
-	};
-	int		crp_buf_type;	/* Which union member describes data. */
+	struct crypto_buffer crp_buf;
+	struct crypto_buffer crp_obuf;
 
 	int		crp_aad_start;	/* Location of AAD. */
 	int		crp_aad_length;	/* 0 => no AAD. */
@@ -447,6 +485,7 @@ struct cryptop {
 					 * the session.
 					 */
 	int		crp_payload_start; /* Location of ciphertext. */
+	int		crp_payload_output_start;
 	int		crp_payload_length;
 	int		crp_digest_start; /* Location of MAC/tag.  Length is
 					   * from the session.
@@ -469,16 +508,72 @@ struct cryptop {
 					 */
 };
 
-#define	CRYPTOP_ASYNC(crp) \
+static __inline void
+_crypto_use_buf(struct crypto_buffer *cb, void *buf, int len)
+{
+	cb->cb_buf = buf;
+	cb->cb_buf_len = len;
+	cb->cb_type = CRYPTO_BUF_CONTIG;
+}
+
+static __inline void
+_crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m)
+{
+	cb->cb_mbuf = m;
+	cb->cb_type = CRYPTO_BUF_MBUF;
+}
+
+static __inline void
+_crypto_use_uio(struct crypto_buffer *cb, struct uio *uio)
+{
+	cb->cb_uio = uio;
+	cb->cb_type = CRYPTO_BUF_UIO;
+}
+
+static __inline void
+crypto_use_buf(struct cryptop *crp, void *buf, int len)
+{
+	_crypto_use_buf(&crp->crp_buf, buf, len);
+}
+
+static __inline void
+crypto_use_mbuf(struct cryptop *crp, struct mbuf *m)
+{
+	_crypto_use_mbuf(&crp->crp_buf, m);
+}
+
+static __inline void
+crypto_use_uio(struct cryptop *crp, struct uio *uio)
+{
+	_crypto_use_uio(&crp->crp_buf, uio);
+}
+
+static __inline void
+crypto_use_output_buf(struct cryptop *crp, void *buf, int len)
+{
+	_crypto_use_buf(&crp->crp_obuf, buf, len);
+}
+
+static __inline void
+crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m)
+{
+	_crypto_use_mbuf(&crp->crp_obuf, m);
+}
+
+static __inline void
+crypto_use_output_uio(struct cryptop *crp, struct uio *uio)
+{
+	_crypto_use_uio(&crp->crp_obuf, uio);
+}
+
+#define	CRYPTOP_ASYNC(crp)			\
 	(((crp)->crp_flags & CRYPTO_F_ASYNC) && \
 	crypto_ses2caps((crp)->crp_session) & CRYPTOCAP_F_SYNC)
 #define	CRYPTOP_ASYNC_KEEPORDER(crp) \
 	(CRYPTOP_ASYNC(crp) && \
 	(crp)->crp_flags & CRYPTO_F_ASYNC_KEEPORDER)
-
-#define	CRYPTO_BUF_CONTIG	0x0
-#define	CRYPTO_BUF_UIO		0x1
-#define	CRYPTO_BUF_MBUF		0x2
+#define	CRYPTO_HAS_OUTPUT_BUFFER(crp)					\
+	((crp)->crp_obuf.cb_type != CRYPTO_BUF_NONE)
 
 /* Flags in crp_op. */
 #define	CRYPTO_OP_DECRYPT		0x0
@@ -559,26 +654,11 @@ void	hmac_init_opad(struct auth_hash *axf, const char *key, int klen,
 /*
  * Crypto-related utility routines used mainly by drivers.
  *
- * XXX these don't really belong here; but for now they're
- *     kept apart from the rest of the system.
- *
  * Similar to m_copyback/data, *_copyback copy data from the 'src'
  * buffer into the crypto request's data buffer while *_copydata copy
  * data from the crypto request's data buffer into the the 'dst'
  * buffer.
  */
-struct uio;
-extern	void cuio_copydata(struct uio* uio, int off, int len, caddr_t cp);
-extern	void cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp);
-extern	int cuio_getptr(struct uio *uio, int loc, int *off);
-extern	int cuio_apply(struct uio *uio, int off, int len,
-	    int (*f)(void *, void *, u_int), void *arg);
-
-struct mbuf;
-struct iovec;
-extern	int crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr,
-	    int *cnt, int *allocated);
-
 void	crypto_copyback(struct cryptop *crp, int off, int size,
 	    const void *src);
 void	crypto_copydata(struct cryptop *crp, int off, int size, void *dst);
@@ -587,6 +667,23 @@ int	crypto_apply(struct cryptop *crp, int off, int len,
 void	*crypto_contiguous_subsegment(struct cryptop *crp, size_t skip,
 	    size_t len);
 
+int	crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
+	    int (*f)(void *, void *, u_int), void *arg);
+void	*crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb,
+	    size_t skip, size_t len);
+size_t	crypto_buffer_len(struct crypto_buffer *cb);
+void	crypto_cursor_init(struct crypto_buffer_cursor *cc,
+	    const struct crypto_buffer *cb);
+void	crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount);
+void	*crypto_cursor_segbase(struct crypto_buffer_cursor *cc);
+size_t	crypto_cursor_seglen(struct crypto_buffer_cursor *cc);
+void	crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
+	    const void *vsrc);
+void	crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size,
+	    void *vdst);
+void	crypto_cursor_copydata_noadv(struct crypto_buffer_cursor *cc, int size,
+	    void *vdst);
+
 static __inline void
 crypto_read_iv(struct cryptop *crp, void *iv)
 {
diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c
index 4d37cd4998e9..db2f611db597 100644
--- a/sys/opencrypto/cryptosoft.c
+++ b/sys/opencrypto/cryptosoft.c
@@ -105,11 +105,10 @@ swcr_encdec(struct swcr_session *ses, struct cryptop *crp)
 	const struct crypto_session_params *csp;
 	struct swcr_encdec *sw;
 	struct enc_xform *exf;
-	int i, j, k, blks, ind, count, ivlen;
-	struct uio *uio, uiolcl;
-	struct iovec iovlcl[4];
-	struct iovec *iov;
-	int iovcnt, iovalloc;
+	int i, blks, inlen, ivlen, outlen, resid;
+	struct crypto_buffer_cursor cc_in, cc_out;
+	const char *inblk;
+	char *outblk;
 	int error;
 	bool encrypting;
 
@@ -142,32 +141,6 @@ swcr_encdec(struct swcr_session *ses, struct cryptop *crp)
 			return (error);
 	}
 
-	iov = iovlcl;
-	iovcnt = nitems(iovlcl);
-	iovalloc = 0;
-	uio = &uiolcl;
-	switch (crp->crp_buf_type) {
-	case CRYPTO_BUF_MBUF:
-		error = crypto_mbuftoiov(crp->crp_mbuf, &iov, &iovcnt,
-		    &iovalloc);
-		if (error)
-			return (error);
-		uio->uio_iov = iov;
-		uio->uio_iovcnt = iovcnt;
-		break;
-	case CRYPTO_BUF_UIO:
-		uio = crp->crp_uio;
-		break;
-	case CRYPTO_BUF_CONTIG:
-		iov[0].iov_base = crp->crp_buf;
-		iov[0].iov_len = crp->crp_ilen;
-		uio->uio_iov = iov;
-		uio->uio_iovcnt = 1;
-		break;
-	}
-
-	ivp = iv;
-
 	if (exf->reinit) {
 		/*
 		 * xforms that provide a reinit method perform all IV
@@ -176,164 +149,135 @@ swcr_encdec(struct swcr_session *ses, struct cryptop *crp)
 		exf->reinit(sw->sw_kschedule, iv);
 	}
 
-	count = crp->crp_payload_start;
-	ind = cuio_getptr(uio, count, &k);
-	if (ind == -1) {
-		error = EINVAL;
-		goto out;
-	}
+	ivp = iv;
+
+	crypto_cursor_init(&cc_in, &crp->crp_buf);
+	crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+	inlen = crypto_cursor_seglen(&cc_in);
+	inblk = crypto_cursor_segbase(&cc_in);
+	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+		crypto_cursor_init(&cc_out, &crp->crp_obuf);
+		crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
+	} else
+		cc_out = cc_in;
+	outlen = crypto_cursor_seglen(&cc_out);
+	outblk = crypto_cursor_segbase(&cc_out);
 
-	i = crp->crp_payload_length;
+	resid = crp->crp_payload_length;
 	encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
 
-	while (i >= blks) {
+	/*
+	 * Loop through encrypting blocks.  'inlen' is the remaining
+	 * length of the current segment in the input buffer.
+	 * 'outlen' is the remaining length of current segment in the
+	 * output buffer.
+	 */
+	while (resid >= blks) {
 		/*
-		 * If there's insufficient data at the end of
-		 * an iovec, we have to do some copying.
+		 * If the current block is not contained within the
+		 * current input/output segment, use 'blk' as a local
+		 * buffer.
 		 */
-		if (uio->uio_iov[ind].iov_len < k + blks &&
-		    uio->uio_iov[ind].iov_len != k) {
-			cuio_copydata(uio, count, blks, blk);
-
-			/* Actual encryption/decryption */
-			if (exf->reinit) {
-				if (encrypting) {
-					exf->encrypt(sw->sw_kschedule, blk,
-					    blk);
-				} else {
-					exf->decrypt(sw->sw_kschedule, blk,
-					    blk);
-				}
-			} else if (encrypting) {
-				/* XOR with previous block */
-				for (j = 0; j < blks; j++)
-					blk[j] ^= ivp[j];
-
-				exf->encrypt(sw->sw_kschedule, blk, blk);
-
-				/*
-				 * Keep encrypted block for XOR'ing
-				 * with next block
-				 */
-				bcopy(blk, iv, blks);
-				ivp = iv;
-			} else {	/* decrypt */
-				/*	
-				 * Keep encrypted block for XOR'ing
-				 * with next block
-				 */
-				nivp = (ivp == iv) ? iv2 : iv;
-				bcopy(blk, nivp, blks);
-
-				exf->decrypt(sw->sw_kschedule, blk, blk);
-
-				/* XOR with previous block */
-				for (j = 0; j < blks; j++)
-					blk[j] ^= ivp[j];
-
-				ivp = nivp;
-			}
-
-			/* Copy back decrypted block */
-			cuio_copyback(uio, count, blks, blk);
-
-			count += blks;
-
-			/* Advance pointer */
-			ind = cuio_getptr(uio, count, &k);
-			if (ind == -1) {
-				error = EINVAL;
-				goto out;
-			}
-
-			i -= blks;
-
-			/* Could be done... */
-			if (i == 0)
-				break;
+		if (inlen < blks) {
+			crypto_cursor_copydata(&cc_in, blks, blk);
+			inblk = blk;
 		}
+		if (outlen < blks)
+			outblk = blk;
 
-		while (uio->uio_iov[ind].iov_len >= k + blks && i >= blks) {
-			uint8_t *idat;
-
-			idat = (uint8_t *)uio->uio_iov[ind].iov_base + k;
+		/*
+		 * Ciphers without a 'reinit' hook are assumed to be
+		 * used in CBC mode where the chaining is done here.
+		 */
+		if (exf->reinit != NULL) {
+			if (encrypting)
+				exf->encrypt(sw->sw_kschedule, inblk, outblk);
+			else
+				exf->decrypt(sw->sw_kschedule, inblk, outblk);
+		} else if (encrypting) {
+			/* XOR with previous block */
+			for (i = 0; i < blks; i++)
+				outblk[i] = inblk[i] ^ ivp[i];
 
-			if (exf->reinit) {
-				if (encrypting)
-					exf->encrypt(sw->sw_kschedule,
-					    idat, idat);
-				else
-					exf->decrypt(sw->sw_kschedule,
-					    idat, idat);
-			} else if (encrypting) {
-				/* XOR with previous block/IV */
-				for (j = 0; j < blks; j++)
-					idat[j] ^= ivp[j];
+			exf->encrypt(sw->sw_kschedule, outblk, outblk);
 
-				exf->encrypt(sw->sw_kschedule, idat, idat);
-				ivp = idat;
-			} else {	/* decrypt */
-				/*
-				 * Keep encrypted block to be used
-				 * in next block's processing.
-				 */
-				nivp = (ivp == iv) ? iv2 : iv;
-				bcopy(idat, nivp, blks);
+			/*
+			 * Keep encrypted block for XOR'ing
+			 * with next block
+			 */
+			memcpy(iv, outblk, blks);
+			ivp = iv;
+		} else {	/* decrypt */
+			/*
+			 * Keep encrypted block for XOR'ing
+			 * with next block
+			 */
+			nivp = (ivp == iv) ? iv2 : iv;
+			memcpy(nivp, inblk, blks);
 
-				exf->decrypt(sw->sw_kschedule, idat, idat);
+			exf->decrypt(sw->sw_kschedule, inblk, outblk);
 
-				/* XOR with previous block/IV */
-				for (j = 0; j < blks; j++)
-					idat[j] ^= ivp[j];
+			/* XOR with previous block */
+			for (i = 0; i < blks; i++)
+				outblk[i] ^= ivp[i];
 
-				ivp = nivp;
-			}
+			ivp = nivp;
+		}
 
-			count += blks;
-			k += blks;
-			i -= blks;
+		if (inlen < blks) {
+			inlen = crypto_cursor_seglen(&cc_in);
+			inblk = crypto_cursor_segbase(&cc_in);
+		} else {
+			crypto_cursor_advance(&cc_in, blks);
+			inlen -= blks;
+			inblk += blks;
 		}
 
-		/*
-		 * Advance to the next iov if the end of the current iov
-		 * is aligned with the end of a cipher block.
-		 * Note that the code is equivalent to calling:
-		 *      ind = cuio_getptr(uio, count, &k);
-		 */
-		if (i > 0 && k == uio->uio_iov[ind].iov_len) {
-			k = 0;
-			ind++;
-			if (ind >= uio->uio_iovcnt) {
-				error = EINVAL;
-				goto out;
-			}
+		if (outlen < blks) {
+			crypto_cursor_copyback(&cc_out, blks, blk);
+			outlen = crypto_cursor_seglen(&cc_out);
+			outblk = crypto_cursor_segbase(&cc_out);
+		} else {
+			crypto_cursor_advance(&cc_out, blks);
+			outlen -= blks;
+			outblk += blks;
 		}
+
+		resid -= blks;
 	}
 
 	/* Handle trailing partial block for stream ciphers. */
-	if (i > 0) {
+	if (resid > 0) {
 		KASSERT(exf->native_blocksize != 0,
 		    ("%s: partial block of %d bytes for cipher %s",
 		    __func__, i, exf->name));
 		KASSERT(exf->reinit != NULL,
 		    ("%s: partial block cipher %s without reinit hook",
 		    __func__, exf->name));
-		KASSERT(i < blks, ("%s: partial block too big", __func__));
+		KASSERT(resid < blks, ("%s: partial block too big", __func__));
 
-		cuio_copydata(uio, count, i, blk);
-		if (encrypting) {
-			exf->encrypt_last(sw->sw_kschedule, blk, blk, i);
-		} else {
-			exf->decrypt_last(sw->sw_kschedule, blk, blk, i);
-		}
-		cuio_copyback(uio, count, i, blk);
+		inlen = crypto_cursor_seglen(&cc_in);
+		outlen = crypto_cursor_seglen(&cc_out);
+		if (inlen < resid) {
+			crypto_cursor_copydata(&cc_in, resid, blk);
+			inblk = blk;
+		} else
+			inblk = crypto_cursor_segbase(&cc_in);
+		if (outlen < resid)
+			outblk = blk;
+		else
+			outblk = crypto_cursor_segbase(&cc_out);
+		if (encrypting)
+			exf->encrypt_last(sw->sw_kschedule, inblk, outblk,
+			    resid);
+		else
+			exf->decrypt_last(sw->sw_kschedule, inblk, outblk,
+			    resid);
+		if (outlen < resid)
+			crypto_cursor_copyback(&cc_out, resid, blk);
 	}
 
-out:
-	if (iovalloc)
-		free(iov, M_CRYPTO_DATA);
-
-	return (error);
+	return (0);
 }
 
 static void
@@ -394,8 +338,15 @@ swcr_authcompute(struct swcr_session *ses, struct cryptop *crp)
 	if (err)
 		return err;
 
-	err = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length,
-	    (int (*)(void *, void *, unsigned int))axf->Update, &ctx);
+	if (CRYPTO_HAS_OUTPUT_BUFFER(crp) &&
+	    CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+		err = crypto_apply_buf(&crp->crp_obuf,
+		    crp->crp_payload_output_start, crp->crp_payload_length,
+		    (int (*)(void *, void *, unsigned int))axf->Update, &ctx);
+	else
+		err = crypto_apply(crp, crp->crp_payload_start,
+		    crp->crp_payload_length,
+		    (int (*)(void *, void *, unsigned int))axf->Update, &ctx);
 	if (err)
 		return err;
 
@@ -453,11 +404,12 @@ swcr_gmac(struct swcr_session *ses, struct cryptop *crp)
 	u_char aalg[AALG_MAX_RESULT_LEN];
 	u_char uaalg[AALG_MAX_RESULT_LEN];
 	u_char iv[EALG_MAX_BLOCK_LEN];
+	struct crypto_buffer_cursor cc;
 	union authctx ctx;
 	struct swcr_auth *swa;
 	struct auth_hash *axf;
 	uint32_t *blkp;
-	int blksz, i, ivlen, len;
+	int blksz, ivlen, len, resid;
 
 	swa = &ses->swcr_auth;
 	axf = swa->sw_axf;
@@ -470,9 +422,11 @@ swcr_gmac(struct swcr_session *ses, struct cryptop *crp)
 	crypto_read_iv(crp, iv);
 
 	axf->Reinit(&ctx, iv, ivlen);
-	for (i = 0; i < crp->crp_payload_length; i += blksz) {
-		len = MIN(crp->crp_payload_length - i, blksz);
-		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+	crypto_cursor_init(&cc, &crp->crp_buf);
+	crypto_cursor_advance(&cc, crp->crp_payload_start);
+	for (resid = crp->crp_payload_length; resid > 0; resid -= len) {
+		len = MIN(resid, blksz);
+		crypto_cursor_copydata(&cc, len, blk);
 		bzero(blk + len, blksz - len);
 		axf->Update(&ctx, blk, blksz);
 	}
@@ -506,13 +460,14 @@ swcr_gcm(struct swcr_session *ses, struct cryptop *crp)
 	u_char aalg[AALG_MAX_RESULT_LEN];
 	u_char uaalg[AALG_MAX_RESULT_LEN];
 	u_char iv[EALG_MAX_BLOCK_LEN];
+	struct crypto_buffer_cursor cc_in, cc_out;
 	union authctx ctx;
 	struct swcr_auth *swa;
 	struct swcr_encdec *swe;
 	struct auth_hash *axf;
 	struct enc_xform *exf;
 	uint32_t *blkp;
-	int blksz, i, ivlen, len, r;
+	int blksz, ivlen, len, r, resid;
 
 	swa = &ses->swcr_auth;
 	axf = swa->sw_axf;
@@ -536,9 +491,11 @@ swcr_gcm(struct swcr_session *ses, struct cryptop *crp)
 	axf->Reinit(&ctx, iv, ivlen);
 
 	/* Supply MAC with AAD */
-	for (i = 0; i < crp->crp_aad_length; i += blksz) {
-		len = MIN(crp->crp_aad_length - i, blksz);
-		crypto_copydata(crp, crp->crp_aad_start + i, len, blk);
+	crypto_cursor_init(&cc_in, &crp->crp_buf);
+	crypto_cursor_advance(&cc_in, crp->crp_aad_start);
+	for (resid = crp->crp_aad_length; resid > 0; resid -= len) {
+		len = MIN(resid, blksz);
+		crypto_cursor_copydata(&cc_in, len, blk);
 		bzero(blk + len, blksz - len);
 		axf->Update(&ctx, blk, blksz);
 	}
@@ -546,16 +503,22 @@ swcr_gcm(struct swcr_session *ses, struct cryptop *crp)
 	exf->reinit(swe->sw_kschedule, iv);
 
 	/* Do encryption with MAC */
-	for (i = 0; i < crp->crp_payload_length; i += len) {
-		len = MIN(crp->crp_payload_length - i, blksz);
+	crypto_cursor_init(&cc_in, &crp->crp_buf);
+	crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+		crypto_cursor_init(&cc_out, &crp->crp_obuf);
+		crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
+	} else
+		cc_out = cc_in;
+	for (resid = crp->crp_payload_length; resid > 0; resid -= len) {
+		len = MIN(resid, blksz);
 		if (len < blksz)
 			bzero(blk, blksz);
-		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+		crypto_cursor_copydata(&cc_in, len, blk);
 		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 			exf->encrypt(swe->sw_kschedule, blk, blk);
 			axf->Update(&ctx, blk, len);
-			crypto_copyback(crp, crp->crp_payload_start + i, len,
-			    blk);
+			crypto_cursor_copyback(&cc_out, len, blk);
 		} else {
 			axf->Update(&ctx, blk, len);
 		}
@@ -582,15 +545,16 @@ swcr_gcm(struct swcr_session *ses, struct cryptop *crp)
 			return (EBADMSG);
 		
 		/* tag matches, decrypt data */
-		for (i = 0; i < crp->crp_payload_length; i += blksz) {
-			len = MIN(crp->crp_payload_length - i, blksz);
+		crypto_cursor_init(&cc_in, &crp->crp_buf);
+		crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+		for (resid = crp->crp_payload_length; resid > 0;
+		     resid -= len) {
+			len = MIN(resid, blksz);
 			if (len < blksz)
 				bzero(blk, blksz);
-			crypto_copydata(crp, crp->crp_payload_start + i, len,
-			    blk);
+			crypto_cursor_copydata(&cc_in, len, blk);
 			exf->decrypt(swe->sw_kschedule, blk, blk);
-			crypto_copyback(crp, crp->crp_payload_start + i, len,
-			    blk);
+			crypto_cursor_copyback(&cc_out, len, blk);
 		}
 	} else {
 		/* Inject the authentication data */
@@ -609,10 +573,11 @@ swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp)
 	u_char aalg[AALG_MAX_RESULT_LEN];
 	u_char uaalg[AALG_MAX_RESULT_LEN];
 	u_char iv[EALG_MAX_BLOCK_LEN];
+	struct crypto_buffer_cursor cc;
 	union authctx ctx;
 	struct swcr_auth *swa;
 	struct auth_hash *axf;
-	int blksz, i, ivlen, len;
+	int blksz, ivlen, len, resid;
 
 	swa = &ses->swcr_auth;
 	axf = swa->sw_axf;
@@ -632,9 +597,11 @@ swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp)
 	ctx.aes_cbc_mac_ctx.cryptDataLength = 0;
 
 	axf->Reinit(&ctx, iv, ivlen);
-	for (i = 0; i < crp->crp_payload_length; i += blksz) {
-		len = MIN(crp->crp_payload_length - i, blksz);
-		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+	crypto_cursor_init(&cc, &crp->crp_buf);
+	crypto_cursor_advance(&cc, crp->crp_aad_start);
+	for (resid = crp->crp_payload_length; resid > 0; resid -= len) {
+		len = MIN(resid, blksz);
+		crypto_cursor_copydata(&cc, len, blk);
 		bzero(blk + len, blksz - len);
 		axf->Update(&ctx, blk, blksz);
 	}
@@ -662,12 +629,13 @@ swcr_ccm(struct swcr_session *ses, struct cryptop *crp)
 	u_char aalg[AALG_MAX_RESULT_LEN];
 	u_char uaalg[AALG_MAX_RESULT_LEN];
 	u_char iv[EALG_MAX_BLOCK_LEN];
+	struct crypto_buffer_cursor cc_in, cc_out;
 	union authctx ctx;
 	struct swcr_auth *swa;
 	struct swcr_encdec *swe;
 	struct auth_hash *axf;
 	struct enc_xform *exf;
-	int blksz, i, ivlen, len, r;
+	int blksz, ivlen, len, r, resid;
 
 	swa = &ses->swcr_auth;
 	axf = swa->sw_axf;
@@ -698,9 +666,11 @@ swcr_ccm(struct swcr_session *ses, struct cryptop *crp)
 	axf->Reinit(&ctx, iv, ivlen);
 
 	/* Supply MAC with AAD */
-	for (i = 0; i < crp->crp_aad_length; i += blksz) {
-		len = MIN(crp->crp_aad_length - i, blksz);
-		crypto_copydata(crp, crp->crp_aad_start + i, len, blk);
+	crypto_cursor_init(&cc_in, &crp->crp_buf);
+	crypto_cursor_advance(&cc_in, crp->crp_aad_start);
+	for (resid = crp->crp_aad_length; resid > 0; resid -= len) {
+		len = MIN(resid, blksz);
+		crypto_cursor_copydata(&cc_in, len, blk);
 		bzero(blk + len, blksz - len);
 		axf->Update(&ctx, blk, blksz);
 	}
@@ -708,16 +678,22 @@ swcr_ccm(struct swcr_session *ses, struct cryptop *crp)
 	exf->reinit(swe->sw_kschedule, iv);
 
 	/* Do encryption/decryption with MAC */
-	for (i = 0; i < crp->crp_payload_length; i += len) {
-		len = MIN(crp->crp_payload_length - i, blksz);
+	crypto_cursor_init(&cc_in, &crp->crp_buf);
+	crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+		crypto_cursor_init(&cc_out, &crp->crp_obuf);
+		crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
+	} else
+		cc_out = cc_in;
+	for (resid = crp->crp_payload_length; resid > 0; resid -= len) {
+		len = MIN(resid, blksz);
 		if (len < blksz)
 			bzero(blk, blksz);
-		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+		crypto_cursor_copydata(&cc_in, len, blk);
 		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 			axf->Update(&ctx, blk, len);
 			exf->encrypt(swe->sw_kschedule, blk, blk);
-			crypto_copyback(crp, crp->crp_payload_start + i, len,
-			    blk);
+			crypto_cursor_copyback(&cc_out, len, blk);
 		} else {
 			/*
 			 * One of the problems with CCM+CBC is that
@@ -746,15 +722,16 @@ swcr_ccm(struct swcr_session *ses, struct cryptop *crp)
 		
 		/* tag matches, decrypt data */
 		exf->reinit(swe->sw_kschedule, iv);
-		for (i = 0; i < crp->crp_payload_length; i += blksz) {
-			len = MIN(crp->crp_payload_length - i, blksz);
+		crypto_cursor_init(&cc_in, &crp->crp_buf);
+		crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+		for (resid = crp->crp_payload_length; resid > 0;
+		     resid -= len) {
+			len = MIN(resid, blksz);
 			if (len < blksz)
 				bzero(blk, blksz);
-			crypto_copydata(crp, crp->crp_payload_start + i, len,
-			    blk);
+			crypto_cursor_copydata(&cc_in, len, blk);
 			exf->decrypt(swe->sw_kschedule, blk, blk);
-			crypto_copyback(crp, crp->crp_payload_start + i, len,
-			    blk);
+			crypto_cursor_copyback(&cc_out, len, blk);
 		}
 	} else {
 		/* Inject the authentication data */
@@ -833,13 +810,13 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
 	 */
 	crypto_copyback(crp, crp->crp_payload_start, result, out);
 	if (result < crp->crp_payload_length) {
-		switch (crp->crp_buf_type) {
+		switch (crp->crp_buf.cb_type) {
 		case CRYPTO_BUF_MBUF:
 			adj = result - crp->crp_payload_length;
-			m_adj(crp->crp_mbuf, adj);
+			m_adj(crp->crp_buf.cb_mbuf, adj);
 			break;
 		case CRYPTO_BUF_UIO: {
-			struct uio *uio = crp->crp_uio;
+			struct uio *uio = crp->crp_buf.cb_uio;
 			int ind;
 
 			adj = crp->crp_payload_length - result;
@@ -858,6 +835,8 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
 			}
 			}
 			break;
+		default:
+			break;
 		}
 	}
 	free(out, M_CRYPTO_DATA);
@@ -1134,7 +1113,7 @@ static int
 swcr_probesession(device_t dev, const struct crypto_session_params *csp)
 {
 
-	if (csp->csp_flags != 0)
+	if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT)) != 0)
 		return (EINVAL);
 	switch (csp->csp_mode) {
 	case CSP_MODE_COMPRESS:
diff --git a/sys/opencrypto/ktls_ocf.c b/sys/opencrypto/ktls_ocf.c
index b607f2eead3d..435b68129d82 100644
--- a/sys/opencrypto/ktls_ocf.c
+++ b/sys/opencrypto/ktls_ocf.c
@@ -155,18 +155,16 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
 
 	crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
 	crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
-	crp->crp_buf_type = CRYPTO_BUF_UIO;
-	crp->crp_uio = &uio;
-	crp->crp_ilen = uio.uio_resid;
+	crypto_use_uio(crp, &uio);
 	crp->crp_opaque = oo;
 	crp->crp_callback = ktls_ocf_callback;
 
 	crp->crp_aad_start = 0;
 	crp->crp_aad_length = sizeof(ad);
 	crp->crp_payload_start = sizeof(ad);
-	crp->crp_payload_length = crp->crp_ilen -
+	crp->crp_payload_length = uio.uio_resid -
 	    (sizeof(ad) + AES_GMAC_HASH_LEN);
-	crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN;
+	crp->crp_digest_start = uio.uio_resid - AES_GMAC_HASH_LEN;
 
 	counter_u64_add(ocf_tls12_gcm_crypts, 1);
 	for (;;) {
@@ -256,18 +254,16 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
 
 	crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
 	crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
-	crp->crp_buf_type = CRYPTO_BUF_UIO;
-	crp->crp_uio = &uio;
-	crp->crp_ilen = uio.uio_resid;
+	crypto_use_uio(crp, &uio);
 	crp->crp_opaque = oo;
 	crp->crp_callback = ktls_ocf_callback;
 
 	crp->crp_aad_start = 0;
 	crp->crp_aad_length = sizeof(ad);
 	crp->crp_payload_start = sizeof(ad);
-	crp->crp_payload_length = crp->crp_ilen -
+	crp->crp_payload_length = uio.uio_resid -
 	    (sizeof(ad) + AES_GMAC_HASH_LEN);
-	crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN;
+	crp->crp_digest_start = uio.uio_resid - AES_GMAC_HASH_LEN;
 	memcpy(crp->crp_iv, nonce, sizeof(nonce));
 
 	counter_u64_add(ocf_tls13_gcm_crypts, 1);