diff options
Diffstat (limited to 'sys/dev')
| -rw-r--r-- | sys/dev/cesa/cesa.c | 599 | ||||
| -rw-r--r-- | sys/dev/cesa/cesa.h | 5 | ||||
| -rw-r--r-- | sys/dev/cxgbe/adapter.h | 2 | ||||
| -rw-r--r-- | sys/dev/cxgbe/crypto/t4_crypto.c | 1299 | ||||
| -rw-r--r-- | sys/dev/cxgbe/crypto/t4_keyctx.c | 40 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_tls.c | 4 | ||||
| -rw-r--r-- | sys/dev/glxsb/glxsb.c | 268 | ||||
| -rw-r--r-- | sys/dev/glxsb/glxsb.h | 6 | ||||
| -rw-r--r-- | sys/dev/glxsb/glxsb_hash.c | 100 | ||||
| -rw-r--r-- | sys/dev/hifn/hifn7751.c | 622 | ||||
| -rw-r--r-- | sys/dev/hifn/hifn7751var.h | 15 | ||||
| -rw-r--r-- | sys/dev/safe/safe.c | 915 | ||||
| -rw-r--r-- | sys/dev/safe/safevar.h | 12 | ||||
| -rw-r--r-- | sys/dev/sec/sec.c | 540 | ||||
| -rw-r--r-- | sys/dev/sec/sec.h | 18 | ||||
| -rw-r--r-- | sys/dev/ubsec/ubsec.c | 585 | ||||
| -rw-r--r-- | sys/dev/ubsec/ubsecvar.h | 10 |
17 files changed, 2191 insertions, 2849 deletions
diff --git a/sys/dev/cesa/cesa.c b/sys/dev/cesa/cesa.c index 6cbac049bb42..d4e056d2a09b 100644 --- a/sys/dev/cesa/cesa.c +++ b/sys/dev/cesa/cesa.c @@ -69,6 +69,7 @@ __FBSDID("$FreeBSD$"); #include <crypto/sha2/sha256.h> #include <crypto/rijndael/rijndael.h> #include <opencrypto/cryptodev.h> +#include <opencrypto/xform.h> #include "cryptodev_if.h" #include <arm/mv/mvreg.h> @@ -80,7 +81,10 @@ static int cesa_attach(device_t); static int cesa_attach_late(device_t); static int cesa_detach(device_t); static void cesa_intr(void *); -static int cesa_newsession(device_t, crypto_session_t, struct cryptoini *); +static int cesa_probesession(device_t, + const struct crypto_session_params *); +static int cesa_newsession(device_t, crypto_session_t, + const struct crypto_session_params *); static int cesa_process(device_t, struct cryptop *, int); static struct resource_spec cesa_res_spec[] = { @@ -97,6 +101,7 @@ static device_method_t cesa_methods[] = { DEVMETHOD(device_detach, cesa_detach), /* Crypto device methods */ + DEVMETHOD(cryptodev_probesession, cesa_probesession), DEVMETHOD(cryptodev_newsession, cesa_newsession), DEVMETHOD(cryptodev_process, cesa_process), @@ -417,78 +422,68 @@ cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, return (0); } -static int +static void cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) { - uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; - uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; - SHA1_CTX sha1ctx; - SHA256_CTX sha256ctx; - MD5_CTX md5ctx; + union authctx auth_ctx; uint32_t *hout; uint32_t *hin; int i; - memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); - memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); - for (i = 0; i < mklen; i++) { - ipad[i] ^= mkey[i]; - opad[i] ^= mkey[i]; - } - hin = (uint32_t *)cs->cs_hiv_in; hout = (uint32_t *)cs->cs_hiv_out; switch (alg) { case CRYPTO_MD5_HMAC: - MD5Init(&md5ctx); - MD5Update(&md5ctx, ipad, MD5_BLOCK_LEN); - memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); - MD5Init(&md5ctx); - MD5Update(&md5ctx, opad, MD5_BLOCK_LEN); - memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); + hmac_init_ipad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx); + memcpy(hin, auth_ctx.md5ctx.state, + sizeof(auth_ctx.md5ctx.state)); + hmac_init_opad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx); + memcpy(hout, auth_ctx.md5ctx.state, + sizeof(auth_ctx.md5ctx.state)); break; case CRYPTO_SHA1_HMAC: - SHA1Init(&sha1ctx); - SHA1Update(&sha1ctx, ipad, SHA1_BLOCK_LEN); - memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); - SHA1Init(&sha1ctx); - SHA1Update(&sha1ctx, opad, SHA1_BLOCK_LEN); - memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); + hmac_init_ipad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx); + memcpy(hin, auth_ctx.sha1ctx.h.b32, + sizeof(auth_ctx.sha1ctx.h.b32)); + hmac_init_opad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx); + memcpy(hout, auth_ctx.sha1ctx.h.b32, + sizeof(auth_ctx.sha1ctx.h.b32)); break; case CRYPTO_SHA2_256_HMAC: - SHA256_Init(&sha256ctx); - SHA256_Update(&sha256ctx, ipad, SHA2_256_BLOCK_LEN); - memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state)); - SHA256_Init(&sha256ctx); - SHA256_Update(&sha256ctx, opad, SHA2_256_BLOCK_LEN); - memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state)); + hmac_init_ipad(&auth_hash_hmac_sha2_256, mkey, mklen, + &auth_ctx); + memcpy(hin, auth_ctx.sha256ctx.state, + sizeof(auth_ctx.sha256ctx.state)); + hmac_init_opad(&auth_hash_hmac_sha2_256, mkey, mklen, + &auth_ctx); + memcpy(hout, auth_ctx.sha256ctx.state, + sizeof(auth_ctx.sha256ctx.state)); break; default: - return (EINVAL); + panic("shouldn't get here"); } for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { hin[i] = htobe32(hin[i]); hout[i] = htobe32(hout[i]); } - - return (0); } static int -cesa_prep_aes_key(struct cesa_session *cs) +cesa_prep_aes_key(struct cesa_session *cs, + const struct crypto_session_params *csp) { uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; uint32_t *dkey; int i; - rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); + rijndaelKeySetupEnc(ek, cs->cs_key, csp->csp_cipher_klen * 8); cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; dkey = (uint32_t *)cs->cs_aes_dkey; - switch (cs->cs_klen) { + switch (csp->csp_cipher_klen) { case 16: cs->cs_config |= CESA_CSH_AES_KLEN_128; for (i = 0; i < 4; i++) @@ -515,22 +510,6 @@ cesa_prep_aes_key(struct cesa_session *cs) return (0); } -static int -cesa_is_hash(int alg) -{ - - switch (alg) { - case CRYPTO_MD5: - case CRYPTO_MD5_HMAC: - case CRYPTO_SHA1: - case CRYPTO_SHA1_HMAC: - case CRYPTO_SHA2_256_HMAC: - return (1); - default: - return (0); - } -} - static void cesa_start_packet(struct cesa_packet *cp, unsigned int size) { @@ -584,6 +563,7 @@ cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) unsigned int skip, len; struct cesa_sa_desc *csd; struct cesa_request *cr; + struct cryptop *crp; struct cesa_softc *sc; struct cesa_packet cp; bus_dma_segment_t seg; @@ -593,73 +573,107 @@ cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) cci = arg; sc = cci->cci_sc; cr = cci->cci_cr; + crp = cr->cr_crp; if (error) { cci->cci_error = error; return; } - elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; - eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; - mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; - mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; - - if (elen && mlen && - ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || - (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || - (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { + /* + * Only do a combined op if the AAD is adjacent to the payload + * and the AAD length is a multiple of the IV length. The + * checks against 'config' are to avoid recursing when the + * logic below invokes separate operations. + */ + config = cci->cci_config; + if (((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC || + (config & CESA_CSHD_OP_MASK) == CESA_CSHD_ENC_AND_MAC) && + crp->crp_aad_length != 0 && + (crp->crp_aad_length & (cr->cr_cs->cs_ivlen - 1)) != 0) { /* * Data alignment in the request does not meet CESA requiremnts * for combined encryption/decryption and hashing. We have to * split the request to separate operations and process them * one by one. */ - config = cci->cci_config; if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { config &= ~CESA_CSHD_OP_MASK; cci->cci_config = config | CESA_CSHD_MAC; - cci->cci_enc = NULL; - cci->cci_mac = cr->cr_mac; - cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + cesa_create_chain_cb(cci, segs, nseg, 0); cci->cci_config = config | CESA_CSHD_ENC; - cci->cci_enc = cr->cr_enc; - cci->cci_mac = NULL; - cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + cesa_create_chain_cb(cci, segs, nseg, 0); } else { config &= ~CESA_CSHD_OP_MASK; cci->cci_config = config | CESA_CSHD_ENC; - cci->cci_enc = cr->cr_enc; - cci->cci_mac = NULL; - cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + cesa_create_chain_cb(cci, segs, nseg, 0); cci->cci_config = config | CESA_CSHD_MAC; - cci->cci_enc = NULL; - cci->cci_mac = cr->cr_mac; - cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + cesa_create_chain_cb(cci, segs, nseg, 0); } return; } + mskip = mlen = eskip = elen = 0; + + if (crp->crp_aad_length == 0) { + skip = crp->crp_payload_start; + len = crp->crp_payload_length; + switch (config & CESA_CSHD_OP_MASK) { + case CESA_CSHD_ENC: + eskip = skip; + elen = len; + break; + case CESA_CSHD_MAC: + mskip = skip; + mlen = len; + break; + default: + eskip = skip; + elen = len; + mskip = skip; + mlen = len; + break; + } + } else { + /* + * For an encryption-only separate request, only + * process the payload. For combined requests and + * hash-only requests, process the entire region. + */ + switch (config & CESA_CSHD_OP_MASK) { + case CESA_CSHD_ENC: + skip = crp->crp_payload_start; + len = crp->crp_payload_length; + eskip = skip; + elen = len; + break; + case CESA_CSHD_MAC: + skip = crp->crp_aad_start; + len = crp->crp_aad_length + crp->crp_payload_length; + mskip = skip; + mlen = len; + break; + default: + skip = crp->crp_aad_start; + len = crp->crp_aad_length + crp->crp_payload_length; + mskip = skip; + mlen = len; + eskip = crp->crp_payload_start; + elen = crp->crp_payload_length; + break; + } + } + tmlen = mlen; fragmented = 0; mpsize = CESA_MAX_PACKET_SIZE; mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); - if (elen && mlen) { - skip = MIN(eskip, mskip); - len = MAX(elen + eskip, mlen + mskip) - skip; - } else if (elen) { - skip = eskip; - len = elen; - } else { - skip = mskip; - len = mlen; - } - /* Start first packet in chain */ cesa_start_packet(&cp, MIN(mpsize, len)); @@ -777,16 +791,9 @@ cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) } } -static void -cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, - bus_size_t size, int error) -{ - - cesa_create_chain_cb(arg, segs, nseg, error); -} - static int -cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) +cesa_create_chain(struct cesa_softc *sc, + const struct crypto_session_params *csp, struct cesa_request *cr) { struct cesa_chain_info cci; struct cesa_tdma_desc *ctd; @@ -797,17 +804,17 @@ cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) CESA_LOCK_ASSERT(sc, sessions); /* Create request metadata */ - if (cr->cr_enc) { - if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && - (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) + if (csp->csp_cipher_klen != 0) { + if (csp->csp_cipher_alg == CRYPTO_AES_CBC && + !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op)) memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, - cr->cr_cs->cs_klen); + csp->csp_cipher_klen); else memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, - cr->cr_cs->cs_klen); + csp->csp_cipher_klen); } - if (cr->cr_mac) { + if (csp->csp_auth_klen != 0) { memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, CESA_MAX_HASH_LEN); memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, @@ -823,37 +830,30 @@ cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) /* Prepare SA configuration */ config = cr->cr_cs->cs_config; - if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) + if (csp->csp_cipher_alg != 0 && + !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op)) config |= CESA_CSHD_DECRYPT; - if (cr->cr_enc && !cr->cr_mac) + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: config |= CESA_CSHD_ENC; - if (!cr->cr_enc && cr->cr_mac) + break; + case CSP_MODE_DIGEST: config |= CESA_CSHD_MAC; - if (cr->cr_enc && cr->cr_mac) + break; + case CSP_MODE_ETA: config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : CESA_CSHD_ENC_AND_MAC; + break; + } /* Create data packets */ cci.cci_sc = sc; cci.cci_cr = cr; - cci.cci_enc = cr->cr_enc; - cci.cci_mac = cr->cr_mac; cci.cci_config = config; cci.cci_error = 0; - if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) - error = bus_dmamap_load_uio(sc->sc_data_dtag, - cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, - cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); - else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) - error = bus_dmamap_load_mbuf(sc->sc_data_dtag, - cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, - cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); - else - error = bus_dmamap_load(sc->sc_data_dtag, - cr->cr_dmap, cr->cr_crp->crp_buf, - cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, - BUS_DMA_NOWAIT); + error = bus_dmamap_load_crp(sc->sc_data_dtag, cr->cr_dmap, cr->cr_crp, + cesa_create_chain_cb, &cci, BUS_DMA_NOWAIT); if (!error) cr->cr_dmap_loaded = 1; @@ -1385,18 +1385,6 @@ cesa_attach_late(device_t dev) goto err8; } - crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); - if (sc->sc_soc_id == MV_DEV_88F6828 || - sc->sc_soc_id == MV_DEV_88F6820 || - sc->sc_soc_id == MV_DEV_88F6810) - crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); - return (0); err8: for (i = 0; i < CESA_REQUESTS; i++) @@ -1487,6 +1475,7 @@ cesa_intr(void *arg) struct cesa_request *cr, *tmp; struct cesa_softc *sc; uint32_t ecr, icr; + uint8_t hash[HASH_MAX_LEN]; int blocked; sc = arg; @@ -1547,11 +1536,19 @@ cesa_intr(void *arg) BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cr->cr_crp->crp_etype = sc->sc_error; - if (cr->cr_mac) - crypto_copyback(cr->cr_crp->crp_flags, - cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, - cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); - + if (cr->cr_cs->cs_hlen != 0 && cr->cr_crp->crp_etype == 0) { + if (cr->cr_crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(cr->cr_crp, + cr->cr_crp->crp_digest_start, + cr->cr_cs->cs_hlen, hash); + if (timingsafe_bcmp(hash, cr->cr_csd->csd_hash, + cr->cr_cs->cs_hlen) != 0) + cr->cr_crp->crp_etype = EBADMSG; + } else + crypto_copyback(cr->cr_crp, + cr->cr_crp->crp_digest_start, + cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); + } crypto_done(cr->cr_crp); cesa_free_request(sc, cr); } @@ -1571,42 +1568,98 @@ cesa_intr(void *arg) crypto_unblock(sc->sc_cid, blocked); } -static int -cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) +static bool +cesa_cipher_supported(const struct crypto_session_params *csp) { - struct cesa_session *cs; - struct cesa_softc *sc; - struct cryptoini *enc; - struct cryptoini *mac; - int error; - - sc = device_get_softc(dev); - enc = NULL; - mac = NULL; - error = 0; - /* Check and parse input */ - if (cesa_is_hash(cri->cri_alg)) - mac = cri; - else - enc = cri; + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + if (csp->csp_ivlen != AES_BLOCK_LEN) + return (false); + break; + case CRYPTO_DES_CBC: + if (csp->csp_ivlen != DES_BLOCK_LEN) + return (false); + break; + case CRYPTO_3DES_CBC: + if (csp->csp_ivlen != DES3_BLOCK_LEN) + return (false); + break; + default: + return (false); + } - cri = cri->cri_next; + if (csp->csp_cipher_klen > CESA_MAX_KEY_LEN) + return (false); - if (cri) { - if (!enc && !cesa_is_hash(cri->cri_alg)) - enc = cri; + return (true); +} - if (!mac && cesa_is_hash(cri->cri_alg)) - mac = cri; +static bool +cesa_auth_supported(struct cesa_softc *sc, + const struct crypto_session_params *csp) +{ - if (cri->cri_next || !(enc && mac)) + switch (csp->csp_auth_alg) { + case CRYPTO_SHA2_256_HMAC: + if (!(sc->sc_soc_id == MV_DEV_88F6828 || + sc->sc_soc_id == MV_DEV_88F6820 || + sc->sc_soc_id == MV_DEV_88F6810)) + return (false); + /* FALLTHROUGH */ + case CRYPTO_MD5: + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + break; + default: + return (false); + } + + if (csp->csp_auth_klen > CESA_MAX_MKEY_LEN) + return (false); + + return (true); +} + +static int +cesa_probesession(device_t dev, const struct crypto_session_params *csp) +{ + struct cesa_softc *sc; + + sc = device_get_softc(dev); + if (csp->csp_flags != 0) + return (EINVAL); + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + if (!cesa_auth_supported(sc, csp)) return (EINVAL); + break; + case CSP_MODE_CIPHER: + if (!cesa_cipher_supported(csp)) + return (EINVAL); + break; + case CSP_MODE_ETA: + if (!cesa_auth_supported(sc, csp) || + !cesa_cipher_supported(csp)) + return (EINVAL); + break; + default: + return (EINVAL); } + return (CRYPTODEV_PROBE_HARDWARE); +} - if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || - (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) - return (E2BIG); +static int +cesa_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct cesa_session *cs; + struct cesa_softc *sc; + int error; + + sc = device_get_softc(dev); + error = 0; /* Allocate session */ cs = crypto_get_driver_session(cses); @@ -1616,106 +1669,89 @@ cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) cs->cs_ivlen = 1; cs->cs_mblen = 1; - if (enc) { - switch (enc->cri_alg) { - case CRYPTO_AES_CBC: - cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; - cs->cs_ivlen = AES_BLOCK_LEN; - break; - case CRYPTO_DES_CBC: - cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; - cs->cs_ivlen = DES_BLOCK_LEN; - break; - case CRYPTO_3DES_CBC: - cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | - CESA_CSHD_CBC; - cs->cs_ivlen = DES3_BLOCK_LEN; - break; - default: - error = EINVAL; - break; - } + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; + cs->cs_ivlen = AES_BLOCK_LEN; + break; + case CRYPTO_DES_CBC: + cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; + cs->cs_ivlen = DES_BLOCK_LEN; + break; + case CRYPTO_3DES_CBC: + cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | + CESA_CSHD_CBC; + cs->cs_ivlen = DES3_BLOCK_LEN; + break; } - if (!error && mac) { - switch (mac->cri_alg) { - case CRYPTO_MD5: - cs->cs_mblen = 1; - cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : - mac->cri_mlen; - cs->cs_config |= CESA_CSHD_MD5; - break; - case CRYPTO_MD5_HMAC: - cs->cs_mblen = MD5_BLOCK_LEN; - cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : - mac->cri_mlen; - cs->cs_config |= CESA_CSHD_MD5_HMAC; - if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) - cs->cs_config |= CESA_CSHD_96_BIT_HMAC; - break; - case CRYPTO_SHA1: - cs->cs_mblen = 1; - cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : - mac->cri_mlen; - cs->cs_config |= CESA_CSHD_SHA1; - break; - case CRYPTO_SHA1_HMAC: - cs->cs_mblen = SHA1_BLOCK_LEN; - cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : - mac->cri_mlen; - cs->cs_config |= CESA_CSHD_SHA1_HMAC; - if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) - cs->cs_config |= CESA_CSHD_96_BIT_HMAC; - break; - case CRYPTO_SHA2_256_HMAC: - cs->cs_mblen = SHA2_256_BLOCK_LEN; - cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN : - mac->cri_mlen; - cs->cs_config |= CESA_CSHD_SHA2_256_HMAC; - break; - default: - error = EINVAL; - break; - } + switch (csp->csp_auth_alg) { + case CRYPTO_MD5: + cs->cs_mblen = 1; + cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN : + csp->csp_auth_mlen; + cs->cs_config |= CESA_CSHD_MD5; + break; + case CRYPTO_MD5_HMAC: + cs->cs_mblen = MD5_BLOCK_LEN; + cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN : + csp->csp_auth_mlen; + cs->cs_config |= CESA_CSHD_MD5_HMAC; + if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) + cs->cs_config |= CESA_CSHD_96_BIT_HMAC; + break; + case CRYPTO_SHA1: + cs->cs_mblen = 1; + cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN : + csp->csp_auth_mlen; + cs->cs_config |= CESA_CSHD_SHA1; + break; + case CRYPTO_SHA1_HMAC: + cs->cs_mblen = SHA1_BLOCK_LEN; + cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN : + csp->csp_auth_mlen; + cs->cs_config |= CESA_CSHD_SHA1_HMAC; + if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) + cs->cs_config |= CESA_CSHD_96_BIT_HMAC; + break; + case CRYPTO_SHA2_256_HMAC: + cs->cs_mblen = SHA2_256_BLOCK_LEN; + cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA2_256_HASH_LEN : + csp->csp_auth_mlen; + cs->cs_config |= CESA_CSHD_SHA2_256_HMAC; + break; } /* Save cipher key */ - if (!error && enc && enc->cri_key) { - cs->cs_klen = enc->cri_klen / 8; - memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); - if (enc->cri_alg == CRYPTO_AES_CBC) - error = cesa_prep_aes_key(cs); + if (csp->csp_cipher_key != NULL) { + memcpy(cs->cs_key, csp->csp_cipher_key, + csp->csp_cipher_klen); + if (csp->csp_cipher_alg == CRYPTO_AES_CBC) + error = cesa_prep_aes_key(cs, csp); } /* Save digest key */ - if (!error && mac && mac->cri_key) - error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, - mac->cri_klen / 8); + if (csp->csp_auth_key != NULL) + cesa_set_mkey(cs, csp->csp_auth_alg, csp->csp_auth_key, + csp->csp_auth_klen); - if (error) - return (error); - - return (0); + return (error); } static int cesa_process(device_t dev, struct cryptop *crp, int hint) { + const struct crypto_session_params *csp; struct cesa_request *cr; struct cesa_session *cs; - struct cryptodesc *crd; - struct cryptodesc *enc; - struct cryptodesc *mac; struct cesa_softc *sc; int error; sc = device_get_softc(dev); - crd = crp->crp_desc; - enc = NULL; - mac = NULL; error = 0; cs = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); /* Check and parse input */ if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { @@ -1724,25 +1760,16 @@ cesa_process(device_t dev, struct cryptop *crp, int hint) return (0); } - if (cesa_is_hash(crd->crd_alg)) - mac = crd; - else - enc = crd; - - crd = crd->crd_next; - - if (crd) { - if (!enc && !cesa_is_hash(crd->crd_alg)) - enc = crd; - - if (!mac && cesa_is_hash(crd->crd_alg)) - mac = crd; - - if (crd->crd_next || !(enc && mac)) { - crp->crp_etype = EINVAL; - crypto_done(crp); - return (0); - } + /* + * For requests with AAD, only requests where the AAD is + * immediately adjacent to the payload are supported. + */ + if (crp->crp_aad_length != 0 && + (crp->crp_aad_start + crp->crp_aad_length) != + crp->crp_payload_start) { + crp->crp_etype = EINVAL; + crypto_done(crp); + return (0); } /* @@ -1759,51 +1786,37 @@ cesa_process(device_t dev, struct cryptop *crp, int hint) /* Prepare request */ cr->cr_crp = crp; - cr->cr_enc = enc; - cr->cr_mac = mac; cr->cr_cs = cs; CESA_LOCK(sc, sessions); cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - if (enc && enc->crd_flags & CRD_F_ENCRYPT) { - if (enc->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); + if (csp->csp_cipher_alg != 0) { + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(cr->cr_csd->csd_iv, csp->csp_ivlen, 0); + crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, + cr->cr_csd->csd_iv); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(cr->cr_csd->csd_iv, crp->crp_iv, csp->csp_ivlen); else - arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); - - if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); - } else if (enc) { - if (enc->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); - else - crypto_copydata(crp->crp_flags, crp->crp_buf, - enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); + crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, + cr->cr_csd->csd_iv); } - if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { - if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { - cs->cs_klen = enc->crd_klen / 8; - memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); - if (enc->crd_alg == CRYPTO_AES_CBC) - error = cesa_prep_aes_key(cs); - } else - error = E2BIG; + if (crp->crp_cipher_key != NULL) { + memcpy(cs->cs_key, crp->crp_cipher_key, + csp->csp_cipher_klen); + if (csp->csp_cipher_alg == CRYPTO_AES_CBC) + error = cesa_prep_aes_key(cs, csp); } - if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { - if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) - error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, - mac->crd_klen / 8); - else - error = E2BIG; - } + if (!error && crp->crp_auth_key != NULL) + cesa_set_mkey(cs, csp->csp_auth_alg, crp->crp_auth_key, + csp->csp_auth_klen); /* Convert request to chain of TDMA and SA descriptors */ if (!error) - error = cesa_create_chain(sc, cr); + error = cesa_create_chain(sc, csp, cr); cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); CESA_UNLOCK(sc, sessions); diff --git a/sys/dev/cesa/cesa.h b/sys/dev/cesa/cesa.h index 9fa35b89b18f..449f4ecce5b2 100644 --- a/sys/dev/cesa/cesa.h +++ b/sys/dev/cesa/cesa.h @@ -194,7 +194,6 @@ struct cesa_sa_desc { struct cesa_session { uint32_t cs_config; - unsigned int cs_klen; unsigned int cs_ivlen; unsigned int cs_hlen; unsigned int cs_mblen; @@ -208,8 +207,6 @@ struct cesa_request { struct cesa_sa_data *cr_csd; bus_addr_t cr_csd_paddr; struct cryptop *cr_crp; - struct cryptodesc *cr_enc; - struct cryptodesc *cr_mac; struct cesa_session *cr_cs; bus_dmamap_t cr_dmap; int cr_dmap_loaded; @@ -272,8 +269,6 @@ struct cesa_softc { struct cesa_chain_info { struct cesa_softc *cci_sc; struct cesa_request *cci_cr; - struct cryptodesc *cci_enc; - struct cryptodesc *cci_mac; uint32_t cci_config; int cci_error; }; diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index fd5487b01179..1f3ccedca9b3 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -1204,7 +1204,7 @@ union authctx; void t4_aes_getdeckey(void *, const void *, unsigned int); void t4_copy_partial_hash(int, union authctx *, void *); void t4_init_gmac_hash(const char *, int, char *); -void t4_init_hmac_digest(struct auth_hash *, u_int, char *, int, char *); +void t4_init_hmac_digest(struct auth_hash *, u_int, const char *, int, char *); #ifdef DEV_NETMAP /* t4_netmap.c */ diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c index 5b924125b6e4..5ab8048ece4d 100644 --- a/sys/dev/cxgbe/crypto/t4_crypto.c +++ b/sys/dev/cxgbe/crypto/t4_crypto.c @@ -165,7 +165,7 @@ struct ccr_session_blkcipher { struct ccr_session { bool active; int pending; - enum { HASH, HMAC, BLKCIPHER, AUTHENC, GCM, CCM } mode; + enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode; union { struct ccr_session_hmac hmac; struct ccr_session_gmac gmac; @@ -208,8 +208,8 @@ struct ccr_softc { uint64_t stats_blkcipher_decrypt; uint64_t stats_hash; uint64_t stats_hmac; - uint64_t stats_authenc_encrypt; - uint64_t stats_authenc_decrypt; + uint64_t stats_eta_encrypt; + uint64_t stats_eta_decrypt; uint64_t stats_gcm_encrypt; uint64_t stats_gcm_decrypt; uint64_t stats_ccm_encrypt; @@ -230,9 +230,9 @@ struct ccr_softc { * Non-hash-only requests require a PHYS_DSGL that describes the * location to store the results of the encryption or decryption * operation. This SGL uses a different format (PHYS_DSGL) and should - * exclude the crd_skip bytes at the start of the data as well as - * any AAD or IV. For authenticated encryption requests it should - * cover include the destination of the hash or tag. + * exclude the skip bytes at the start of the data as well as any AAD + * or IV. For authenticated encryption requests it should include the + * destination of the hash or tag. * * The input payload may either be supplied inline as immediate data, * or via a standard ULP_TX SGL. This SGL should include AAD, @@ -251,12 +251,19 @@ ccr_populate_sglist(struct sglist *sg, struct cryptop *crp) int error; sglist_reset(sg); - if (crp->crp_flags & CRYPTO_F_IMBUF) - error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf); - else if (crp->crp_flags & CRYPTO_F_IOV) - error = sglist_append_uio(sg, (struct uio *)crp->crp_buf); - else + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + error = sglist_append_mbuf(sg, crp->crp_mbuf); + break; + case CRYPTO_BUF_UIO: + error = sglist_append_uio(sg, crp->crp_uio); + break; + case CRYPTO_BUF_CONTIG: error = sglist_append(sg, crp->crp_buf, crp->crp_ilen); + break; + default: + error = EINVAL; + } return (error); } @@ -436,16 +443,13 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) struct chcr_wr *crwr; struct wrqe *wr; struct auth_hash *axf; - struct cryptodesc *crd; char *dst; u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len; u_int hmac_ctrl, imm_len, iopad_size; int error, sgl_nsegs, sgl_len, use_opad; - crd = crp->crp_desc; - /* Reject requests with too large of an input buffer. */ - if (crd->crd_len > MAX_REQUEST_SIZE) + if (crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); axf = s->hmac.auth_hash; @@ -471,19 +475,19 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) hash_size_in_response = axf->hashsize; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); - if (crd->crd_len == 0) { + if (crp->crp_payload_length == 0) { imm_len = axf->blocksize; sgl_nsegs = 0; sgl_len = 0; - } else if (ccr_use_imm_data(transhdr_len, crd->crd_len)) { - imm_len = crd->crd_len; + } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) { + imm_len = crp->crp_payload_length; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(sc->sg_ulptx); error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crd->crd_skip, crd->crd_len); + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); sgl_nsegs = sc->sg_ulptx->sg_nseg; @@ -512,8 +516,8 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(0)); - crwr->sec_cpl.pldlen = htobe32(crd->crd_len == 0 ? axf->blocksize : - crd->crd_len); + crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ? + axf->blocksize : crp->crp_payload_length); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0)); @@ -527,7 +531,8 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) V_SCMD_HMAC_CTRL(hmac_ctrl)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_LAST_FRAG(0) | - V_SCMD_MORE_FRAGS(crd->crd_len == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1)); + V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) | + V_SCMD_MAC_ONLY(1)); memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len); @@ -540,14 +545,14 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1)); dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES; - if (crd->crd_len == 0) { + if (crp->crp_payload_length == 0) { dst[0] = 0x80; if (s->mode == HMAC) *(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) = htobe64(axf->blocksize << 3); } else if (imm_len != 0) - crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip, - crd->crd_len, dst); + crypto_copydata(crp, crp->crp_payload_start, + crp->crp_payload_length, dst); else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); @@ -561,15 +566,20 @@ static int ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { - struct cryptodesc *crd; + uint8_t hash[HASH_MAX_LEN]; - crd = crp->crp_desc; - if (error == 0) { - crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, - s->hmac.hash_len, (c_caddr_t)(cpl + 1)); - } + if (error) + return (error); - return (error); + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len, + hash); + if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0) + return (EBADMSG); + } else + crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len, + (cpl + 1)); + return (0); } static int @@ -578,34 +588,31 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; - struct cryptodesc *crd; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; - u_int imm_len; + u_int imm_len, iv_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; - crd = crp->crp_desc; - - if (s->blkcipher.key_len == 0 || crd->crd_len == 0) + if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) return (EINVAL); - if (crd->crd_alg == CRYPTO_AES_CBC && - (crd->crd_len % AES_BLOCK_LEN) != 0) + if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && + (crp->crp_payload_length % AES_BLOCK_LEN) != 0) return (EINVAL); /* Reject requests with too large of an input buffer. */ - if (crd->crd_len > MAX_REQUEST_SIZE) + if (crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); - if (crd->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; sglist_reset(sc->sg_dsgl); - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crd->crd_skip, - crd->crd_len); + error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); @@ -617,23 +624,28 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) kctx_len = roundup2(s->blkcipher.key_len, 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); - if (ccr_use_imm_data(transhdr_len, crd->crd_len + - s->blkcipher.iv_len)) { - imm_len = crd->crd_len; + /* For AES-XTS we send a 16-byte IV in the work request. */ + if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) + iv_len = AES_BLOCK_LEN; + else + iv_len = s->blkcipher.iv_len; + + if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) { + imm_len = crp->crp_payload_length; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(sc->sg_ulptx); error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crd->crd_skip, crd->crd_len); + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } - wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len + + wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); @@ -647,24 +659,20 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) /* * Read the existing IV from the request or generate a random - * one if none is provided. Optionally copy the generated IV - * into the output buffer if requested. + * one if none is provided. */ - if (op_type == CHCR_ENCRYPT_OP) { - if (crd->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crd->crd_iv, s->blkcipher.iv_len); - else - arc4rand(iv, s->blkcipher.iv_len, 0); - if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - crd->crd_inject, s->blkcipher.iv_len, iv); - } else { - if (crd->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crd->crd_iv, s->blkcipher.iv_len); - else - crypto_copydata(crp->crp_flags, crp->crp_buf, - crd->crd_inject, s->blkcipher.iv_len, iv); - } + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(iv, s->blkcipher.iv_len, 0); + crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len, + iv); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(iv, crp->crp_iv, s->blkcipher.iv_len); + else + crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len, + iv); + + /* Zero the remainder of the IV for AES-XTS. */ + memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); @@ -677,10 +685,10 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); - crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + crd->crd_len); + crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( - V_CPL_TX_SEC_PDU_CIPHERSTART(s->blkcipher.iv_len + 1) | + V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0)); @@ -693,7 +701,7 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | - V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) | + V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | @@ -701,24 +709,24 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; - switch (crd->crd_alg) { - case CRYPTO_AES_CBC: - if (crd->crd_flags & CRD_F_ENCRYPT) + switch (s->blkcipher.cipher_mode) { + case SCMD_CIPH_MODE_AES_CBC: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; - case CRYPTO_AES_ICM: + case SCMD_CIPH_MODE_AES_CTR: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; - case CRYPTO_AES_XTS: + case SCMD_CIPH_MODE_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); - if (crd->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else @@ -730,11 +738,11 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(sc, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; - memcpy(dst, iv, s->blkcipher.iv_len); - dst += s->blkcipher.iv_len; + memcpy(dst, iv, iv_len); + dst += iv_len; if (imm_len != 0) - crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip, - crd->crd_len, dst); + crypto_copydata(crp, crp->crp_payload_start, + crp->crp_payload_length, dst); else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); @@ -775,8 +783,7 @@ ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize) } static int -ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, - struct cryptodesc *crda, struct cryptodesc *crde) +ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; @@ -784,9 +791,9 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, struct auth_hash *axf; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; - u_int hash_size_in_response, imm_len, iopad_size; - u_int aad_start, aad_len, aad_stop; - u_int auth_start, auth_stop, auth_insert; + u_int hash_size_in_response, imm_len, iopad_size, iv_len; + u_int aad_start, aad_stop; + u_int auth_insert; u_int cipher_start, cipher_stop; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; @@ -797,34 +804,24 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * If there is a need in the future, requests with an empty * payload could be supported as HMAC-only requests. */ - if (s->blkcipher.key_len == 0 || crde->crd_len == 0) + if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) return (EINVAL); - if (crde->crd_alg == CRYPTO_AES_CBC && - (crde->crd_len % AES_BLOCK_LEN) != 0) + if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && + (crp->crp_payload_length % AES_BLOCK_LEN) != 0) return (EINVAL); - /* - * Compute the length of the AAD (data covered by the - * authentication descriptor but not the encryption - * descriptor). To simplify the logic, AAD is only permitted - * before the cipher/plain text, not after. This is true of - * all currently-generated requests. - */ - if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip) - return (EINVAL); - if (crda->crd_skip < crde->crd_skip) { - if (crda->crd_skip + crda->crd_len > crde->crd_skip) - aad_len = (crde->crd_skip - crda->crd_skip); - else - aad_len = crda->crd_len; - } else - aad_len = 0; - if (aad_len + s->blkcipher.iv_len > MAX_AAD_LEN) + /* For AES-XTS we send a 16-byte IV in the work request. */ + if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) + iv_len = AES_BLOCK_LEN; + else + iv_len = s->blkcipher.iv_len; + + if (crp->crp_aad_length + iv_len > MAX_AAD_LEN) return (EINVAL); axf = s->hmac.auth_hash; hash_size_in_response = s->hmac.hash_len; - if (crde->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; @@ -839,26 +836,26 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { - if (s->blkcipher.iv_len + aad_len + crde->crd_len + + if (iv_len + crp->crp_aad_length + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { - if (s->blkcipher.iv_len + aad_len + crde->crd_len > + if (iv_len + crp->crp_aad_length + crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(sc->sg_dsgl); error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, - s->blkcipher.iv_len + aad_len); + iv_len + crp->crp_aad_length); if (error) return (error); - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip, - crde->crd_len); + error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, - crda->crd_inject, hash_size_in_response); + crp->crp_digest_start, hash_size_in_response); if (error) return (error); } @@ -888,7 +885,7 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * inside of the AAD region, so a second copy is always * required. */ - input_len = aad_len + crde->crd_len; + input_len = crp->crp_aad_length + crp->crp_payload_length; /* * The firmware hangs if sent a request which is a @@ -902,26 +899,27 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, return (EFBIG); if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; - if (ccr_use_imm_data(transhdr_len, s->blkcipher.iv_len + input_len)) { + + if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(sc->sg_ulptx); - if (aad_len != 0) { + if (crp->crp_aad_length != 0) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crda->crd_skip, aad_len); + crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crde->crd_skip, crde->crd_len); + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crda->crd_inject, hash_size_in_response); + crp->crp_digest_start, hash_size_in_response); if (error) return (error); } @@ -934,37 +932,25 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * Auth-data that overlaps with the cipher region is placed in * the auth section. */ - if (aad_len != 0) { - aad_start = s->blkcipher.iv_len + 1; - aad_stop = aad_start + aad_len - 1; + if (crp->crp_aad_length != 0) { + aad_start = iv_len + 1; + aad_stop = aad_start + crp->crp_aad_length - 1; } else { aad_start = 0; aad_stop = 0; } - cipher_start = s->blkcipher.iv_len + aad_len + 1; + cipher_start = iv_len + crp->crp_aad_length + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; - if (aad_len == crda->crd_len) { - auth_start = 0; - auth_stop = 0; - } else { - if (aad_len != 0) - auth_start = cipher_start; - else - auth_start = s->blkcipher.iv_len + crda->crd_skip - - crde->crd_skip + 1; - auth_stop = (crde->crd_skip + crde->crd_len) - - (crda->crd_skip + crda->crd_len) + cipher_stop; - } if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; - wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len + - roundup2(imm_len, 16) + sgl_len; + wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, sc->txq); @@ -977,24 +963,20 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, /* * Read the existing IV from the request or generate a random - * one if none is provided. Optionally copy the generated IV - * into the output buffer if requested. + * one if none is provided. */ - if (op_type == CHCR_ENCRYPT_OP) { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); - else - arc4rand(iv, s->blkcipher.iv_len, 0); - if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_inject, s->blkcipher.iv_len, iv); - } else { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); - else - crypto_copydata(crp->crp_flags, crp->crp_buf, - crde->crd_inject, s->blkcipher.iv_len, iv); - } + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(iv, s->blkcipher.iv_len, 0); + crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len, + iv); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(iv, crp->crp_iv, s->blkcipher.iv_len); + else + crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len, + iv); + + /* Zero the remainder of the IV for AES-XTS. */ + memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp); @@ -1007,7 +989,7 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); - crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + input_len); + crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | @@ -1016,8 +998,8 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) | - V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) | - V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) | + V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | + V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ @@ -1030,7 +1012,7 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | V_SCMD_AUTH_MODE(s->hmac.auth_mode) | V_SCMD_HMAC_CTRL(hmac_ctrl) | - V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) | + V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | @@ -1038,24 +1020,24 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; - switch (crde->crd_alg) { - case CRYPTO_AES_CBC: - if (crde->crd_flags & CRD_F_ENCRYPT) + switch (s->blkcipher.cipher_mode) { + case SCMD_CIPH_MODE_AES_CBC: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; - case CRYPTO_AES_ICM: + case SCMD_CIPH_MODE_AES_CTR: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; - case CRYPTO_AES_XTS: + case SCMD_CIPH_MODE_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); - if (crde->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else @@ -1070,20 +1052,20 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(sc, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; - memcpy(dst, iv, s->blkcipher.iv_len); - dst += s->blkcipher.iv_len; + memcpy(dst, iv, iv_len); + dst += iv_len; if (imm_len != 0) { - if (aad_len != 0) { - crypto_copydata(crp->crp_flags, crp->crp_buf, - crda->crd_skip, aad_len, dst); - dst += aad_len; + if (crp->crp_aad_length != 0) { + crypto_copydata(crp, crp->crp_aad_start, + crp->crp_aad_length, dst); + dst += crp->crp_aad_length; } - crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip, - crde->crd_len, dst); - dst += crde->crd_len; + crypto_copydata(crp, crp->crp_payload_start, + crp->crp_payload_length, dst); + dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) - crypto_copydata(crp->crp_flags, crp->crp_buf, - crda->crd_inject, hash_size_in_response, dst); + crypto_copydata(crp, crp->crp_digest_start, + hash_size_in_response, dst); } else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); @@ -1094,38 +1076,19 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, } static int -ccr_authenc_done(struct ccr_softc *sc, struct ccr_session *s, +ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { - struct cryptodesc *crd; /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. - * - * For a decryption request, the hardware may do a verification - * of the HMAC which will fail if the existing HMAC isn't in the - * buffer. If that happens, clear the error and copy the HMAC - * from the CPL reply into the buffer. - * - * For encryption requests, crd should be the cipher request - * which will have CRD_F_ENCRYPT set. For decryption - * requests, crp_desc will be the HMAC request which should - * not have this flag set. */ - crd = crp->crp_desc; - if (error == EBADMSG && !CHK_PAD_ERR_BIT(be64toh(cpl->data[0])) && - !(crd->crd_flags & CRD_F_ENCRYPT)) { - crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, - s->hmac.hash_len, (c_caddr_t)(cpl + 1)); - error = 0; - } return (error); } static int -ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, - struct cryptodesc *crda, struct cryptodesc *crde) +ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; @@ -1146,21 +1109,14 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * The crypto engine doesn't handle GCM requests with an empty * payload, so handle those in software instead. */ - if (crde->crd_len == 0) + if (crp->crp_payload_length == 0) return (EMSGSIZE); - /* - * AAD is only permitted before the cipher/plain text, not - * after. - */ - if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip) - return (EMSGSIZE); - - if (crda->crd_len + AES_BLOCK_LEN > MAX_AAD_LEN) + if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN) return (EMSGSIZE); hash_size_in_response = s->gmac.hash_len; - if (crde->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; @@ -1187,6 +1143,12 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, iv_len = s->blkcipher.iv_len; /* + * GCM requests should always provide an explicit IV. + */ + if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) + return (EINVAL); + + /* * The output buffer consists of the cipher text followed by * the tag when encrypting. For decryption it only contains * the plain text. @@ -1196,25 +1158,26 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { - if (iv_len + crda->crd_len + crde->crd_len + + if (iv_len + crp->crp_aad_length + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { - if (iv_len + crda->crd_len + crde->crd_len > MAX_REQUEST_SIZE) + if (iv_len + crp->crp_aad_length + crp->crp_payload_length > + MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(sc->sg_dsgl); error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + - crda->crd_len); + crp->crp_aad_length); if (error) return (error); - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip, - crde->crd_len); + error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, - crda->crd_inject, hash_size_in_response); + crp->crp_digest_start, hash_size_in_response); if (error) return (error); } @@ -1241,7 +1204,7 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * inside of the AAD region, so a second copy is always * required. */ - input_len = crda->crd_len + crde->crd_len; + input_len = crp->crp_aad_length + crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (input_len > MAX_REQUEST_SIZE) @@ -1253,19 +1216,19 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, } else { imm_len = 0; sglist_reset(sc->sg_ulptx); - if (crda->crd_len != 0) { + if (crp->crp_aad_length != 0) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crda->crd_skip, crda->crd_len); + crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crde->crd_skip, crde->crd_len); + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crda->crd_inject, hash_size_in_response); + crp->crp_digest_start, hash_size_in_response); if (error) return (error); } @@ -1273,14 +1236,14 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } - if (crda->crd_len != 0) { + if (crp->crp_aad_length != 0) { aad_start = iv_len + 1; - aad_stop = aad_start + crda->crd_len - 1; + aad_stop = aad_start + crp->crp_aad_length - 1; } else { aad_start = 0; aad_stop = 0; } - cipher_start = iv_len + crda->crd_len + 1; + cipher_start = iv_len + crp->crp_aad_length + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else @@ -1302,29 +1265,7 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, crwr = wrtod(wr); memset(crwr, 0, wr_len); - /* - * Read the existing IV from the request or generate a random - * one if none is provided. Optionally copy the generated IV - * into the output buffer if requested. - * - * If the input IV is 12 bytes, append an explicit 4-byte - * counter of 1. - */ - if (op_type == CHCR_ENCRYPT_OP) { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); - else - arc4rand(iv, s->blkcipher.iv_len, 0); - if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_inject, s->blkcipher.iv_len, iv); - } else { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); - else - crypto_copydata(crp->crp_flags, crp->crp_buf, - crde->crd_inject, s->blkcipher.iv_len, iv); - } + memcpy(iv, crp->crp_iv, s->blkcipher.iv_len); if (s->blkcipher.iv_len == 12) *(uint32_t *)&iv[12] = htobe32(1); @@ -1343,13 +1284,12 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, /* * NB: cipherstop is explicitly set to 0. On encrypt it - * should normally be set to 0 anyway (as the encrypt crd ends - * at the end of the input). However, for decrypt the cipher - * ends before the tag in the AUTHENC case (and authstop is - * set to stop before the tag), but for GCM the cipher still - * runs to the end of the buffer. Not sure if this is - * intentional or a firmware quirk, but it is required for - * working tag validation with GCM decryption. + * should normally be set to 0 anyway. However, for decrypt + * the cipher ends before the tag in the ETA case (and + * authstop is set to stop before the tag), but for GCM the + * cipher still runs to the end of the buffer. Not sure if + * this is intentional or a firmware quirk, but it is required + * for working tag validation with GCM decryption. */ crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | @@ -1390,17 +1330,17 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) { - if (crda->crd_len != 0) { - crypto_copydata(crp->crp_flags, crp->crp_buf, - crda->crd_skip, crda->crd_len, dst); - dst += crda->crd_len; + if (crp->crp_aad_length != 0) { + crypto_copydata(crp, crp->crp_aad_start, + crp->crp_aad_length, dst); + dst += crp->crp_aad_length; } - crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip, - crde->crd_len, dst); - dst += crde->crd_len; + crypto_copydata(crp, crp->crp_payload_start, + crp->crp_payload_length, dst); + dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) - crypto_copydata(crp->crp_flags, crp->crp_buf, - crda->crd_inject, hash_size_in_response, dst); + crypto_copydata(crp, crp->crp_digest_start, + hash_size_in_response, dst); } else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); @@ -1429,8 +1369,7 @@ ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s, * performing the operation in software. Derived from swcr_authenc(). */ static void -ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp, - struct cryptodesc *crda, struct cryptodesc *crde) +ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp) { struct auth_hash *axf; struct enc_xform *exf; @@ -1478,30 +1417,19 @@ ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp, * This assumes a 12-byte IV from the crp. See longer comment * above in ccr_gcm() for more details. */ - if (crde->crd_flags & CRD_F_ENCRYPT) { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, 12); - else - arc4rand(iv, 12, 0); - if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_inject, 12, iv); - } else { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, 12); - else - crypto_copydata(crp->crp_flags, crp->crp_buf, - crde->crd_inject, 12, iv); + if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { + error = EINVAL; + goto out; } + memcpy(iv, crp->crp_iv, 12); *(uint32_t *)&iv[12] = htobe32(1); axf->Reinit(auth_ctx, iv, sizeof(iv)); /* MAC the AAD. */ - for (i = 0; i < crda->crd_len; i += sizeof(block)) { - len = imin(crda->crd_len - i, sizeof(block)); - crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip + - i, len, block); + for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { + len = imin(crp->crp_aad_length - i, sizeof(block)); + crypto_copydata(crp, crp->crp_aad_start + i, len, block); bzero(block + len, sizeof(block) - len); axf->Update(auth_ctx, block, sizeof(block)); } @@ -1509,16 +1437,15 @@ ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp, exf->reinit(kschedule, iv); /* Do encryption with MAC */ - for (i = 0; i < crde->crd_len; i += sizeof(block)) { - len = imin(crde->crd_len - i, sizeof(block)); - crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip + - i, len, block); + for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { + len = imin(crp->crp_payload_length - i, sizeof(block)); + crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); - if (crde->crd_flags & CRD_F_ENCRYPT) { + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { exf->encrypt(kschedule, block); axf->Update(auth_ctx, block, len); - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_skip + i, len, block); + crypto_copyback(crp, crp->crp_payload_start + i, len, + block); } else { axf->Update(auth_ctx, block, len); } @@ -1526,35 +1453,37 @@ ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp, /* Length block. */ bzero(block, sizeof(block)); - ((uint32_t *)block)[1] = htobe32(crda->crd_len * 8); - ((uint32_t *)block)[3] = htobe32(crde->crd_len * 8); + ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8); + ((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8); axf->Update(auth_ctx, block, sizeof(block)); /* Finalize MAC. */ axf->Final(digest, auth_ctx); /* Inject or validate tag. */ - if (crde->crd_flags & CRD_F_ENCRYPT) { - crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject, - sizeof(digest), digest); + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), + digest); error = 0; } else { char digest2[GMAC_DIGEST_LEN]; - crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject, - sizeof(digest2), digest2); + crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), + digest2); if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { error = 0; /* Tag matches, decrypt data. */ - for (i = 0; i < crde->crd_len; i += sizeof(block)) { - len = imin(crde->crd_len - i, sizeof(block)); - crypto_copydata(crp->crp_flags, crp->crp_buf, - crde->crd_skip + i, len, block); + for (i = 0; i < crp->crp_payload_length; + i += sizeof(block)) { + len = imin(crp->crp_payload_length - i, + sizeof(block)); + crypto_copydata(crp, crp->crp_payload_start + i, + len, block); bzero(block + len, sizeof(block) - len); exf->decrypt(kschedule, block); - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_skip + i, len, block); + crypto_copyback(crp, crp->crp_payload_start + i, + len, block); } } else error = EBADMSG; @@ -1571,8 +1500,8 @@ out: } static void -generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde, - u_int hash_size_in_response, const char *iv, char *b0) +generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response, + const char *iv, char *b0) { u_int i, payload_len; @@ -1583,7 +1512,7 @@ generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde, b0[0] |= (((hash_size_in_response - 2) / 2) << 3); /* Store the payload length as a big-endian value. */ - payload_len = crde->crd_len; + payload_len = crp->crp_payload_length; for (i = 0; i < iv[0]; i++) { b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len; payload_len >>= 8; @@ -1595,15 +1524,14 @@ generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde, * start of block 1. This only assumes a 16-bit AAD length * since T6 doesn't support large AAD sizes. */ - if (crda->crd_len != 0) { + if (crp->crp_aad_length != 0) { b0[0] |= (1 << 6); - *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crda->crd_len); + *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length); } } static int -ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, - struct cryptodesc *crda, struct cryptodesc *crde) +ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct ulptx_idata *idata; @@ -1625,14 +1553,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * The crypto engine doesn't handle CCM requests with an empty * payload, so handle those in software instead. */ - if (crde->crd_len == 0) - return (EMSGSIZE); - - /* - * AAD is only permitted before the cipher/plain text, not - * after. - */ - if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip) + if (crp->crp_payload_length == 0) return (EMSGSIZE); /* @@ -1640,14 +1561,21 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * request. */ b0_len = CCM_B0_SIZE; - if (crda->crd_len != 0) + if (crp->crp_aad_length != 0) b0_len += CCM_AAD_FIELD_SIZE; - aad_len = b0_len + crda->crd_len; + aad_len = b0_len + crp->crp_aad_length; + + /* + * CCM requests should always provide an explicit IV (really + * the nonce). + */ + if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) + return (EINVAL); /* - * Always assume a 12 byte input IV for now since that is what - * OCF always generates. The full IV in the work request is - * 16 bytes. + * Always assume a 12 byte input nonce for now since that is + * what OCF always generates. The full IV in the work request + * is 16 bytes. */ iv_len = AES_BLOCK_LEN; @@ -1655,7 +1583,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, return (EMSGSIZE); hash_size_in_response = s->ccm_mac.hash_len; - if (crde->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; @@ -1670,11 +1598,12 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { - if (iv_len + aad_len + crde->crd_len + hash_size_in_response > - MAX_REQUEST_SIZE) + if (iv_len + aad_len + crp->crp_payload_length + + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { - if (iv_len + aad_len + crde->crd_len > MAX_REQUEST_SIZE) + if (iv_len + aad_len + crp->crp_payload_length > + MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(sc->sg_dsgl); @@ -1682,13 +1611,13 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, aad_len); if (error) return (error); - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip, - crde->crd_len); + error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, - crda->crd_inject, hash_size_in_response); + crp->crp_digest_start, hash_size_in_response); if (error) return (error); } @@ -1715,7 +1644,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, * inside of the AAD region, so a second copy is always * required. */ - input_len = aad_len + crde->crd_len; + input_len = aad_len + crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (input_len > MAX_REQUEST_SIZE) @@ -1729,19 +1658,19 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, imm_len = b0_len; sglist_reset(sc->sg_ulptx); - if (crda->crd_len != 0) { + if (crp->crp_aad_length != 0) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crda->crd_skip, crda->crd_len); + crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crde->crd_skip, crde->crd_len); + crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, - crda->crd_inject, hash_size_in_response); + crp->crp_digest_start, hash_size_in_response); if (error) return (error); } @@ -1774,27 +1703,12 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, memset(crwr, 0, wr_len); /* - * Read the nonce from the request or generate a random one if - * none is provided. Use the nonce to generate the full IV - * with the counter set to 0. + * Read the nonce from the request. Use the nonce to generate + * the full IV with the counter set to 0. */ memset(iv, 0, iv_len); iv[0] = (15 - AES_CCM_IV_LEN) - 1; - if (op_type == CHCR_ENCRYPT_OP) { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN); - else - arc4rand(iv + 1, AES_CCM_IV_LEN, 0); - if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_inject, AES_CCM_IV_LEN, iv + 1); - } else { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN); - else - crypto_copydata(crp->crp_flags, crp->crp_buf, - crde->crd_inject, AES_CCM_IV_LEN, iv + 1); - } + memcpy(iv + 1, crp->crp_iv, AES_CCM_IV_LEN); ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); @@ -1851,20 +1765,20 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; - generate_ccm_b0(crda, crde, hash_size_in_response, iv, dst); + generate_ccm_b0(crp, hash_size_in_response, iv, dst); if (sgl_nsegs == 0) { dst += b0_len; - if (crda->crd_len != 0) { - crypto_copydata(crp->crp_flags, crp->crp_buf, - crda->crd_skip, crda->crd_len, dst); - dst += crda->crd_len; + if (crp->crp_aad_length != 0) { + crypto_copydata(crp, crp->crp_aad_start, + crp->crp_aad_length, dst); + dst += crp->crp_aad_length; } - crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip, - crde->crd_len, dst); - dst += crde->crd_len; + crypto_copydata(crp, crp->crp_payload_start, + crp->crp_payload_length, dst); + dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) - crypto_copydata(crp->crp_flags, crp->crp_buf, - crda->crd_inject, hash_size_in_response, dst); + crypto_copydata(crp, crp->crp_digest_start, + hash_size_in_response, dst); } else { dst += CCM_B0_SIZE; if (b0_len > CCM_B0_SIZE) { @@ -1911,8 +1825,7 @@ ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s, * performing the operation in software. Derived from swcr_authenc(). */ static void -ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp, - struct cryptodesc *crda, struct cryptodesc *crde) +ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp) { struct auth_hash *axf; struct enc_xform *exf; @@ -1956,31 +1869,20 @@ ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp, if (error) goto out; - if (crde->crd_flags & CRD_F_ENCRYPT) { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN); - else - arc4rand(iv, AES_CCM_IV_LEN, 0); - if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_inject, AES_CCM_IV_LEN, iv); - } else { - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN); - else - crypto_copydata(crp->crp_flags, crp->crp_buf, - crde->crd_inject, AES_CCM_IV_LEN, iv); + if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { + error = EINVAL; + goto out; } + memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN); - auth_ctx->aes_cbc_mac_ctx.authDataLength = crda->crd_len; - auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crde->crd_len; + auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; + auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; axf->Reinit(auth_ctx, iv, sizeof(iv)); /* MAC the AAD. */ - for (i = 0; i < crda->crd_len; i += sizeof(block)) { - len = imin(crda->crd_len - i, sizeof(block)); - crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip + - i, len, block); + for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { + len = imin(crp->crp_aad_length - i, sizeof(block)); + crypto_copydata(crp, crp->crp_aad_start + i, len, block); bzero(block + len, sizeof(block) - len); axf->Update(auth_ctx, block, sizeof(block)); } @@ -1988,16 +1890,15 @@ ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp, exf->reinit(kschedule, iv); /* Do encryption/decryption with MAC */ - for (i = 0; i < crde->crd_len; i += sizeof(block)) { - len = imin(crde->crd_len - i, sizeof(block)); - crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip + - i, len, block); + for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { + len = imin(crp->crp_payload_length - i, sizeof(block)); + crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); - if (crde->crd_flags & CRD_F_ENCRYPT) { + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { axf->Update(auth_ctx, block, len); exf->encrypt(kschedule, block); - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_skip + i, len, block); + crypto_copyback(crp, crp->crp_payload_start + i, len, + block); } else { exf->decrypt(kschedule, block); axf->Update(auth_ctx, block, len); @@ -2008,28 +1909,30 @@ ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp, axf->Final(digest, auth_ctx); /* Inject or validate tag. */ - if (crde->crd_flags & CRD_F_ENCRYPT) { - crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject, - sizeof(digest), digest); + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), + digest); error = 0; } else { - char digest2[GMAC_DIGEST_LEN]; + char digest2[AES_CBC_MAC_HASH_LEN]; - crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject, - sizeof(digest2), digest2); + crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), + digest2); if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { error = 0; /* Tag matches, decrypt data. */ exf->reinit(kschedule, iv); - for (i = 0; i < crde->crd_len; i += sizeof(block)) { - len = imin(crde->crd_len - i, sizeof(block)); - crypto_copydata(crp->crp_flags, crp->crp_buf, - crde->crd_skip + i, len, block); + for (i = 0; i < crp->crp_payload_length; + i += sizeof(block)) { + len = imin(crp->crp_payload_length - i, + sizeof(block)); + crypto_copydata(crp, crp->crp_payload_start + i, + len, block); bzero(block + len, sizeof(block) - len); exf->decrypt(kschedule, block); - crypto_copyback(crp->crp_flags, crp->crp_buf, - crde->crd_skip + i, len, block); + crypto_copyback(crp, crp->crp_payload_start + i, + len, block); } } else error = EBADMSG; @@ -2096,11 +1999,11 @@ ccr_sysctls(struct ccr_softc *sc) SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD, &sc->stats_blkcipher_decrypt, 0, "Cipher decryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_encrypt", CTLFLAG_RD, - &sc->stats_authenc_encrypt, 0, + SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD, + &sc->stats_eta_encrypt, 0, "Combined AES+HMAC encryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_decrypt", CTLFLAG_RD, - &sc->stats_authenc_decrypt, 0, + SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD, + &sc->stats_eta_decrypt, 0, "Combined AES+HMAC decryption requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD, &sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted"); @@ -2161,25 +2064,6 @@ ccr_attach(device_t dev) sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); ccr_sysctls(sc); - crypto_register(cid, CRYPTO_SHA1, 0, 0); - crypto_register(cid, CRYPTO_SHA2_224, 0, 0); - crypto_register(cid, CRYPTO_SHA2_256, 0, 0); - crypto_register(cid, CRYPTO_SHA2_384, 0, 0); - crypto_register(cid, CRYPTO_SHA2_512, 0, 0); - crypto_register(cid, CRYPTO_SHA1_HMAC, 0, 0); - crypto_register(cid, CRYPTO_SHA2_224_HMAC, 0, 0); - crypto_register(cid, CRYPTO_SHA2_256_HMAC, 0, 0); - crypto_register(cid, CRYPTO_SHA2_384_HMAC, 0, 0); - crypto_register(cid, CRYPTO_SHA2_512_HMAC, 0, 0); - crypto_register(cid, CRYPTO_AES_CBC, 0, 0); - crypto_register(cid, CRYPTO_AES_ICM, 0, 0); - crypto_register(cid, CRYPTO_AES_NIST_GCM_16, 0, 0); - crypto_register(cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); - crypto_register(cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); - crypto_register(cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); - crypto_register(cid, CRYPTO_AES_XTS, 0, 0); - crypto_register(cid, CRYPTO_AES_CCM_16, 0, 0); - crypto_register(cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0); return (0); } @@ -2207,48 +2091,48 @@ ccr_detach(device_t dev) } static void -ccr_init_hash_digest(struct ccr_session *s, int cri_alg) +ccr_init_hash_digest(struct ccr_session *s) { union authctx auth_ctx; struct auth_hash *axf; axf = s->hmac.auth_hash; axf->Init(&auth_ctx); - t4_copy_partial_hash(cri_alg, &auth_ctx, s->hmac.pads); + t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads); } -static int +static bool ccr_aes_check_keylen(int alg, int klen) { - switch (klen) { + switch (klen * 8) { case 128: case 192: if (alg == CRYPTO_AES_XTS) - return (EINVAL); + return (false); break; case 256: break; case 512: if (alg != CRYPTO_AES_XTS) - return (EINVAL); + return (false); break; default: - return (EINVAL); + return (false); } - return (0); + return (true); } static void -ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen) +ccr_aes_setkey(struct ccr_session *s, const void *key, int klen) { unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size; unsigned int opad_present; - if (alg == CRYPTO_AES_XTS) - kbits = klen / 2; + if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) + kbits = (klen / 2) * 8; else - kbits = klen; + kbits = klen * 8; switch (kbits) { case 128: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; @@ -2263,18 +2147,18 @@ ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen) panic("should not get here"); } - s->blkcipher.key_len = klen / 8; + s->blkcipher.key_len = klen; memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); - switch (alg) { - case CRYPTO_AES_CBC: - case CRYPTO_AES_XTS: + switch (s->blkcipher.cipher_mode) { + case SCMD_CIPH_MODE_AES_CBC: + case SCMD_CIPH_MODE_AES_XTS: t4_aes_getdeckey(s->blkcipher.deckey, key, kbits); break; } kctx_len = roundup2(s->blkcipher.key_len, 16); switch (s->mode) { - case AUTHENC: + case ETA: mk_size = s->hmac.mk_size; opad_present = 1; iopad_size = roundup2(s->hmac.partial_digest_len, 16); @@ -2309,171 +2193,220 @@ ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen) } kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | - V_KEY_CONTEXT_DUAL_CK(alg == CRYPTO_AES_XTS) | + V_KEY_CONTEXT_DUAL_CK(s->blkcipher.cipher_mode == + SCMD_CIPH_MODE_AES_XTS) | V_KEY_CONTEXT_OPAD_PRESENT(opad_present) | V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) | V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1)); } +static bool +ccr_auth_supported(const struct crypto_session_params *csp) +{ + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_512: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + break; + default: + return (false); + } + return (true); +} + +static bool +ccr_cipher_supported(const struct crypto_session_params *csp) +{ + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + if (csp->csp_ivlen != AES_BLOCK_LEN) + return (false); + break; + case CRYPTO_AES_ICM: + if (csp->csp_ivlen != AES_BLOCK_LEN) + return (false); + break; + case CRYPTO_AES_XTS: + if (csp->csp_ivlen != AES_XTS_IV_LEN) + return (false); + break; + default: + return (false); + } + return (ccr_aes_check_keylen(csp->csp_cipher_alg, + csp->csp_cipher_klen)); +} + static int -ccr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) +ccr_cipher_mode(const struct crypto_session_params *csp) { - struct ccr_softc *sc; - struct ccr_session *s; - struct auth_hash *auth_hash; - struct cryptoini *c, *hash, *cipher; - unsigned int auth_mode, cipher_mode, iv_len, mk_size; - unsigned int partial_digest_len; - int error; - bool gcm_hash, hmac; - if (cri == NULL) - return (EINVAL); + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + return (SCMD_CIPH_MODE_AES_CBC); + case CRYPTO_AES_ICM: + return (SCMD_CIPH_MODE_AES_CTR); + case CRYPTO_AES_NIST_GCM_16: + return (SCMD_CIPH_MODE_AES_GCM); + case CRYPTO_AES_XTS: + return (SCMD_CIPH_MODE_AES_XTS); + case CRYPTO_AES_CCM_16: + return (SCMD_CIPH_MODE_AES_CCM); + default: + return (SCMD_CIPH_MODE_NOP); + } +} + +static int +ccr_probesession(device_t dev, const struct crypto_session_params *csp) +{ + unsigned int cipher_mode; - gcm_hash = false; - hmac = false; - cipher = NULL; - hash = NULL; - auth_hash = NULL; - auth_mode = SCMD_AUTH_MODE_NOP; - cipher_mode = SCMD_CIPH_MODE_NOP; - iv_len = 0; - mk_size = 0; - partial_digest_len = 0; - for (c = cri; c != NULL; c = c->cri_next) { - switch (c->cri_alg) { - case CRYPTO_SHA1: - case CRYPTO_SHA2_224: - case CRYPTO_SHA2_256: - case CRYPTO_SHA2_384: - case CRYPTO_SHA2_512: - case CRYPTO_SHA1_HMAC: - case CRYPTO_SHA2_224_HMAC: - case CRYPTO_SHA2_256_HMAC: - case CRYPTO_SHA2_384_HMAC: - case CRYPTO_SHA2_512_HMAC: - case CRYPTO_AES_128_NIST_GMAC: - case CRYPTO_AES_192_NIST_GMAC: - case CRYPTO_AES_256_NIST_GMAC: - case CRYPTO_AES_CCM_CBC_MAC: - if (hash) + if (csp->csp_flags != 0) + return (EINVAL); + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + if (!ccr_auth_supported(csp)) + return (EINVAL); + break; + case CSP_MODE_CIPHER: + if (!ccr_cipher_supported(csp)) + return (EINVAL); + break; + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (EINVAL); + if (csp->csp_auth_mlen < 0 || + csp->csp_auth_mlen > AES_GMAC_HASH_LEN) return (EINVAL); - hash = c; - switch (c->cri_alg) { - case CRYPTO_SHA1: - case CRYPTO_SHA1_HMAC: - auth_hash = &auth_hash_hmac_sha1; - auth_mode = SCMD_AUTH_MODE_SHA1; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; - partial_digest_len = SHA1_HASH_LEN; - break; - case CRYPTO_SHA2_224: - case CRYPTO_SHA2_224_HMAC: - auth_hash = &auth_hash_hmac_sha2_224; - auth_mode = SCMD_AUTH_MODE_SHA224; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; - partial_digest_len = SHA2_256_HASH_LEN; - break; - case CRYPTO_SHA2_256: - case CRYPTO_SHA2_256_HMAC: - auth_hash = &auth_hash_hmac_sha2_256; - auth_mode = SCMD_AUTH_MODE_SHA256; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; - partial_digest_len = SHA2_256_HASH_LEN; - break; - case CRYPTO_SHA2_384: - case CRYPTO_SHA2_384_HMAC: - auth_hash = &auth_hash_hmac_sha2_384; - auth_mode = SCMD_AUTH_MODE_SHA512_384; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; - partial_digest_len = SHA2_512_HASH_LEN; - break; - case CRYPTO_SHA2_512: - case CRYPTO_SHA2_512_HMAC: - auth_hash = &auth_hash_hmac_sha2_512; - auth_mode = SCMD_AUTH_MODE_SHA512_512; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; - partial_digest_len = SHA2_512_HASH_LEN; - break; - case CRYPTO_AES_128_NIST_GMAC: - case CRYPTO_AES_192_NIST_GMAC: - case CRYPTO_AES_256_NIST_GMAC: - gcm_hash = true; - auth_mode = SCMD_AUTH_MODE_GHASH; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; - break; - case CRYPTO_AES_CCM_CBC_MAC: - auth_mode = SCMD_AUTH_MODE_CBCMAC; - break; - } - switch (c->cri_alg) { - case CRYPTO_SHA1_HMAC: - case CRYPTO_SHA2_224_HMAC: - case CRYPTO_SHA2_256_HMAC: - case CRYPTO_SHA2_384_HMAC: - case CRYPTO_SHA2_512_HMAC: - hmac = true; - break; - } break; - case CRYPTO_AES_CBC: - case CRYPTO_AES_ICM: - case CRYPTO_AES_NIST_GCM_16: - case CRYPTO_AES_XTS: case CRYPTO_AES_CCM_16: - if (cipher) + if (csp->csp_ivlen != AES_CCM_IV_LEN) + return (EINVAL); + if (csp->csp_auth_mlen < 0 || + csp->csp_auth_mlen > AES_CBC_MAC_HASH_LEN) return (EINVAL); - cipher = c; - switch (c->cri_alg) { - case CRYPTO_AES_CBC: - cipher_mode = SCMD_CIPH_MODE_AES_CBC; - iv_len = AES_BLOCK_LEN; - break; - case CRYPTO_AES_ICM: - cipher_mode = SCMD_CIPH_MODE_AES_CTR; - iv_len = AES_BLOCK_LEN; - break; - case CRYPTO_AES_NIST_GCM_16: - cipher_mode = SCMD_CIPH_MODE_AES_GCM; - iv_len = AES_GCM_IV_LEN; - break; - case CRYPTO_AES_XTS: - cipher_mode = SCMD_CIPH_MODE_AES_XTS; - iv_len = AES_BLOCK_LEN; - break; - case CRYPTO_AES_CCM_16: - cipher_mode = SCMD_CIPH_MODE_AES_CCM; - iv_len = AES_CCM_IV_LEN; - break; - } - if (c->cri_key != NULL) { - error = ccr_aes_check_keylen(c->cri_alg, - c->cri_klen); - if (error) - return (error); - } break; default: return (EINVAL); } - } - if (gcm_hash != (cipher_mode == SCMD_CIPH_MODE_AES_GCM)) - return (EINVAL); - if ((auth_mode == SCMD_AUTH_MODE_CBCMAC) != - (cipher_mode == SCMD_CIPH_MODE_AES_CCM)) - return (EINVAL); - if (hash == NULL && cipher == NULL) + break; + case CSP_MODE_ETA: + if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp)) + return (EINVAL); + break; + default: return (EINVAL); - if (hash != NULL) { - if (hmac || gcm_hash || auth_mode == SCMD_AUTH_MODE_CBCMAC) { - if (hash->cri_key == NULL) - return (EINVAL); - } else { - if (hash->cri_key != NULL) - return (EINVAL); - } } + if (csp->csp_cipher_klen != 0) { + cipher_mode = ccr_cipher_mode(csp); + if (cipher_mode == SCMD_CIPH_MODE_NOP) + return (EINVAL); + } + + return (CRYPTODEV_PROBE_HARDWARE); +} + +static int +ccr_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct ccr_softc *sc; + struct ccr_session *s; + struct auth_hash *auth_hash; + unsigned int auth_mode, cipher_mode, mk_size; + unsigned int partial_digest_len; + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + auth_hash = &auth_hash_hmac_sha1; + auth_mode = SCMD_AUTH_MODE_SHA1; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; + partial_digest_len = SHA1_HASH_LEN; + break; + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + auth_hash = &auth_hash_hmac_sha2_224; + auth_mode = SCMD_AUTH_MODE_SHA224; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; + partial_digest_len = SHA2_256_HASH_LEN; + break; + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + auth_hash = &auth_hash_hmac_sha2_256; + auth_mode = SCMD_AUTH_MODE_SHA256; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; + partial_digest_len = SHA2_256_HASH_LEN; + break; + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + auth_hash = &auth_hash_hmac_sha2_384; + auth_mode = SCMD_AUTH_MODE_SHA512_384; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; + partial_digest_len = SHA2_512_HASH_LEN; + break; + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + auth_hash = &auth_hash_hmac_sha2_512; + auth_mode = SCMD_AUTH_MODE_SHA512_512; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; + partial_digest_len = SHA2_512_HASH_LEN; + break; + default: + auth_hash = NULL; + auth_mode = SCMD_AUTH_MODE_NOP; + mk_size = 0; + partial_digest_len = 0; + break; + } + + cipher_mode = ccr_cipher_mode(csp); + +#ifdef INVARIANTS + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + if (cipher_mode == SCMD_CIPH_MODE_NOP || + cipher_mode == SCMD_CIPH_MODE_AES_GCM || + cipher_mode == SCMD_CIPH_MODE_AES_CCM) + panic("invalid cipher algo"); + break; + case CSP_MODE_DIGEST: + if (auth_mode == SCMD_AUTH_MODE_NOP) + panic("invalid auth algo"); + break; + case CSP_MODE_AEAD: + if (cipher_mode != SCMD_CIPH_MODE_AES_GCM && + cipher_mode != SCMD_CIPH_MODE_AES_CCM) + panic("invalid aead cipher algo"); + if (auth_mode != SCMD_AUTH_MODE_NOP) + panic("invalid aead auth aglo"); + break; + case CSP_MODE_ETA: + if (cipher_mode == SCMD_CIPH_MODE_NOP || + cipher_mode == SCMD_CIPH_MODE_AES_GCM || + cipher_mode == SCMD_CIPH_MODE_AES_CCM) + panic("invalid cipher algo"); + if (auth_mode == SCMD_AUTH_MODE_NOP) + panic("invalid auth algo"); + break; + default: + panic("invalid csp mode"); + } +#endif + sc = device_get_softc(dev); /* @@ -2493,54 +2426,61 @@ ccr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) s = crypto_get_driver_session(cses); - if (gcm_hash) - s->mode = GCM; - else if (cipher_mode == SCMD_CIPH_MODE_AES_CCM) - s->mode = CCM; - else if (hash != NULL && cipher != NULL) - s->mode = AUTHENC; - else if (hash != NULL) { - if (hmac) + switch (csp->csp_mode) { + case CSP_MODE_AEAD: + if (cipher_mode == SCMD_CIPH_MODE_AES_CCM) + s->mode = CCM; + else + s->mode = GCM; + break; + case CSP_MODE_ETA: + s->mode = ETA; + break; + case CSP_MODE_DIGEST: + if (csp->csp_auth_klen != 0) s->mode = HMAC; else s->mode = HASH; - } else { - MPASS(cipher != NULL); + break; + case CSP_MODE_CIPHER: s->mode = BLKCIPHER; + break; } - if (gcm_hash) { - if (hash->cri_mlen == 0) + + if (s->mode == GCM) { + if (csp->csp_auth_mlen == 0) s->gmac.hash_len = AES_GMAC_HASH_LEN; else - s->gmac.hash_len = hash->cri_mlen; - t4_init_gmac_hash(hash->cri_key, hash->cri_klen, + s->gmac.hash_len = csp->csp_auth_mlen; + t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen, s->gmac.ghash_h); - } else if (auth_mode == SCMD_AUTH_MODE_CBCMAC) { - if (hash->cri_mlen == 0) + } else if (s->mode == CCM) { + if (csp->csp_auth_mlen == 0) s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN; else - s->ccm_mac.hash_len = hash->cri_mlen; - } else if (hash != NULL) { + s->ccm_mac.hash_len = csp->csp_auth_mlen; + } else if (auth_mode != SCMD_AUTH_MODE_NOP) { s->hmac.auth_hash = auth_hash; s->hmac.auth_mode = auth_mode; s->hmac.mk_size = mk_size; s->hmac.partial_digest_len = partial_digest_len; - if (hash->cri_mlen == 0) + if (csp->csp_auth_mlen == 0) s->hmac.hash_len = auth_hash->hashsize; else - s->hmac.hash_len = hash->cri_mlen; - if (hmac) + s->hmac.hash_len = csp->csp_auth_mlen; + if (csp->csp_auth_key != NULL) t4_init_hmac_digest(auth_hash, partial_digest_len, - hash->cri_key, hash->cri_klen, s->hmac.pads); + csp->csp_auth_key, csp->csp_auth_klen, + s->hmac.pads); else - ccr_init_hash_digest(s, hash->cri_alg); + ccr_init_hash_digest(s); } - if (cipher != NULL) { + if (cipher_mode != SCMD_CIPH_MODE_NOP) { s->blkcipher.cipher_mode = cipher_mode; - s->blkcipher.iv_len = iv_len; - if (cipher->cri_key != NULL) - ccr_aes_setkey(s, cipher->cri_alg, cipher->cri_key, - cipher->cri_klen); + s->blkcipher.iv_len = csp->csp_ivlen; + if (csp->csp_cipher_key != NULL) + ccr_aes_setkey(s, csp->csp_cipher_key, + csp->csp_cipher_klen); } s->active = true; @@ -2568,15 +2508,12 @@ ccr_freesession(device_t dev, crypto_session_t cses) static int ccr_process(device_t dev, struct cryptop *crp, int hint) { + const struct crypto_session_params *csp; struct ccr_softc *sc; struct ccr_session *s; - struct cryptodesc *crd, *crda, *crde; int error; - if (crp == NULL) - return (EINVAL); - - crd = crp->crp_desc; + csp = crypto_get_params(crp->crp_session); s = crypto_get_driver_session(crp->crp_session); sc = device_get_softc(dev); @@ -2594,141 +2531,82 @@ ccr_process(device_t dev, struct cryptop *crp, int hint) sc->stats_hash++; break; case HMAC: - if (crd->crd_flags & CRD_F_KEY_EXPLICIT) + if (crp->crp_auth_key != NULL) t4_init_hmac_digest(s->hmac.auth_hash, - s->hmac.partial_digest_len, crd->crd_key, - crd->crd_klen, s->hmac.pads); + s->hmac.partial_digest_len, crp->crp_auth_key, + csp->csp_auth_klen, s->hmac.pads); error = ccr_hash(sc, s, crp); if (error == 0) sc->stats_hmac++; break; case BLKCIPHER: - if (crd->crd_flags & CRD_F_KEY_EXPLICIT) { - error = ccr_aes_check_keylen(crd->crd_alg, - crd->crd_klen); - if (error) - break; - ccr_aes_setkey(s, crd->crd_alg, crd->crd_key, - crd->crd_klen); - } + if (crp->crp_cipher_key != NULL) + ccr_aes_setkey(s, crp->crp_cipher_key, + csp->csp_cipher_klen); error = ccr_blkcipher(sc, s, crp); if (error == 0) { - if (crd->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) sc->stats_blkcipher_encrypt++; else sc->stats_blkcipher_decrypt++; } break; - case AUTHENC: - error = 0; - switch (crd->crd_alg) { - case CRYPTO_AES_CBC: - case CRYPTO_AES_ICM: - case CRYPTO_AES_XTS: - /* Only encrypt-then-authenticate supported. */ - crde = crd; - crda = crd->crd_next; - if (!(crde->crd_flags & CRD_F_ENCRYPT)) { - error = EINVAL; - break; - } - break; - default: - crda = crd; - crde = crd->crd_next; - if (crde->crd_flags & CRD_F_ENCRYPT) { - error = EINVAL; - break; - } - break; - } - if (error) - break; - if (crda->crd_flags & CRD_F_KEY_EXPLICIT) + case ETA: + if (crp->crp_auth_key != NULL) t4_init_hmac_digest(s->hmac.auth_hash, - s->hmac.partial_digest_len, crda->crd_key, - crda->crd_klen, s->hmac.pads); - if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { - error = ccr_aes_check_keylen(crde->crd_alg, - crde->crd_klen); - if (error) - break; - ccr_aes_setkey(s, crde->crd_alg, crde->crd_key, - crde->crd_klen); - } - error = ccr_authenc(sc, s, crp, crda, crde); + s->hmac.partial_digest_len, crp->crp_auth_key, + csp->csp_auth_klen, s->hmac.pads); + if (crp->crp_cipher_key != NULL) + ccr_aes_setkey(s, crp->crp_cipher_key, + csp->csp_cipher_klen); + error = ccr_eta(sc, s, crp); if (error == 0) { - if (crde->crd_flags & CRD_F_ENCRYPT) - sc->stats_authenc_encrypt++; + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + sc->stats_eta_encrypt++; else - sc->stats_authenc_decrypt++; + sc->stats_eta_decrypt++; } break; case GCM: - error = 0; - if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) { - crde = crd; - crda = crd->crd_next; - } else { - crda = crd; - crde = crd->crd_next; - } - if (crda->crd_flags & CRD_F_KEY_EXPLICIT) - t4_init_gmac_hash(crda->crd_key, crda->crd_klen, - s->gmac.ghash_h); - if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { - error = ccr_aes_check_keylen(crde->crd_alg, - crde->crd_klen); - if (error) - break; - ccr_aes_setkey(s, crde->crd_alg, crde->crd_key, - crde->crd_klen); + if (crp->crp_cipher_key != NULL) { + t4_init_gmac_hash(crp->crp_cipher_key, + csp->csp_cipher_klen, s->gmac.ghash_h); + ccr_aes_setkey(s, crp->crp_cipher_key, + csp->csp_cipher_klen); } - if (crde->crd_len == 0) { + if (crp->crp_payload_length == 0) { mtx_unlock(&sc->lock); - ccr_gcm_soft(s, crp, crda, crde); + ccr_gcm_soft(s, crp); return (0); } - error = ccr_gcm(sc, s, crp, crda, crde); + error = ccr_gcm(sc, s, crp); if (error == EMSGSIZE) { sc->stats_sw_fallback++; mtx_unlock(&sc->lock); - ccr_gcm_soft(s, crp, crda, crde); + ccr_gcm_soft(s, crp); return (0); } if (error == 0) { - if (crde->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) sc->stats_gcm_encrypt++; else sc->stats_gcm_decrypt++; } break; case CCM: - error = 0; - if (crd->crd_alg == CRYPTO_AES_CCM_16) { - crde = crd; - crda = crd->crd_next; - } else { - crda = crd; - crde = crd->crd_next; - } - if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { - error = ccr_aes_check_keylen(crde->crd_alg, - crde->crd_klen); - if (error) - break; - ccr_aes_setkey(s, crde->crd_alg, crde->crd_key, - crde->crd_klen); + if (crp->crp_cipher_key != NULL) { + ccr_aes_setkey(s, crp->crp_cipher_key, + csp->csp_cipher_klen); } - error = ccr_ccm(sc, s, crp, crda, crde); + error = ccr_ccm(sc, s, crp); if (error == EMSGSIZE) { sc->stats_sw_fallback++; mtx_unlock(&sc->lock); - ccr_ccm_soft(s, crp, crda, crde); + ccr_ccm_soft(s, crp); return (0); } if (error == 0) { - if (crde->crd_flags & CRD_F_ENCRYPT) + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) sc->stats_ccm_encrypt++; else sc->stats_ccm_decrypt++; @@ -2789,8 +2667,8 @@ do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss, case BLKCIPHER: error = ccr_blkcipher_done(sc, s, crp, cpl, error); break; - case AUTHENC: - error = ccr_authenc_done(sc, s, crp, cpl, error); + case ETA: + error = ccr_eta_done(sc, s, crp, cpl, error); break; case GCM: error = ccr_gcm_done(sc, s, crp, cpl, error); @@ -2835,6 +2713,7 @@ static device_method_t ccr_methods[] = { DEVMETHOD(device_attach, ccr_attach), DEVMETHOD(device_detach, ccr_detach), + DEVMETHOD(cryptodev_probesession, ccr_probesession), DEVMETHOD(cryptodev_newsession, ccr_newsession), DEVMETHOD(cryptodev_freesession, ccr_freesession), DEVMETHOD(cryptodev_process, ccr_process), diff --git a/sys/dev/cxgbe/crypto/t4_keyctx.c b/sys/dev/cxgbe/crypto/t4_keyctx.c index bceade0ec810..0f034f1be334 100644 --- a/sys/dev/cxgbe/crypto/t4_keyctx.c +++ b/sys/dev/cxgbe/crypto/t4_keyctx.c @@ -73,7 +73,7 @@ t4_init_gmac_hash(const char *key, int klen, char *ghash) uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)]; int rounds; - rounds = rijndaelKeySetupEnc(keysched, key, klen); + rounds = rijndaelKeySetupEnc(keysched, key, klen * 8); rijndaelEncrypt(keysched, rounds, zeroes, ghash); } @@ -118,45 +118,19 @@ t4_copy_partial_hash(int alg, union authctx *auth_ctx, void *dst) void t4_init_hmac_digest(struct auth_hash *axf, u_int partial_digest_len, - char *key, int klen, char *dst) + const char *key, int klen, char *dst) { union authctx auth_ctx; - char ipad[SHA2_512_BLOCK_LEN], opad[SHA2_512_BLOCK_LEN]; - u_int i; - - /* - * If the key is larger than the block size, use the digest of - * the key as the key instead. - */ - klen /= 8; - if (klen > axf->blocksize) { - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, key, klen); - axf->Final(ipad, &auth_ctx); - klen = axf->hashsize; - } else - memcpy(ipad, key, klen); - memset(ipad + klen, 0, axf->blocksize - klen); - memcpy(opad, ipad, axf->blocksize); - - for (i = 0; i < axf->blocksize; i++) { - ipad[i] ^= HMAC_IPAD_VAL; - opad[i] ^= HMAC_OPAD_VAL; - } - - /* - * Hash the raw ipad and opad and store the partial results in - * the key context. - */ - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, ipad, axf->blocksize); + hmac_init_ipad(axf, key, klen, &auth_ctx); t4_copy_partial_hash(axf->type, &auth_ctx, dst); dst += roundup2(partial_digest_len, 16); - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, opad, axf->blocksize); + + hmac_init_opad(axf, key, klen, &auth_ctx); t4_copy_partial_hash(axf->type, &auth_ctx, dst); + + explicit_bzero(&auth_ctx, sizeof(auth_ctx)); } /* diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c index 57bae811f0f6..a82edd29bc3f 100644 --- a/sys/dev/cxgbe/tom/t4_tls.c +++ b/sys/dev/cxgbe/tom/t4_tls.c @@ -892,7 +892,7 @@ init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx) k_ctx->tx_key_info_size += GMAC_BLOCK_LEN; memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE); t4_init_gmac_hash(tls->params.cipher_key, - tls->params.cipher_key_len * 8, hash); + tls->params.cipher_key_len, hash); } else { switch (tls->params.auth_algorithm) { case CRYPTO_SHA1_HMAC: @@ -920,7 +920,7 @@ init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx) k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2; k_ctx->mac_secret_size = mac_key_size; t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, - tls->params.auth_key_len * 8, hash); + tls->params.auth_key_len, hash); } k_ctx->frag_size = tls->params.max_frame_len; diff --git a/sys/dev/glxsb/glxsb.c b/sys/dev/glxsb/glxsb.c index 4d89e7d6756a..0e80b1dba2aa 100644 --- a/sys/dev/glxsb/glxsb.c +++ b/sys/dev/glxsb/glxsb.c @@ -51,7 +51,6 @@ __FBSDID("$FreeBSD$"); #include <dev/pci/pcireg.h> #include <opencrypto/cryptodev.h> -#include <opencrypto/cryptosoft.h> #include <opencrypto/xform.h> #include "cryptodev_if.h" @@ -172,8 +171,6 @@ struct glxsb_dma_map { struct glxsb_taskop { struct glxsb_session *to_ses; /* crypto session */ struct cryptop *to_crp; /* cryptop to perfom */ - struct cryptodesc *to_enccrd; /* enccrd to perform */ - struct cryptodesc *to_maccrd; /* maccrd to perform */ }; struct glxsb_softc { @@ -204,13 +201,16 @@ static void glxsb_dma_free(struct glxsb_softc *, struct glxsb_dma_map *); static void glxsb_rnd(void *); static int glxsb_crypto_setup(struct glxsb_softc *); -static int glxsb_crypto_newsession(device_t, crypto_session_t, struct cryptoini *); +static int glxsb_crypto_probesession(device_t, + const struct crypto_session_params *); +static int glxsb_crypto_newsession(device_t, crypto_session_t, + const struct crypto_session_params *); static void glxsb_crypto_freesession(device_t, crypto_session_t); static int glxsb_aes(struct glxsb_softc *, uint32_t, uint32_t, - uint32_t, void *, int, void *); + uint32_t, const void *, int, const void *); -static int glxsb_crypto_encdec(struct cryptop *, struct cryptodesc *, - struct glxsb_session *, struct glxsb_softc *); +static int glxsb_crypto_encdec(struct cryptop *, struct glxsb_session *, + struct glxsb_softc *); static void glxsb_crypto_task(void *, int); static int glxsb_crypto_process(device_t, struct cryptop *, int); @@ -222,6 +222,7 @@ static device_method_t glxsb_methods[] = { DEVMETHOD(device_detach, glxsb_detach), /* crypto device methods */ + DEVMETHOD(cryptodev_probesession, glxsb_crypto_probesession), DEVMETHOD(cryptodev_newsession, glxsb_crypto_newsession), DEVMETHOD(cryptodev_freesession, glxsb_crypto_freesession), DEVMETHOD(cryptodev_process, glxsb_crypto_process), @@ -477,47 +478,24 @@ glxsb_crypto_setup(struct glxsb_softc *sc) mtx_init(&sc->sc_task_mtx, "glxsb_crypto_mtx", NULL, MTX_DEF); - if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0) - goto crypto_fail; - if (crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0) != 0) - goto crypto_fail; - if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0) - goto crypto_fail; - if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0) - goto crypto_fail; - if (crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0) != 0) - goto crypto_fail; - if (crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0) != 0) - goto crypto_fail; - if (crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0) != 0) - goto crypto_fail; - if (crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0) != 0) - goto crypto_fail; - return (0); - -crypto_fail: - device_printf(sc->sc_dev, "cannot register crypto\n"); - crypto_unregister_all(sc->sc_cid); - mtx_destroy(&sc->sc_task_mtx); - return (ENOMEM); } static int -glxsb_crypto_newsession(device_t dev, crypto_session_t cses, - struct cryptoini *cri) +glxsb_crypto_probesession(device_t dev, const struct crypto_session_params *csp) { - struct glxsb_softc *sc = device_get_softc(dev); - struct glxsb_session *ses; - struct cryptoini *encini, *macini; - int error; - if (sc == NULL || cri == NULL) + if (csp->csp_flags != 0) return (EINVAL); - encini = macini = NULL; - for (; cri != NULL; cri = cri->cri_next) { - switch(cri->cri_alg) { + /* + * We only support HMAC algorithms to be able to work with + * ipsec(4), so if we are asked only for authentication without + * encryption, don't pretend we can accelerate it. + */ + switch (csp->csp_mode) { + case CSP_MODE_ETA: + switch (csp->csp_auth_alg) { case CRYPTO_NULL_HMAC: case CRYPTO_MD5_HMAC: case CRYPTO_SHA1_HMAC: @@ -525,43 +503,42 @@ glxsb_crypto_newsession(device_t dev, crypto_session_t cses, case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: - if (macini != NULL) - return (EINVAL); - macini = cri; break; + default: + return (EINVAL); + } + /* FALLTHROUGH */ + case CSP_MODE_CIPHER: + switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: - if (encini != NULL) + if (csp->csp_cipher_klen * 8 != 128) return (EINVAL); - encini = cri; break; default: return (EINVAL); } + default: + return (EINVAL); } + return (CRYPTODEV_PROBE_HARDWARE); +} - /* - * We only support HMAC algorithms to be able to work with - * ipsec(4), so if we are asked only for authentication without - * encryption, don't pretend we can accellerate it. - */ - if (encini == NULL) - return (EINVAL); +static int +glxsb_crypto_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct glxsb_softc *sc = device_get_softc(dev); + struct glxsb_session *ses; + int error; ses = crypto_get_driver_session(cses); - if (encini->cri_alg == CRYPTO_AES_CBC) { - if (encini->cri_klen != 128) { - glxsb_crypto_freesession(sc->sc_dev, cses); - return (EINVAL); - } - arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0); - ses->ses_klen = encini->cri_klen; - /* Copy the key (Geode LX wants the primary key only) */ - bcopy(encini->cri_key, ses->ses_key, sizeof(ses->ses_key)); - } + /* Copy the key (Geode LX wants the primary key only) */ + if (csp->csp_cipher_key != NULL) + bcopy(csp->csp_cipher_key, ses->ses_key, sizeof(ses->ses_key)); - if (macini != NULL) { - error = glxsb_hash_setup(ses, macini); + if (csp->csp_auth_alg != 0) { + error = glxsb_hash_setup(ses, csp); if (error != 0) { glxsb_crypto_freesession(sc->sc_dev, cses); return (error); @@ -574,19 +551,15 @@ glxsb_crypto_newsession(device_t dev, crypto_session_t cses, static void glxsb_crypto_freesession(device_t dev, crypto_session_t cses) { - struct glxsb_softc *sc = device_get_softc(dev); struct glxsb_session *ses; - if (sc == NULL) - return; - ses = crypto_get_driver_session(cses); glxsb_hash_free(ses); } static int glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc, - uint32_t pdst, void *key, int len, void *iv) + uint32_t pdst, const void *key, int len, const void *iv) { uint32_t status; int i; @@ -652,23 +625,24 @@ glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc, } static int -glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd, - struct glxsb_session *ses, struct glxsb_softc *sc) +glxsb_crypto_encdec(struct cryptop *crp, struct glxsb_session *ses, + struct glxsb_softc *sc) { char *op_src, *op_dst; + const void *key; uint32_t op_psrc, op_pdst; - uint8_t op_iv[SB_AES_BLOCK_SIZE], *piv; + uint8_t op_iv[SB_AES_BLOCK_SIZE]; int error; int len, tlen, xlen; int offset; uint32_t control; - if (crd == NULL || (crd->crd_len % SB_AES_BLOCK_SIZE) != 0) + if ((crp->crp_payload_length % SB_AES_BLOCK_SIZE) != 0) return (EINVAL); /* How much of our buffer will we need to use? */ - xlen = crd->crd_len > GLXSB_MAX_AES_LEN ? - GLXSB_MAX_AES_LEN : crd->crd_len; + xlen = crp->crp_payload_length > GLXSB_MAX_AES_LEN ? + GLXSB_MAX_AES_LEN : crp->crp_payload_length; /* * XXX Check if we can have input == output on Geode LX. @@ -680,73 +654,57 @@ glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd, op_psrc = sc->sc_dma.dma_paddr; op_pdst = sc->sc_dma.dma_paddr + xlen; - if (crd->crd_flags & CRD_F_ENCRYPT) { + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) control = SB_CTL_ENC; - if (crd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crd->crd_iv, op_iv, sizeof(op_iv)); - else - bcopy(ses->ses_iv, op_iv, sizeof(op_iv)); - - if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { - crypto_copyback(crp->crp_flags, crp->crp_buf, - crd->crd_inject, sizeof(op_iv), op_iv); - } - } else { + else control = SB_CTL_DEC; - if (crd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crd->crd_iv, op_iv, sizeof(op_iv)); - else { - crypto_copydata(crp->crp_flags, crp->crp_buf, - crd->crd_inject, sizeof(op_iv), op_iv); - } - } + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(op_iv, sizeof(op_iv), 0); + crypto_copyback(crp, crp->crp_iv_start, sizeof(op_iv), op_iv); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(op_iv, crp->crp_iv, sizeof(op_iv)); + else + crypto_copydata(crp, crp->crp_iv_start, sizeof(op_iv), op_iv); + offset = 0; - tlen = crd->crd_len; - piv = op_iv; + tlen = crp->crp_payload_length; + + if (crp->crp_cipher_key != NULL) + key = crp->crp_cipher_key; + else + key = ses->ses_key; /* Process the data in GLXSB_MAX_AES_LEN chunks */ while (tlen > 0) { len = (tlen > GLXSB_MAX_AES_LEN) ? GLXSB_MAX_AES_LEN : tlen; - crypto_copydata(crp->crp_flags, crp->crp_buf, - crd->crd_skip + offset, len, op_src); + crypto_copydata(crp, crp->crp_payload_start + offset, len, + op_src); glxsb_dma_pre_op(sc, &sc->sc_dma); - error = glxsb_aes(sc, control, op_psrc, op_pdst, ses->ses_key, - len, op_iv); + error = glxsb_aes(sc, control, op_psrc, op_pdst, key, len, + op_iv); glxsb_dma_post_op(sc, &sc->sc_dma); if (error != 0) return (error); - crypto_copyback(crp->crp_flags, crp->crp_buf, - crd->crd_skip + offset, len, op_dst); + crypto_copyback(crp, crp->crp_payload_start + offset, len, + op_dst); offset += len; tlen -= len; - if (tlen <= 0) { /* Ideally, just == 0 */ - /* Finished - put the IV in session IV */ - piv = ses->ses_iv; - } - /* - * Copy out last block for use as next iteration/session IV. - * - * piv is set to op_iv[] before the loop starts, but is - * set to ses->ses_iv if we're going to exit the loop this - * time. + * Copy out last block for use as next iteration IV. */ - if (crd->crd_flags & CRD_F_ENCRYPT) - bcopy(op_dst + len - sizeof(op_iv), piv, sizeof(op_iv)); - else { - /* Decryption, only need this if another iteration */ - if (tlen > 0) { - bcopy(op_src + len - sizeof(op_iv), piv, - sizeof(op_iv)); - } - } + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + bcopy(op_dst + len - sizeof(op_iv), op_iv, + sizeof(op_iv)); + else + bcopy(op_src + len - sizeof(op_iv), op_iv, + sizeof(op_iv)); } /* while */ /* All AES processing has now been done. */ @@ -759,30 +717,31 @@ static void glxsb_crypto_task(void *arg, int pending) { struct glxsb_softc *sc = arg; + const struct crypto_session_params *csp; struct glxsb_session *ses; struct cryptop *crp; - struct cryptodesc *enccrd, *maccrd; int error; - maccrd = sc->sc_to.to_maccrd; - enccrd = sc->sc_to.to_enccrd; crp = sc->sc_to.to_crp; ses = sc->sc_to.to_ses; + csp = crypto_get_params(crp->crp_session); /* Perform data authentication if requested before encryption */ - if (maccrd != NULL && maccrd->crd_next == enccrd) { - error = glxsb_hash_process(ses, maccrd, crp); + if (csp->csp_mode == CSP_MODE_ETA && + !CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + error = glxsb_hash_process(ses, csp, crp); if (error != 0) goto out; } - error = glxsb_crypto_encdec(crp, enccrd, ses, sc); + error = glxsb_crypto_encdec(crp, ses, sc); if (error != 0) goto out; /* Perform data authentication if requested after encryption */ - if (maccrd != NULL && enccrd->crd_next == maccrd) { - error = glxsb_hash_process(ses, maccrd, crp); + if (csp->csp_mode == CSP_MODE_ETA && + CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + error = glxsb_hash_process(ses, csp, crp); if (error != 0) goto out; } @@ -801,52 +760,6 @@ glxsb_crypto_process(device_t dev, struct cryptop *crp, int hint) { struct glxsb_softc *sc = device_get_softc(dev); struct glxsb_session *ses; - struct cryptodesc *crd, *enccrd, *maccrd; - int error = 0; - - enccrd = maccrd = NULL; - - /* Sanity check. */ - if (crp == NULL) - return (EINVAL); - - if (crp->crp_callback == NULL || crp->crp_desc == NULL) { - error = EINVAL; - goto fail; - } - - for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { - switch (crd->crd_alg) { - case CRYPTO_NULL_HMAC: - case CRYPTO_MD5_HMAC: - case CRYPTO_SHA1_HMAC: - case CRYPTO_RIPEMD160_HMAC: - case CRYPTO_SHA2_256_HMAC: - case CRYPTO_SHA2_384_HMAC: - case CRYPTO_SHA2_512_HMAC: - if (maccrd != NULL) { - error = EINVAL; - goto fail; - } - maccrd = crd; - break; - case CRYPTO_AES_CBC: - if (enccrd != NULL) { - error = EINVAL; - goto fail; - } - enccrd = crd; - break; - default: - error = EINVAL; - goto fail; - } - } - - if (enccrd == NULL || enccrd->crd_len % AES_BLOCK_LEN != 0) { - error = EINVAL; - goto fail; - } ses = crypto_get_driver_session(crp->crp_session); @@ -857,17 +770,10 @@ glxsb_crypto_process(device_t dev, struct cryptop *crp, int hint) } sc->sc_task_count++; - sc->sc_to.to_maccrd = maccrd; - sc->sc_to.to_enccrd = enccrd; sc->sc_to.to_crp = crp; sc->sc_to.to_ses = ses; mtx_unlock(&sc->sc_task_mtx); taskqueue_enqueue(sc->sc_tq, &sc->sc_cryptotask); return(0); - -fail: - crp->crp_etype = error; - crypto_done(crp); - return (error); } diff --git a/sys/dev/glxsb/glxsb.h b/sys/dev/glxsb/glxsb.h index fe5128a744c6..27e5bb44709c 100644 --- a/sys/dev/glxsb/glxsb.h +++ b/sys/dev/glxsb/glxsb.h @@ -37,8 +37,6 @@ struct glxsb_session { uint32_t ses_key[4]; /* key */ - uint8_t ses_iv[SB_AES_BLOCK_SIZE]; /* initialization vector */ - int ses_klen; /* key len */ struct auth_hash *ses_axf; uint8_t *ses_ictx; uint8_t *ses_octx; @@ -46,10 +44,10 @@ struct glxsb_session { }; int glxsb_hash_setup(struct glxsb_session *ses, - struct cryptoini *macini); + const struct crypto_session_params *csp); int glxsb_hash_process(struct glxsb_session *ses, - struct cryptodesc *maccrd, struct cryptop *crp); + const struct crypto_session_params *csp, struct cryptop *crp); void glxsb_hash_free(struct glxsb_session *ses); diff --git a/sys/dev/glxsb/glxsb_hash.c b/sys/dev/glxsb/glxsb_hash.c index c5c2028103f2..73d9896ffc01 100644 --- a/sys/dev/glxsb/glxsb_hash.c +++ b/sys/dev/glxsb/glxsb_hash.c @@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/malloc.h> -#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */ #include <opencrypto/xform.h> #include "glxsb.h" @@ -51,92 +50,66 @@ __FBSDID("$FreeBSD$"); MALLOC_DECLARE(M_GLXSB); static void -glxsb_hash_key_setup(struct glxsb_session *ses, caddr_t key, int klen) +glxsb_hash_key_setup(struct glxsb_session *ses, const char *key, int klen) { struct auth_hash *axf; - int i; - klen /= 8; axf = ses->ses_axf; - - for (i = 0; i < klen; i++) - key[i] ^= HMAC_IPAD_VAL; - - axf->Init(ses->ses_ictx); - axf->Update(ses->ses_ictx, key, klen); - axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen); - - for (i = 0; i < klen; i++) - key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); - - axf->Init(ses->ses_octx); - axf->Update(ses->ses_octx, key, klen); - axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen); - - for (i = 0; i < klen; i++) - key[i] ^= HMAC_OPAD_VAL; + hmac_init_ipad(axf, key, klen, ses->ses_ictx); + hmac_init_opad(axf, key, klen, ses->ses_octx); } /* * Compute keyed-hash authenticator. */ static int -glxsb_authcompute(struct glxsb_session *ses, struct cryptodesc *crd, - caddr_t buf, int flags) +glxsb_authcompute(struct glxsb_session *ses, struct cryptop *crp) { - u_char hash[HASH_MAX_LEN]; + u_char hash[HASH_MAX_LEN], hash2[HASH_MAX_LEN]; struct auth_hash *axf; union authctx ctx; int error; axf = ses->ses_axf; bcopy(ses->ses_ictx, &ctx, axf->ctxsize); - error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len, + error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx); if (error != 0) return (error); + error = crypto_apply(crp, crp->crp_payload_start, + crp->crp_payload_length, + (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx); + if (error != 0) + return (error); + axf->Final(hash, &ctx); bcopy(ses->ses_octx, &ctx, axf->ctxsize); axf->Update(&ctx, hash, axf->hashsize); axf->Final(hash, &ctx); - /* Inject the authentication data */ - crypto_copyback(flags, buf, crd->crd_inject, - ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash); + /* Verify or inject the authentication data */ + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen, + hash2); + if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0) + return (EBADMSG); + } else + crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen, + hash); return (0); } int -glxsb_hash_setup(struct glxsb_session *ses, struct cryptoini *macini) +glxsb_hash_setup(struct glxsb_session *ses, + const struct crypto_session_params *csp) { - ses->ses_mlen = macini->cri_mlen; - - /* Find software structure which describes HMAC algorithm. */ - switch (macini->cri_alg) { - case CRYPTO_NULL_HMAC: - ses->ses_axf = &auth_hash_null; - break; - case CRYPTO_MD5_HMAC: - ses->ses_axf = &auth_hash_hmac_md5; - break; - case CRYPTO_SHA1_HMAC: - ses->ses_axf = &auth_hash_hmac_sha1; - break; - case CRYPTO_RIPEMD160_HMAC: - ses->ses_axf = &auth_hash_hmac_ripemd_160; - break; - case CRYPTO_SHA2_256_HMAC: - ses->ses_axf = &auth_hash_hmac_sha2_256; - break; - case CRYPTO_SHA2_384_HMAC: - ses->ses_axf = &auth_hash_hmac_sha2_384; - break; - case CRYPTO_SHA2_512_HMAC: - ses->ses_axf = &auth_hash_hmac_sha2_512; - break; - } + ses->ses_axf = crypto_auth_hash(csp); + if (csp->csp_auth_mlen == 0) + ses->ses_mlen = ses->ses_axf->hashsize; + else + ses->ses_mlen = csp->csp_auth_mlen; /* Allocate memory for HMAC inner and outer contexts. */ ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_GLXSB, @@ -147,23 +120,24 @@ glxsb_hash_setup(struct glxsb_session *ses, struct cryptoini *macini) return (ENOMEM); /* Setup key if given. */ - if (macini->cri_key != NULL) { - glxsb_hash_key_setup(ses, macini->cri_key, - macini->cri_klen); + if (csp->csp_auth_key != NULL) { + glxsb_hash_key_setup(ses, csp->csp_auth_key, + csp->csp_auth_klen); } return (0); } int -glxsb_hash_process(struct glxsb_session *ses, struct cryptodesc *maccrd, - struct cryptop *crp) +glxsb_hash_process(struct glxsb_session *ses, + const struct crypto_session_params *csp, struct cryptop *crp) { int error; - if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) - glxsb_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen); + if (crp->crp_auth_key != NULL) + glxsb_hash_key_setup(ses, crp->crp_auth_key, + csp->csp_auth_klen); - error = glxsb_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags); + error = glxsb_authcompute(ses, crp); return (error); } diff --git a/sys/dev/hifn/hifn7751.c b/sys/dev/hifn/hifn7751.c index ce0f060aa7d2..7f1889767090 100644 --- a/sys/dev/hifn/hifn7751.c +++ b/sys/dev/hifn/hifn7751.c @@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/mutex.h> #include <sys/sysctl.h> +#include <sys/uio.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -71,6 +72,7 @@ __FBSDID("$FreeBSD$"); #include <sys/rman.h> #include <opencrypto/cryptodev.h> +#include <opencrypto/xform_auth.h> #include <sys/random.h> #include <sys/kobj.h> @@ -102,7 +104,9 @@ static int hifn_suspend(device_t); static int hifn_resume(device_t); static int hifn_shutdown(device_t); -static int hifn_newsession(device_t, crypto_session_t, struct cryptoini *); +static int hifn_probesession(device_t, const struct crypto_session_params *); +static int hifn_newsession(device_t, crypto_session_t, + const struct crypto_session_params *); static int hifn_process(device_t, struct cryptop *, int); static device_method_t hifn_methods[] = { @@ -115,6 +119,7 @@ static device_method_t hifn_methods[] = { DEVMETHOD(device_shutdown, hifn_shutdown), /* crypto device methods */ + DEVMETHOD(cryptodev_probesession, hifn_probesession), DEVMETHOD(cryptodev_newsession, hifn_newsession), DEVMETHOD(cryptodev_process, hifn_process), @@ -356,7 +361,7 @@ hifn_attach(device_t dev) caddr_t kva; int rseg, rid; char rbase; - u_int16_t ena, rev; + uint16_t rev; sc->sc_dev = dev; @@ -558,33 +563,22 @@ hifn_attach(device_t dev) 2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11)); printf("\n"); - sc->sc_cid = crypto_get_driverid(dev, sizeof(struct hifn_session), - CRYPTOCAP_F_HARDWARE); - if (sc->sc_cid < 0) { - device_printf(dev, "could not get crypto driver id\n"); - goto fail_intr; - } - WRITE_REG_0(sc, HIFN_0_PUCNFG, READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID); - ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; + sc->sc_ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; - switch (ena) { + switch (sc->sc_ena) { case HIFN_PUSTAT_ENA_2: - crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0); - if (sc->sc_flags & HIFN_HAS_AES) - crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); - /*FALLTHROUGH*/ case HIFN_PUSTAT_ENA_1: - crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); + sc->sc_cid = crypto_get_driverid(dev, + sizeof(struct hifn_session), CRYPTOCAP_F_HARDWARE); + if (sc->sc_cid < 0) { + device_printf(dev, "could not get crypto driver id\n"); + goto fail_intr; + } break; } - + bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); @@ -1547,6 +1541,7 @@ hifn_init_dma(struct hifn_softc *sc) static u_int hifn_write_command(struct hifn_command *cmd, u_int8_t *buf) { + struct cryptop *crp; u_int8_t *buf_pos; hifn_base_command_t *base_cmd; hifn_mac_command_t *mac_cmd; @@ -1554,6 +1549,7 @@ hifn_write_command(struct hifn_command *cmd, u_int8_t *buf) int using_mac, using_crypt, len, ivlen; u_int32_t dlen, slen; + crp = cmd->crp; buf_pos = buf; using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC; using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT; @@ -1576,24 +1572,27 @@ hifn_write_command(struct hifn_command *cmd, u_int8_t *buf) if (using_mac) { mac_cmd = (hifn_mac_command_t *)buf_pos; - dlen = cmd->maccrd->crd_len; + dlen = crp->crp_aad_length + crp->crp_payload_length; mac_cmd->source_count = htole16(dlen & 0xffff); dlen >>= 16; mac_cmd->masks = htole16(cmd->mac_masks | ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M)); - mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip); + if (crp->crp_aad_length != 0) + mac_cmd->header_skip = htole16(crp->crp_aad_start); + else + mac_cmd->header_skip = htole16(crp->crp_payload_start); mac_cmd->reserved = 0; buf_pos += sizeof(hifn_mac_command_t); } if (using_crypt) { cry_cmd = (hifn_crypt_command_t *)buf_pos; - dlen = cmd->enccrd->crd_len; + dlen = crp->crp_payload_length; cry_cmd->source_count = htole16(dlen & 0xffff); dlen >>= 16; cry_cmd->masks = htole16(cmd->cry_masks | ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M)); - cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip); + cry_cmd->header_skip = htole16(crp->crp_payload_length); cry_cmd->reserved = 0; buf_pos += sizeof(hifn_crypt_command_t); } @@ -1782,15 +1781,30 @@ hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd) return (idx); } +static bus_size_t +hifn_crp_length(struct cryptop *crp) +{ + + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + return (crp->crp_mbuf->m_pkthdr.len); + case CRYPTO_BUF_UIO: + return (crp->crp_uio->uio_resid); + case CRYPTO_BUF_CONTIG: + return (crp->crp_ilen); + default: + panic("bad crp buffer type"); + } +} + static void -hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) +hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, int error) { struct hifn_operand *op = arg; KASSERT(nsegs <= MAX_SCATTER, ("hifn_op_cb: too many DMA segments (%u > %u) " "returned when mapping operand", nsegs, MAX_SCATTER)); - op->mapsize = mapsize; op->nsegs = nsegs; bcopy(seg, op->segs, nsegs * sizeof (seg[0])); } @@ -1832,130 +1846,110 @@ hifn_crypto( return (ENOMEM); } - if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, - cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) { - hifnstats.hst_nomem_load++; - err = ENOMEM; - goto err_srcmap1; - } - } else if (crp->crp_flags & CRYPTO_F_IOV) { - if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, - cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) { - hifnstats.hst_nomem_load++; - err = ENOMEM; - goto err_srcmap1; - } - } else { - err = EINVAL; + if (bus_dmamap_load_crp(sc->sc_dmat, cmd->src_map, crp, hifn_op_cb, + &cmd->src, BUS_DMA_NOWAIT)) { + hifnstats.hst_nomem_load++; + err = ENOMEM; goto err_srcmap1; } + cmd->src_mapsize = hifn_crp_length(crp); if (hifn_dmamap_aligned(&cmd->src)) { cmd->sloplen = cmd->src_mapsize & 3; cmd->dst = cmd->src; - } else { - if (crp->crp_flags & CRYPTO_F_IOV) { - err = EINVAL; - goto err_srcmap; - } else if (crp->crp_flags & CRYPTO_F_IMBUF) { - int totlen, len; - struct mbuf *m, *m0, *mlast; + } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) { + int totlen, len; + struct mbuf *m, *m0, *mlast; - KASSERT(cmd->dst_m == cmd->src_m, - ("hifn_crypto: dst_m initialized improperly")); - hifnstats.hst_unaligned++; - /* - * Source is not aligned on a longword boundary. - * Copy the data to insure alignment. If we fail - * to allocate mbufs or clusters while doing this - * we return ERESTART so the operation is requeued - * at the crypto later, but only if there are - * ops already posted to the hardware; otherwise we - * have no guarantee that we'll be re-entered. - */ - totlen = cmd->src_mapsize; - if (cmd->src_m->m_flags & M_PKTHDR) { - len = MHLEN; - MGETHDR(m0, M_NOWAIT, MT_DATA); - if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) { - m_free(m0); - m0 = NULL; - } - } else { - len = MLEN; - MGET(m0, M_NOWAIT, MT_DATA); + KASSERT(cmd->dst_m == NULL, + ("hifn_crypto: dst_m initialized improperly")); + hifnstats.hst_unaligned++; + + /* + * Source is not aligned on a longword boundary. + * Copy the data to insure alignment. If we fail + * to allocate mbufs or clusters while doing this + * we return ERESTART so the operation is requeued + * at the crypto later, but only if there are + * ops already posted to the hardware; otherwise we + * have no guarantee that we'll be re-entered. + */ + totlen = cmd->src_mapsize; + if (crp->crp_mbuf->m_flags & M_PKTHDR) { + len = MHLEN; + MGETHDR(m0, M_NOWAIT, MT_DATA); + if (m0 && !m_dup_pkthdr(m0, crp->crp_mbuf, M_NOWAIT)) { + m_free(m0); + m0 = NULL; } - if (m0 == NULL) { + } else { + len = MLEN; + MGET(m0, M_NOWAIT, MT_DATA); + } + if (m0 == NULL) { + hifnstats.hst_nomem_mbuf++; + err = sc->sc_cmdu ? ERESTART : ENOMEM; + goto err_srcmap; + } + if (totlen >= MINCLSIZE) { + if (!(MCLGET(m0, M_NOWAIT))) { + hifnstats.hst_nomem_mcl++; + err = sc->sc_cmdu ? ERESTART : ENOMEM; + m_freem(m0); + goto err_srcmap; + } + len = MCLBYTES; + } + totlen -= len; + m0->m_pkthdr.len = m0->m_len = len; + mlast = m0; + + while (totlen > 0) { + MGET(m, M_NOWAIT, MT_DATA); + if (m == NULL) { hifnstats.hst_nomem_mbuf++; err = sc->sc_cmdu ? ERESTART : ENOMEM; + m_freem(m0); goto err_srcmap; } + len = MLEN; if (totlen >= MINCLSIZE) { - if (!(MCLGET(m0, M_NOWAIT))) { + if (!(MCLGET(m, M_NOWAIT))) { hifnstats.hst_nomem_mcl++; err = sc->sc_cmdu ? ERESTART : ENOMEM; + mlast->m_next = m; m_freem(m0); goto err_srcmap; } len = MCLBYTES; } - totlen -= len; - m0->m_pkthdr.len = m0->m_len = len; - mlast = m0; - while (totlen > 0) { - MGET(m, M_NOWAIT, MT_DATA); - if (m == NULL) { - hifnstats.hst_nomem_mbuf++; - err = sc->sc_cmdu ? ERESTART : ENOMEM; - m_freem(m0); - goto err_srcmap; - } - len = MLEN; - if (totlen >= MINCLSIZE) { - if (!(MCLGET(m, M_NOWAIT))) { - hifnstats.hst_nomem_mcl++; - err = sc->sc_cmdu ? ERESTART : ENOMEM; - mlast->m_next = m; - m_freem(m0); - goto err_srcmap; - } - len = MCLBYTES; - } - - m->m_len = len; - m0->m_pkthdr.len += len; - totlen -= len; + m->m_len = len; + m0->m_pkthdr.len += len; + totlen -= len; - mlast->m_next = m; - mlast = m; - } - cmd->dst_m = m0; + mlast->m_next = m; + mlast = m; } - } + cmd->dst_m = m0; - if (cmd->dst_map == NULL) { - if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) { + if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, + &cmd->dst_map)) { hifnstats.hst_nomem_map++; err = ENOMEM; goto err_srcmap; } - if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, - cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) { - hifnstats.hst_nomem_map++; - err = ENOMEM; - goto err_dstmap1; - } - } else if (crp->crp_flags & CRYPTO_F_IOV) { - if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, - cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) { - hifnstats.hst_nomem_load++; - err = ENOMEM; - goto err_dstmap1; - } + + if (bus_dmamap_load_mbuf_sg(sc->sc_dmat, cmd->dst_map, m0, + cmd->dst_segs, &cmd->dst_nsegs, 0)) { + hifnstats.hst_nomem_map++; + err = ENOMEM; + goto err_dstmap1; } + cmd->dst_mapsize = m0->m_pkthdr.len; + } else { + err = EINVAL; + goto err_srcmap; } #ifdef HIFN_DEBUG @@ -2111,8 +2105,8 @@ err_dstmap1: if (cmd->src_map != cmd->dst_map) bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); err_srcmap: - if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (cmd->src_m != cmd->dst_m) + if (crp->crp_buf_type == CRYPTO_BUF_MBUF) { + if (cmd->dst_m != NULL) m_freem(cmd->dst_m); } bus_dmamap_unload(sc->sc_dmat, cmd->src_map); @@ -2307,67 +2301,121 @@ hifn_intr(void *arg) } } -/* - * Allocate a new 'session' and return an encoded session id. 'sidp' - * contains our registration id, and should contain an encoded session - * id on successful allocation. - */ -static int -hifn_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) +static bool +hifn_auth_supported(struct hifn_softc *sc, + const struct crypto_session_params *csp) { - struct hifn_softc *sc = device_get_softc(dev); - struct cryptoini *c; - int mac = 0, cry = 0; - struct hifn_session *ses; - KASSERT(sc != NULL, ("hifn_newsession: null softc")); - if (cri == NULL || sc == NULL) - return (EINVAL); + switch (sc->sc_ena) { + case HIFN_PUSTAT_ENA_2: + case HIFN_PUSTAT_ENA_1: + break; + default: + return (false); + } + + switch (csp->csp_auth_alg) { + case CRYPTO_MD5: + case CRYPTO_SHA1: + break; + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + if (csp->csp_auth_klen > HIFN_MAC_KEY_LENGTH) + return (false); + break; + default: + return (false); + } - ses = crypto_get_driver_session(cses); + return (true); +} - for (c = cri; c != NULL; c = c->cri_next) { - switch (c->cri_alg) { - case CRYPTO_MD5: - case CRYPTO_SHA1: - case CRYPTO_MD5_HMAC: - case CRYPTO_SHA1_HMAC: - if (mac) - return (EINVAL); - mac = 1; - ses->hs_mlen = c->cri_mlen; - if (ses->hs_mlen == 0) { - switch (c->cri_alg) { - case CRYPTO_MD5: - case CRYPTO_MD5_HMAC: - ses->hs_mlen = 16; - break; - case CRYPTO_SHA1: - case CRYPTO_SHA1_HMAC: - ses->hs_mlen = 20; - break; - } - } - break; - case CRYPTO_DES_CBC: +static bool +hifn_cipher_supported(struct hifn_softc *sc, + const struct crypto_session_params *csp) +{ + + if (csp->csp_cipher_klen == 0) + return (false); + if (csp->csp_ivlen > HIFN_MAX_IV_LENGTH) + return (false); + switch (sc->sc_ena) { + case HIFN_PUSTAT_ENA_2: + switch (csp->csp_cipher_alg) { case CRYPTO_3DES_CBC: - case CRYPTO_AES_CBC: - /* XXX this may read fewer, does it matter? */ - read_random(ses->hs_iv, - c->cri_alg == CRYPTO_AES_CBC ? - HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); - /*FALLTHROUGH*/ case CRYPTO_ARC4: - if (cry) - return (EINVAL); - cry = 1; break; - default: - return (EINVAL); + case CRYPTO_AES_CBC: + if ((sc->sc_flags & HIFN_HAS_AES) == 0) + return (false); + switch (csp->csp_cipher_klen) { + case 128: + case 192: + case 256: + break; + default: + return (false); + } + return (true); } + /*FALLTHROUGH*/ + case HIFN_PUSTAT_ENA_1: + switch (csp->csp_cipher_alg) { + case CRYPTO_DES_CBC: + return (true); + } + break; } - if (mac == 0 && cry == 0) + return (false); +} + +static int +hifn_probesession(device_t dev, const struct crypto_session_params *csp) +{ + struct hifn_softc *sc; + + sc = device_get_softc(dev); + if (csp->csp_flags != 0) + return (EINVAL); + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + if (!hifn_auth_supported(sc, csp)) + return (EINVAL); + break; + case CSP_MODE_CIPHER: + if (!hifn_cipher_supported(sc, csp)) + return (EINVAL); + break; + case CSP_MODE_ETA: + if (!hifn_auth_supported(sc, csp) || + !hifn_cipher_supported(sc, csp)) + return (EINVAL); + break; + default: return (EINVAL); + } + + return (CRYPTODEV_PROBE_HARDWARE); +} + +/* + * Allocate a new 'session'. + */ +static int +hifn_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct hifn_session *ses; + + ses = crypto_get_driver_session(cses); + + if (csp->csp_auth_alg != 0) { + if (csp->csp_auth_mlen == 0) + ses->hs_mlen = crypto_auth_hash(csp)->hashsize; + else + ses->hs_mlen = csp->csp_auth_mlen; + } + return (0); } @@ -2379,18 +2427,15 @@ hifn_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) static int hifn_process(device_t dev, struct cryptop *crp, int hint) { + const struct crypto_session_params *csp; struct hifn_softc *sc = device_get_softc(dev); struct hifn_command *cmd = NULL; - int err, ivlen; - struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; + const void *mackey; + int err, ivlen, keylen; struct hifn_session *ses; - if (crp == NULL || crp->crp_callback == NULL) { - hifnstats.hst_invalid++; - return (EINVAL); - } - ses = crypto_get_driver_session(crp->crp_session); + cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO); if (cmd == NULL) { hifnstats.hst_nomem++; @@ -2398,80 +2443,26 @@ hifn_process(device_t dev, struct cryptop *crp, int hint) goto errout; } - if (crp->crp_flags & CRYPTO_F_IMBUF) { - cmd->src_m = (struct mbuf *)crp->crp_buf; - cmd->dst_m = (struct mbuf *)crp->crp_buf; - } else if (crp->crp_flags & CRYPTO_F_IOV) { - cmd->src_io = (struct uio *)crp->crp_buf; - cmd->dst_io = (struct uio *)crp->crp_buf; - } else { - err = EINVAL; - goto errout; /* XXX we don't handle contiguous buffers! */ - } + csp = crypto_get_params(crp->crp_session); - crd1 = crp->crp_desc; - if (crd1 == NULL) { + /* + * The driver only supports ETA requests where there is no + * gap between the AAD and payload. + */ + if (csp->csp_mode == CSP_MODE_ETA && crp->crp_aad_length != 0 && + crp->crp_aad_start + crp->crp_aad_length != + crp->crp_payload_start) { err = EINVAL; goto errout; } - crd2 = crd1->crd_next; - - if (crd2 == NULL) { - if (crd1->crd_alg == CRYPTO_MD5_HMAC || - crd1->crd_alg == CRYPTO_SHA1_HMAC || - crd1->crd_alg == CRYPTO_SHA1 || - crd1->crd_alg == CRYPTO_MD5) { - maccrd = crd1; - enccrd = NULL; - } else if (crd1->crd_alg == CRYPTO_DES_CBC || - crd1->crd_alg == CRYPTO_3DES_CBC || - crd1->crd_alg == CRYPTO_AES_CBC || - crd1->crd_alg == CRYPTO_ARC4) { - if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0) - cmd->base_masks |= HIFN_BASE_CMD_DECODE; - maccrd = NULL; - enccrd = crd1; - } else { - err = EINVAL; - goto errout; - } - } else { - if ((crd1->crd_alg == CRYPTO_MD5_HMAC || - crd1->crd_alg == CRYPTO_SHA1_HMAC || - crd1->crd_alg == CRYPTO_MD5 || - crd1->crd_alg == CRYPTO_SHA1) && - (crd2->crd_alg == CRYPTO_DES_CBC || - crd2->crd_alg == CRYPTO_3DES_CBC || - crd2->crd_alg == CRYPTO_AES_CBC || - crd2->crd_alg == CRYPTO_ARC4) && - ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { - cmd->base_masks = HIFN_BASE_CMD_DECODE; - maccrd = crd1; - enccrd = crd2; - } else if ((crd1->crd_alg == CRYPTO_DES_CBC || - crd1->crd_alg == CRYPTO_ARC4 || - crd1->crd_alg == CRYPTO_3DES_CBC || - crd1->crd_alg == CRYPTO_AES_CBC) && - (crd2->crd_alg == CRYPTO_MD5_HMAC || - crd2->crd_alg == CRYPTO_SHA1_HMAC || - crd2->crd_alg == CRYPTO_MD5 || - crd2->crd_alg == CRYPTO_SHA1) && - (crd1->crd_flags & CRD_F_ENCRYPT)) { - enccrd = crd1; - maccrd = crd2; - } else { - /* - * We cannot order the 7751 as requested - */ - err = EINVAL; - goto errout; - } - } - if (enccrd) { - cmd->enccrd = enccrd; + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + case CSP_MODE_ETA: + if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + cmd->base_masks |= HIFN_BASE_CMD_DECODE; cmd->base_masks |= HIFN_BASE_CMD_CRYPT; - switch (enccrd->crd_alg) { + switch (csp->csp_cipher_alg) { case CRYPTO_ARC4: cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4; break; @@ -2494,36 +2485,24 @@ hifn_process(device_t dev, struct cryptop *crp, int hint) err = EINVAL; goto errout; } - if (enccrd->crd_alg != CRYPTO_ARC4) { - ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ? - HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); - if (enccrd->crd_flags & CRD_F_ENCRYPT) { - if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(enccrd->crd_iv, cmd->iv, ivlen); - else - bcopy(ses->hs_iv, cmd->iv, ivlen); - - if ((enccrd->crd_flags & CRD_F_IV_PRESENT) - == 0) { - crypto_copyback(crp->crp_flags, - crp->crp_buf, enccrd->crd_inject, - ivlen, cmd->iv); - } - } else { - if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(enccrd->crd_iv, cmd->iv, ivlen); - else { - crypto_copydata(crp->crp_flags, - crp->crp_buf, enccrd->crd_inject, - ivlen, cmd->iv); - } - } + if (csp->csp_cipher_alg != CRYPTO_ARC4) { + ivlen = csp->csp_ivlen; + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(cmd->iv, ivlen, 0); + crypto_copyback(crp, crp->crp_iv_start, ivlen, + cmd->iv); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(cmd->iv, crp->crp_iv, ivlen); + else + crypto_copydata(crp, crp->crp_iv_start, ivlen, + cmd->iv); } - if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) - cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY; - cmd->ck = enccrd->crd_key; - cmd->cklen = enccrd->crd_klen >> 3; + if (crp->crp_cipher_key != NULL) + cmd->ck = crp->crp_cipher_key; + else + cmd->ck = csp->csp_cipher_key; + cmd->cklen = csp->csp_cipher_klen; cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY; /* @@ -2546,13 +2525,15 @@ hifn_process(device_t dev, struct cryptop *crp, int hint) goto errout; } } + break; } - if (maccrd) { - cmd->maccrd = maccrd; + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + case CSP_MODE_ETA: cmd->base_masks |= HIFN_BASE_CMD_MAC; - switch (maccrd->crd_alg) { + switch (csp->csp_auth_alg) { case CRYPTO_MD5: cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | @@ -2575,12 +2556,16 @@ hifn_process(device_t dev, struct cryptop *crp, int hint) break; } - if (maccrd->crd_alg == CRYPTO_SHA1_HMAC || - maccrd->crd_alg == CRYPTO_MD5_HMAC) { + if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC || + csp->csp_auth_alg == CRYPTO_MD5_HMAC) { cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY; - bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3); - bzero(cmd->mac + (maccrd->crd_klen >> 3), - HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3)); + if (crp->crp_auth_key != NULL) + mackey = crp->crp_auth_key; + else + mackey = csp->csp_auth_key; + keylen = csp->csp_auth_klen; + bcopy(mackey, cmd->mac, keylen); + bzero(cmd->mac + keylen, HIFN_MAC_KEY_LENGTH - keylen); } } @@ -2655,9 +2640,8 @@ hifn_abort(struct hifn_softc *sc) BUS_DMASYNC_POSTREAD); } - if (cmd->src_m != cmd->dst_m) { - m_freem(cmd->src_m); - crp->crp_buf = (caddr_t)cmd->dst_m; + if (cmd->dst_m != NULL) { + m_freem(cmd->dst_m); } /* non-shared buffers cannot be restarted */ @@ -2696,9 +2680,9 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf) { struct hifn_dma *dma = sc->sc_dma; struct cryptop *crp = cmd->crp; - struct cryptodesc *crd; + uint8_t macbuf2[SHA1_HASH_LEN]; struct mbuf *m; - int totlen, i, u, ivlen; + int totlen, i, u; if (cmd->src_map == cmd->dst_map) { bus_dmamap_sync(sc->sc_dmat, cmd->src_map, @@ -2710,9 +2694,8 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf) BUS_DMASYNC_POSTREAD); } - if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (cmd->src_m != cmd->dst_m) { - crp->crp_buf = (caddr_t)cmd->dst_m; + if (crp->crp_buf_type == CRYPTO_BUF_MBUF) { + if (cmd->dst_m != NULL) { totlen = cmd->src_mapsize; for (m = cmd->dst_m; m != NULL; m = m->m_next) { if (totlen < m->m_len) { @@ -2721,15 +2704,15 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf) } else totlen -= m->m_len; } - cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len; - m_freem(cmd->src_m); + cmd->dst_m->m_pkthdr.len = crp->crp_mbuf->m_pkthdr.len; + m_freem(crp->crp_mbuf); + crp->crp_mbuf = cmd->dst_m; } } if (cmd->sloplen != 0) { - crypto_copyback(crp->crp_flags, crp->crp_buf, - cmd->src_mapsize - cmd->sloplen, cmd->sloplen, - (caddr_t)&dma->slop[cmd->slopidx]); + crypto_copyback(crp, cmd->src_mapsize - cmd->sloplen, + cmd->sloplen, &dma->slop[cmd->slopidx]); } i = sc->sc_dstk; u = sc->sc_dstu; @@ -2749,37 +2732,16 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf) hifnstats.hst_obytes += cmd->dst_mapsize; - if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) == - HIFN_BASE_CMD_CRYPT) { - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - if (crd->crd_alg != CRYPTO_DES_CBC && - crd->crd_alg != CRYPTO_3DES_CBC && - crd->crd_alg != CRYPTO_AES_CBC) - continue; - ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ? - HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); - crypto_copydata(crp->crp_flags, crp->crp_buf, - crd->crd_skip + crd->crd_len - ivlen, ivlen, - cmd->session->hs_iv); - break; - } - } - if (macbuf != NULL) { - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - int len; - - if (crd->crd_alg != CRYPTO_MD5 && - crd->crd_alg != CRYPTO_SHA1 && - crd->crd_alg != CRYPTO_MD5_HMAC && - crd->crd_alg != CRYPTO_SHA1_HMAC) { - continue; - } - len = cmd->session->hs_mlen; - crypto_copyback(crp->crp_flags, crp->crp_buf, - crd->crd_inject, len, macbuf); - break; - } + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, + cmd->session->hs_mlen, macbuf2); + if (timingsafe_bcmp(macbuf, macbuf2, + cmd->session->hs_mlen) != 0) + crp->crp_etype = EBADMSG; + } else + crypto_copyback(crp, crp->crp_digest_start, + cmd->session->hs_mlen, macbuf); } if (cmd->src_map != cmd->dst_map) { diff --git a/sys/dev/hifn/hifn7751var.h b/sys/dev/hifn/hifn7751var.h index e7ace8bcc977..ec0d96ec20a4 100644 --- a/sys/dev/hifn/hifn7751var.h +++ b/sys/dev/hifn/hifn7751var.h @@ -105,7 +105,6 @@ struct hifn_dma { struct hifn_session { - u_int8_t hs_iv[HIFN_MAX_IV_LENGTH]; int hs_mlen; }; @@ -160,6 +159,7 @@ struct hifn_softc { int sc_cmdk, sc_srck, sc_dstk, sc_resk; int32_t sc_cid; + uint16_t sc_ena; int sc_maxses; int sc_ramsize; int sc_flags; @@ -257,10 +257,6 @@ struct hifn_softc { * */ struct hifn_operand { - union { - struct mbuf *m; - struct uio *io; - } u; bus_dmamap_t map; bus_size_t mapsize; int nsegs; @@ -269,27 +265,24 @@ struct hifn_operand { struct hifn_command { struct hifn_session *session; u_int16_t base_masks, cry_masks, mac_masks; - u_int8_t iv[HIFN_MAX_IV_LENGTH], *ck, mac[HIFN_MAC_KEY_LENGTH]; + u_int8_t iv[HIFN_MAX_IV_LENGTH], mac[HIFN_MAC_KEY_LENGTH]; + const uint8_t *ck; int cklen; int sloplen, slopidx; struct hifn_operand src; struct hifn_operand dst; + struct mbuf *dst_m; struct hifn_softc *softc; struct cryptop *crp; - struct cryptodesc *enccrd, *maccrd; }; -#define src_m src.u.m -#define src_io src.u.io #define src_map src.map #define src_mapsize src.mapsize #define src_segs src.segs #define src_nsegs src.nsegs -#define dst_m dst.u.m -#define dst_io dst.u.io #define dst_map dst.map #define dst_mapsize dst.mapsize #define dst_segs dst.segs diff --git a/sys/dev/safe/safe.c b/sys/dev/safe/safe.c index 7a577dfd0a8c..99f16de56c50 100644 --- a/sys/dev/safe/safe.c +++ b/sys/dev/safe/safe.c @@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/sysctl.h> #include <sys/endian.h> +#include <sys/uio.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -56,10 +57,8 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include <sys/rman.h> -#include <crypto/sha1.h> #include <opencrypto/cryptodev.h> -#include <opencrypto/cryptosoft.h> -#include <sys/md5.h> +#include <opencrypto/xform_auth.h> #include <sys/random.h> #include <sys/kobj.h> @@ -88,7 +87,9 @@ static int safe_suspend(device_t); static int safe_resume(device_t); static int safe_shutdown(device_t); -static int safe_newsession(device_t, crypto_session_t, struct cryptoini *); +static int safe_probesession(device_t, const struct crypto_session_params *); +static int safe_newsession(device_t, crypto_session_t, + const struct crypto_session_params *); static int safe_process(device_t, struct cryptop *, int); static device_method_t safe_methods[] = { @@ -101,6 +102,7 @@ static device_method_t safe_methods[] = { DEVMETHOD(device_shutdown, safe_shutdown), /* crypto device methods */ + DEVMETHOD(cryptodev_probesession, safe_probesession), DEVMETHOD(cryptodev_newsession, safe_newsession), DEVMETHOD(cryptodev_process, safe_process), @@ -221,7 +223,7 @@ safe_attach(device_t dev) { struct safe_softc *sc = device_get_softc(dev); u_int32_t raddr; - u_int32_t i, devinfo; + u_int32_t i; int rid; bzero(sc, sizeof (*sc)); @@ -374,12 +376,12 @@ safe_attach(device_t dev) device_printf(sc->sc_dev, "%s", safe_partname(sc)); - devinfo = READ_REG(sc, SAFE_DEVINFO); - if (devinfo & SAFE_DEVINFO_RNG) { + sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO); + if (sc->sc_devinfo & SAFE_DEVINFO_RNG) { sc->sc_flags |= SAFE_FLAGS_RNG; printf(" rng"); } - if (devinfo & SAFE_DEVINFO_PKEY) { + if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) { #if 0 printf(" key"); sc->sc_flags |= SAFE_FLAGS_KEY; @@ -387,26 +389,18 @@ safe_attach(device_t dev) crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0); #endif } - if (devinfo & SAFE_DEVINFO_DES) { + if (sc->sc_devinfo & SAFE_DEVINFO_DES) { printf(" des/3des"); - crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); } - if (devinfo & SAFE_DEVINFO_AES) { + if (sc->sc_devinfo & SAFE_DEVINFO_AES) { printf(" aes"); - crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); } - if (devinfo & SAFE_DEVINFO_MD5) { + if (sc->sc_devinfo & SAFE_DEVINFO_MD5) { printf(" md5"); - crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); } - if (devinfo & SAFE_DEVINFO_SHA1) { + if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) { printf(" sha1"); - crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); } - printf(" null"); - crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0); /* XXX other supported algorithms */ printf("\n"); @@ -629,11 +623,11 @@ safe_feed(struct safe_softc *sc, struct safe_ringentry *re) #define N(a) (sizeof(a) / sizeof (a[0])) static void -safe_setup_enckey(struct safe_session *ses, caddr_t key) +safe_setup_enckey(struct safe_session *ses, const void *key) { int i; - bcopy(key, ses->ses_key, ses->ses_klen / 8); + bcopy(key, ses->ses_key, ses->ses_klen); /* PE is little-endian, insure proper byte order */ for (i = 0; i < N(ses->ses_key); i++) @@ -641,47 +635,30 @@ safe_setup_enckey(struct safe_session *ses, caddr_t key) } static void -safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen) +safe_setup_mackey(struct safe_session *ses, int algo, const uint8_t *key, + int klen) { MD5_CTX md5ctx; SHA1_CTX sha1ctx; int i; - - for (i = 0; i < klen; i++) - key[i] ^= HMAC_IPAD_VAL; - if (algo == CRYPTO_MD5_HMAC) { - MD5Init(&md5ctx); - MD5Update(&md5ctx, key, klen); - MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen); + hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx); bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state)); - } else { - SHA1Init(&sha1ctx); - SHA1Update(&sha1ctx, key, klen); - SHA1Update(&sha1ctx, hmac_ipad_buffer, - SHA1_BLOCK_LEN - klen); - bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); - } - - for (i = 0; i < klen; i++) - key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); - if (algo == CRYPTO_MD5_HMAC) { - MD5Init(&md5ctx); - MD5Update(&md5ctx, key, klen); - MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen); + hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx); bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); + + explicit_bzero(&md5ctx, sizeof(md5ctx)); } else { - SHA1Init(&sha1ctx); - SHA1Update(&sha1ctx, key, klen); - SHA1Update(&sha1ctx, hmac_opad_buffer, - SHA1_BLOCK_LEN - klen); + hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx); + bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); + + hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx); bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); - } - for (i = 0; i < klen; i++) - key[i] ^= HMAC_OPAD_VAL; + explicit_bzero(&sha1ctx, sizeof(sha1ctx)); + } /* PE is little-endian, insure proper byte order */ for (i = 0; i < N(ses->ses_hminner); i++) { @@ -691,98 +668,147 @@ safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen) } #undef N -/* - * Allocate a new 'session' and return an encoded session id. 'sidp' - * contains our registration id, and should contain an encoded session - * id on successful allocation. - */ +static bool +safe_auth_supported(struct safe_softc *sc, + const struct crypto_session_params *csp) +{ + + switch (csp->csp_auth_alg) { + case CRYPTO_MD5_HMAC: + if ((sc->sc_devinfo & SAFE_DEVINFO_MD5) == 0) + return (false); + break; + case CRYPTO_SHA1_HMAC: + if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0) + return (false); + break; + default: + return (false); + } + return (true); +} + +static bool +safe_cipher_supported(struct safe_softc *sc, + const struct crypto_session_params *csp) +{ + + switch (csp->csp_cipher_alg) { + case CRYPTO_DES_CBC: + case CRYPTO_3DES_CBC: + if ((sc->sc_devinfo & SAFE_DEVINFO_DES) == 0) + return (false); + if (csp->csp_ivlen != 8) + return (false); + if (csp->csp_cipher_alg == CRYPTO_DES_CBC) { + if (csp->csp_cipher_klen != 8) + return (false); + } else { + if (csp->csp_cipher_klen != 24) + return (false); + } + break; + case CRYPTO_AES_CBC: + if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0) + return (false); + if (csp->csp_ivlen != 16) + return (false); + if (csp->csp_cipher_klen != 16 && + csp->csp_cipher_klen != 24 && + csp->csp_cipher_klen != 32) + return (false); + break; + } + return (true); +} + static int -safe_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) +safe_probesession(device_t dev, const struct crypto_session_params *csp) { struct safe_softc *sc = device_get_softc(dev); - struct cryptoini *c, *encini = NULL, *macini = NULL; - struct safe_session *ses = NULL; - if (cri == NULL || sc == NULL) + if (csp->csp_flags != 0) return (EINVAL); - - for (c = cri; c != NULL; c = c->cri_next) { - if (c->cri_alg == CRYPTO_MD5_HMAC || - c->cri_alg == CRYPTO_SHA1_HMAC || - c->cri_alg == CRYPTO_NULL_HMAC) { - if (macini) - return (EINVAL); - macini = c; - } else if (c->cri_alg == CRYPTO_DES_CBC || - c->cri_alg == CRYPTO_3DES_CBC || - c->cri_alg == CRYPTO_AES_CBC || - c->cri_alg == CRYPTO_NULL_CBC) { - if (encini) - return (EINVAL); - encini = c; - } else + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + if (!safe_auth_supported(sc, csp)) return (EINVAL); - } - if (encini == NULL && macini == NULL) + break; + case CSP_MODE_CIPHER: + if (!safe_cipher_supported(sc, csp)) + return (EINVAL); + break; + case CSP_MODE_ETA: + if (!safe_auth_supported(sc, csp) || + !safe_cipher_supported(sc, csp)) + return (EINVAL); + break; + default: return (EINVAL); - if (encini) { /* validate key length */ - switch (encini->cri_alg) { - case CRYPTO_DES_CBC: - if (encini->cri_klen != 64) - return (EINVAL); - break; - case CRYPTO_3DES_CBC: - if (encini->cri_klen != 192) - return (EINVAL); - break; - case CRYPTO_AES_CBC: - if (encini->cri_klen != 128 && - encini->cri_klen != 192 && - encini->cri_klen != 256) - return (EINVAL); - break; - } } - ses = crypto_get_driver_session(cses); - if (encini) { - /* get an IV */ - /* XXX may read fewer than requested */ - read_random(ses->ses_iv, sizeof(ses->ses_iv)); + return (CRYPTODEV_PROBE_HARDWARE); +} + +/* + * Allocate a new 'session'. + */ +static int +safe_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct safe_session *ses; - ses->ses_klen = encini->cri_klen; - if (encini->cri_key != NULL) - safe_setup_enckey(ses, encini->cri_key); + ses = crypto_get_driver_session(cses); + if (csp->csp_cipher_alg != 0) { + ses->ses_klen = csp->csp_cipher_klen; + if (csp->csp_cipher_key != NULL) + safe_setup_enckey(ses, csp->csp_cipher_key); } - if (macini) { - ses->ses_mlen = macini->cri_mlen; + if (csp->csp_auth_alg != 0) { + ses->ses_mlen = csp->csp_auth_mlen; if (ses->ses_mlen == 0) { - if (macini->cri_alg == CRYPTO_MD5_HMAC) + if (csp->csp_auth_alg == CRYPTO_MD5_HMAC) ses->ses_mlen = MD5_HASH_LEN; else ses->ses_mlen = SHA1_HASH_LEN; } - if (macini->cri_key != NULL) { - safe_setup_mackey(ses, macini->cri_alg, macini->cri_key, - macini->cri_klen / 8); + if (csp->csp_auth_key != NULL) { + safe_setup_mackey(ses, csp->csp_auth_alg, + csp->csp_auth_key, csp->csp_auth_klen); } } return (0); } +static bus_size_t +safe_crp_length(struct cryptop *crp) +{ + + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + return (crp->crp_mbuf->m_pkthdr.len); + case CRYPTO_BUF_UIO: + return (crp->crp_uio->uio_resid); + case CRYPTO_BUF_CONTIG: + return (crp->crp_ilen); + default: + panic("bad crp buffer type"); + } +} + static void -safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) +safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error) { struct safe_operand *op = arg; - DPRINTF(("%s: mapsize %u nsegs %d error %d\n", __func__, - (u_int) mapsize, nsegs, error)); + DPRINTF(("%s: nsegs %d error %d\n", __func__, + nsegs, error)); if (error != 0) return; - op->mapsize = mapsize; op->nsegs = nsegs; bcopy(seg, op->segs, nsegs * sizeof (seg[0])); } @@ -790,11 +816,10 @@ safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int static int safe_process(device_t dev, struct cryptop *crp, int hint) { - struct safe_softc *sc = device_get_softc(dev); + struct safe_softc *sc = device_get_softc(dev); + const struct crypto_session_params *csp; int err = 0, i, nicealign, uniform; - struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; - int bypass, oplen, ivsize; - caddr_t iv; + int bypass, oplen; int16_t coffset; struct safe_session *ses; struct safe_ringentry *re; @@ -802,11 +827,6 @@ safe_process(device_t dev, struct cryptop *crp, int hint) struct safe_pdesc *pd; u_int32_t cmd0, cmd1, staterec; - if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { - safestats.st_invalid++; - return (EINVAL); - } - mtx_lock(&sc->sc_ringmtx); if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) { safestats.st_ringfull++; @@ -823,104 +843,46 @@ safe_process(device_t dev, struct cryptop *crp, int hint) re->re_crp = crp; - if (crp->crp_flags & CRYPTO_F_IMBUF) { - re->re_src_m = (struct mbuf *)crp->crp_buf; - re->re_dst_m = (struct mbuf *)crp->crp_buf; - } else if (crp->crp_flags & CRYPTO_F_IOV) { - re->re_src_io = (struct uio *)crp->crp_buf; - re->re_dst_io = (struct uio *)crp->crp_buf; - } else { - safestats.st_badflags++; - err = EINVAL; - goto errout; /* XXX we don't handle contiguous blocks! */ - } - sa = &re->re_sa; ses = crypto_get_driver_session(crp->crp_session); - - crd1 = crp->crp_desc; - if (crd1 == NULL) { - safestats.st_nodesc++; - err = EINVAL; - goto errout; - } - crd2 = crd1->crd_next; + csp = crypto_get_params(crp->crp_session); cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */ cmd1 = 0; - if (crd2 == NULL) { - if (crd1->crd_alg == CRYPTO_MD5_HMAC || - crd1->crd_alg == CRYPTO_SHA1_HMAC || - crd1->crd_alg == CRYPTO_NULL_HMAC) { - maccrd = crd1; - enccrd = NULL; - cmd0 |= SAFE_SA_CMD0_OP_HASH; - } else if (crd1->crd_alg == CRYPTO_DES_CBC || - crd1->crd_alg == CRYPTO_3DES_CBC || - crd1->crd_alg == CRYPTO_AES_CBC || - crd1->crd_alg == CRYPTO_NULL_CBC) { - maccrd = NULL; - enccrd = crd1; - cmd0 |= SAFE_SA_CMD0_OP_CRYPT; - } else { - safestats.st_badalg++; - err = EINVAL; - goto errout; - } - } else { - if ((crd1->crd_alg == CRYPTO_MD5_HMAC || - crd1->crd_alg == CRYPTO_SHA1_HMAC || - crd1->crd_alg == CRYPTO_NULL_HMAC) && - (crd2->crd_alg == CRYPTO_DES_CBC || - crd2->crd_alg == CRYPTO_3DES_CBC || - crd2->crd_alg == CRYPTO_AES_CBC || - crd2->crd_alg == CRYPTO_NULL_CBC) && - ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { - maccrd = crd1; - enccrd = crd2; - } else if ((crd1->crd_alg == CRYPTO_DES_CBC || - crd1->crd_alg == CRYPTO_3DES_CBC || - crd1->crd_alg == CRYPTO_AES_CBC || - crd1->crd_alg == CRYPTO_NULL_CBC) && - (crd2->crd_alg == CRYPTO_MD5_HMAC || - crd2->crd_alg == CRYPTO_SHA1_HMAC || - crd2->crd_alg == CRYPTO_NULL_HMAC) && - (crd1->crd_flags & CRD_F_ENCRYPT)) { - enccrd = crd1; - maccrd = crd2; - } else { - safestats.st_badalg++; - err = EINVAL; - goto errout; - } + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + cmd0 |= SAFE_SA_CMD0_OP_HASH; + break; + case CSP_MODE_CIPHER: + cmd0 |= SAFE_SA_CMD0_OP_CRYPT; + break; + case CSP_MODE_ETA: cmd0 |= SAFE_SA_CMD0_OP_BOTH; + break; } - if (enccrd) { - if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) - safe_setup_enckey(ses, enccrd->crd_key); + if (csp->csp_cipher_alg != 0) { + if (crp->crp_cipher_key != NULL) + safe_setup_enckey(ses, crp->crp_cipher_key); - if (enccrd->crd_alg == CRYPTO_DES_CBC) { + switch (csp->csp_cipher_alg) { + case CRYPTO_DES_CBC: cmd0 |= SAFE_SA_CMD0_DES; cmd1 |= SAFE_SA_CMD1_CBC; - ivsize = 2*sizeof(u_int32_t); - } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) { + break; + case CRYPTO_3DES_CBC: cmd0 |= SAFE_SA_CMD0_3DES; cmd1 |= SAFE_SA_CMD1_CBC; - ivsize = 2*sizeof(u_int32_t); - } else if (enccrd->crd_alg == CRYPTO_AES_CBC) { + break; + case CRYPTO_AES_CBC: cmd0 |= SAFE_SA_CMD0_AES; cmd1 |= SAFE_SA_CMD1_CBC; - if (ses->ses_klen == 128) + if (ses->ses_klen * 8 == 128) cmd1 |= SAFE_SA_CMD1_AES128; - else if (ses->ses_klen == 192) + else if (ses->ses_klen * 8 == 192) cmd1 |= SAFE_SA_CMD1_AES192; else cmd1 |= SAFE_SA_CMD1_AES256; - ivsize = 4*sizeof(u_int32_t); - } else { - cmd0 |= SAFE_SA_CMD0_CRYPT_NULL; - ivsize = 0; } /* @@ -932,32 +894,28 @@ safe_process(device_t dev, struct cryptop *crp, int hint) * in the state record and set the hash/crypt offset to * copy both the header+IV. */ - if (enccrd->crd_flags & CRD_F_ENCRYPT) { + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(re->re_sastate.sa_saved_iv, csp->csp_ivlen, 0); + crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, + re->re_sastate.sa_saved_iv); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(re->re_sastate.sa_saved_iv, crp->crp_iv, + csp->csp_ivlen); + else + crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, + re->re_sastate.sa_saved_iv); + cmd0 |= SAFE_SA_CMD0_IVLD_STATE; + + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { cmd0 |= SAFE_SA_CMD0_OUTBOUND; - if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) - iv = enccrd->crd_iv; - else - iv = (caddr_t) ses->ses_iv; - if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { - crypto_copyback(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, ivsize, iv); - } - bcopy(iv, re->re_sastate.sa_saved_iv, ivsize); - cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV; - re->re_flags |= SAFE_QFLAGS_COPYOUTIV; + /* + * XXX: I suspect we don't need this since we + * don't save the returned IV. + */ + cmd0 |= SAFE_SA_CMD0_SAVEIV; } else { cmd0 |= SAFE_SA_CMD0_INBOUND; - - if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) { - bcopy(enccrd->crd_iv, - re->re_sastate.sa_saved_iv, ivsize); - } else { - crypto_copydata(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, ivsize, - (caddr_t)re->re_sastate.sa_saved_iv); - } - cmd0 |= SAFE_SA_CMD0_IVLD_STATE; } /* * For basic encryption use the zero pad algorithm. @@ -973,21 +931,23 @@ safe_process(device_t dev, struct cryptop *crp, int hint) bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key)); } - if (maccrd) { - if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { - safe_setup_mackey(ses, maccrd->crd_alg, - maccrd->crd_key, maccrd->crd_klen / 8); + if (csp->csp_auth_alg != 0) { + if (crp->crp_auth_key != NULL) { + safe_setup_mackey(ses, csp->csp_auth_alg, + crp->crp_auth_key, csp->csp_auth_klen); } - if (maccrd->crd_alg == CRYPTO_MD5_HMAC) { + switch (csp->csp_auth_alg) { + case CRYPTO_MD5_HMAC: cmd0 |= SAFE_SA_CMD0_MD5; cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */ - } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) { + break; + case CRYPTO_SHA1_HMAC: cmd0 |= SAFE_SA_CMD0_SHA1; cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */ - } else { - cmd0 |= SAFE_SA_CMD0_HASH_NULL; + break; } + /* * Digest data is loaded from the SA and the hash * result is saved to the state block where we @@ -1003,38 +963,32 @@ safe_process(device_t dev, struct cryptop *crp, int hint) re->re_flags |= SAFE_QFLAGS_COPYOUTICV; } - if (enccrd && maccrd) { + if (csp->csp_mode == CSP_MODE_ETA) { /* - * The offset from hash data to the start of - * crypt data is the difference in the skips. + * The driver only supports ETA requests where there + * is no gap between the AAD and payload. */ - bypass = maccrd->crd_skip; - coffset = enccrd->crd_skip - maccrd->crd_skip; - if (coffset < 0) { - DPRINTF(("%s: hash does not precede crypt; " - "mac skip %u enc skip %u\n", - __func__, maccrd->crd_skip, enccrd->crd_skip)); - safestats.st_skipmismatch++; - err = EINVAL; - goto errout; - } - oplen = enccrd->crd_skip + enccrd->crd_len; - if (maccrd->crd_skip + maccrd->crd_len != oplen) { - DPRINTF(("%s: hash amount %u != crypt amount %u\n", - __func__, maccrd->crd_skip + maccrd->crd_len, - oplen)); + if (crp->crp_aad_length != 0 && + crp->crp_aad_start + crp->crp_aad_length != + crp->crp_payload_start) { safestats.st_lenmismatch++; err = EINVAL; goto errout; } + if (crp->crp_aad_length != 0) + bypass = crp->crp_aad_start; + else + bypass = crp->crp_payload_start; + coffset = crp->crp_aad_length; + oplen = crp->crp_payload_start + crp->crp_payload_length; #ifdef SAFE_DEBUG if (safe_debug) { - printf("mac: skip %d, len %d, inject %d\n", - maccrd->crd_skip, maccrd->crd_len, - maccrd->crd_inject); - printf("enc: skip %d, len %d, inject %d\n", - enccrd->crd_skip, enccrd->crd_len, - enccrd->crd_inject); + printf("AAD: skip %d, len %d, digest %d\n", + crp->crp_aad_start, crp->crp_aad_length, + crp->crp_digest_start); + printf("payload: skip %d, len %d, IV %d\n", + crp->crp_payload_start, crp->crp_payload_length, + crp->crp_iv_start); printf("bypass %d coffset %d oplen %d\n", bypass, coffset, oplen); } @@ -1070,13 +1024,8 @@ safe_process(device_t dev, struct cryptop *crp, int hint) */ cmd1 |= SAFE_SA_CMD1_MUTABLE; } else { - if (enccrd) { - bypass = enccrd->crd_skip; - oplen = bypass + enccrd->crd_len; - } else { - bypass = maccrd->crd_skip; - oplen = bypass + maccrd->crd_len; - } + bypass = crp->crp_payload_start; + oplen = bypass + crp->crp_payload_length; coffset = 0; } /* XXX verify multiple of 4 when using s/g */ @@ -1092,27 +1041,15 @@ safe_process(device_t dev, struct cryptop *crp, int hint) err = ENOMEM; goto errout; } - if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (bus_dmamap_load_mbuf(sc->sc_srcdmat, re->re_src_map, - re->re_src_m, safe_op_cb, - &re->re_src, BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); - re->re_src_map = NULL; - safestats.st_noload++; - err = ENOMEM; - goto errout; - } - } else if (crp->crp_flags & CRYPTO_F_IOV) { - if (bus_dmamap_load_uio(sc->sc_srcdmat, re->re_src_map, - re->re_src_io, safe_op_cb, - &re->re_src, BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); - re->re_src_map = NULL; - safestats.st_noload++; - err = ENOMEM; - goto errout; - } + if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb, + &re->re_src, BUS_DMA_NOWAIT) != 0) { + bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); + re->re_src_map = NULL; + safestats.st_noload++; + err = ENOMEM; + goto errout; } + re->re_src_mapsize = safe_crp_length(crp); nicealign = safe_dmamap_aligned(&re->re_src); uniform = safe_dmamap_uniform(&re->re_src); @@ -1143,211 +1080,175 @@ safe_process(device_t dev, struct cryptop *crp, int hint) re->re_desc.d_src = re->re_src_segs[0].ds_addr; } - if (enccrd == NULL && maccrd != NULL) { + if (csp->csp_mode == CSP_MODE_DIGEST) { /* * Hash op; no destination needed. */ } else { - if (crp->crp_flags & CRYPTO_F_IOV) { - if (!nicealign) { - safestats.st_iovmisaligned++; - err = EINVAL; + if (nicealign && uniform == 1) { + /* + * Source layout is suitable for direct + * sharing of the DMA map and segment list. + */ + re->re_dst = re->re_src; + } else if (nicealign && uniform == 2) { + /* + * The source is properly aligned but requires a + * different particle list to handle DMA of the + * result. Create a new map and do the load to + * create the segment list. The particle + * descriptor setup code below will handle the + * rest. + */ + if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT, + &re->re_dst_map)) { + safestats.st_nomap++; + err = ENOMEM; goto errout; } - if (uniform != 1) { - /* - * Source is not suitable for direct use as - * the destination. Create a new scatter/gather - * list based on the destination requirements - * and check if that's ok. - */ - if (bus_dmamap_create(sc->sc_dstdmat, - BUS_DMA_NOWAIT, &re->re_dst_map)) { - safestats.st_nomap++; - err = ENOMEM; - goto errout; - } - if (bus_dmamap_load_uio(sc->sc_dstdmat, - re->re_dst_map, re->re_dst_io, - safe_op_cb, &re->re_dst, - BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_dstdmat, - re->re_dst_map); - re->re_dst_map = NULL; - safestats.st_noload++; - err = ENOMEM; - goto errout; - } - uniform = safe_dmamap_uniform(&re->re_dst); - if (!uniform) { - /* - * There's no way to handle the DMA - * requirements with this uio. We - * could create a separate DMA area for - * the result and then copy it back, - * but for now we just bail and return - * an error. Note that uio requests - * > SAFE_MAX_DSIZE are handled because - * the DMA map and segment list for the - * destination wil result in a - * destination particle list that does - * the necessary scatter DMA. - */ - safestats.st_iovnotuniform++; - err = EINVAL; - goto errout; - } - } else - re->re_dst = re->re_src; - } else if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (nicealign && uniform == 1) { - /* - * Source layout is suitable for direct - * sharing of the DMA map and segment list. - */ - re->re_dst = re->re_src; - } else if (nicealign && uniform == 2) { - /* - * The source is properly aligned but requires a - * different particle list to handle DMA of the - * result. Create a new map and do the load to - * create the segment list. The particle - * descriptor setup code below will handle the - * rest. - */ - if (bus_dmamap_create(sc->sc_dstdmat, - BUS_DMA_NOWAIT, &re->re_dst_map)) { - safestats.st_nomap++; - err = ENOMEM; - goto errout; + if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map, + crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) != + 0) { + bus_dmamap_destroy(sc->sc_dstdmat, + re->re_dst_map); + re->re_dst_map = NULL; + safestats.st_noload++; + err = ENOMEM; + goto errout; + } + } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) { + int totlen, len; + struct mbuf *m, *top, **mp; + + /* + * DMA constraints require that we allocate a + * new mbuf chain for the destination. We + * allocate an entire new set of mbufs of + * optimal/required size and then tell the + * hardware to copy any bits that are not + * created as a byproduct of the operation. + */ + if (!nicealign) + safestats.st_unaligned++; + if (!uniform) + safestats.st_notuniform++; + totlen = re->re_src_mapsize; + if (crp->crp_mbuf->m_flags & M_PKTHDR) { + len = MHLEN; + MGETHDR(m, M_NOWAIT, MT_DATA); + if (m && !m_dup_pkthdr(m, crp->crp_mbuf, + M_NOWAIT)) { + m_free(m); + m = NULL; } - if (bus_dmamap_load_mbuf(sc->sc_dstdmat, - re->re_dst_map, re->re_dst_m, - safe_op_cb, &re->re_dst, - BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_dstdmat, - re->re_dst_map); - re->re_dst_map = NULL; - safestats.st_noload++; - err = ENOMEM; + } else { + len = MLEN; + MGET(m, M_NOWAIT, MT_DATA); + } + if (m == NULL) { + safestats.st_nombuf++; + err = sc->sc_nqchip ? ERESTART : ENOMEM; + goto errout; + } + if (totlen >= MINCLSIZE) { + if (!(MCLGET(m, M_NOWAIT))) { + m_free(m); + safestats.st_nomcl++; + err = sc->sc_nqchip ? + ERESTART : ENOMEM; goto errout; } - } else { /* !(aligned and/or uniform) */ - int totlen, len; - struct mbuf *m, *top, **mp; + len = MCLBYTES; + } + m->m_len = len; + top = NULL; + mp = ⊤ - /* - * DMA constraints require that we allocate a - * new mbuf chain for the destination. We - * allocate an entire new set of mbufs of - * optimal/required size and then tell the - * hardware to copy any bits that are not - * created as a byproduct of the operation. - */ - if (!nicealign) - safestats.st_unaligned++; - if (!uniform) - safestats.st_notuniform++; - totlen = re->re_src_mapsize; - if (re->re_src_m->m_flags & M_PKTHDR) { - len = MHLEN; - MGETHDR(m, M_NOWAIT, MT_DATA); - if (m && !m_dup_pkthdr(m, re->re_src_m, - M_NOWAIT)) { - m_free(m); - m = NULL; + while (totlen > 0) { + if (top) { + MGET(m, M_NOWAIT, MT_DATA); + if (m == NULL) { + m_freem(top); + safestats.st_nombuf++; + err = sc->sc_nqchip ? + ERESTART : ENOMEM; + goto errout; } - } else { len = MLEN; - MGET(m, M_NOWAIT, MT_DATA); - } - if (m == NULL) { - safestats.st_nombuf++; - err = sc->sc_nqchip ? ERESTART : ENOMEM; - goto errout; } - if (totlen >= MINCLSIZE) { + if (top && totlen >= MINCLSIZE) { if (!(MCLGET(m, M_NOWAIT))) { - m_free(m); + *mp = m; + m_freem(top); safestats.st_nomcl++; err = sc->sc_nqchip ? - ERESTART : ENOMEM; + ERESTART : ENOMEM; goto errout; } len = MCLBYTES; } - m->m_len = len; - top = NULL; - mp = ⊤ - - while (totlen > 0) { - if (top) { - MGET(m, M_NOWAIT, MT_DATA); - if (m == NULL) { - m_freem(top); - safestats.st_nombuf++; - err = sc->sc_nqchip ? - ERESTART : ENOMEM; - goto errout; - } - len = MLEN; - } - if (top && totlen >= MINCLSIZE) { - if (!(MCLGET(m, M_NOWAIT))) { - *mp = m; - m_freem(top); - safestats.st_nomcl++; - err = sc->sc_nqchip ? - ERESTART : ENOMEM; - goto errout; - } - len = MCLBYTES; - } - m->m_len = len = min(totlen, len); - totlen -= len; - *mp = m; - mp = &m->m_next; - } - re->re_dst_m = top; - if (bus_dmamap_create(sc->sc_dstdmat, - BUS_DMA_NOWAIT, &re->re_dst_map) != 0) { - safestats.st_nomap++; - err = ENOMEM; - goto errout; - } - if (bus_dmamap_load_mbuf(sc->sc_dstdmat, - re->re_dst_map, re->re_dst_m, - safe_op_cb, &re->re_dst, - BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_dstdmat, - re->re_dst_map); - re->re_dst_map = NULL; - safestats.st_noload++; - err = ENOMEM; - goto errout; - } - if (re->re_src.mapsize > oplen) { - /* - * There's data following what the - * hardware will copy for us. If this - * isn't just the ICV (that's going to - * be written on completion), copy it - * to the new mbufs - */ - if (!(maccrd && - (re->re_src.mapsize-oplen) == 12 && - maccrd->crd_inject == oplen)) - safe_mcopy(re->re_src_m, - re->re_dst_m, - oplen); - else - safestats.st_noicvcopy++; - } + m->m_len = len = min(totlen, len); + totlen -= len; + *mp = m; + mp = &m->m_next; + } + re->re_dst_m = top; + if (bus_dmamap_create(sc->sc_dstdmat, + BUS_DMA_NOWAIT, &re->re_dst_map) != 0) { + safestats.st_nomap++; + err = ENOMEM; + goto errout; + } + if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat, + re->re_dst_map, top, re->re_dst_segs, + &re->re_dst_nsegs, 0) != 0) { + bus_dmamap_destroy(sc->sc_dstdmat, + re->re_dst_map); + re->re_dst_map = NULL; + safestats.st_noload++; + err = ENOMEM; + goto errout; + } + re->re_dst_mapsize = re->re_src_mapsize; + if (re->re_src.mapsize > oplen) { + /* + * There's data following what the + * hardware will copy for us. If this + * isn't just the ICV (that's going to + * be written on completion), copy it + * to the new mbufs + */ + if (!(csp->csp_mode == CSP_MODE_ETA && + (re->re_src.mapsize-oplen) == ses->ses_mlen && + crp->crp_digest_start == oplen)) + safe_mcopy(crp->crp_mbuf, re->re_dst_m, + oplen); + else + safestats.st_noicvcopy++; } } else { - safestats.st_badflags++; - err = EINVAL; - goto errout; + if (!nicealign) { + safestats.st_iovmisaligned++; + err = EINVAL; + goto errout; + } else { + /* + * There's no way to handle the DMA + * requirements with this uio. We + * could create a separate DMA area for + * the result and then copy it back, + * but for now we just bail and return + * an error. Note that uio requests + * > SAFE_MAX_DSIZE are handled because + * the DMA map and segment list for the + * destination wil result in a + * destination particle list that does + * the necessary scatter DMA. + */ + safestats.st_iovnotuniform++; + err = EINVAL; + goto errout; + } } if (re->re_dst.nsegs > 1) { @@ -1393,7 +1294,7 @@ safe_process(device_t dev, struct cryptop *crp, int hint) * ready for processing. */ re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI; - if (maccrd) + if (csp->csp_auth_alg != 0) re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL; re->re_desc.d_len = oplen | SAFE_PE_LEN_READY @@ -1412,7 +1313,7 @@ safe_process(device_t dev, struct cryptop *crp, int hint) return (0); errout: - if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m)) + if (re->re_dst_m != NULL) m_freem(re->re_dst_m); if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) { @@ -1436,11 +1337,13 @@ errout: static void safe_callback(struct safe_softc *sc, struct safe_ringentry *re) { + const struct crypto_session_params *csp; struct cryptop *crp = (struct cryptop *)re->re_crp; struct safe_session *ses; - struct cryptodesc *crd; + uint8_t hash[HASH_MAX_LEN]; ses = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); safestats.st_opackets++; safestats.st_obytes += re->re_dst.mapsize; @@ -1454,6 +1357,9 @@ safe_callback(struct safe_softc *sc, struct safe_ringentry *re) safestats.st_peoperr++; crp->crp_etype = EIO; /* something more meaningful? */ } + + /* XXX: Should crp_mbuf be updated to re->re_dst_m if it is non-NULL? */ + if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) { bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map, BUS_DMASYNC_POSTREAD); @@ -1464,58 +1370,29 @@ safe_callback(struct safe_softc *sc, struct safe_ringentry *re) bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map); bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); - /* - * If result was written to a differet mbuf chain, swap - * it in as the return value and reclaim the original. - */ - if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) { - m_freem(re->re_src_m); - crp->crp_buf = (caddr_t)re->re_dst_m; - } - - if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) { - /* copy out IV for future use */ - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - int ivsize; - - if (crd->crd_alg == CRYPTO_DES_CBC || - crd->crd_alg == CRYPTO_3DES_CBC) { - ivsize = 2*sizeof(u_int32_t); - } else if (crd->crd_alg == CRYPTO_AES_CBC) { - ivsize = 4*sizeof(u_int32_t); - } else - continue; - crypto_copydata(crp->crp_flags, crp->crp_buf, - crd->crd_skip + crd->crd_len - ivsize, ivsize, - (caddr_t)ses->ses_iv); - break; - } - } - if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) { - /* copy out ICV result */ - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - if (!(crd->crd_alg == CRYPTO_MD5_HMAC || - crd->crd_alg == CRYPTO_SHA1_HMAC || - crd->crd_alg == CRYPTO_NULL_HMAC)) - continue; - if (crd->crd_alg == CRYPTO_SHA1_HMAC) { - /* - * SHA-1 ICV's are byte-swapped; fix 'em up - * before copy them to their destination. - */ - re->re_sastate.sa_saved_indigest[0] = - bswap32(re->re_sastate.sa_saved_indigest[0]); - re->re_sastate.sa_saved_indigest[1] = - bswap32(re->re_sastate.sa_saved_indigest[1]); - re->re_sastate.sa_saved_indigest[2] = - bswap32(re->re_sastate.sa_saved_indigest[2]); - } - crypto_copyback(crp->crp_flags, crp->crp_buf, - crd->crd_inject, ses->ses_mlen, - (caddr_t)re->re_sastate.sa_saved_indigest); - break; + if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) { + /* + * SHA-1 ICV's are byte-swapped; fix 'em up + * before copying them to their destination. + */ + re->re_sastate.sa_saved_indigest[0] = + bswap32(re->re_sastate.sa_saved_indigest[0]); + re->re_sastate.sa_saved_indigest[1] = + bswap32(re->re_sastate.sa_saved_indigest[1]); + re->re_sastate.sa_saved_indigest[2] = + bswap32(re->re_sastate.sa_saved_indigest[2]); } + + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, + ses->ses_mlen, hash); + if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest, + hash, ses->ses_mlen) != 0) + crp->crp_etype = EBADMSG; + } else + crypto_copyback(crp, crp->crp_digest_start, + ses->ses_mlen, re->re_sastate.sa_saved_indigest); } crypto_done(crp); } @@ -1921,7 +1798,7 @@ safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re) /* * Free header MCR */ - if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m)) + if (re->re_dst_m != NULL) m_freem(re->re_dst_m); crp = (struct cryptop *)re->re_crp; diff --git a/sys/dev/safe/safevar.h b/sys/dev/safe/safevar.h index 024d00564562..e7d238669ed6 100644 --- a/sys/dev/safe/safevar.h +++ b/sys/dev/safe/safevar.h @@ -75,10 +75,6 @@ struct safe_dma_alloc { * where each is mapped for DMA. */ struct safe_operand { - union { - struct mbuf *m; - struct uio *io; - } u; bus_dmamap_t map; bus_size_t mapsize; int nsegs; @@ -109,22 +105,18 @@ struct safe_ringentry { struct safe_operand re_src; /* source operand */ struct safe_operand re_dst; /* destination operand */ + struct mbuf *re_dst_m; int unused; int re_flags; -#define SAFE_QFLAGS_COPYOUTIV 0x1 /* copy back on completion */ #define SAFE_QFLAGS_COPYOUTICV 0x2 /* copy back on completion */ }; -#define re_src_m re_src.u.m -#define re_src_io re_src.u.io #define re_src_map re_src.map #define re_src_nsegs re_src.nsegs #define re_src_segs re_src.segs #define re_src_mapsize re_src.mapsize -#define re_dst_m re_dst.u.m -#define re_dst_io re_dst.u.io #define re_dst_map re_dst.map #define re_dst_nsegs re_dst.nsegs #define re_dst_segs re_dst.segs @@ -138,7 +130,6 @@ struct safe_session { u_int32_t ses_mlen; /* hmac length in bytes */ u_int32_t ses_hminner[5]; /* hmac inner state */ u_int32_t ses_hmouter[5]; /* hmac outer state */ - u_int32_t ses_iv[4]; /* DES/3DES/AES iv */ }; struct safe_softc { @@ -157,6 +148,7 @@ struct safe_softc { int sc_suspended; int sc_needwakeup; /* notify crypto layer */ int32_t sc_cid; /* crypto tag */ + uint32_t sc_devinfo; struct safe_dma_alloc sc_ringalloc; /* PE ring allocation state */ struct safe_ringentry *sc_ring; /* PE ring */ struct safe_ringentry *sc_ringtop; /* PE ring top */ diff --git a/sys/dev/sec/sec.c b/sys/dev/sec/sec.c index 76f808757845..3b3ea0018060 100644 --- a/sys/dev/sec/sec.c +++ b/sys/dev/sec/sec.c @@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$"); #include <machine/resource.h> #include <opencrypto/cryptodev.h> +#include <opencrypto/xform_auth.h> #include "cryptodev_if.h" #include <dev/ofw/ofw_bus_subr.h> @@ -74,7 +75,7 @@ static int sec_init(struct sec_softc *sc); static int sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, bus_size_t size); static int sec_desc_map_dma(struct sec_softc *sc, - struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type, + struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi); static void sec_free_dma_mem(struct sec_dma_mem *dma_mem); static void sec_enqueue(struct sec_softc *sc); @@ -82,48 +83,43 @@ static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel); static int sec_eu_channel(struct sec_softc *sc, int eu); static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, - u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype); + u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize); static int sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); +static int sec_probesession(device_t dev, + const struct crypto_session_params *csp); static int sec_newsession(device_t dev, crypto_session_t cses, - struct cryptoini *cri); + const struct crypto_session_params *csp); static int sec_process(device_t dev, struct cryptop *crp, int hint); -static int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, - struct cryptoini **mac); -static int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, - struct cryptodesc **mac); static int sec_build_common_ns_desc(struct sec_softc *sc, - struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, - struct cryptodesc *enc, int buftype); + struct sec_desc *desc, const struct crypto_session_params *csp, + struct cryptop *crp); static int sec_build_common_s_desc(struct sec_softc *sc, - struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, - struct cryptodesc *enc, struct cryptodesc *mac, int buftype); + struct sec_desc *desc, const struct crypto_session_params *csp, + struct cryptop *crp); static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); /* AESU */ -static int sec_aesu_newsession(struct sec_softc *sc, - struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); +static bool sec_aesu_newsession(const struct crypto_session_params *csp); static int sec_aesu_make_desc(struct sec_softc *sc, - struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, - int buftype); + const struct crypto_session_params *csp, struct sec_desc *desc, + struct cryptop *crp); /* DEU */ -static int sec_deu_newsession(struct sec_softc *sc, - struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); +static bool sec_deu_newsession(const struct crypto_session_params *csp); static int sec_deu_make_desc(struct sec_softc *sc, - struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, - int buftype); + const struct crypto_session_params *csp, struct sec_desc *desc, + struct cryptop *crp); /* MDEU */ -static int sec_mdeu_can_handle(u_int alg); -static int sec_mdeu_config(struct cryptodesc *crd, +static bool sec_mdeu_can_handle(u_int alg); +static int sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode, u_int *hashlen); -static int sec_mdeu_newsession(struct sec_softc *sc, - struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); +static bool sec_mdeu_newsession(const struct crypto_session_params *csp); static int sec_mdeu_make_desc(struct sec_softc *sc, - struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, - int buftype); + const struct crypto_session_params *csp, struct sec_desc *desc, + struct cryptop *crp); static device_method_t sec_methods[] = { /* Device interface */ @@ -136,6 +132,7 @@ static device_method_t sec_methods[] = { DEVMETHOD(device_shutdown, sec_shutdown), /* Crypto methods */ + DEVMETHOD(cryptodev_probesession, sec_probesession), DEVMETHOD(cryptodev_newsession, sec_newsession), DEVMETHOD(cryptodev_process, sec_process), @@ -362,24 +359,6 @@ sec_attach(device_t dev) if (error) goto fail6; - /* Register in OCF (AESU) */ - crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); - - /* Register in OCF (DEU) */ - crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); - - /* Register in OCF (MDEU) */ - crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); - if (sc->sc_version >= 3) { - crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); - } - return (0); fail6: @@ -545,9 +524,12 @@ sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, static void sec_primary_intr(void *arg) { + struct sec_session *ses; struct sec_softc *sc = arg; struct sec_desc *desc; + struct cryptop *crp; uint64_t isr; + uint8_t hash[HASH_MAX_LEN]; int i, wakeup = 0; SEC_LOCK(sc, controller); @@ -595,7 +577,26 @@ sec_primary_intr(void *arg) SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - desc->sd_crp->crp_etype = desc->sd_error; + crp = desc->sd_crp; + crp->crp_etype = desc->sd_error; + if (crp->crp_etype == 0) { + ses = crypto_get_driver_session(crp->crp_session); + if (ses->ss_mlen != 0) { + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, + crp->crp_digest_start, + ses->ss_mlen, hash); + if (timingsafe_bcmp( + desc->sd_desc->shd_digest, + hash, ses->ss_mlen) != 0) + crp->crp_etype = EBADMSG; + } else + crypto_copyback(crp, + crp->crp_digest_start, + ses->ss_mlen, + desc->sd_desc->shd_digest); + } + } crypto_done(desc->sd_crp); SEC_DESC_FREE_POINTERS(desc); @@ -786,14 +787,6 @@ sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, sdmi->sdmi_lt_last = lt; } -static void -sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, - bus_size_t size, int error) -{ - - sec_dma_map_desc_cb(arg, segs, nseg, error); -} - static int sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, bus_size_t size) @@ -851,22 +844,22 @@ err1: } static int -sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, - bus_size_t size, int type, struct sec_desc_map_info *sdmi) +sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, + struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi) { int error; if (dma_mem->dma_vaddr != NULL) return (EBUSY); - switch (type) { - case SEC_MEMORY: + switch (crp->crp_buf_type) { + case CRYPTO_BUF_CONTIG: break; - case SEC_UIO: + case CRYPTO_BUF_UIO: size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; break; - case SEC_MBUF: - size = m_length((struct mbuf*)mem, NULL); + case CRYPTO_BUF_MBUF: + size = m_length(crp->crp_mbuf, NULL); break; default: return (EINVAL); @@ -899,20 +892,8 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, return (error); } - switch (type) { - case SEC_MEMORY: - error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, - mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); - break; - case SEC_UIO: - error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, - mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); - break; - case SEC_MBUF: - error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, - mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); - break; - } + error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp, + sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); if (error) { device_printf(sc->sc_dev, "cannot get address of the DMA" @@ -923,7 +904,7 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, } dma_mem->dma_is_map = 1; - dma_mem->dma_vaddr = mem; + dma_mem->dma_vaddr = crp; return (0); } @@ -1130,7 +1111,7 @@ sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, - u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) + u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize) { struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; struct sec_hw_desc_ptr *ptr; @@ -1138,14 +1119,8 @@ sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, SEC_LOCK_ASSERT(sc, descriptors); - /* For flat memory map only requested region */ - if (dtype == SEC_MEMORY) { - data = (uint8_t*)(data) + doffset; - sdmi.sdmi_offset = 0; - } - - error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, - dtype, &sdmi); + error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize, + &sdmi); if (error) return (error); @@ -1162,115 +1137,116 @@ sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, return (0); } -static int -sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, - struct cryptoini **mac) +static bool +sec_cipher_supported(const struct crypto_session_params *csp) { - struct cryptoini *e, *m; - - e = cri; - m = cri->cri_next; - /* We can haldle only two operations */ - if (m && m->cri_next) - return (EINVAL); - - if (sec_mdeu_can_handle(e->cri_alg)) { - cri = m; - m = e; - e = cri; + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + /* AESU */ + if (csp->csp_ivlen != AES_BLOCK_LEN) + return (false); + break; + case CRYPTO_DES_CBC: + case CRYPTO_3DES_CBC: + /* DEU */ + if (csp->csp_ivlen != DES_BLOCK_LEN) + return (false); + break; + default: + return (false); } - if (m && !sec_mdeu_can_handle(m->cri_alg)) - return (EINVAL); + if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN) + return (false); - *enc = e; - *mac = m; + return (true); +} - return (0); +static bool +sec_auth_supported(struct sec_softc *sc, + const struct crypto_session_params *csp) +{ + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + if (sc->sc_version < 3) + return (false); + /* FALLTHROUGH */ + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: + if (csp->csp_auth_klen > SEC_MAX_KEY_LEN) + return (false); + break; + case CRYPTO_MD5: + case CRYPTO_SHA1: + break; + default: + return (false); + } + return (true); } static int -sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, - struct cryptodesc **mac) +sec_probesession(device_t dev, const struct crypto_session_params *csp) { - struct cryptodesc *e, *m, *t; - - e = crp->crp_desc; - m = e->crd_next; + struct sec_softc *sc = device_get_softc(dev); - /* We can haldle only two operations */ - if (m && m->crd_next) + if (csp->csp_flags != 0) return (EINVAL); - - if (sec_mdeu_can_handle(e->crd_alg)) { - t = m; - m = e; - e = t; - } - - if (m && !sec_mdeu_can_handle(m->crd_alg)) + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + if (!sec_auth_supported(sc, csp)) + return (EINVAL); + break; + case CSP_MODE_CIPHER: + if (!sec_cipher_supported(csp)) + return (EINVAL); + break; + case CSP_MODE_ETA: + if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp)) + return (EINVAL); + break; + default: return (EINVAL); - - *enc = e; - *mac = m; - - return (0); + } + return (CRYPTODEV_PROBE_HARDWARE); } static int -sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) +sec_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) { - struct sec_softc *sc = device_get_softc(dev); struct sec_eu_methods *eu = sec_eus; - struct cryptoini *enc = NULL; - struct cryptoini *mac = NULL; struct sec_session *ses; - int error = -1; - - error = sec_split_cri(cri, &enc, &mac); - if (error) - return (error); - - /* Check key lengths */ - if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) - return (E2BIG); - - if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) - return (E2BIG); - - /* Only SEC 3.0 supports digests larger than 256 bits */ - if (sc->sc_version < 3 && mac && mac->cri_klen > 256) - return (E2BIG); ses = crypto_get_driver_session(cses); /* Find EU for this session */ while (eu->sem_make_desc != NULL) { - error = eu->sem_newsession(sc, ses, enc, mac); - if (error >= 0) + if (eu->sem_newsession(csp)) break; - eu++; } - - /* If not found, return EINVAL */ - if (error < 0) - return (EINVAL); + KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session")); /* Save cipher key */ - if (enc && enc->cri_key) { - ses->ss_klen = enc->cri_klen / 8; - memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); - } + if (csp->csp_cipher_key != NULL) + memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen); /* Save digest key */ - if (mac && mac->cri_key) { - ses->ss_mklen = mac->cri_klen / 8; - memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); + if (csp->csp_auth_key != NULL) + memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen); + + if (csp->csp_auth_alg != 0) { + if (csp->csp_auth_mlen == 0) + ses->ss_mlen = crypto_auth_hash(csp)->hashsize; + else + ses->ss_mlen = csp->csp_auth_mlen; } - ses->ss_eu = eu; return (0); } @@ -1279,11 +1255,12 @@ sec_process(device_t dev, struct cryptop *crp, int hint) { struct sec_softc *sc = device_get_softc(dev); struct sec_desc *desc = NULL; - struct cryptodesc *mac, *enc; + const struct crypto_session_params *csp; struct sec_session *ses; - int buftype, error = 0; + int error = 0; ses = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); /* Check for input length */ if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { @@ -1292,13 +1269,6 @@ sec_process(device_t dev, struct cryptop *crp, int hint) return (0); } - /* Get descriptors */ - if (sec_split_crp(crp, &enc, &mac)) { - crp->crp_etype = EINVAL; - crypto_done(crp); - return (0); - } - SEC_LOCK(sc, descriptors); SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); @@ -1315,56 +1285,29 @@ sec_process(device_t dev, struct cryptop *crp, int hint) desc->sd_error = 0; desc->sd_crp = crp; - if (crp->crp_flags & CRYPTO_F_IOV) - buftype = SEC_UIO; - else if (crp->crp_flags & CRYPTO_F_IMBUF) - buftype = SEC_MBUF; - else - buftype = SEC_MEMORY; - - if (enc && enc->crd_flags & CRD_F_ENCRYPT) { - if (enc->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(desc->sd_desc->shd_iv, enc->crd_iv, - ses->ss_ivlen); - else - arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); - - if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - enc->crd_inject, ses->ss_ivlen, + if (csp->csp_cipher_alg != 0) { + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(desc->sd_desc->shd_iv, csp->csp_ivlen, 0); + crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, desc->sd_desc->shd_iv); - } else if (enc) { - if (enc->crd_flags & CRD_F_IV_EXPLICIT) - memcpy(desc->sd_desc->shd_iv, enc->crd_iv, - ses->ss_ivlen); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(desc->sd_desc->shd_iv, crp->crp_iv, + csp->csp_ivlen); else - crypto_copydata(crp->crp_flags, crp->crp_buf, - enc->crd_inject, ses->ss_ivlen, + crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, desc->sd_desc->shd_iv); } - if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { - if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { - ses->ss_klen = enc->crd_klen / 8; - memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); - } else - error = E2BIG; - } + if (crp->crp_cipher_key != NULL) + memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen); - if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { - if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { - ses->ss_mklen = mac->crd_klen / 8; - memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); - } else - error = E2BIG; - } + if (crp->crp_auth_key != NULL) + memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen); - if (!error) { - memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); - memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); + memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen); + memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen); - error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); - } + error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp); if (error) { SEC_DESC_FREE_POINTERS(desc); @@ -1400,8 +1343,7 @@ sec_process(device_t dev, struct cryptop *crp, int hint) static int sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, - struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, - int buftype) + const struct crypto_session_params *csp, struct cryptop *crp) { struct sec_hw_desc *hd = desc->sd_desc; int error; @@ -1417,25 +1359,25 @@ sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, /* Pointer 1: IV IN */ error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + - offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); + offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); if (error) return (error); /* Pointer 2: Cipher Key */ error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + - offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); + offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); if (error) return (error); /* Pointer 3: Data IN */ - error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, - enc->crd_len, buftype); + error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, + crp->crp_payload_length); if (error) return (error); /* Pointer 4: Data OUT */ - error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, - enc->crd_len, buftype); + error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, + crp->crp_payload_length); if (error) return (error); @@ -1452,20 +1394,13 @@ sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, static int sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, - struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, - struct cryptodesc *mac, int buftype) + const struct crypto_session_params *csp, struct cryptop *crp) { struct sec_hw_desc *hd = desc->sd_desc; u_int eu, mode, hashlen; int error; - if (mac->crd_len < enc->crd_len) - return (EINVAL); - - if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) - return (EINVAL); - - error = sec_mdeu_config(mac, &eu, &mode, &hashlen); + error = sec_mdeu_config(csp, &eu, &mode, &hashlen); if (error) return (error); @@ -1475,144 +1410,107 @@ sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, /* Pointer 0: HMAC Key */ error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + - offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); + offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen); if (error) return (error); /* Pointer 1: HMAC-Only Data IN */ - error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, - mac->crd_len - enc->crd_len, buftype); + error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start, + crp->crp_aad_length); if (error) return (error); /* Pointer 2: Cipher Key */ error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + - offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); + offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); if (error) return (error); /* Pointer 3: IV IN */ error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + - offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); + offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); if (error) return (error); /* Pointer 4: Data IN */ - error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, - enc->crd_len, buftype); + error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, + crp->crp_payload_length); if (error) return (error); /* Pointer 5: Data OUT */ - error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, - enc->crd_len, buftype); + error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start, + crp->crp_payload_length); if (error) return (error); /* Pointer 6: HMAC OUT */ - error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, - hashlen, buftype); + error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr + + offsetof(struct sec_hw_desc, shd_digest), hashlen); return (error); } /* AESU */ -static int -sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, - struct cryptoini *enc, struct cryptoini *mac) +static bool +sec_aesu_newsession(const struct crypto_session_params *csp) { - if (enc == NULL) - return (-1); - - if (enc->cri_alg != CRYPTO_AES_CBC) - return (-1); - - ses->ss_ivlen = AES_BLOCK_LEN; - - return (0); + return (csp->csp_cipher_alg == CRYPTO_AES_CBC); } static int -sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, - struct sec_desc *desc, struct cryptop *crp, int buftype) +sec_aesu_make_desc(struct sec_softc *sc, + const struct crypto_session_params *csp, struct sec_desc *desc, + struct cryptop *crp) { struct sec_hw_desc *hd = desc->sd_desc; - struct cryptodesc *enc, *mac; int error; - error = sec_split_crp(crp, &enc, &mac); - if (error) - return (error); - - if (!enc) - return (EINVAL); - hd->shd_eu_sel0 = SEC_EU_AESU; hd->shd_mode0 = SEC_AESU_MODE_CBC; - if (enc->crd_alg != CRYPTO_AES_CBC) - return (EINVAL); - - if (enc->crd_flags & CRD_F_ENCRYPT) { + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { hd->shd_mode0 |= SEC_AESU_MODE_ED; hd->shd_dir = 0; } else hd->shd_dir = 1; - if (mac) - error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, - buftype); + if (csp->csp_mode == CSP_MODE_ETA) + error = sec_build_common_s_desc(sc, desc, csp, crp); else - error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, - buftype); + error = sec_build_common_ns_desc(sc, desc, csp, crp); return (error); } /* DEU */ -static int -sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, - struct cryptoini *enc, struct cryptoini *mac) +static bool +sec_deu_newsession(const struct crypto_session_params *csp) { - if (enc == NULL) - return (-1); - - switch (enc->cri_alg) { + switch (csp->csp_cipher_alg) { case CRYPTO_DES_CBC: case CRYPTO_3DES_CBC: - break; + return (true); default: - return (-1); + return (false); } - - ses->ss_ivlen = DES_BLOCK_LEN; - - return (0); } static int -sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, - struct sec_desc *desc, struct cryptop *crp, int buftype) +sec_deu_make_desc(struct sec_softc *sc, const struct crypto_session_params *csp, + struct sec_desc *desc, struct cryptop *crp) { struct sec_hw_desc *hd = desc->sd_desc; - struct cryptodesc *enc, *mac; int error; - error = sec_split_crp(crp, &enc, &mac); - if (error) - return (error); - - if (!enc) - return (EINVAL); - hd->shd_eu_sel0 = SEC_EU_DEU; hd->shd_mode0 = SEC_DEU_MODE_CBC; - switch (enc->crd_alg) { + switch (csp->csp_cipher_alg) { case CRYPTO_3DES_CBC: hd->shd_mode0 |= SEC_DEU_MODE_TS; break; @@ -1622,25 +1520,23 @@ sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, return (EINVAL); } - if (enc->crd_flags & CRD_F_ENCRYPT) { + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { hd->shd_mode0 |= SEC_DEU_MODE_ED; hd->shd_dir = 0; } else hd->shd_dir = 1; - if (mac) - error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, - buftype); + if (csp->csp_mode == CSP_MODE_ETA) + error = sec_build_common_s_desc(sc, desc, csp, crp); else - error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, - buftype); + error = sec_build_common_ns_desc(sc, desc, csp, crp); return (error); } /* MDEU */ -static int +static bool sec_mdeu_can_handle(u_int alg) { switch (alg) { @@ -1651,20 +1547,21 @@ sec_mdeu_can_handle(u_int alg) case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: - return (1); + return (true); default: - return (0); + return (false); } } static int -sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) +sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode, + u_int *hashlen) { *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; *eu = SEC_EU_NONE; - switch (crd->crd_alg) { + switch (csp->csp_auth_alg) { case CRYPTO_MD5_HMAC: *mode |= SEC_MDEU_MODE_HMAC; /* FALLTHROUGH */ @@ -1703,34 +1600,23 @@ sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) return (0); } -static int -sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, - struct cryptoini *enc, struct cryptoini *mac) +static bool +sec_mdeu_newsession(const struct crypto_session_params *csp) { - if (mac && sec_mdeu_can_handle(mac->cri_alg)) - return (0); - - return (-1); + return (sec_mdeu_can_handle(csp->csp_auth_alg)); } static int -sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, - struct sec_desc *desc, struct cryptop *crp, int buftype) +sec_mdeu_make_desc(struct sec_softc *sc, + const struct crypto_session_params *csp, + struct sec_desc *desc, struct cryptop *crp) { - struct cryptodesc *enc, *mac; struct sec_hw_desc *hd = desc->sd_desc; u_int eu, mode, hashlen; int error; - error = sec_split_crp(crp, &enc, &mac); - if (error) - return (error); - - if (enc) - return (EINVAL); - - error = sec_mdeu_config(mac, &eu, &mode, &hashlen); + error = sec_mdeu_config(csp, &eu, &mode, &hashlen); if (error) return (error); @@ -1754,7 +1640,7 @@ sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + offsetof(struct sec_hw_desc, - shd_mkey), ses->ss_mklen); + shd_mkey), csp->csp_auth_klen); else error = sec_make_pointer_direct(sc, desc, 2, 0, 0); @@ -1762,8 +1648,8 @@ sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, return (error); /* Pointer 3: Input Data */ - error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, - mac->crd_len, buftype); + error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, + crp->crp_payload_length); if (error) return (error); @@ -1773,8 +1659,8 @@ sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, return (error); /* Pointer 5: Hash out */ - error = sec_make_pointer(sc, desc, 5, crp->crp_buf, - mac->crd_inject, hashlen, buftype); + error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr + + offsetof(struct sec_hw_desc, shd_digest), hashlen); if (error) return (error); diff --git a/sys/dev/sec/sec.h b/sys/dev/sec/sec.h index 05b15039ad64..6ad482316a54 100644 --- a/sys/dev/sec/sec.h +++ b/sys/dev/sec/sec.h @@ -98,6 +98,7 @@ struct sec_hw_desc { uint8_t shd_iv[SEC_MAX_IV_LEN]; uint8_t shd_key[SEC_MAX_KEY_LEN]; uint8_t shd_mkey[SEC_MAX_KEY_LEN]; + uint8_t shd_digest[HASH_MAX_LEN]; } __packed__; #define shd_eu_sel0 shd_control.request.eu_sel0 @@ -144,21 +145,17 @@ struct sec_lt { }; struct sec_eu_methods { - int (*sem_newsession)(struct sec_softc *sc, - struct sec_session *ses, struct cryptoini *enc, - struct cryptoini *mac); + bool (*sem_newsession)(const struct crypto_session_params *csp); int (*sem_make_desc)(struct sec_softc *sc, - struct sec_session *ses, struct sec_desc *desc, - struct cryptop *crp, int buftype); + const struct crypto_session_params *csp, struct sec_desc *desc, + struct cryptop *crp); }; struct sec_session { struct sec_eu_methods *ss_eu; uint8_t ss_key[SEC_MAX_KEY_LEN]; uint8_t ss_mkey[SEC_MAX_KEY_LEN]; - u_int ss_klen; - u_int ss_mklen; - u_int ss_ivlen; + int ss_mlen; }; struct sec_desc_map_info { @@ -319,11 +316,6 @@ struct sec_softc { (((sc)->sc_lt_free_cnt - (sc)->sc_lt_alloc_cnt - 1) \ & (SEC_LT_ENTRIES - 1)) -/* DMA Maping defines */ -#define SEC_MEMORY 0 -#define SEC_UIO 1 -#define SEC_MBUF 2 - /* Size of SEC registers area */ #define SEC_IO_SIZE 0x10000 diff --git a/sys/dev/ubsec/ubsec.c b/sys/dev/ubsec/ubsec.c index 19f46458ac3b..e4b324e05f86 100644 --- a/sys/dev/ubsec/ubsec.c +++ b/sys/dev/ubsec/ubsec.c @@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/sysctl.h> #include <sys/endian.h> +#include <sys/uio.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -70,10 +71,8 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include <sys/rman.h> -#include <crypto/sha1.h> #include <opencrypto/cryptodev.h> -#include <opencrypto/cryptosoft.h> -#include <sys/md5.h> +#include <opencrypto/xform_auth.h> #include <sys/random.h> #include <sys/kobj.h> @@ -111,7 +110,9 @@ static int ubsec_suspend(device_t); static int ubsec_resume(device_t); static int ubsec_shutdown(device_t); -static int ubsec_newsession(device_t, crypto_session_t, struct cryptoini *); +static int ubsec_probesession(device_t, const struct crypto_session_params *); +static int ubsec_newsession(device_t, crypto_session_t, + const struct crypto_session_params *); static int ubsec_process(device_t, struct cryptop *, int); static int ubsec_kprocess(device_t, struct cryptkop *, int); @@ -125,6 +126,7 @@ static device_method_t ubsec_methods[] = { DEVMETHOD(device_shutdown, ubsec_shutdown), /* crypto device methods */ + DEVMETHOD(cryptodev_probesession, ubsec_probesession), DEVMETHOD(cryptodev_newsession, ubsec_newsession), DEVMETHOD(cryptodev_process, ubsec_process), DEVMETHOD(cryptodev_kprocess, ubsec_kprocess), @@ -348,13 +350,6 @@ ubsec_attach(device_t dev) goto bad2; } - sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session), - CRYPTOCAP_F_HARDWARE); - if (sc->sc_cid < 0) { - device_printf(dev, "could not get crypto driver id\n"); - goto bad3; - } - /* * Setup DMA descriptor area. */ @@ -370,7 +365,7 @@ ubsec_attach(device_t dev) NULL, NULL, /* lockfunc, lockarg */ &sc->sc_dmat)) { device_printf(dev, "cannot allocate DMA tag\n"); - goto bad4; + goto bad3; } SIMPLEQ_INIT(&sc->sc_freequeue); dmap = sc->sc_dmaa; @@ -404,11 +399,6 @@ ubsec_attach(device_t dev) device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); - crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); - crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); - /* * Reset Broadcom chip */ @@ -424,6 +414,13 @@ ubsec_attach(device_t dev) */ ubsec_init_board(sc); + sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session), + CRYPTOCAP_F_HARDWARE); + if (sc->sc_cid < 0) { + device_printf(dev, "could not get crypto driver id\n"); + goto bad4; + } + #ifndef UBSEC_NO_RNG if (sc->sc_flags & UBS_FLAGS_RNG) { sc->sc_statmask |= BS_STAT_MCR2_DONE; @@ -477,7 +474,15 @@ skip_rng: } return (0); bad4: - crypto_unregister_all(sc->sc_cid); + while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { + struct ubsec_q *q; + + q = SIMPLEQ_FIRST(&sc->sc_freequeue); + SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next); + ubsec_dma_free(sc, &q->q_dma->d_alloc); + free(q, M_DEVBUF); + } + bus_dma_tag_destroy(sc->sc_dmat); bad3: bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); bad2: @@ -498,13 +503,14 @@ ubsec_detach(device_t dev) /* XXX wait/abort active ops */ + crypto_unregister_all(sc->sc_cid); + /* disable interrupts */ WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~ (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR)); callout_stop(&sc->sc_rngto); - - crypto_unregister_all(sc->sc_cid); + bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); #ifdef UBSEC_RNDTEST if (sc->sc_rndtest) @@ -531,7 +537,6 @@ ubsec_detach(device_t dev) mtx_destroy(&sc->sc_mcr2lock); bus_generic_detach(dev); - bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); bus_dma_tag_destroy(sc->sc_dmat); @@ -826,7 +831,7 @@ feed1: } static void -ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key) +ubsec_setup_enckey(struct ubsec_session *ses, int algo, const void *key) { /* Go ahead and compute key in ubsec's byte order */ @@ -846,112 +851,134 @@ ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key) } static void -ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen) +ubsec_setup_mackey(struct ubsec_session *ses, int algo, const char *key, + int klen) { MD5_CTX md5ctx; SHA1_CTX sha1ctx; - int i; - - for (i = 0; i < klen; i++) - key[i] ^= HMAC_IPAD_VAL; if (algo == CRYPTO_MD5_HMAC) { - MD5Init(&md5ctx); - MD5Update(&md5ctx, key, klen); - MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen); + hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx); bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state)); + + hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx); + bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); + + explicit_bzero(&md5ctx, sizeof(md5ctx)); } else { - SHA1Init(&sha1ctx); - SHA1Update(&sha1ctx, key, klen); - SHA1Update(&sha1ctx, hmac_ipad_buffer, - SHA1_BLOCK_LEN - klen); + hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx); bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); + + hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx); + bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); + + explicit_bzero(&sha1ctx, sizeof(sha1ctx)); } +} - for (i = 0; i < klen; i++) - key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); +static bool +ubsec_auth_supported(const struct crypto_session_params *csp) +{ - if (algo == CRYPTO_MD5_HMAC) { - MD5Init(&md5ctx); - MD5Update(&md5ctx, key, klen); - MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen); - bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state)); - } else { - SHA1Init(&sha1ctx); - SHA1Update(&sha1ctx, key, klen); - SHA1Update(&sha1ctx, hmac_opad_buffer, - SHA1_BLOCK_LEN - klen); - bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); + switch (csp->csp_auth_alg) { + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + return (true); + default: + return (false); } +} + +static bool +ubsec_cipher_supported(const struct crypto_session_params *csp) +{ - for (i = 0; i < klen; i++) - key[i] ^= HMAC_OPAD_VAL; + switch (csp->csp_cipher_alg) { + case CRYPTO_DES_CBC: + case CRYPTO_3DES_CBC: + return (csp->csp_ivlen == 8); + default: + return (false); + } } -/* - * Allocate a new 'session' and return an encoded session id. 'sidp' - * contains our registration id, and should contain an encoded session - * id on successful allocation. - */ static int -ubsec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) +ubsec_probesession(device_t dev, const struct crypto_session_params *csp) { - struct ubsec_softc *sc = device_get_softc(dev); - struct cryptoini *c, *encini = NULL, *macini = NULL; - struct ubsec_session *ses = NULL; - if (cri == NULL || sc == NULL) + if (csp->csp_flags != 0) return (EINVAL); - - for (c = cri; c != NULL; c = c->cri_next) { - if (c->cri_alg == CRYPTO_MD5_HMAC || - c->cri_alg == CRYPTO_SHA1_HMAC) { - if (macini) - return (EINVAL); - macini = c; - } else if (c->cri_alg == CRYPTO_DES_CBC || - c->cri_alg == CRYPTO_3DES_CBC) { - if (encini) - return (EINVAL); - encini = c; - } else + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + if (!ubsec_auth_supported(csp)) return (EINVAL); - } - if (encini == NULL && macini == NULL) + break; + case CSP_MODE_CIPHER: + if (!ubsec_cipher_supported(csp)) + return (EINVAL); + break; + case CSP_MODE_ETA: + if (!ubsec_auth_supported(csp) || + !ubsec_cipher_supported(csp)) + return (EINVAL); + break; + default: return (EINVAL); + } - ses = crypto_get_driver_session(cses); - if (encini) { - /* get an IV, network byte order */ - /* XXX may read fewer than requested */ - read_random(ses->ses_iv, sizeof(ses->ses_iv)); + return (CRYPTODEV_PROBE_HARDWARE); +} - if (encini->cri_key != NULL) { - ubsec_setup_enckey(ses, encini->cri_alg, - encini->cri_key); - } - } +/* + * Allocate a new 'session'. + */ +static int +ubsec_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct ubsec_session *ses; + + ses = crypto_get_driver_session(cses); + if (csp->csp_cipher_alg != 0 && csp->csp_cipher_key != NULL) + ubsec_setup_enckey(ses, csp->csp_cipher_alg, + csp->csp_cipher_key); - if (macini) { - ses->ses_mlen = macini->cri_mlen; + if (csp->csp_auth_alg != 0) { + ses->ses_mlen = csp->csp_auth_mlen; if (ses->ses_mlen == 0) { - if (macini->cri_alg == CRYPTO_MD5_HMAC) + if (csp->csp_auth_alg == CRYPTO_MD5_HMAC) ses->ses_mlen = MD5_HASH_LEN; else ses->ses_mlen = SHA1_HASH_LEN; } - if (macini->cri_key != NULL) { - ubsec_setup_mackey(ses, macini->cri_alg, - macini->cri_key, macini->cri_klen / 8); + if (csp->csp_auth_key != NULL) { + ubsec_setup_mackey(ses, csp->csp_auth_alg, + csp->csp_auth_key, csp->csp_auth_klen); } } return (0); } +static bus_size_t +ubsec_crp_length(struct cryptop *crp) +{ + + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + return (crp->crp_mbuf->m_pkthdr.len); + case CRYPTO_BUF_UIO: + return (crp->crp_uio->uio_resid); + case CRYPTO_BUF_CONTIG: + return (crp->crp_ilen); + default: + panic("bad crp buffer type"); + } +} + static void -ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) +ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error) { struct ubsec_operand *op = arg; @@ -959,12 +986,11 @@ ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, in ("Too many DMA segments returned when mapping operand")); #ifdef UBSEC_DEBUG if (ubsec_debug) - printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n", - (u_int) mapsize, nsegs, error); + printf("ubsec_op_cb: nsegs %d error %d\n", + nsegs, error); #endif if (error != 0) return; - op->mapsize = mapsize; op->nsegs = nsegs; bcopy(seg, op->segs, nsegs * sizeof (seg[0])); } @@ -972,22 +998,17 @@ ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, in static int ubsec_process(device_t dev, struct cryptop *crp, int hint) { + const struct crypto_session_params *csp; struct ubsec_softc *sc = device_get_softc(dev); struct ubsec_q *q = NULL; int err = 0, i, j, nicealign; - struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; - int encoffset = 0, macoffset = 0, cpskip, cpoffset; + int cpskip, cpoffset; int sskip, dskip, stheend, dtheend; int16_t coffset; struct ubsec_session *ses; struct ubsec_pktctx ctx; struct ubsec_dma *dmap = NULL; - if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { - ubsecstats.hst_invalid++; - return (EINVAL); - } - mtx_lock(&sc->sc_freeqlock); if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { ubsecstats.hst_queuefull++; @@ -1006,103 +1027,34 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) q->q_dma = dmap; ses = crypto_get_driver_session(crp->crp_session); - if (crp->crp_flags & CRYPTO_F_IMBUF) { - q->q_src_m = (struct mbuf *)crp->crp_buf; - q->q_dst_m = (struct mbuf *)crp->crp_buf; - } else if (crp->crp_flags & CRYPTO_F_IOV) { - q->q_src_io = (struct uio *)crp->crp_buf; - q->q_dst_io = (struct uio *)crp->crp_buf; - } else { - ubsecstats.hst_badflags++; - err = EINVAL; - goto errout; /* XXX we don't handle contiguous blocks! */ - } - bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); dmap->d_dma->d_mcr.mcr_pkts = htole16(1); dmap->d_dma->d_mcr.mcr_flags = 0; q->q_crp = crp; - crd1 = crp->crp_desc; - if (crd1 == NULL) { - ubsecstats.hst_nodesc++; - err = EINVAL; - goto errout; - } - crd2 = crd1->crd_next; + csp = crypto_get_params(crp->crp_session); - if (crd2 == NULL) { - if (crd1->crd_alg == CRYPTO_MD5_HMAC || - crd1->crd_alg == CRYPTO_SHA1_HMAC) { - maccrd = crd1; - enccrd = NULL; - } else if (crd1->crd_alg == CRYPTO_DES_CBC || - crd1->crd_alg == CRYPTO_3DES_CBC) { - maccrd = NULL; - enccrd = crd1; - } else { - ubsecstats.hst_badalg++; - err = EINVAL; - goto errout; - } - } else { - if ((crd1->crd_alg == CRYPTO_MD5_HMAC || - crd1->crd_alg == CRYPTO_SHA1_HMAC) && - (crd2->crd_alg == CRYPTO_DES_CBC || - crd2->crd_alg == CRYPTO_3DES_CBC) && - ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { - maccrd = crd1; - enccrd = crd2; - } else if ((crd1->crd_alg == CRYPTO_DES_CBC || - crd1->crd_alg == CRYPTO_3DES_CBC) && - (crd2->crd_alg == CRYPTO_MD5_HMAC || - crd2->crd_alg == CRYPTO_SHA1_HMAC) && - (crd1->crd_flags & CRD_F_ENCRYPT)) { - enccrd = crd1; - maccrd = crd2; - } else { - /* - * We cannot order the ubsec as requested - */ - ubsecstats.hst_badalg++; - err = EINVAL; - goto errout; + if (csp->csp_cipher_alg != 0) { + if (crp->crp_cipher_key != NULL) { + ubsec_setup_enckey(ses, csp->csp_cipher_alg, + crp->crp_cipher_key); } - } - if (enccrd) { - if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) { - ubsec_setup_enckey(ses, enccrd->crd_alg, - enccrd->crd_key); - } - - encoffset = enccrd->crd_skip; ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); - if (enccrd->crd_flags & CRD_F_ENCRYPT) { - q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; - - if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(enccrd->crd_iv, ctx.pc_iv, 8); - else { - ctx.pc_iv[0] = ses->ses_iv[0]; - ctx.pc_iv[1] = ses->ses_iv[1]; - } + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(ctx.pc_iv, csp->csp_ivlen, 0); + crypto_copyback(crp, crp->crp_iv_start, + csp->csp_ivlen, ctx.pc_iv); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + memcpy(ctx.pc_iv, crp->crp_iv, csp->csp_ivlen); + else + crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, + ctx.pc_iv); - if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { - crypto_copyback(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); - } - } else { + if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); - - if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(enccrd->crd_iv, ctx.pc_iv, 8); - else { - crypto_copydata(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv); - } } ctx.pc_deskey[0] = ses->ses_deskey[0]; @@ -1115,15 +1067,13 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) SWAP32(ctx.pc_iv[1]); } - if (maccrd) { - if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { - ubsec_setup_mackey(ses, maccrd->crd_alg, - maccrd->crd_key, maccrd->crd_klen / 8); + if (csp->csp_auth_alg != 0) { + if (crp->crp_auth_key != NULL) { + ubsec_setup_mackey(ses, csp->csp_auth_alg, + crp->crp_auth_key, csp->csp_auth_klen); } - macoffset = maccrd->crd_skip; - - if (maccrd->crd_alg == CRYPTO_MD5_HMAC) + if (csp->csp_auth_alg == CRYPTO_MD5_HMAC) ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); else ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); @@ -1137,35 +1087,37 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) } } - if (enccrd && maccrd) { + if (csp->csp_mode == CSP_MODE_ETA) { /* - * ubsec cannot handle packets where the end of encryption - * and authentication are not the same, or where the - * encrypted part begins before the authenticated part. + * ubsec only supports ETA requests where there is no + * gap between the AAD and payload. */ - if ((encoffset + enccrd->crd_len) != - (macoffset + maccrd->crd_len)) { + if (crp->crp_aad_length != 0 && + crp->crp_aad_start + crp->crp_aad_length != + crp->crp_payload_start) { ubsecstats.hst_lenmismatch++; err = EINVAL; goto errout; } - if (enccrd->crd_skip < maccrd->crd_skip) { - ubsecstats.hst_skipmismatch++; - err = EINVAL; - goto errout; + + if (crp->crp_aad_length != 0) { + sskip = crp->crp_aad_start; + } else { + sskip = crp->crp_payload_start; } - sskip = maccrd->crd_skip; - cpskip = dskip = enccrd->crd_skip; - stheend = maccrd->crd_len; - dtheend = enccrd->crd_len; - coffset = enccrd->crd_skip - maccrd->crd_skip; + cpskip = dskip = crp->crp_payload_start; + stheend = crp->crp_aad_length + crp->crp_payload_length; + dtheend = crp->crp_payload_length; + coffset = crp->crp_aad_length; cpoffset = cpskip + dtheend; #ifdef UBSEC_DEBUG if (ubsec_debug) { - printf("mac: skip %d, len %d, inject %d\n", - maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); - printf("enc: skip %d, len %d, inject %d\n", - enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); + printf("AAD: start %d, len %d, digest %d\n", + crp->crp_aad_start, crp->crp_aad_length, + crp->crp_digest_start); + printf("payload: start %d, len %d, IV %d\n", + crp->crp_payload_start, crp->crp_payload_length, + crp->crp_iv_start); printf("src: skip %d, len %d\n", sskip, stheend); printf("dst: skip %d, len %d\n", dskip, dtheend); printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", @@ -1173,8 +1125,8 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) } #endif } else { - cpskip = dskip = sskip = macoffset + encoffset; - dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; + cpskip = dskip = sskip = crp->crp_payload_start; + dtheend = stheend = crp->crp_payload_length; cpoffset = cpskip + dtheend; coffset = 0; } @@ -1185,25 +1137,15 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) err = ENOMEM; goto errout; } - if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, - q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); - q->q_src_map = NULL; - ubsecstats.hst_noload++; - err = ENOMEM; - goto errout; - } - } else if (crp->crp_flags & CRYPTO_F_IOV) { - if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, - q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); - q->q_src_map = NULL; - ubsecstats.hst_noload++; - err = ENOMEM; - goto errout; - } + if (bus_dmamap_load_crp(sc->sc_dmat, q->q_src_map, crp, ubsec_op_cb, + &q->q_src, BUS_DMA_NOWAIT) != 0) { + bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); + q->q_src_map = NULL; + ubsecstats.hst_noload++; + err = ENOMEM; + goto errout; } + q->q_src_mapsize = ubsec_crp_length(crp); nicealign = ubsec_dmamap_aligned(&q->q_src); dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); @@ -1257,7 +1199,7 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) j++; } - if (enccrd == NULL && maccrd != NULL) { + if (csp->csp_mode == CSP_MODE_DIGEST) { dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + @@ -1270,104 +1212,79 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); #endif } else { - if (crp->crp_flags & CRYPTO_F_IOV) { - if (!nicealign) { - ubsecstats.hst_iovmisaligned++; - err = EINVAL; - goto errout; + if (nicealign) { + q->q_dst = q->q_src; + } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) { + int totlen, len; + struct mbuf *m, *top, **mp; + + ubsecstats.hst_unaligned++; + totlen = q->q_src_mapsize; + if (totlen >= MINCLSIZE) { + m = m_getcl(M_NOWAIT, MT_DATA, + crp->crp_mbuf->m_flags & M_PKTHDR); + len = MCLBYTES; + } else if (crp->crp_mbuf->m_flags & M_PKTHDR) { + m = m_gethdr(M_NOWAIT, MT_DATA); + len = MHLEN; + } else { + m = m_get(M_NOWAIT, MT_DATA); + len = MLEN; } - if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, - &q->q_dst_map)) { - ubsecstats.hst_nomap++; - err = ENOMEM; - goto errout; + if (m && crp->crp_mbuf->m_flags & M_PKTHDR && + !m_dup_pkthdr(m, crp->crp_mbuf, M_NOWAIT)) { + m_free(m); + m = NULL; } - if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, - q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); - q->q_dst_map = NULL; - ubsecstats.hst_noload++; - err = ENOMEM; + if (m == NULL) { + ubsecstats.hst_nombuf++; + err = sc->sc_nqueue ? ERESTART : ENOMEM; goto errout; } - } else if (crp->crp_flags & CRYPTO_F_IMBUF) { - if (nicealign) { - q->q_dst = q->q_src; - } else { - int totlen, len; - struct mbuf *m, *top, **mp; + m->m_len = len = min(totlen, len); + totlen -= len; + top = m; + mp = ⊤ - ubsecstats.hst_unaligned++; - totlen = q->q_src_mapsize; + while (totlen > 0) { if (totlen >= MINCLSIZE) { - m = m_getcl(M_NOWAIT, MT_DATA, - q->q_src_m->m_flags & M_PKTHDR); + m = m_getcl(M_NOWAIT, MT_DATA, 0); len = MCLBYTES; - } else if (q->q_src_m->m_flags & M_PKTHDR) { - m = m_gethdr(M_NOWAIT, MT_DATA); - len = MHLEN; } else { m = m_get(M_NOWAIT, MT_DATA); len = MLEN; } - if (m && q->q_src_m->m_flags & M_PKTHDR && - !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) { - m_free(m); - m = NULL; - } if (m == NULL) { + m_freem(top); ubsecstats.hst_nombuf++; err = sc->sc_nqueue ? ERESTART : ENOMEM; goto errout; } m->m_len = len = min(totlen, len); totlen -= len; - top = m; - mp = ⊤ - - while (totlen > 0) { - if (totlen >= MINCLSIZE) { - m = m_getcl(M_NOWAIT, - MT_DATA, 0); - len = MCLBYTES; - } else { - m = m_get(M_NOWAIT, MT_DATA); - len = MLEN; - } - if (m == NULL) { - m_freem(top); - ubsecstats.hst_nombuf++; - err = sc->sc_nqueue ? ERESTART : ENOMEM; - goto errout; - } - m->m_len = len = min(totlen, len); - totlen -= len; - *mp = m; - mp = &m->m_next; - } - q->q_dst_m = top; - ubsec_mcopy(q->q_src_m, q->q_dst_m, - cpskip, cpoffset); - if (bus_dmamap_create(sc->sc_dmat, - BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { - ubsecstats.hst_nomap++; - err = ENOMEM; - goto errout; - } - if (bus_dmamap_load_mbuf(sc->sc_dmat, - q->q_dst_map, q->q_dst_m, - ubsec_op_cb, &q->q_dst, - BUS_DMA_NOWAIT) != 0) { - bus_dmamap_destroy(sc->sc_dmat, - q->q_dst_map); - q->q_dst_map = NULL; - ubsecstats.hst_noload++; - err = ENOMEM; - goto errout; - } + *mp = m; + mp = &m->m_next; + } + q->q_dst_m = top; + ubsec_mcopy(crp->crp_mbuf, q->q_dst_m, cpskip, cpoffset); + if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, + &q->q_dst_map) != 0) { + ubsecstats.hst_nomap++; + err = ENOMEM; + goto errout; + } + if (bus_dmamap_load_mbuf_sg(sc->sc_dmat, + q->q_dst_map, q->q_dst_m, q->q_dst_segs, + &q->q_dst_nsegs, 0) != 0) { + bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); + q->q_dst_map = NULL; + ubsecstats.hst_noload++; + err = ENOMEM; + goto errout; } + q->q_dst_mapsize = q->q_src_mapsize; } else { - ubsecstats.hst_badflags++; + ubsecstats.hst_iovmisaligned++; err = EINVAL; goto errout; } @@ -1414,7 +1331,7 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) pb->pb_len = htole32(packl); if ((i + 1) == q->q_dst_nsegs) { - if (maccrd) + if (csp->csp_auth_alg != 0) pb->pb_next = htole32(dmap->d_alloc.dma_paddr + offsetof(struct ubsec_dmachunk, d_macbuf[0])); else @@ -1465,7 +1382,7 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint) errout: if (q != NULL) { - if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) + if (q->q_dst_m != NULL) m_freem(q->q_dst_m); if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { @@ -1495,12 +1412,14 @@ errout: static void ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) { + const struct crypto_session_params *csp; struct cryptop *crp = (struct cryptop *)q->q_crp; struct ubsec_session *ses; - struct cryptodesc *crd; struct ubsec_dma *dmap = q->q_dma; + char hash[SHA1_HASH_LEN]; ses = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); ubsecstats.hst_opackets++; ubsecstats.hst_obytes += dmap->d_alloc.dma_size; @@ -1517,31 +1436,21 @@ ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) bus_dmamap_unload(sc->sc_dmat, q->q_src_map); bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); - if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { - m_freem(q->q_src_m); - crp->crp_buf = (caddr_t)q->q_dst_m; - } - - /* copy out IV for future use */ - if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - if (crd->crd_alg != CRYPTO_DES_CBC && - crd->crd_alg != CRYPTO_3DES_CBC) - continue; - crypto_copydata(crp->crp_flags, crp->crp_buf, - crd->crd_skip + crd->crd_len - 8, 8, - (caddr_t)ses->ses_iv); - break; - } + if (q->q_dst_m != NULL) { + m_freem(crp->crp_mbuf); + crp->crp_mbuf = q->q_dst_m; } - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - if (crd->crd_alg != CRYPTO_MD5_HMAC && - crd->crd_alg != CRYPTO_SHA1_HMAC) - continue; - crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, - ses->ses_mlen, (caddr_t)dmap->d_dma->d_macbuf); - break; + if (csp->csp_auth_alg != 0) { + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, + ses->ses_mlen, hash); + if (timingsafe_bcmp(dmap->d_dma->d_macbuf, hash, + ses->ses_mlen) != 0) + crp->crp_etype = EBADMSG; + } else + crypto_copyback(crp, crp->crp_digest_start, + ses->ses_mlen, dmap->d_dma->d_macbuf); } mtx_lock(&sc->sc_freeqlock); SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); @@ -1942,7 +1851,7 @@ ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) if(q->q_stacked_mcr[i]) { q2 = q->q_stacked_mcr[i]; - if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) + if (q2->q_dst_m != NULL) m_freem(q2->q_dst_m); crp = (struct cryptop *)q2->q_crp; @@ -1959,7 +1868,7 @@ ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) /* * Free header MCR */ - if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) + if (q->q_dst_m != NULL) m_freem(q->q_dst_m); crp = (struct cryptop *)q->q_crp; diff --git a/sys/dev/ubsec/ubsecvar.h b/sys/dev/ubsec/ubsecvar.h index ae6d5e2cb6bc..b857061be5c0 100644 --- a/sys/dev/ubsec/ubsecvar.h +++ b/sys/dev/ubsec/ubsecvar.h @@ -134,10 +134,6 @@ struct ubsec_dma { #define UBS_FLAGS_RNG 0x10 /* hardware rng */ struct ubsec_operand { - union { - struct mbuf *m; - struct uio *io; - } u; bus_dmamap_t map; bus_size_t mapsize; int nsegs; @@ -153,19 +149,16 @@ struct ubsec_q { struct ubsec_operand q_src; struct ubsec_operand q_dst; + struct mbuf *q_dst_m; int q_flags; }; -#define q_src_m q_src.u.m -#define q_src_io q_src.u.io #define q_src_map q_src.map #define q_src_nsegs q_src.nsegs #define q_src_segs q_src.segs #define q_src_mapsize q_src.mapsize -#define q_dst_m q_dst.u.m -#define q_dst_io q_dst.u.io #define q_dst_map q_dst.map #define q_dst_nsegs q_dst.nsegs #define q_dst_segs q_dst.segs @@ -215,7 +208,6 @@ struct ubsec_session { u_int32_t ses_mlen; /* hmac length */ u_int32_t ses_hminner[5]; /* hmac inner state */ u_int32_t ses_hmouter[5]; /* hmac outer state */ - u_int32_t ses_iv[2]; /* [3]DES iv */ }; #endif /* _KERNEL */ |