From e9fcefce9bb70f20c272a996443928c5f6ab8cd8 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Fri, 23 Oct 2015 19:46:02 +0000 Subject: Import OpenSSL 1.0.2d. --- crypto/Makefile | 8 +- crypto/aes/Makefile | 20 +- crypto/aes/aes_wrap.c | 186 +- crypto/aes/aes_x86core.c | 6 +- crypto/aes/asm/aes-586.pl | 283 +- crypto/aes/asm/aes-armv4.pl | 139 +- crypto/aes/asm/aes-mips.pl | 813 ++- crypto/aes/asm/aes-ppc.pl | 113 +- crypto/aes/asm/aes-x86_64.pl | 250 +- crypto/aes/asm/aesni-mb-x86_64.pl | 1395 +++++ crypto/aes/asm/aesni-sha1-x86_64.pl | 1121 +++- crypto/aes/asm/aesni-sha256-x86_64.pl | 1708 ++++++ crypto/aes/asm/aesni-x86.pl | 644 ++- crypto/aes/asm/aesni-x86_64.pl | 2917 ++++++---- crypto/aes/asm/aesp8-ppc.pl | 1942 +++++++ crypto/aes/asm/aest4-sparcv9.pl | 919 ++++ crypto/aes/asm/aesv8-armx.pl | 989 ++++ crypto/aes/asm/bsaes-armv7.pl | 2469 +++++++++ crypto/aes/asm/bsaes-x86_64.pl | 74 +- crypto/aes/asm/vpaes-ppc.pl | 1512 ++++++ crypto/aes/asm/vpaes-x86.pl | 110 +- crypto/aes/asm/vpaes-x86_64.pl | 100 +- crypto/arm64cpuid.S | 46 + crypto/arm_arch.h | 35 +- crypto/armcap.c | 97 +- crypto/armv4cpuid.S | 109 +- crypto/asn1/Makefile | 5 +- crypto/asn1/a_gentm.c | 46 +- crypto/asn1/a_time.c | 30 + crypto/asn1/a_utctm.c | 92 +- crypto/asn1/ameth_lib.c | 24 +- crypto/asn1/asn1.h | 10 +- crypto/asn1/asn1_locl.h | 3 + crypto/asn1/t_x509.c | 15 + crypto/asn1/x_crl.c | 4 +- crypto/asn1/x_x509.c | 20 + crypto/bio/b_dump.c | 25 + crypto/bio/b_sock.c | 8 +- crypto/bio/bio.h | 8 +- crypto/bio/bio_err.c | 2 +- crypto/bio/bss_acpt.c | 2 +- crypto/bio/bss_conn.c | 2 +- crypto/bio/bss_dgram.c | 84 +- crypto/bio/bss_fd.c | 22 +- crypto/bn/Makefile | 20 +- crypto/bn/asm/armv4-gf2m.pl | 169 +- crypto/bn/asm/armv4-mont.pl | 484 +- crypto/bn/asm/mips-mont.pl | 2 +- crypto/bn/asm/mips.pl | 2 +- crypto/bn/asm/mips3.s | 2201 ++++++++ crypto/bn/asm/modexp512-x86_64.pl | 1497 ------ crypto/bn/asm/ppc-mont.pl | 1 + crypto/bn/asm/ppc.pl | 10 + crypto/bn/asm/ppc64-mont.pl | 660 ++- crypto/bn/asm/rsaz-avx2.pl | 1898 +++++++ crypto/bn/asm/rsaz-x86_64.pl | 2144 ++++++++ crypto/bn/asm/sparct4-mont.pl | 1222 +++++ crypto/bn/asm/sparcv9-gf2m.pl | 190 + crypto/bn/asm/vis3-mont.pl | 373 ++ crypto/bn/asm/x86_64-gcc.c | 110 +- crypto/bn/asm/x86_64-mont.pl | 1289 ++--- crypto/bn/asm/x86_64-mont5.pl | 2908 +++++++++- crypto/bn/bn.h | 18 - crypto/bn/bn_asm.c | 243 +- crypto/bn/bn_exp.c | 326 +- crypto/bn/bn_gf2m.c | 3 +- crypto/bn/bn_lcl.h | 27 + crypto/bn/bntest.c | 3 +- crypto/bn/rsaz_exp.c | 346 ++ crypto/bn/rsaz_exp.h | 56 + crypto/buffer/buf_str.c | 11 + crypto/buffer/buffer.h | 1 + crypto/camellia/Makefile | 2 + crypto/camellia/asm/cmll-x86_64.pl | 4 +- crypto/camellia/asm/cmllt4-sparcv9.pl | 929 ++++ crypto/cast/cast_lcl.h | 2 + crypto/cms/Makefile | 49 +- crypto/cms/cms.h | 52 +- crypto/cms/cms_asn1.c | 86 +- crypto/cms/cms_env.c | 320 +- crypto/cms/cms_err.c | 18 +- crypto/cms/cms_kari.c | 465 ++ crypto/cms/cms_lcl.h | 32 +- crypto/cms/cms_lib.c | 59 +- crypto/cms/cms_sd.c | 205 +- crypto/cms/cms_smime.c | 54 +- crypto/cryptlib.c | 35 +- crypto/cversion.c | 4 + crypto/des/Makefile | 2 + crypto/des/asm/des-586.pl | 4 +- crypto/des/asm/des_enc.m4 | 2 + crypto/des/asm/dest4-sparcv9.pl | 617 +++ crypto/des/des_locl.h | 4 +- crypto/des/read_pwd.c | 2 +- crypto/dh/Makefile | 50 +- crypto/dh/dh.h | 105 + crypto/dh/dh_ameth.c | 519 +- crypto/dh/dh_asn1.c | 98 +- crypto/dh/dh_check.c | 46 +- crypto/dh/dh_err.c | 8 +- crypto/dh/dh_kdf.c | 187 + crypto/dh/dh_key.c | 14 + crypto/dh/dh_pmeth.c | 322 +- crypto/dh/dh_rfc5114.c | 285 + crypto/dh/dhtest.c | 323 ++ crypto/dsa/dsa.h | 3 + crypto/dsa/dsa_ameth.c | 6 +- crypto/dsa/dsa_err.c | 5 +- crypto/dsa/dsa_gen.c | 369 ++ crypto/dsa/dsa_locl.h | 8 + crypto/dsa/dsa_ossl.c | 6 +- crypto/dsa/dsa_pmeth.c | 4 + crypto/ebcdic.c | 2 +- crypto/ec/Makefile | 41 +- crypto/ec/asm/ecp_nistz256-avx2.pl | 2093 ++++++++ crypto/ec/asm/ecp_nistz256-x86_64.pl | 2997 +++++++++++ crypto/ec/ec.h | 91 +- crypto/ec/ec_ameth.c | 340 +- crypto/ec/ec_curve.c | 643 ++- crypto/ec/ec_cvt.c | 12 + crypto/ec/ec_err.c | 15 +- crypto/ec/ec_lcl.h | 25 + crypto/ec/ec_lib.c | 81 +- crypto/ec/ec_pmeth.c | 220 +- crypto/ec/eck_prn.c | 8 + crypto/ec/ecp_nistp521.c | 10 +- crypto/ec/ecp_nistz256.c | 1521 ++++++ crypto/ec/ecp_nistz256_table.c | 9533 +++++++++++++++++++++++++++++++++ crypto/ecdh/Makefile | 12 +- crypto/ecdh/ecdh.h | 7 + crypto/ecdh/ecdhtest.c | 170 + crypto/ecdh/ech_kdf.c | 111 + crypto/ecdh/ech_ossl.c | 10 + crypto/ecdsa/ecdsa.h | 75 + crypto/ecdsa/ecs_err.c | 1 + crypto/ecdsa/ecs_lib.c | 77 + crypto/ecdsa/ecs_locl.h | 6 +- crypto/ecdsa/ecs_ossl.c | 28 +- crypto/engine/Makefile | 18 +- crypto/engine/eng_all.c | 3 - crypto/engine/eng_cryptodev.c | 70 +- crypto/engine/eng_rsax.c | 701 --- crypto/engine/engine.h | 1 - crypto/evp/Makefile | 52 +- crypto/evp/c_allc.c | 11 + crypto/evp/digest.c | 12 + crypto/evp/e_aes.c | 962 +++- crypto/evp/e_aes_cbc_hmac_sha1.c | 460 +- crypto/evp/e_aes_cbc_hmac_sha256.c | 973 ++++ crypto/evp/e_camellia.c | 321 +- crypto/evp/e_des.c | 59 +- crypto/evp/e_des3.c | 273 +- crypto/evp/e_null.c | 3 - crypto/evp/encode.c | 2 +- crypto/evp/evp.h | 56 +- crypto/evp/evp_enc.c | 17 +- crypto/evp/evp_err.c | 5 +- crypto/evp/evp_extra_test.c | 2 +- crypto/evp/evp_fips.c | 310 -- crypto/evp/evp_lib.c | 62 +- crypto/evp/evp_locl.h | 3 + crypto/evp/evp_test.c | 202 +- crypto/evp/evptests.txt | 67 + crypto/evp/m_dss.c | 2 - crypto/evp/m_dss1.c | 3 - crypto/evp/m_ecdsa.c | 2 - crypto/evp/m_sha1.c | 28 +- crypto/evp/m_sigver.c | 44 +- crypto/evp/p_lib.c | 2 +- crypto/evp/pmeth_lib.c | 6 +- crypto/hmac/hm_ameth.c | 2 +- crypto/hmac/hmac.c | 10 + crypto/hmac/hmactest.c | 3 + crypto/jpake/jpake.c | 1 + crypto/md32_common.h | 38 +- crypto/md5/Makefile | 3 + crypto/md5/asm/md5-sparcv9.pl | 430 ++ crypto/md5/md5_locl.h | 2 + crypto/modes/Makefile | 24 +- crypto/modes/asm/aesni-gcm-x86_64.pl | 1057 ++++ crypto/modes/asm/ghash-armv4.pl | 232 +- crypto/modes/asm/ghash-s390x.pl | 6 +- crypto/modes/asm/ghash-sparcv9.pl | 247 +- crypto/modes/asm/ghash-x86.pl | 199 +- crypto/modes/asm/ghash-x86_64.pl | 1149 +++- crypto/modes/asm/ghashp8-ppc.pl | 234 + crypto/modes/asm/ghashv8-armx.pl | 409 ++ crypto/modes/cbc128.c | 2 +- crypto/modes/gcm128.c | 98 +- crypto/modes/modes.h | 10 + crypto/modes/modes_lcl.h | 50 +- crypto/modes/wrap128.c | 138 + crypto/o_str.c | 2 +- crypto/o_time.c | 126 +- crypto/o_time.h | 2 + crypto/objects/obj_dat.h | 234 +- crypto/objects/obj_mac.h | 163 + crypto/objects/obj_mac.num | 38 + crypto/objects/obj_xref.h | 25 + crypto/objects/obj_xref.txt | 12 + crypto/objects/objects.txt | 58 + crypto/objects/objxref.pl | 3 +- crypto/ocsp/ocsp.h | 17 +- crypto/ocsp/ocsp_ht.c | 151 +- crypto/ocsp/ocsp_lib.c | 2 +- crypto/opensslconf.h | 12 + crypto/opensslv.h | 6 +- crypto/ossl_typ.h | 2 + crypto/pem/Makefile | 13 +- crypto/pem/pem.h | 12 +- crypto/pem/pem_all.c | 5 +- crypto/pem/pem_err.c | 4 +- crypto/pem/pem_lib.c | 17 +- crypto/pem/pem_pkey.c | 50 + crypto/perlasm/ppc-xlate.pl | 81 +- crypto/perlasm/sparcv9_modes.pl | 1687 ++++++ crypto/perlasm/x86_64-xlate.pl | 86 +- crypto/perlasm/x86asm.pl | 38 + crypto/perlasm/x86gas.pl | 13 +- crypto/perlasm/x86masm.pl | 4 +- crypto/perlasm/x86nasm.pl | 4 +- crypto/pkcs12/p12_decr.c | 12 +- crypto/pkcs12/p12_p8e.c | 6 +- crypto/ppc_arch.h | 10 + crypto/ppccap.c | 54 +- crypto/ppccpuid.pl | 16 + crypto/rc4/Makefile | 4 +- crypto/rc4/asm/rc4-586.pl | 8 +- crypto/rc4/rc4_enc.c | 2 +- crypto/rc5/rc5_locl.h | 5 +- crypto/rsa/Makefile | 15 +- crypto/rsa/rsa.h | 58 +- crypto/rsa/rsa_ameth.c | 538 +- crypto/rsa/rsa_asn1.c | 8 + crypto/rsa/rsa_err.c | 18 +- crypto/rsa/rsa_oaep.c | 110 +- crypto/rsa/rsa_pmeth.c | 159 +- crypto/rsa/rsa_sign.c | 15 +- crypto/sha/Makefile | 17 +- crypto/sha/asm/sha1-586.pl | 357 +- crypto/sha/asm/sha1-armv4-large.pl | 451 +- crypto/sha/asm/sha1-armv8.pl | 338 ++ crypto/sha/asm/sha1-mb-x86_64.pl | 1574 ++++++ crypto/sha/asm/sha1-mips.pl | 98 +- crypto/sha/asm/sha1-ppc.pl | 46 +- crypto/sha/asm/sha1-sparcv9.pl | 165 +- crypto/sha/asm/sha1-x86_64.pl | 1210 ++++- crypto/sha/asm/sha256-586.pl | 1266 ++++- crypto/sha/asm/sha256-armv4.pl | 644 ++- crypto/sha/asm/sha256-mb-x86_64.pl | 1560 ++++++ crypto/sha/asm/sha512-586.pl | 517 +- crypto/sha/asm/sha512-armv4.pl | 69 +- crypto/sha/asm/sha512-armv8.pl | 422 ++ crypto/sha/asm/sha512-ia64.pl | 351 +- crypto/sha/asm/sha512-mips.pl | 79 +- crypto/sha/asm/sha512-ppc.pl | 460 +- crypto/sha/asm/sha512-sparcv9.pl | 320 +- crypto/sha/asm/sha512-x86_64.pl | 2149 +++++++- crypto/sha/asm/sha512p8-ppc.pl | 424 ++ crypto/sha/sha512.c | 13 + crypto/sparc_arch.h | 101 + crypto/sparccpuid.S | 127 + crypto/sparcv9cap.c | 157 +- crypto/srp/Makefile | 3 + crypto/srp/srptest.c | 1 + crypto/stack/safestack.h | 148 +- crypto/stack/stack.c | 34 + crypto/stack/stack.h | 1 + crypto/symhacks.h | 34 +- crypto/ts/ts_rsp_sign.c | 7 +- crypto/ts/ts_rsp_verify.c | 2 +- crypto/ui/ui_openssl.c | 2 +- crypto/whrlpool/asm/wp-mmx.pl | 42 +- crypto/whrlpool/asm/wp-x86_64.pl | 43 +- crypto/x509/Makefile | 22 +- crypto/x509/verify_extra_test.c | 3 +- crypto/x509/vpm_int.h | 70 + crypto/x509/x509.h | 26 + crypto/x509/x509_cmp.c | 159 +- crypto/x509/x509_err.c | 10 +- crypto/x509/x509_lu.c | 26 + crypto/x509/x509_set.c | 5 + crypto/x509/x509_trs.c | 8 + crypto/x509/x509_txt.c | 20 + crypto/x509/x509_vfy.c | 338 +- crypto/x509/x509_vfy.h | 56 +- crypto/x509/x509_vpm.c | 246 +- crypto/x509/x_all.c | 14 + crypto/x509v3/Makefile | 20 +- crypto/x509v3/ext_dat.h | 3 + crypto/x509v3/v3_lib.c | 22 + crypto/x509v3/v3_purp.c | 29 +- crypto/x509v3/v3_scts.c | 332 ++ crypto/x509v3/v3_utl.c | 427 ++ crypto/x509v3/v3err.c | 9 +- crypto/x509v3/v3nametest.c | 336 ++ crypto/x509v3/x509v3.h | 44 +- crypto/x86_64cpuid.pl | 29 +- crypto/x86cpuid.pl | 31 +- 299 files changed, 78000 insertions(+), 8553 deletions(-) create mode 100755 crypto/aes/asm/aesni-mb-x86_64.pl create mode 100755 crypto/aes/asm/aesni-sha256-x86_64.pl create mode 100755 crypto/aes/asm/aesp8-ppc.pl create mode 100755 crypto/aes/asm/aest4-sparcv9.pl create mode 100755 crypto/aes/asm/aesv8-armx.pl create mode 100755 crypto/aes/asm/bsaes-armv7.pl create mode 100755 crypto/aes/asm/vpaes-ppc.pl create mode 100644 crypto/arm64cpuid.S create mode 100644 crypto/bn/asm/mips3.s delete mode 100755 crypto/bn/asm/modexp512-x86_64.pl create mode 100755 crypto/bn/asm/rsaz-avx2.pl create mode 100755 crypto/bn/asm/rsaz-x86_64.pl create mode 100755 crypto/bn/asm/sparct4-mont.pl create mode 100755 crypto/bn/asm/sparcv9-gf2m.pl create mode 100755 crypto/bn/asm/vis3-mont.pl create mode 100644 crypto/bn/rsaz_exp.c create mode 100644 crypto/bn/rsaz_exp.h create mode 100755 crypto/camellia/asm/cmllt4-sparcv9.pl create mode 100644 crypto/cms/cms_kari.c create mode 100755 crypto/des/asm/dest4-sparcv9.pl create mode 100644 crypto/dh/dh_kdf.c create mode 100644 crypto/dh/dh_rfc5114.c create mode 100755 crypto/ec/asm/ecp_nistz256-avx2.pl create mode 100755 crypto/ec/asm/ecp_nistz256-x86_64.pl create mode 100644 crypto/ec/ecp_nistz256.c create mode 100644 crypto/ec/ecp_nistz256_table.c create mode 100644 crypto/ecdh/ech_kdf.c delete mode 100644 crypto/engine/eng_rsax.c create mode 100644 crypto/evp/e_aes_cbc_hmac_sha256.c delete mode 100644 crypto/evp/evp_fips.c create mode 100755 crypto/md5/asm/md5-sparcv9.pl create mode 100755 crypto/modes/asm/aesni-gcm-x86_64.pl create mode 100755 crypto/modes/asm/ghashp8-ppc.pl create mode 100755 crypto/modes/asm/ghashv8-armx.pl create mode 100644 crypto/modes/wrap128.c create mode 100755 crypto/perlasm/sparcv9_modes.pl create mode 100644 crypto/ppc_arch.h create mode 100755 crypto/sha/asm/sha1-armv8.pl create mode 100755 crypto/sha/asm/sha1-mb-x86_64.pl create mode 100755 crypto/sha/asm/sha256-mb-x86_64.pl create mode 100755 crypto/sha/asm/sha512-armv8.pl create mode 100755 crypto/sha/asm/sha512p8-ppc.pl create mode 100644 crypto/sparc_arch.h create mode 100644 crypto/x509/vpm_int.h create mode 100644 crypto/x509v3/v3_scts.c create mode 100644 crypto/x509v3/v3nametest.c (limited to 'crypto') diff --git a/crypto/Makefile b/crypto/Makefile index 618c95878ce4..7869996a9c07 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -74,9 +74,9 @@ ia64cpuid.s: ia64cpuid.S; $(CC) $(CFLAGS) -E ia64cpuid.S > $@ ppccpuid.s: ppccpuid.pl; $(PERL) ppccpuid.pl $(PERLASM_SCHEME) $@ pariscid.s: pariscid.pl; $(PERL) pariscid.pl $(PERLASM_SCHEME) $@ alphacpuid.s: alphacpuid.pl - (preproc=/tmp/$$$$.$@; trap "rm $$preproc" INT; \ + (preproc=$$$$.$@.S; trap "rm $$preproc" INT; \ $(PERL) alphacpuid.pl > $$preproc && \ - $(CC) -E $$preproc > $@ && rm $$preproc) + $(CC) -E -P $$preproc > $@ && rm $$preproc) testapps: [ -z "$(THIS)" ] || ( if echo $(SDIRS) | fgrep ' des '; \ @@ -88,7 +88,7 @@ subdirs: @target=all; $(RECURSIVE_MAKE) files: - $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO + $(PERL) $(TOP)/util/files.pl "CPUID_OBJ=$(CPUID_OBJ)" Makefile >> $(TOP)/MINFO @target=files; $(RECURSIVE_MAKE) links: @@ -102,7 +102,7 @@ lib: $(LIB) @touch lib $(LIB): $(LIBOBJ) $(AR) $(LIB) $(LIBOBJ) - [ -z "$(FIPSLIBDIR)" ] || $(AR) $(LIB) $(FIPSLIBDIR)fipscanister.o + test -z "$(FIPSLIBDIR)" || $(AR) $(LIB) $(FIPSLIBDIR)fipscanister.o $(RANLIB) $(LIB) || echo Never mind. shared: buildinf.h lib subdirs diff --git a/crypto/aes/Makefile b/crypto/aes/Makefile index b3a95812af3b..e825c140194f 100644 --- a/crypto/aes/Makefile +++ b/crypto/aes/Makefile @@ -65,12 +65,22 @@ aesni-x86_64.s: asm/aesni-x86_64.pl $(PERL) asm/aesni-x86_64.pl $(PERLASM_SCHEME) > $@ aesni-sha1-x86_64.s: asm/aesni-sha1-x86_64.pl $(PERL) asm/aesni-sha1-x86_64.pl $(PERLASM_SCHEME) > $@ +aesni-sha256-x86_64.s: asm/aesni-sha256-x86_64.pl + $(PERL) asm/aesni-sha256-x86_64.pl $(PERLASM_SCHEME) > $@ +aesni-mb-x86_64.s: asm/aesni-mb-x86_64.pl + $(PERL) asm/aesni-mb-x86_64.pl $(PERLASM_SCHEME) > $@ aes-sparcv9.s: asm/aes-sparcv9.pl $(PERL) asm/aes-sparcv9.pl $(CFLAGS) > $@ +aest4-sparcv9.s: asm/aest4-sparcv9.pl ../perlasm/sparcv9_modes.pl + $(PERL) asm/aest4-sparcv9.pl $(CFLAGS) > $@ aes-ppc.s: asm/aes-ppc.pl $(PERL) asm/aes-ppc.pl $(PERLASM_SCHEME) $@ +vpaes-ppc.s: asm/vpaes-ppc.pl + $(PERL) asm/vpaes-ppc.pl $(PERLASM_SCHEME) $@ +aesp8-ppc.s: asm/aesp8-ppc.pl + $(PERL) asm/aesp8-ppc.pl $(PERLASM_SCHEME) $@ aes-parisc.s: asm/aes-parisc.pl $(PERL) asm/aes-parisc.pl $(PERLASM_SCHEME) $@ @@ -78,12 +88,18 @@ aes-parisc.s: asm/aes-parisc.pl aes-mips.S: asm/aes-mips.pl $(PERL) asm/aes-mips.pl $(PERLASM_SCHEME) $@ +aesv8-armx.S: asm/aesv8-armx.pl + $(PERL) asm/aesv8-armx.pl $(PERLASM_SCHEME) $@ +aesv8-armx.o: aesv8-armx.S + # GNU make "catch all" aes-%.S: asm/aes-%.pl; $(PERL) $< $(PERLASM_SCHEME) > $@ aes-armv4.o: aes-armv4.S +bsaes-%.S: asm/bsaes-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +bsaes-armv7.o: bsaes-armv7.S files: - $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO + $(PERL) $(TOP)/util/files.pl "AES_ENC=$(AES_ENC)" Makefile >> $(TOP)/MINFO links: @$(PERL) $(TOP)/util/mklink.pl ../../include/openssl $(EXHEADER) @@ -149,7 +165,7 @@ aes_wrap.o: ../../e_os.h ../../include/openssl/aes.h aes_wrap.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h aes_wrap.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h aes_wrap.o: ../../include/openssl/err.h ../../include/openssl/lhash.h -aes_wrap.o: ../../include/openssl/opensslconf.h +aes_wrap.o: ../../include/openssl/modes.h ../../include/openssl/opensslconf.h aes_wrap.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h aes_wrap.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h aes_wrap.o: ../../include/openssl/symhacks.h ../cryptlib.h aes_wrap.c diff --git a/crypto/aes/aes_wrap.c b/crypto/aes/aes_wrap.c index b1ab8e25ef37..b7b64d57a487 100644 --- a/crypto/aes/aes_wrap.c +++ b/crypto/aes/aes_wrap.c @@ -54,197 +54,19 @@ #include "cryptlib.h" #include -#include - -static const unsigned char default_iv[] = { - 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, -}; +#include int AES_wrap_key(AES_KEY *key, const unsigned char *iv, unsigned char *out, const unsigned char *in, unsigned int inlen) { - unsigned char *A, B[16], *R; - unsigned int i, j, t; - if ((inlen & 0x7) || (inlen < 8)) - return -1; - A = B; - t = 1; - memcpy(out + 8, in, inlen); - if (!iv) - iv = default_iv; - - memcpy(A, iv, 8); - - for (j = 0; j < 6; j++) { - R = out + 8; - for (i = 0; i < inlen; i += 8, t++, R += 8) { - memcpy(B + 8, R, 8); - AES_encrypt(B, B, key); - A[7] ^= (unsigned char)(t & 0xff); - if (t > 0xff) { - A[6] ^= (unsigned char)((t >> 8) & 0xff); - A[5] ^= (unsigned char)((t >> 16) & 0xff); - A[4] ^= (unsigned char)((t >> 24) & 0xff); - } - memcpy(R, B + 8, 8); - } - } - memcpy(out, A, 8); - return inlen + 8; + return CRYPTO_128_wrap(key, iv, out, in, inlen, (block128_f) AES_encrypt); } int AES_unwrap_key(AES_KEY *key, const unsigned char *iv, unsigned char *out, const unsigned char *in, unsigned int inlen) { - unsigned char *A, B[16], *R; - unsigned int i, j, t; - inlen -= 8; - if (inlen & 0x7) - return -1; - if (inlen < 8) - return -1; - A = B; - t = 6 * (inlen >> 3); - memcpy(A, in, 8); - memcpy(out, in + 8, inlen); - for (j = 0; j < 6; j++) { - R = out + inlen - 8; - for (i = 0; i < inlen; i += 8, t--, R -= 8) { - A[7] ^= (unsigned char)(t & 0xff); - if (t > 0xff) { - A[6] ^= (unsigned char)((t >> 8) & 0xff); - A[5] ^= (unsigned char)((t >> 16) & 0xff); - A[4] ^= (unsigned char)((t >> 24) & 0xff); - } - memcpy(B + 8, R, 8); - AES_decrypt(B, B, key); - memcpy(R, B + 8, 8); - } - } - if (!iv) - iv = default_iv; - if (memcmp(A, iv, 8)) { - OPENSSL_cleanse(out, inlen); - return 0; - } - return inlen; -} - -#ifdef AES_WRAP_TEST - -int AES_wrap_unwrap_test(const unsigned char *kek, int keybits, - const unsigned char *iv, - const unsigned char *eout, - const unsigned char *key, int keylen) -{ - unsigned char *otmp = NULL, *ptmp = NULL; - int r, ret = 0; - AES_KEY wctx; - otmp = OPENSSL_malloc(keylen + 8); - ptmp = OPENSSL_malloc(keylen); - if (!otmp || !ptmp) - return 0; - if (AES_set_encrypt_key(kek, keybits, &wctx)) - goto err; - r = AES_wrap_key(&wctx, iv, otmp, key, keylen); - if (r <= 0) - goto err; - - if (eout && memcmp(eout, otmp, keylen)) - goto err; - - if (AES_set_decrypt_key(kek, keybits, &wctx)) - goto err; - r = AES_unwrap_key(&wctx, iv, ptmp, otmp, r); - - if (memcmp(key, ptmp, keylen)) - goto err; - - ret = 1; - - err: - if (otmp) - OPENSSL_free(otmp); - if (ptmp) - OPENSSL_free(ptmp); - - return ret; - + return CRYPTO_128_unwrap(key, iv, out, in, inlen, + (block128_f) AES_decrypt); } - -int main(int argc, char **argv) -{ - - static const unsigned char kek[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f - }; - - static const unsigned char key[] = { - 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, - 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f - }; - - static const unsigned char e1[] = { - 0x1f, 0xa6, 0x8b, 0x0a, 0x81, 0x12, 0xb4, 0x47, - 0xae, 0xf3, 0x4b, 0xd8, 0xfb, 0x5a, 0x7b, 0x82, - 0x9d, 0x3e, 0x86, 0x23, 0x71, 0xd2, 0xcf, 0xe5 - }; - - static const unsigned char e2[] = { - 0x96, 0x77, 0x8b, 0x25, 0xae, 0x6c, 0xa4, 0x35, - 0xf9, 0x2b, 0x5b, 0x97, 0xc0, 0x50, 0xae, 0xd2, - 0x46, 0x8a, 0xb8, 0xa1, 0x7a, 0xd8, 0x4e, 0x5d - }; - - static const unsigned char e3[] = { - 0x64, 0xe8, 0xc3, 0xf9, 0xce, 0x0f, 0x5b, 0xa2, - 0x63, 0xe9, 0x77, 0x79, 0x05, 0x81, 0x8a, 0x2a, - 0x93, 0xc8, 0x19, 0x1e, 0x7d, 0x6e, 0x8a, 0xe7 - }; - - static const unsigned char e4[] = { - 0x03, 0x1d, 0x33, 0x26, 0x4e, 0x15, 0xd3, 0x32, - 0x68, 0xf2, 0x4e, 0xc2, 0x60, 0x74, 0x3e, 0xdc, - 0xe1, 0xc6, 0xc7, 0xdd, 0xee, 0x72, 0x5a, 0x93, - 0x6b, 0xa8, 0x14, 0x91, 0x5c, 0x67, 0x62, 0xd2 - }; - - static const unsigned char e5[] = { - 0xa8, 0xf9, 0xbc, 0x16, 0x12, 0xc6, 0x8b, 0x3f, - 0xf6, 0xe6, 0xf4, 0xfb, 0xe3, 0x0e, 0x71, 0xe4, - 0x76, 0x9c, 0x8b, 0x80, 0xa3, 0x2c, 0xb8, 0x95, - 0x8c, 0xd5, 0xd1, 0x7d, 0x6b, 0x25, 0x4d, 0xa1 - }; - - static const unsigned char e6[] = { - 0x28, 0xc9, 0xf4, 0x04, 0xc4, 0xb8, 0x10, 0xf4, - 0xcb, 0xcc, 0xb3, 0x5c, 0xfb, 0x87, 0xf8, 0x26, - 0x3f, 0x57, 0x86, 0xe2, 0xd8, 0x0e, 0xd3, 0x26, - 0xcb, 0xc7, 0xf0, 0xe7, 0x1a, 0x99, 0xf4, 0x3b, - 0xfb, 0x98, 0x8b, 0x9b, 0x7a, 0x02, 0xdd, 0x21 - }; - - AES_KEY wctx, xctx; - int ret; - ret = AES_wrap_unwrap_test(kek, 128, NULL, e1, key, 16); - fprintf(stderr, "Key test result %d\n", ret); - ret = AES_wrap_unwrap_test(kek, 192, NULL, e2, key, 16); - fprintf(stderr, "Key test result %d\n", ret); - ret = AES_wrap_unwrap_test(kek, 256, NULL, e3, key, 16); - fprintf(stderr, "Key test result %d\n", ret); - ret = AES_wrap_unwrap_test(kek, 192, NULL, e4, key, 24); - fprintf(stderr, "Key test result %d\n", ret); - ret = AES_wrap_unwrap_test(kek, 256, NULL, e5, key, 24); - fprintf(stderr, "Key test result %d\n", ret); - ret = AES_wrap_unwrap_test(kek, 256, NULL, e6, key, 32); - fprintf(stderr, "Key test result %d\n", ret); -} - -#endif diff --git a/crypto/aes/aes_x86core.c b/crypto/aes/aes_x86core.c index 1defbb1abfb2..c869ed719852 100644 --- a/crypto/aes/aes_x86core.c +++ b/crypto/aes/aes_x86core.c @@ -89,8 +89,10 @@ typedef unsigned long long u64; #endif #undef ROTATE -#if defined(_MSC_VER) || defined(__ICC) -# define ROTATE(a,n) _lrotl(a,n) +#if defined(_MSC_VER) +# define ROTATE(a,n) _lrotl(a,n) +#elif defined(__ICC) +# define ROTATE(a,n) _rotl(a,n) #elif defined(__GNUC__) && __GNUC__>=2 # if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) # define ROTATE(a,n) ({ register unsigned int ret; \ diff --git a/crypto/aes/asm/aes-586.pl b/crypto/aes/asm/aes-586.pl index 687ed811be47..451d0e0ed1e2 100755 --- a/crypto/aes/asm/aes-586.pl +++ b/crypto/aes/asm/aes-586.pl @@ -39,7 +39,7 @@ # but exhibits up to 10% improvement on other cores. # # Second version is "monolithic" replacement for aes_core.c, which in -# addition to AES_[de|en]crypt implements private_AES_set_[de|en]cryption_key. +# addition to AES_[de|en]crypt implements AES_set_[de|en]cryption_key. # This made it possible to implement little-endian variant of the # algorithm without modifying the base C code. Motivating factor for # the undertaken effort was that it appeared that in tight IA-32 @@ -103,11 +103,12 @@ # byte for 128-bit key. # # ECB encrypt ECB decrypt CBC large chunk -# P4 56[60] 84[100] 23 -# AMD K8 48[44] 70[79] 18 -# PIII 41[50] 61[91] 24 -# Core 2 32[38] 45[70] 18.5 -# Pentium 120 160 77 +# P4 52[54] 83[95] 23 +# AMD K8 46[41] 66[70] 18 +# PIII 41[50] 60[77] 24 +# Core 2 31[36] 45[64] 18.5 +# Atom 76[100] 96[138] 60 +# Pentium 115 150 77 # # Version 4.1 switches to compact S-box even in key schedule setup. # @@ -242,7 +243,7 @@ $vertical_spin=0; # shift "verticaly" defaults to 0, because of sub encvert() { my ($te,@s) = @_; - my $v0 = $acc, $v1 = $key; + my ($v0,$v1) = ($acc,$key); &mov ($v0,$s[3]); # copy s3 &mov (&DWP(4,"esp"),$s[2]); # save s2 @@ -299,7 +300,7 @@ sub encvert() # Another experimental routine, which features "horizontal spin," but # eliminates one reference to stack. Strangely enough runs slower... sub enchoriz() -{ my $v0 = $key, $v1 = $acc; +{ my ($v0,$v1) = ($key,$acc); &movz ($v0,&LB($s0)); # 3, 2, 1, 0* &rotr ($s2,8); # 8,11,10, 9 @@ -427,7 +428,7 @@ sub sse_encbody() ###################################################################### sub enccompact() -{ my $Fn = mov; +{ my $Fn = \&mov; while ($#_>5) { pop(@_); $Fn=sub{}; } my ($i,$te,@s)=@_; my $tmp = $key; @@ -476,24 +477,25 @@ sub enctransform() my $tmp = $tbl; my $r2 = $key ; - &mov ($acc,$s[$i]); - &and ($acc,0x80808080); - &mov ($tmp,$acc); - &shr ($tmp,7); + &and ($tmp,$s[$i]); &lea ($r2,&DWP(0,$s[$i],$s[$i])); - &sub ($acc,$tmp); + &mov ($acc,$tmp); + &shr ($tmp,7); &and ($r2,0xfefefefe); - &and ($acc,0x1b1b1b1b); + &sub ($acc,$tmp); &mov ($tmp,$s[$i]); + &and ($acc,0x1b1b1b1b); + &rotr ($tmp,16); &xor ($acc,$r2); # r2 + &mov ($r2,$s[$i]); &xor ($s[$i],$acc); # r0 ^ r2 + &rotr ($r2,16+8); + &xor ($acc,$tmp); &rotl ($s[$i],24); - &xor ($s[$i],$acc) # ROTATE(r2^r0,24) ^ r2 - &rotr ($tmp,16); - &xor ($s[$i],$tmp); - &rotr ($tmp,8); - &xor ($s[$i],$tmp); + &xor ($acc,$r2); + &mov ($tmp,0x80808080) if ($i!=1); + &xor ($s[$i],$acc); # ROTATE(r2^r0,24) ^ r2 } &function_begin_B("_x86_AES_encrypt_compact"); @@ -526,6 +528,7 @@ sub enctransform() &enccompact(1,$tbl,$s1,$s2,$s3,$s0,1); &enccompact(2,$tbl,$s2,$s3,$s0,$s1,1); &enccompact(3,$tbl,$s3,$s0,$s1,$s2,1); + &mov ($tbl,0x80808080); &enctransform(2); &enctransform(3); &enctransform(0); @@ -607,82 +610,84 @@ sub sse_enccompact() &pshufw ("mm5","mm4",0x0d); # 15,14,11,10 &movd ("eax","mm1"); # 5, 4, 1, 0 &movd ("ebx","mm5"); # 15,14,11,10 + &mov ($__key,$key); &movz ($acc,&LB("eax")); # 0 - &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 0 - &pshufw ("mm2","mm0",0x0d); # 7, 6, 3, 2 &movz ("edx",&HB("eax")); # 1 + &pshufw ("mm2","mm0",0x0d); # 7, 6, 3, 2 + &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 0 + &movz ($key,&LB("ebx")); # 10 &movz ("edx",&BP(-128,$tbl,"edx",1)); # 1 - &shl ("edx",8); # 1 &shr ("eax",16); # 5, 4 + &shl ("edx",8); # 1 - &movz ($acc,&LB("ebx")); # 10 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 10 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 10 + &movz ($key,&HB("ebx")); # 11 &shl ($acc,16); # 10 - &or ("ecx",$acc); # 10 &pshufw ("mm6","mm4",0x08); # 13,12, 9, 8 - &movz ($acc,&HB("ebx")); # 11 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 11 + &or ("ecx",$acc); # 10 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 11 + &movz ($key,&HB("eax")); # 5 &shl ($acc,24); # 11 - &or ("edx",$acc); # 11 &shr ("ebx",16); # 15,14 + &or ("edx",$acc); # 11 - &movz ($acc,&HB("eax")); # 5 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 5 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 5 + &movz ($key,&HB("ebx")); # 15 &shl ($acc,8); # 5 &or ("ecx",$acc); # 5 - &movz ($acc,&HB("ebx")); # 15 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 15 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 15 + &movz ($key,&LB("eax")); # 4 &shl ($acc,24); # 15 &or ("ecx",$acc); # 15 - &movd ("mm0","ecx"); # t[0] collected - &movz ($acc,&LB("eax")); # 4 - &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 4 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 4 + &movz ($key,&LB("ebx")); # 14 &movd ("eax","mm2"); # 7, 6, 3, 2 - &movz ($acc,&LB("ebx")); # 14 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 14 - &shl ($acc,16); # 14 + &movd ("mm0","ecx"); # t[0] collected + &movz ("ecx",&BP(-128,$tbl,$key,1)); # 14 + &movz ($key,&HB("eax")); # 3 + &shl ("ecx",16); # 14 + &movd ("ebx","mm6"); # 13,12, 9, 8 &or ("ecx",$acc); # 14 - &movd ("ebx","mm6"); # 13,12, 9, 8 - &movz ($acc,&HB("eax")); # 3 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 3 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 3 + &movz ($key,&HB("ebx")); # 9 &shl ($acc,24); # 3 &or ("ecx",$acc); # 3 - &movz ($acc,&HB("ebx")); # 9 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 9 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 9 + &movz ($key,&LB("ebx")); # 8 &shl ($acc,8); # 9 + &shr ("ebx",16); # 13,12 &or ("ecx",$acc); # 9 - &movd ("mm1","ecx"); # t[1] collected - &movz ($acc,&LB("ebx")); # 8 - &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 8 - &shr ("ebx",16); # 13,12 - &movz ($acc,&LB("eax")); # 2 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 2 - &shl ($acc,16); # 2 - &or ("ecx",$acc); # 2 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 8 + &movz ($key,&LB("eax")); # 2 &shr ("eax",16); # 7, 6 + &movd ("mm1","ecx"); # t[1] collected + &movz ("ecx",&BP(-128,$tbl,$key,1)); # 2 + &movz ($key,&HB("eax")); # 7 + &shl ("ecx",16); # 2 + &and ("eax",0xff); # 6 + &or ("ecx",$acc); # 2 &punpckldq ("mm0","mm1"); # t[0,1] collected - &movz ($acc,&HB("eax")); # 7 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 7 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 7 + &movz ($key,&HB("ebx")); # 13 &shl ($acc,24); # 7 - &or ("ecx",$acc); # 7 - &and ("eax",0xff); # 6 + &and ("ebx",0xff); # 12 &movz ("eax",&BP(-128,$tbl,"eax",1)); # 6 + &or ("ecx",$acc); # 7 &shl ("eax",16); # 6 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 13 &or ("edx","eax"); # 6 - &movz ($acc,&HB("ebx")); # 13 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 13 &shl ($acc,8); # 13 - &or ("ecx",$acc); # 13 - &movd ("mm4","ecx"); # t[2] collected - &and ("ebx",0xff); # 12 &movz ("ebx",&BP(-128,$tbl,"ebx",1)); # 12 + &or ("ecx",$acc); # 13 &or ("edx","ebx"); # 12 + &mov ($key,$__key); + &movd ("mm4","ecx"); # t[2] collected &movd ("mm5","edx"); # t[3] collected &punpckldq ("mm4","mm5"); # t[2,3] collected @@ -1222,7 +1227,7 @@ sub enclast() ###################################################################### sub deccompact() -{ my $Fn = mov; +{ my $Fn = \&mov; while ($#_>5) { pop(@_); $Fn=sub{}; } my ($i,$td,@s)=@_; my $tmp = $key; @@ -1270,30 +1275,30 @@ sub dectransform() my $tp4 = @s[($i+3)%4]; $tp4 = @s[3] if ($i==1); my $tp8 = $tbl; - &mov ($acc,$s[$i]); - &and ($acc,0x80808080); - &mov ($tmp,$acc); + &mov ($tmp,0x80808080); + &and ($tmp,$s[$i]); + &mov ($acc,$tmp); &shr ($tmp,7); &lea ($tp2,&DWP(0,$s[$i],$s[$i])); &sub ($acc,$tmp); &and ($tp2,0xfefefefe); &and ($acc,0x1b1b1b1b); - &xor ($acc,$tp2); - &mov ($tp2,$acc); + &xor ($tp2,$acc); + &mov ($tmp,0x80808080); - &and ($acc,0x80808080); - &mov ($tmp,$acc); + &and ($tmp,$tp2); + &mov ($acc,$tmp); &shr ($tmp,7); &lea ($tp4,&DWP(0,$tp2,$tp2)); &sub ($acc,$tmp); &and ($tp4,0xfefefefe); &and ($acc,0x1b1b1b1b); &xor ($tp2,$s[$i]); # tp2^tp1 - &xor ($acc,$tp4); - &mov ($tp4,$acc); + &xor ($tp4,$acc); + &mov ($tmp,0x80808080); - &and ($acc,0x80808080); - &mov ($tmp,$acc); + &and ($tmp,$tp4); + &mov ($acc,$tmp); &shr ($tmp,7); &lea ($tp8,&DWP(0,$tp4,$tp4)); &sub ($acc,$tmp); @@ -1305,13 +1310,13 @@ sub dectransform() &xor ($s[$i],$tp2); &xor ($tp2,$tp8); - &rotl ($tp2,24); &xor ($s[$i],$tp4); &xor ($tp4,$tp8); - &rotl ($tp4,16); + &rotl ($tp2,24); &xor ($s[$i],$tp8); # ^= tp8^(tp4^tp1)^(tp2^tp1) - &rotl ($tp8,8); + &rotl ($tp4,16); &xor ($s[$i],$tp2); # ^= ROTATE(tp8^tp2^tp1,24) + &rotl ($tp8,8); &xor ($s[$i],$tp4); # ^= ROTATE(tp8^tp4^tp1,16) &mov ($s[0],$__s0) if($i==2); #prefetch $s0 &mov ($s[1],$__s1) if($i==3); #prefetch $s1 @@ -1389,85 +1394,87 @@ sub dectransform() sub sse_deccompact() { &pshufw ("mm1","mm0",0x0c); # 7, 6, 1, 0 + &pshufw ("mm5","mm4",0x09); # 13,12,11,10 &movd ("eax","mm1"); # 7, 6, 1, 0 + &movd ("ebx","mm5"); # 13,12,11,10 + &mov ($__key,$key); - &pshufw ("mm5","mm4",0x09); # 13,12,11,10 &movz ($acc,&LB("eax")); # 0 - &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 0 - &movd ("ebx","mm5"); # 13,12,11,10 &movz ("edx",&HB("eax")); # 1 + &pshufw ("mm2","mm0",0x06); # 3, 2, 5, 4 + &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 0 + &movz ($key,&LB("ebx")); # 10 &movz ("edx",&BP(-128,$tbl,"edx",1)); # 1 + &shr ("eax",16); # 7, 6 &shl ("edx",8); # 1 - &pshufw ("mm2","mm0",0x06); # 3, 2, 5, 4 - &movz ($acc,&LB("ebx")); # 10 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 10 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 10 + &movz ($key,&HB("ebx")); # 11 &shl ($acc,16); # 10 + &pshufw ("mm6","mm4",0x03); # 9, 8,15,14 &or ("ecx",$acc); # 10 - &shr ("eax",16); # 7, 6 - &movz ($acc,&HB("ebx")); # 11 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 11 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 11 + &movz ($key,&HB("eax")); # 7 &shl ($acc,24); # 11 - &or ("edx",$acc); # 11 &shr ("ebx",16); # 13,12 + &or ("edx",$acc); # 11 - &pshufw ("mm6","mm4",0x03); # 9, 8,15,14 - &movz ($acc,&HB("eax")); # 7 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 7 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 7 + &movz ($key,&HB("ebx")); # 13 &shl ($acc,24); # 7 &or ("ecx",$acc); # 7 - &movz ($acc,&HB("ebx")); # 13 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 13 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 13 + &movz ($key,&LB("eax")); # 6 &shl ($acc,8); # 13 + &movd ("eax","mm2"); # 3, 2, 5, 4 &or ("ecx",$acc); # 13 - &movd ("mm0","ecx"); # t[0] collected - &movz ($acc,&LB("eax")); # 6 - &movd ("eax","mm2"); # 3, 2, 5, 4 - &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 6 - &shl ("ecx",16); # 6 - &movz ($acc,&LB("ebx")); # 12 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 6 + &movz ($key,&LB("ebx")); # 12 + &shl ($acc,16); # 6 &movd ("ebx","mm6"); # 9, 8,15,14 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 12 + &movd ("mm0","ecx"); # t[0] collected + &movz ("ecx",&BP(-128,$tbl,$key,1)); # 12 + &movz ($key,&LB("eax")); # 4 &or ("ecx",$acc); # 12 - &movz ($acc,&LB("eax")); # 4 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 4 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 4 + &movz ($key,&LB("ebx")); # 14 &or ("edx",$acc); # 4 - &movz ($acc,&LB("ebx")); # 14 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 14 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 14 + &movz ($key,&HB("eax")); # 5 &shl ($acc,16); # 14 + &shr ("eax",16); # 3, 2 &or ("edx",$acc); # 14 - &movd ("mm1","edx"); # t[1] collected - &movz ($acc,&HB("eax")); # 5 - &movz ("edx",&BP(-128,$tbl,$acc,1)); # 5 - &shl ("edx",8); # 5 - &movz ($acc,&HB("ebx")); # 15 - &shr ("eax",16); # 3, 2 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 15 - &shl ($acc,24); # 15 - &or ("edx",$acc); # 15 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 5 + &movz ($key,&HB("ebx")); # 15 &shr ("ebx",16); # 9, 8 + &shl ($acc,8); # 5 + &movd ("mm1","edx"); # t[1] collected + &movz ("edx",&BP(-128,$tbl,$key,1)); # 15 + &movz ($key,&HB("ebx")); # 9 + &shl ("edx",24); # 15 + &and ("ebx",0xff); # 8 + &or ("edx",$acc); # 15 &punpckldq ("mm0","mm1"); # t[0,1] collected - &movz ($acc,&HB("ebx")); # 9 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 9 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 9 + &movz ($key,&LB("eax")); # 2 &shl ($acc,8); # 9 - &or ("ecx",$acc); # 9 - &and ("ebx",0xff); # 8 + &movz ("eax",&HB("eax")); # 3 &movz ("ebx",&BP(-128,$tbl,"ebx",1)); # 8 + &or ("ecx",$acc); # 9 + &movz ($acc,&BP(-128,$tbl,$key,1)); # 2 &or ("edx","ebx"); # 8 - &movz ($acc,&LB("eax")); # 2 - &movz ($acc,&BP(-128,$tbl,$acc,1)); # 2 &shl ($acc,16); # 2 - &or ("edx",$acc); # 2 - &movd ("mm4","edx"); # t[2] collected - &movz ("eax",&HB("eax")); # 3 &movz ("eax",&BP(-128,$tbl,"eax",1)); # 3 + &or ("edx",$acc); # 2 &shl ("eax",24); # 3 &or ("ecx","eax"); # 3 + &mov ($key,$__key); + &movd ("mm4","edx"); # t[2] collected &movd ("mm5","ecx"); # t[3] collected &punpckldq ("mm4","mm5"); # t[2,3] collected @@ -2181,8 +2188,8 @@ my $mark=&DWP(76+240,"esp"); # copy of aes_key->rounds &mov ("ecx",240/4); &xor ("eax","eax"); &align (4); - &data_word(0xABF3F689); # rep stosd - &set_label("skip_ezero") + &data_word(0xABF3F689); # rep stosd + &set_label("skip_ezero"); &mov ("esp",$_esp); &popf (); &set_label("drop_out"); @@ -2301,8 +2308,8 @@ my $mark=&DWP(76+240,"esp"); # copy of aes_key->rounds &mov ("ecx",240/4); &xor ("eax","eax"); &align (4); - &data_word(0xABF3F689); # rep stosd - &set_label("skip_dzero") + &data_word(0xABF3F689); # rep stosd + &set_label("skip_dzero"); &mov ("esp",$_esp); &popf (); &function_end_A(); @@ -2865,32 +2872,32 @@ sub deckey() { my ($i,$key,$tp1,$tp2,$tp4,$tp8) = @_; my $tmp = $tbl; - &mov ($acc,$tp1); - &and ($acc,0x80808080); - &mov ($tmp,$acc); - &shr ($tmp,7); + &mov ($tmp,0x80808080); + &and ($tmp,$tp1); &lea ($tp2,&DWP(0,$tp1,$tp1)); + &mov ($acc,$tmp); + &shr ($tmp,7); &sub ($acc,$tmp); &and ($tp2,0xfefefefe); &and ($acc,0x1b1b1b1b); - &xor ($acc,$tp2); - &mov ($tp2,$acc); + &xor ($tp2,$acc); + &mov ($tmp,0x80808080); - &and ($acc,0x80808080); - &mov ($tmp,$acc); - &shr ($tmp,7); + &and ($tmp,$tp2); &lea ($tp4,&DWP(0,$tp2,$tp2)); + &mov ($acc,$tmp); + &shr ($tmp,7); &sub ($acc,$tmp); &and ($tp4,0xfefefefe); &and ($acc,0x1b1b1b1b); &xor ($tp2,$tp1); # tp2^tp1 - &xor ($acc,$tp4); - &mov ($tp4,$acc); + &xor ($tp4,$acc); + &mov ($tmp,0x80808080); - &and ($acc,0x80808080); - &mov ($tmp,$acc); - &shr ($tmp,7); + &and ($tmp,$tp4); &lea ($tp8,&DWP(0,$tp4,$tp4)); + &mov ($acc,$tmp); + &shr ($tmp,7); &xor ($tp4,$tp1); # tp4^tp1 &sub ($acc,$tmp); &and ($tp8,0xfefefefe); diff --git a/crypto/aes/asm/aes-armv4.pl b/crypto/aes/asm/aes-armv4.pl index 86b86c4a0fbd..4f8917089f6c 100755 --- a/crypto/aes/asm/aes-armv4.pl +++ b/crypto/aes/asm/aes-armv4.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -51,9 +51,23 @@ $key="r11"; $rounds="r12"; $code=<<___; -#include "arm_arch.h" +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +#endif + .text +#if __ARM_ARCH__<7 +.code 32 +#else +.syntax unified +# ifdef __thumb2__ +.thumb +# else .code 32 +# endif +#endif .type AES_Te,%object .align 5 @@ -167,7 +181,11 @@ AES_Te: .type AES_encrypt,%function .align 5 AES_encrypt: +#if __ARM_ARCH__<7 sub r3,pc,#8 @ AES_encrypt +#else + adr r3,AES_encrypt +#endif stmdb sp!,{r1,r4-r12,lr} mov $rounds,r0 @ inp mov $key,r2 @@ -409,11 +427,21 @@ _armv4_AES_encrypt: .align 5 private_AES_set_encrypt_key: _armv4_AES_set_encrypt_key: +#if __ARM_ARCH__<7 sub r3,pc,#8 @ AES_set_encrypt_key +#else + adr r3,private_AES_set_encrypt_key +#endif teq r0,#0 +#if __ARM_ARCH__>=7 + itt eq @ Thumb2 thing, sanity check in ARM +#endif moveq r0,#-1 beq .Labrt teq r2,#0 +#if __ARM_ARCH__>=7 + itt eq @ Thumb2 thing, sanity check in ARM +#endif moveq r0,#-1 beq .Labrt @@ -422,6 +450,9 @@ _armv4_AES_set_encrypt_key: teq r1,#192 beq .Lok teq r1,#256 +#if __ARM_ARCH__>=7 + itt ne @ Thumb2 thing, sanity check in ARM +#endif movne r0,#-1 bne .Labrt @@ -576,6 +607,9 @@ _armv4_AES_set_encrypt_key: str $s2,[$key,#-16] subs $rounds,$rounds,#1 str $s3,[$key,#-12] +#if __ARM_ARCH__>=7 + itt eq @ Thumb2 thing, sanity check in ARM +#endif subeq r2,$key,#216 beq .Ldone @@ -645,6 +679,9 @@ _armv4_AES_set_encrypt_key: str $s2,[$key,#-24] subs $rounds,$rounds,#1 str $s3,[$key,#-20] +#if __ARM_ARCH__>=7 + itt eq @ Thumb2 thing, sanity check in ARM +#endif subeq r2,$key,#256 beq .Ldone @@ -674,11 +711,17 @@ _armv4_AES_set_encrypt_key: str $i3,[$key,#-4] b .L256_loop +.align 2 .Ldone: mov r0,#0 ldmia sp!,{r4-r12,lr} -.Labrt: tst lr,#1 +.Labrt: +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .size private_AES_set_encrypt_key,.-private_AES_set_encrypt_key .global private_AES_set_decrypt_key @@ -688,34 +731,57 @@ private_AES_set_decrypt_key: str lr,[sp,#-4]! @ push lr bl _armv4_AES_set_encrypt_key teq r0,#0 - ldrne lr,[sp],#4 @ pop lr + ldr lr,[sp],#4 @ pop lr bne .Labrt - stmdb sp!,{r4-r12} + mov r0,r2 @ AES_set_encrypt_key preserves r2, + mov r1,r2 @ which is AES_KEY *key + b _armv4_AES_set_enc2dec_key +.size private_AES_set_decrypt_key,.-private_AES_set_decrypt_key - ldr $rounds,[r2,#240] @ AES_set_encrypt_key preserves r2, - mov $key,r2 @ which is AES_KEY *key - mov $i1,r2 - add $i2,r2,$rounds,lsl#4 +@ void AES_set_enc2dec_key(const AES_KEY *inp,AES_KEY *out) +.global AES_set_enc2dec_key +.type AES_set_enc2dec_key,%function +.align 5 +AES_set_enc2dec_key: +_armv4_AES_set_enc2dec_key: + stmdb sp!,{r4-r12,lr} + + ldr $rounds,[r0,#240] + mov $i1,r0 @ input + add $i2,r0,$rounds,lsl#4 + mov $key,r1 @ ouput + add $tbl,r1,$rounds,lsl#4 + str $rounds,[r1,#240] + +.Linv: ldr $s0,[$i1],#16 + ldr $s1,[$i1,#-12] + ldr $s2,[$i1,#-8] + ldr $s3,[$i1,#-4] + ldr $t1,[$i2],#-16 + ldr $t2,[$i2,#16+4] + ldr $t3,[$i2,#16+8] + ldr $i3,[$i2,#16+12] + str $s0,[$tbl],#-16 + str $s1,[$tbl,#16+4] + str $s2,[$tbl,#16+8] + str $s3,[$tbl,#16+12] + str $t1,[$key],#16 + str $t2,[$key,#-12] + str $t3,[$key,#-8] + str $i3,[$key,#-4] + teq $i1,$i2 + bne .Linv -.Linv: ldr $s0,[$i1] + ldr $s0,[$i1] ldr $s1,[$i1,#4] ldr $s2,[$i1,#8] ldr $s3,[$i1,#12] - ldr $t1,[$i2] - ldr $t2,[$i2,#4] - ldr $t3,[$i2,#8] - ldr $i3,[$i2,#12] - str $s0,[$i2],#-16 - str $s1,[$i2,#16+4] - str $s2,[$i2,#16+8] - str $s3,[$i2,#16+12] - str $t1,[$i1],#16 - str $t2,[$i1,#-12] - str $t3,[$i1,#-8] - str $i3,[$i1,#-4] - teq $i1,$i2 - bne .Linv + str $s0,[$key] + str $s1,[$key,#4] + str $s2,[$key,#8] + str $s3,[$key,#12] + sub $key,$key,$rounds,lsl#3 ___ $mask80=$i1; $mask1b=$i2; @@ -773,7 +839,7 @@ $code.=<<___; moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif -.size private_AES_set_decrypt_key,.-private_AES_set_decrypt_key +.size AES_set_enc2dec_key,.-AES_set_enc2dec_key .type AES_Td,%object .align 5 @@ -883,7 +949,11 @@ AES_Td: .type AES_decrypt,%function .align 5 AES_decrypt: +#if __ARM_ARCH__<7 sub r3,pc,#8 @ AES_decrypt +#else + adr r3,AES_decrypt +#endif stmdb sp!,{r1,r4-r12,lr} mov $rounds,r0 @ inp mov $key,r2 @@ -1080,8 +1150,9 @@ _armv4_AES_decrypt: ldrb $t3,[$tbl,$i3] @ Td4[s0>>0] and $i3,lr,$s1,lsr#8 + add $s1,$tbl,$s1,lsr#24 ldrb $i1,[$tbl,$i1] @ Td4[s1>>0] - ldrb $s1,[$tbl,$s1,lsr#24] @ Td4[s1>>24] + ldrb $s1,[$s1] @ Td4[s1>>24] ldrb $i2,[$tbl,$i2] @ Td4[s1>>16] eor $s0,$i1,$s0,lsl#24 ldrb $i3,[$tbl,$i3] @ Td4[s1>>8] @@ -1094,7 +1165,8 @@ _armv4_AES_decrypt: ldrb $i2,[$tbl,$i2] @ Td4[s2>>0] and $i3,lr,$s2,lsr#16 - ldrb $s2,[$tbl,$s2,lsr#24] @ Td4[s2>>24] + add $s2,$tbl,$s2,lsr#24 + ldrb $s2,[$s2] @ Td4[s2>>24] eor $s0,$s0,$i1,lsl#8 ldrb $i3,[$tbl,$i3] @ Td4[s2>>16] eor $s1,$i2,$s1,lsl#16 @@ -1106,8 +1178,9 @@ _armv4_AES_decrypt: ldrb $i2,[$tbl,$i2] @ Td4[s3>>8] and $i3,lr,$s3 @ i2 + add $s3,$tbl,$s3,lsr#24 ldrb $i3,[$tbl,$i3] @ Td4[s3>>0] - ldrb $s3,[$tbl,$s3,lsr#24] @ Td4[s3>>24] + ldrb $s3,[$s3] @ Td4[s3>>24] eor $s0,$s0,$i1,lsl#16 ldr $i1,[$key,#0] eor $s1,$s1,$i2,lsl#8 @@ -1130,5 +1203,15 @@ _armv4_AES_decrypt: ___ $code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 +$code =~ s/\bret\b/bx\tlr/gm; + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + print $code; close STDOUT; # enforce flush diff --git a/crypto/aes/asm/aes-mips.pl b/crypto/aes/asm/aes-mips.pl index 537c8d3172b6..4de3ee26bb74 100755 --- a/crypto/aes/asm/aes-mips.pl +++ b/crypto/aes/asm/aes-mips.pl @@ -20,6 +20,13 @@ # thing about this module is its endian neutrality, which means that # it processes data without ever changing byte order... +# September 2012 +# +# Add MIPS32R2 (~10% less instructions) and SmartMIPS ASE (further +# ~25% less instructions) code. Note that there is no run-time switch, +# instead, code path is chosen upon pre-process time, pass -mips32r2 +# or/and -msmartmips. + ###################################################################### # There is a number of MIPS ABI in use, O32 and N32/64 are most # widely used. Then there is a new contender: NUBI. It appears that if @@ -47,11 +54,12 @@ # ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); # ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); # -$flavour = shift; # supported flavours are o32,n32,64,nubi32,nubi64 +$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64 if ($flavour =~ /64|n32/i) { $PTR_ADD="dadd"; # incidentally works even on n32 $PTR_SUB="dsub"; # incidentally works even on n32 + $PTR_INS="dins"; $REG_S="sd"; $REG_L="ld"; $PTR_SLL="dsll"; # incidentally works even on n32 @@ -59,6 +67,7 @@ if ($flavour =~ /64|n32/i) { } else { $PTR_ADD="add"; $PTR_SUB="sub"; + $PTR_INS="ins"; $REG_S="sw"; $REG_L="lw"; $PTR_SLL="sll"; @@ -89,7 +98,11 @@ $code.=<<___; # include #endif -#if !defined(__vxworks) || defined(__pic__) +#if defined(__mips_smartmips) && !defined(_MIPS_ARCH_MIPS32R2) +#define _MIPS_ARCH_MIPS32R2 +#endif + +#if !defined(__mips_eabi) && (!defined(__vxworks) || defined(__pic__)) .option pic2 #endif .set noat @@ -125,6 +138,89 @@ _mips_AES_encrypt: xor $s3,$t3 sub $cnt,1 +#if defined(__mips_smartmips) + ext $i0,$s1,16,8 +.Loop_enc: + ext $i1,$s2,16,8 + ext $i2,$s3,16,8 + ext $i3,$s0,16,8 + lwxs $t0,$i0($Tbl) # Te1[s1>>16] + ext $i0,$s2,8,8 + lwxs $t1,$i1($Tbl) # Te1[s2>>16] + ext $i1,$s3,8,8 + lwxs $t2,$i2($Tbl) # Te1[s3>>16] + ext $i2,$s0,8,8 + lwxs $t3,$i3($Tbl) # Te1[s0>>16] + ext $i3,$s1,8,8 + + lwxs $t4,$i0($Tbl) # Te2[s2>>8] + ext $i0,$s3,0,8 + lwxs $t5,$i1($Tbl) # Te2[s3>>8] + ext $i1,$s0,0,8 + lwxs $t6,$i2($Tbl) # Te2[s0>>8] + ext $i2,$s1,0,8 + lwxs $t7,$i3($Tbl) # Te2[s1>>8] + ext $i3,$s2,0,8 + + lwxs $t8,$i0($Tbl) # Te3[s3] + ext $i0,$s0,24,8 + lwxs $t9,$i1($Tbl) # Te3[s0] + ext $i1,$s1,24,8 + lwxs $t10,$i2($Tbl) # Te3[s1] + ext $i2,$s2,24,8 + lwxs $t11,$i3($Tbl) # Te3[s2] + ext $i3,$s3,24,8 + + rotr $t0,$t0,8 + rotr $t1,$t1,8 + rotr $t2,$t2,8 + rotr $t3,$t3,8 + + rotr $t4,$t4,16 + rotr $t5,$t5,16 + rotr $t6,$t6,16 + rotr $t7,$t7,16 + + xor $t0,$t4 + lwxs $t4,$i0($Tbl) # Te0[s0>>24] + xor $t1,$t5 + lwxs $t5,$i1($Tbl) # Te0[s1>>24] + xor $t2,$t6 + lwxs $t6,$i2($Tbl) # Te0[s2>>24] + xor $t3,$t7 + lwxs $t7,$i3($Tbl) # Te0[s3>>24] + + rotr $t8,$t8,24 + lw $s0,0($key0) + rotr $t9,$t9,24 + lw $s1,4($key0) + rotr $t10,$t10,24 + lw $s2,8($key0) + rotr $t11,$t11,24 + lw $s3,12($key0) + + xor $t0,$t8 + xor $t1,$t9 + xor $t2,$t10 + xor $t3,$t11 + + xor $t0,$t4 + xor $t1,$t5 + xor $t2,$t6 + xor $t3,$t7 + + sub $cnt,1 + $PTR_ADD $key0,16 + xor $s0,$t0 + xor $s1,$t1 + xor $s2,$t2 + xor $s3,$t3 + .set noreorder + bnez $cnt,.Loop_enc + ext $i0,$s1,16,8 + + _xtr $i0,$s1,16-2 +#else _xtr $i0,$s1,16-2 .Loop_enc: _xtr $i1,$s2,16-2 @@ -138,19 +234,29 @@ _mips_AES_encrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + lw $t0,0($i0) # Te1[s1>>16] + _xtr $i0,$s2,8-2 + lw $t1,0($i1) # Te1[s2>>16] + _xtr $i1,$s3,8-2 + lw $t2,0($i2) # Te1[s3>>16] + _xtr $i2,$s0,8-2 + lw $t3,0($i3) # Te1[s0>>16] + _xtr $i3,$s1,8-2 +#else lwl $t0,3($i0) # Te1[s1>>16] lwl $t1,3($i1) # Te1[s2>>16] lwl $t2,3($i2) # Te1[s3>>16] lwl $t3,3($i3) # Te1[s0>>16] lwr $t0,2($i0) # Te1[s1>>16] - lwr $t1,2($i1) # Te1[s2>>16] - lwr $t2,2($i2) # Te1[s3>>16] - lwr $t3,2($i3) # Te1[s0>>16] - _xtr $i0,$s2,8-2 + lwr $t1,2($i1) # Te1[s2>>16] _xtr $i1,$s3,8-2 + lwr $t2,2($i2) # Te1[s3>>16] _xtr $i2,$s0,8-2 + lwr $t3,2($i3) # Te1[s0>>16] _xtr $i3,$s1,8-2 +#endif and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -159,19 +265,88 @@ _mips_AES_encrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + rotr $t0,$t0,8 + rotr $t1,$t1,8 + rotr $t2,$t2,8 + rotr $t3,$t3,8 +# if defined(_MIPSEL) + lw $t4,0($i0) # Te2[s2>>8] + _xtr $i0,$s3,0-2 + lw $t5,0($i1) # Te2[s3>>8] + _xtr $i1,$s0,0-2 + lw $t6,0($i2) # Te2[s0>>8] + _xtr $i2,$s1,0-2 + lw $t7,0($i3) # Te2[s1>>8] + _xtr $i3,$s2,0-2 + + and $i0,0x3fc + and $i1,0x3fc + and $i2,0x3fc + and $i3,0x3fc + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl + lw $t8,0($i0) # Te3[s3] + $PTR_INS $i0,$s0,2,8 + lw $t9,0($i1) # Te3[s0] + $PTR_INS $i1,$s1,2,8 + lw $t10,0($i2) # Te3[s1] + $PTR_INS $i2,$s2,2,8 + lw $t11,0($i3) # Te3[s2] + $PTR_INS $i3,$s3,2,8 +# else + lw $t4,0($i0) # Te2[s2>>8] + $PTR_INS $i0,$s3,2,8 + lw $t5,0($i1) # Te2[s3>>8] + $PTR_INS $i1,$s0,2,8 + lw $t6,0($i2) # Te2[s0>>8] + $PTR_INS $i2,$s1,2,8 + lw $t7,0($i3) # Te2[s1>>8] + $PTR_INS $i3,$s2,2,8 + + lw $t8,0($i0) # Te3[s3] + _xtr $i0,$s0,24-2 + lw $t9,0($i1) # Te3[s0] + _xtr $i1,$s1,24-2 + lw $t10,0($i2) # Te3[s1] + _xtr $i2,$s2,24-2 + lw $t11,0($i3) # Te3[s2] + _xtr $i3,$s3,24-2 + + and $i0,0x3fc + and $i1,0x3fc + and $i2,0x3fc + and $i3,0x3fc + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl +# endif + rotr $t4,$t4,16 + rotr $t5,$t5,16 + rotr $t6,$t6,16 + rotr $t7,$t7,16 + + rotr $t8,$t8,24 + rotr $t9,$t9,24 + rotr $t10,$t10,24 + rotr $t11,$t11,24 +#else lwl $t4,2($i0) # Te2[s2>>8] lwl $t5,2($i1) # Te2[s3>>8] lwl $t6,2($i2) # Te2[s0>>8] lwl $t7,2($i3) # Te2[s1>>8] lwr $t4,1($i0) # Te2[s2>>8] - lwr $t5,1($i1) # Te2[s3>>8] - lwr $t6,1($i2) # Te2[s0>>8] - lwr $t7,1($i3) # Te2[s1>>8] - _xtr $i0,$s3,0-2 + lwr $t5,1($i1) # Te2[s3>>8] _xtr $i1,$s0,0-2 + lwr $t6,1($i2) # Te2[s0>>8] _xtr $i2,$s1,0-2 + lwr $t7,1($i3) # Te2[s1>>8] _xtr $i3,$s2,0-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -185,14 +360,14 @@ _mips_AES_encrypt: lwl $t10,1($i2) # Te3[s1] lwl $t11,1($i3) # Te3[s2] lwr $t8,0($i0) # Te3[s3] - lwr $t9,0($i1) # Te3[s0] - lwr $t10,0($i2) # Te3[s1] - lwr $t11,0($i3) # Te3[s2] - _xtr $i0,$s0,24-2 + lwr $t9,0($i1) # Te3[s0] _xtr $i1,$s1,24-2 + lwr $t10,0($i2) # Te3[s1] _xtr $i2,$s2,24-2 + lwr $t11,0($i3) # Te3[s2] _xtr $i3,$s3,24-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -201,24 +376,24 @@ _mips_AES_encrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#endif xor $t0,$t4 - xor $t1,$t5 - xor $t2,$t6 - xor $t3,$t7 lw $t4,0($i0) # Te0[s0>>24] + xor $t1,$t5 lw $t5,0($i1) # Te0[s1>>24] + xor $t2,$t6 lw $t6,0($i2) # Te0[s2>>24] + xor $t3,$t7 lw $t7,0($i3) # Te0[s3>>24] - lw $s0,0($key0) - lw $s1,4($key0) - lw $s2,8($key0) - lw $s3,12($key0) - xor $t0,$t8 + lw $s0,0($key0) xor $t1,$t9 + lw $s1,4($key0) xor $t2,$t10 + lw $s2,8($key0) xor $t3,$t11 + lw $s3,12($key0) xor $t0,$t4 xor $t1,$t5 @@ -234,6 +409,7 @@ _mips_AES_encrypt: .set noreorder bnez $cnt,.Loop_enc _xtr $i0,$s1,16-2 +#endif .set reorder _xtr $i1,$s2,16-2 @@ -248,14 +424,14 @@ _mips_AES_encrypt: $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl lbu $t0,2($i0) # Te4[s1>>16] - lbu $t1,2($i1) # Te4[s2>>16] - lbu $t2,2($i2) # Te4[s3>>16] - lbu $t3,2($i3) # Te4[s0>>16] - _xtr $i0,$s2,8-2 + lbu $t1,2($i1) # Te4[s2>>16] _xtr $i1,$s3,8-2 + lbu $t2,2($i2) # Te4[s3>>16] _xtr $i2,$s0,8-2 + lbu $t3,2($i3) # Te4[s0>>16] _xtr $i3,$s1,8-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -264,15 +440,44 @@ _mips_AES_encrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) +# if defined(_MIPSEL) lbu $t4,2($i0) # Te4[s2>>8] + $PTR_INS $i0,$s0,2,8 lbu $t5,2($i1) # Te4[s3>>8] + $PTR_INS $i1,$s1,2,8 lbu $t6,2($i2) # Te4[s0>>8] + $PTR_INS $i2,$s2,2,8 lbu $t7,2($i3) # Te4[s1>>8] + $PTR_INS $i3,$s3,2,8 + lbu $t8,2($i0) # Te4[s0>>24] + _xtr $i0,$s3,0-2 + lbu $t9,2($i1) # Te4[s1>>24] + _xtr $i1,$s0,0-2 + lbu $t10,2($i2) # Te4[s2>>24] + _xtr $i2,$s1,0-2 + lbu $t11,2($i3) # Te4[s3>>24] + _xtr $i3,$s2,0-2 + + and $i0,0x3fc + and $i1,0x3fc + and $i2,0x3fc + and $i3,0x3fc + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl +# else + lbu $t4,2($i0) # Te4[s2>>8] _xtr $i0,$s0,24-2 + lbu $t5,2($i1) # Te4[s3>>8] _xtr $i1,$s1,24-2 + lbu $t6,2($i2) # Te4[s0>>8] _xtr $i2,$s2,24-2 + lbu $t7,2($i3) # Te4[s1>>8] _xtr $i3,$s3,24-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -282,18 +487,76 @@ _mips_AES_encrypt: $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl lbu $t8,2($i0) # Te4[s0>>24] + $PTR_INS $i0,$s3,2,8 lbu $t9,2($i1) # Te4[s1>>24] + $PTR_INS $i1,$s0,2,8 lbu $t10,2($i2) # Te4[s2>>24] + $PTR_INS $i2,$s1,2,8 lbu $t11,2($i3) # Te4[s3>>24] + $PTR_INS $i3,$s2,2,8 +# endif + _ins $t0,16 + _ins $t1,16 + _ins $t2,16 + _ins $t3,16 + _ins2 $t0,$t4,8 + lbu $t4,2($i0) # Te4[s3] + _ins2 $t1,$t5,8 + lbu $t5,2($i1) # Te4[s0] + _ins2 $t2,$t6,8 + lbu $t6,2($i2) # Te4[s1] + _ins2 $t3,$t7,8 + lbu $t7,2($i3) # Te4[s2] + + _ins2 $t0,$t8,24 + lw $s0,0($key0) + _ins2 $t1,$t9,24 + lw $s1,4($key0) + _ins2 $t2,$t10,24 + lw $s2,8($key0) + _ins2 $t3,$t11,24 + lw $s3,12($key0) + + _ins2 $t0,$t4,0 + _ins2 $t1,$t5,0 + _ins2 $t2,$t6,0 + _ins2 $t3,$t7,0 +#else + lbu $t4,2($i0) # Te4[s2>>8] + _xtr $i0,$s0,24-2 + lbu $t5,2($i1) # Te4[s3>>8] + _xtr $i1,$s1,24-2 + lbu $t6,2($i2) # Te4[s0>>8] + _xtr $i2,$s2,24-2 + lbu $t7,2($i3) # Te4[s1>>8] + _xtr $i3,$s3,24-2 + + and $i0,0x3fc + and $i1,0x3fc + and $i2,0x3fc + and $i3,0x3fc + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl + lbu $t8,2($i0) # Te4[s0>>24] _xtr $i0,$s3,0-2 + lbu $t9,2($i1) # Te4[s1>>24] _xtr $i1,$s0,0-2 + lbu $t10,2($i2) # Te4[s2>>24] _xtr $i2,$s1,0-2 + lbu $t11,2($i3) # Te4[s3>>24] _xtr $i3,$s2,0-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc and $i3,0x3fc + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl _ins $t0,16 _ins $t1,16 @@ -306,27 +569,21 @@ _mips_AES_encrypt: _ins $t7,8 xor $t0,$t4 - xor $t1,$t5 - xor $t2,$t6 - xor $t3,$t7 - - $PTR_ADD $i0,$Tbl - $PTR_ADD $i1,$Tbl - $PTR_ADD $i2,$Tbl - $PTR_ADD $i3,$Tbl lbu $t4,2($i0) # Te4[s3] + xor $t1,$t5 lbu $t5,2($i1) # Te4[s0] + xor $t2,$t6 lbu $t6,2($i2) # Te4[s1] + xor $t3,$t7 lbu $t7,2($i3) # Te4[s2] _ins $t8,24 - _ins $t9,24 - _ins $t10,24 - _ins $t11,24 - lw $s0,0($key0) + _ins $t9,24 lw $s1,4($key0) + _ins $t10,24 lw $s2,8($key0) + _ins $t11,24 lw $s3,12($key0) xor $t0,$t8 @@ -343,7 +600,7 @@ _mips_AES_encrypt: xor $t1,$t5 xor $t2,$t6 xor $t3,$t7 - +#endif xor $s0,$t0 xor $s1,$t1 xor $s2,$t2 @@ -455,6 +712,89 @@ _mips_AES_decrypt: xor $s3,$t3 sub $cnt,1 +#if defined(__mips_smartmips) + ext $i0,$s3,16,8 +.Loop_dec: + ext $i1,$s0,16,8 + ext $i2,$s1,16,8 + ext $i3,$s2,16,8 + lwxs $t0,$i0($Tbl) # Td1[s3>>16] + ext $i0,$s2,8,8 + lwxs $t1,$i1($Tbl) # Td1[s0>>16] + ext $i1,$s3,8,8 + lwxs $t2,$i2($Tbl) # Td1[s1>>16] + ext $i2,$s0,8,8 + lwxs $t3,$i3($Tbl) # Td1[s2>>16] + ext $i3,$s1,8,8 + + lwxs $t4,$i0($Tbl) # Td2[s2>>8] + ext $i0,$s1,0,8 + lwxs $t5,$i1($Tbl) # Td2[s3>>8] + ext $i1,$s2,0,8 + lwxs $t6,$i2($Tbl) # Td2[s0>>8] + ext $i2,$s3,0,8 + lwxs $t7,$i3($Tbl) # Td2[s1>>8] + ext $i3,$s0,0,8 + + lwxs $t8,$i0($Tbl) # Td3[s1] + ext $i0,$s0,24,8 + lwxs $t9,$i1($Tbl) # Td3[s2] + ext $i1,$s1,24,8 + lwxs $t10,$i2($Tbl) # Td3[s3] + ext $i2,$s2,24,8 + lwxs $t11,$i3($Tbl) # Td3[s0] + ext $i3,$s3,24,8 + + rotr $t0,$t0,8 + rotr $t1,$t1,8 + rotr $t2,$t2,8 + rotr $t3,$t3,8 + + rotr $t4,$t4,16 + rotr $t5,$t5,16 + rotr $t6,$t6,16 + rotr $t7,$t7,16 + + xor $t0,$t4 + lwxs $t4,$i0($Tbl) # Td0[s0>>24] + xor $t1,$t5 + lwxs $t5,$i1($Tbl) # Td0[s1>>24] + xor $t2,$t6 + lwxs $t6,$i2($Tbl) # Td0[s2>>24] + xor $t3,$t7 + lwxs $t7,$i3($Tbl) # Td0[s3>>24] + + rotr $t8,$t8,24 + lw $s0,0($key0) + rotr $t9,$t9,24 + lw $s1,4($key0) + rotr $t10,$t10,24 + lw $s2,8($key0) + rotr $t11,$t11,24 + lw $s3,12($key0) + + xor $t0,$t8 + xor $t1,$t9 + xor $t2,$t10 + xor $t3,$t11 + + xor $t0,$t4 + xor $t1,$t5 + xor $t2,$t6 + xor $t3,$t7 + + sub $cnt,1 + $PTR_ADD $key0,16 + xor $s0,$t0 + xor $s1,$t1 + xor $s2,$t2 + xor $s3,$t3 + .set noreorder + bnez $cnt,.Loop_dec + ext $i0,$s3,16,8 + + _xtr $i0,$s3,16-2 +#else _xtr $i0,$s3,16-2 .Loop_dec: _xtr $i1,$s0,16-2 @@ -468,19 +808,88 @@ _mips_AES_decrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + lw $t0,0($i0) # Td1[s3>>16] + _xtr $i0,$s2,8-2 + lw $t1,0($i1) # Td1[s0>>16] + _xtr $i1,$s3,8-2 + lw $t2,0($i2) # Td1[s1>>16] + _xtr $i2,$s0,8-2 + lw $t3,0($i3) # Td1[s2>>16] + _xtr $i3,$s1,8-2 +#else lwl $t0,3($i0) # Td1[s3>>16] lwl $t1,3($i1) # Td1[s0>>16] lwl $t2,3($i2) # Td1[s1>>16] lwl $t3,3($i3) # Td1[s2>>16] lwr $t0,2($i0) # Td1[s3>>16] - lwr $t1,2($i1) # Td1[s0>>16] - lwr $t2,2($i2) # Td1[s1>>16] - lwr $t3,2($i3) # Td1[s2>>16] - _xtr $i0,$s2,8-2 + lwr $t1,2($i1) # Td1[s0>>16] _xtr $i1,$s3,8-2 + lwr $t2,2($i2) # Td1[s1>>16] _xtr $i2,$s0,8-2 + lwr $t3,2($i3) # Td1[s2>>16] _xtr $i3,$s1,8-2 +#endif + + and $i0,0x3fc + and $i1,0x3fc + and $i2,0x3fc + and $i3,0x3fc + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + rotr $t0,$t0,8 + rotr $t1,$t1,8 + rotr $t2,$t2,8 + rotr $t3,$t3,8 +# if defined(_MIPSEL) + lw $t4,0($i0) # Td2[s2>>8] + _xtr $i0,$s1,0-2 + lw $t5,0($i1) # Td2[s3>>8] + _xtr $i1,$s2,0-2 + lw $t6,0($i2) # Td2[s0>>8] + _xtr $i2,$s3,0-2 + lw $t7,0($i3) # Td2[s1>>8] + _xtr $i3,$s0,0-2 + + and $i0,0x3fc + and $i1,0x3fc + and $i2,0x3fc + and $i3,0x3fc + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl + lw $t8,0($i0) # Td3[s1] + $PTR_INS $i0,$s0,2,8 + lw $t9,0($i1) # Td3[s2] + $PTR_INS $i1,$s1,2,8 + lw $t10,0($i2) # Td3[s3] + $PTR_INS $i2,$s2,2,8 + lw $t11,0($i3) # Td3[s0] + $PTR_INS $i3,$s3,2,8 +#else + lw $t4,0($i0) # Td2[s2>>8] + $PTR_INS $i0,$s1,2,8 + lw $t5,0($i1) # Td2[s3>>8] + $PTR_INS $i1,$s2,2,8 + lw $t6,0($i2) # Td2[s0>>8] + $PTR_INS $i2,$s3,2,8 + lw $t7,0($i3) # Td2[s1>>8] + $PTR_INS $i3,$s0,2,8 + + lw $t8,0($i0) # Td3[s1] + _xtr $i0,$s0,24-2 + lw $t9,0($i1) # Td3[s2] + _xtr $i1,$s1,24-2 + lw $t10,0($i2) # Td3[s3] + _xtr $i2,$s2,24-2 + lw $t11,0($i3) # Td3[s0] + _xtr $i3,$s3,24-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -489,19 +898,30 @@ _mips_AES_decrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#endif + rotr $t4,$t4,16 + rotr $t5,$t5,16 + rotr $t6,$t6,16 + rotr $t7,$t7,16 + + rotr $t8,$t8,24 + rotr $t9,$t9,24 + rotr $t10,$t10,24 + rotr $t11,$t11,24 +#else lwl $t4,2($i0) # Td2[s2>>8] lwl $t5,2($i1) # Td2[s3>>8] lwl $t6,2($i2) # Td2[s0>>8] lwl $t7,2($i3) # Td2[s1>>8] lwr $t4,1($i0) # Td2[s2>>8] - lwr $t5,1($i1) # Td2[s3>>8] - lwr $t6,1($i2) # Td2[s0>>8] - lwr $t7,1($i3) # Td2[s1>>8] - _xtr $i0,$s1,0-2 + lwr $t5,1($i1) # Td2[s3>>8] _xtr $i1,$s2,0-2 + lwr $t6,1($i2) # Td2[s0>>8] _xtr $i2,$s3,0-2 + lwr $t7,1($i3) # Td2[s1>>8] _xtr $i3,$s0,0-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -515,14 +935,14 @@ _mips_AES_decrypt: lwl $t10,1($i2) # Td3[s3] lwl $t11,1($i3) # Td3[s0] lwr $t8,0($i0) # Td3[s1] - lwr $t9,0($i1) # Td3[s2] - lwr $t10,0($i2) # Td3[s3] - lwr $t11,0($i3) # Td3[s0] - _xtr $i0,$s0,24-2 + lwr $t9,0($i1) # Td3[s2] _xtr $i1,$s1,24-2 + lwr $t10,0($i2) # Td3[s3] _xtr $i2,$s2,24-2 + lwr $t11,0($i3) # Td3[s0] _xtr $i3,$s3,24-2 + and $i0,0x3fc and $i1,0x3fc and $i2,0x3fc @@ -531,27 +951,25 @@ _mips_AES_decrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#endif xor $t0,$t4 - xor $t1,$t5 - xor $t2,$t6 - xor $t3,$t7 - - lw $t4,0($i0) # Td0[s0>>24] + xor $t1,$t5 lw $t5,0($i1) # Td0[s1>>24] + xor $t2,$t6 lw $t6,0($i2) # Td0[s2>>24] + xor $t3,$t7 lw $t7,0($i3) # Td0[s3>>24] - lw $s0,0($key0) - lw $s1,4($key0) - lw $s2,8($key0) - lw $s3,12($key0) - xor $t0,$t8 + lw $s0,0($key0) xor $t1,$t9 + lw $s1,4($key0) xor $t2,$t10 + lw $s2,8($key0) xor $t3,$t11 + lw $s3,12($key0) xor $t0,$t4 xor $t1,$t5 @@ -567,38 +985,39 @@ _mips_AES_decrypt: .set noreorder bnez $cnt,.Loop_dec _xtr $i0,$s3,16-2 +#endif .set reorder lw $t4,1024($Tbl) # prefetch Td4 - lw $t5,1024+32($Tbl) - lw $t6,1024+64($Tbl) - lw $t7,1024+96($Tbl) - lw $t8,1024+128($Tbl) - lw $t9,1024+160($Tbl) - lw $t10,1024+192($Tbl) - lw $t11,1024+224($Tbl) - _xtr $i0,$s3,16 + lw $t5,1024+32($Tbl) _xtr $i1,$s0,16 + lw $t6,1024+64($Tbl) _xtr $i2,$s1,16 + lw $t7,1024+96($Tbl) _xtr $i3,$s2,16 + lw $t8,1024+128($Tbl) and $i0,0xff + lw $t9,1024+160($Tbl) and $i1,0xff + lw $t10,1024+192($Tbl) and $i2,0xff + lw $t11,1024+224($Tbl) and $i3,0xff + $PTR_ADD $i0,$Tbl $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl lbu $t0,1024($i0) # Td4[s3>>16] - lbu $t1,1024($i1) # Td4[s0>>16] - lbu $t2,1024($i2) # Td4[s1>>16] - lbu $t3,1024($i3) # Td4[s2>>16] - _xtr $i0,$s2,8 + lbu $t1,1024($i1) # Td4[s0>>16] _xtr $i1,$s3,8 + lbu $t2,1024($i2) # Td4[s1>>16] _xtr $i2,$s0,8 + lbu $t3,1024($i3) # Td4[s2>>16] _xtr $i3,$s1,8 + and $i0,0xff and $i1,0xff and $i2,0xff @@ -607,29 +1026,108 @@ _mips_AES_decrypt: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) +# if defined(_MIPSEL) lbu $t4,1024($i0) # Td4[s2>>8] + $PTR_INS $i0,$s0,0,8 lbu $t5,1024($i1) # Td4[s3>>8] + $PTR_INS $i1,$s1,0,8 lbu $t6,1024($i2) # Td4[s0>>8] + $PTR_INS $i2,$s2,0,8 lbu $t7,1024($i3) # Td4[s1>>8] + $PTR_INS $i3,$s3,0,8 + lbu $t8,1024($i0) # Td4[s0>>24] + _xtr $i0,$s1,0 + lbu $t9,1024($i1) # Td4[s1>>24] + _xtr $i1,$s2,0 + lbu $t10,1024($i2) # Td4[s2>>24] + _xtr $i2,$s3,0 + lbu $t11,1024($i3) # Td4[s3>>24] + _xtr $i3,$s0,0 + + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl +# else + lbu $t4,1024($i0) # Td4[s2>>8] _xtr $i0,$s0,24 + lbu $t5,1024($i1) # Td4[s3>>8] _xtr $i1,$s1,24 + lbu $t6,1024($i2) # Td4[s0>>8] _xtr $i2,$s2,24 + lbu $t7,1024($i3) # Td4[s1>>8] _xtr $i3,$s3,24 + $PTR_ADD $i0,$Tbl $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl lbu $t8,1024($i0) # Td4[s0>>24] + $PTR_INS $i0,$s1,0,8 lbu $t9,1024($i1) # Td4[s1>>24] + $PTR_INS $i1,$s2,0,8 lbu $t10,1024($i2) # Td4[s2>>24] + $PTR_INS $i2,$s3,0,8 lbu $t11,1024($i3) # Td4[s3>>24] + $PTR_INS $i3,$s0,0,8 +# endif + _ins $t0,16 + _ins $t1,16 + _ins $t2,16 + _ins $t3,16 + + _ins2 $t0,$t4,8 + lbu $t4,1024($i0) # Td4[s1] + _ins2 $t1,$t5,8 + lbu $t5,1024($i1) # Td4[s2] + _ins2 $t2,$t6,8 + lbu $t6,1024($i2) # Td4[s3] + _ins2 $t3,$t7,8 + lbu $t7,1024($i3) # Td4[s0] + + _ins2 $t0,$t8,24 + lw $s0,0($key0) + _ins2 $t1,$t9,24 + lw $s1,4($key0) + _ins2 $t2,$t10,24 + lw $s2,8($key0) + _ins2 $t3,$t11,24 + lw $s3,12($key0) + _ins2 $t0,$t4,0 + _ins2 $t1,$t5,0 + _ins2 $t2,$t6,0 + _ins2 $t3,$t7,0 +#else + lbu $t4,1024($i0) # Td4[s2>>8] + _xtr $i0,$s0,24 + lbu $t5,1024($i1) # Td4[s3>>8] + _xtr $i1,$s1,24 + lbu $t6,1024($i2) # Td4[s0>>8] + _xtr $i2,$s2,24 + lbu $t7,1024($i3) # Td4[s1>>8] + _xtr $i3,$s3,24 + + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl + lbu $t8,1024($i0) # Td4[s0>>24] _xtr $i0,$s1,0 + lbu $t9,1024($i1) # Td4[s1>>24] _xtr $i1,$s2,0 + lbu $t10,1024($i2) # Td4[s2>>24] _xtr $i2,$s3,0 + lbu $t11,1024($i3) # Td4[s3>>24] _xtr $i3,$s0,0 + $PTR_ADD $i0,$Tbl + $PTR_ADD $i1,$Tbl + $PTR_ADD $i2,$Tbl + $PTR_ADD $i3,$Tbl + _ins $t0,16 _ins $t1,16 _ins $t2,16 @@ -641,44 +1139,38 @@ _mips_AES_decrypt: _ins $t7,8 xor $t0,$t4 - xor $t1,$t5 - xor $t2,$t6 - xor $t3,$t7 - - $PTR_ADD $i0,$Tbl - $PTR_ADD $i1,$Tbl - $PTR_ADD $i2,$Tbl - $PTR_ADD $i3,$Tbl lbu $t4,1024($i0) # Td4[s1] + xor $t1,$t5 lbu $t5,1024($i1) # Td4[s2] + xor $t2,$t6 lbu $t6,1024($i2) # Td4[s3] + xor $t3,$t7 lbu $t7,1024($i3) # Td4[s0] _ins $t8,24 - _ins $t9,24 - _ins $t10,24 - _ins $t11,24 - lw $s0,0($key0) + _ins $t9,24 lw $s1,4($key0) + _ins $t10,24 lw $s2,8($key0) + _ins $t11,24 lw $s3,12($key0) - _ins $t4,0 - _ins $t5,0 - _ins $t6,0 - _ins $t7,0 - - xor $t0,$t8 xor $t1,$t9 xor $t2,$t10 xor $t3,$t11 + _ins $t4,0 + _ins $t5,0 + _ins $t6,0 + _ins $t7,0 + xor $t0,$t4 xor $t1,$t5 xor $t2,$t6 xor $t3,$t7 +#endif xor $s0,$t0 xor $s1,$t1 @@ -791,7 +1283,7 @@ _mips_AES_set_encrypt_key: beqz $inp,.Lekey_done li $t0,-1 beqz $key,.Lekey_done - $PTR_ADD $rcon,$Tbl,1024+256 + $PTR_ADD $rcon,$Tbl,256 .set reorder lwl $rk0,0+$MSB($inp) # load 128 bits @@ -843,10 +1335,10 @@ _mips_AES_set_encrypt_key: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl - lbu $i0,1024($i0) - lbu $i1,1024($i1) - lbu $i2,1024($i2) - lbu $i3,1024($i3) + lbu $i0,0($i0) + lbu $i1,0($i1) + lbu $i2,0($i2) + lbu $i3,0($i3) sw $rk0,0($key) sw $rk1,4($key) @@ -898,10 +1390,10 @@ _mips_AES_set_encrypt_key: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl - lbu $i0,1024($i0) - lbu $i1,1024($i1) - lbu $i2,1024($i2) - lbu $i3,1024($i3) + lbu $i0,0($i0) + lbu $i1,0($i1) + lbu $i2,0($i2) + lbu $i3,0($i3) sw $rk0,0($key) sw $rk1,4($key) @@ -957,10 +1449,10 @@ _mips_AES_set_encrypt_key: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl - lbu $i0,1024($i0) - lbu $i1,1024($i1) - lbu $i2,1024($i2) - lbu $i3,1024($i3) + lbu $i0,0($i0) + lbu $i1,0($i1) + lbu $i2,0($i2) + lbu $i3,0($i3) sw $rk0,0($key) sw $rk1,4($key) @@ -999,10 +1491,10 @@ _mips_AES_set_encrypt_key: $PTR_ADD $i1,$Tbl $PTR_ADD $i2,$Tbl $PTR_ADD $i3,$Tbl - lbu $i0,1024($i0) - lbu $i1,1024($i1) - lbu $i2,1024($i2) - lbu $i3,1024($i3) + lbu $i0,0($i0) + lbu $i1,0($i1) + lbu $i2,0($i2) + lbu $i3,0($i3) sll $i0,24 sll $i1,16 sll $i2,8 @@ -1064,7 +1556,7 @@ $code.=<<___ if ($flavour !~ /o32/i); # non-o32 PIC-ification ___ $code.=<<___; .set reorder - la $Tbl,AES_Te # PIC-ified 'load address' + la $Tbl,AES_Te4 # PIC-ified 'load address' bal _mips_AES_set_encrypt_key @@ -1119,7 +1611,7 @@ $code.=<<___ if ($flavour !~ /o32/i); # non-o32 PIC-ification ___ $code.=<<___; .set reorder - la $Tbl,AES_Te # PIC-ified 'load address' + la $Tbl,AES_Te4 # PIC-ified 'load address' bal _mips_AES_set_encrypt_key @@ -1190,6 +1682,16 @@ $code.=<<___; xor $tpb,$tp9,$tp2 xor $tpd,$tp9,$tp4 +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + rotr $tp1,$tpd,16 + xor $tpe,$tp2 + rotr $tp2,$tp9,8 + xor $tpe,$tp1 + rotr $tp4,$tpb,24 + xor $tpe,$tp2 + lw $tp1,4($key) # modulo-scheduled + xor $tpe,$tp4 +#else _ror $tp1,$tpd,16 xor $tpe,$tp2 _ror $tp2,$tpd,-16 @@ -1204,6 +1706,7 @@ $code.=<<___; xor $tpe,$tp1 lw $tp1,4($key) # modulo-scheduled xor $tpe,$tp2 +#endif sub $cnt,1 sw $tpe,0($key) $PTR_ADD $key,4 @@ -1234,7 +1737,7 @@ ___ # Tables are kept in endian-neutral manner $code.=<<___; .rdata -.align 6 +.align 10 AES_Te: .byte 0xc6,0x63,0x63,0xa5, 0xf8,0x7c,0x7c,0x84 # Te0 .byte 0xee,0x77,0x77,0x99, 0xf6,0x7b,0x7b,0x8d @@ -1365,46 +1868,6 @@ AES_Te: .byte 0x7b,0xb0,0xb0,0xcb, 0xa8,0x54,0x54,0xfc .byte 0x6d,0xbb,0xbb,0xd6, 0x2c,0x16,0x16,0x3a -.byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 # Te4 -.byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76 -.byte 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0 -.byte 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0 -.byte 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc -.byte 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15 -.byte 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a -.byte 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75 -.byte 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0 -.byte 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84 -.byte 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b -.byte 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf -.byte 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85 -.byte 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8 -.byte 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5 -.byte 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2 -.byte 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17 -.byte 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73 -.byte 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88 -.byte 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb -.byte 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c -.byte 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79 -.byte 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9 -.byte 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08 -.byte 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6 -.byte 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a -.byte 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e -.byte 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e -.byte 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94 -.byte 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf -.byte 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68 -.byte 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 - -.byte 0x01,0x00,0x00,0x00, 0x02,0x00,0x00,0x00 # rcon -.byte 0x04,0x00,0x00,0x00, 0x08,0x00,0x00,0x00 -.byte 0x10,0x00,0x00,0x00, 0x20,0x00,0x00,0x00 -.byte 0x40,0x00,0x00,0x00, 0x80,0x00,0x00,0x00 -.byte 0x1B,0x00,0x00,0x00, 0x36,0x00,0x00,0x00 - -.align 6 AES_Td: .byte 0x51,0xf4,0xa7,0x50, 0x7e,0x41,0x65,0x53 # Td0 .byte 0x1a,0x17,0xa4,0xc3, 0x3a,0x27,0x5e,0x96 @@ -1567,6 +2030,46 @@ AES_Td: .byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d + +AES_Te4: +.byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 # Te4 +.byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76 +.byte 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0 +.byte 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0 +.byte 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc +.byte 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15 +.byte 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a +.byte 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75 +.byte 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0 +.byte 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84 +.byte 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b +.byte 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf +.byte 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85 +.byte 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8 +.byte 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5 +.byte 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2 +.byte 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17 +.byte 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73 +.byte 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88 +.byte 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb +.byte 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c +.byte 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79 +.byte 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9 +.byte 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08 +.byte 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6 +.byte 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a +.byte 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e +.byte 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e +.byte 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94 +.byte 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf +.byte 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68 +.byte 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 + +.byte 0x01,0x00,0x00,0x00, 0x02,0x00,0x00,0x00 # rcon +.byte 0x04,0x00,0x00,0x00, 0x08,0x00,0x00,0x00 +.byte 0x10,0x00,0x00,0x00, 0x20,0x00,0x00,0x00 +.byte 0x40,0x00,0x00,0x00, 0x80,0x00,0x00,0x00 +.byte 0x1B,0x00,0x00,0x00, 0x36,0x00,0x00,0x00 ___ foreach (split("\n",$code)) { @@ -1583,6 +2086,9 @@ foreach (split("\n",$code)) { s/_ins\s+(\$[0-9]+),(\$[0-9]+),([0-9]+)/ sprintf("sll\t$1,$2,%d",$big_endian ? eval($3) : eval("24-$3"))/e or + s/_ins2\s+(\$[0-9]+),(\$[0-9]+),([0-9]+)/ + sprintf("ins\t$1,$2,%d,8",$big_endian ? eval($3) + : eval("24-$3"))/e or s/_ror\s+(\$[0-9]+),(\$[0-9]+),(\-?[0-9]+)/ sprintf("srl\t$1,$2,%d",$big_endian ? eval($3) : eval("$3*-1"))/e or @@ -1605,6 +2111,11 @@ foreach (split("\n",$code)) { sprintf("$1%d($3)",eval("$2-$2%4+($2%4+1)&3"))/e; } + if (!$big_endian) { + s/(rotr\s+\$[0-9]+,\$[0-9]+),([0-9]+)/sprintf("$1,%d",32-$2)/e; + s/(ext\s+\$[0-9]+,\$[0-9]+),([0-9]+),8/sprintf("$1,%d,8",24-$2)/e; + } + print $_,"\n"; } diff --git a/crypto/aes/asm/aes-ppc.pl b/crypto/aes/asm/aes-ppc.pl index 7c52cbe5f9fa..7a99fc3d0452 100755 --- a/crypto/aes/asm/aes-ppc.pl +++ b/crypto/aes/asm/aes-ppc.pl @@ -45,6 +45,8 @@ if ($flavour =~ /64/) { $PUSH ="stw"; } else { die "nonsense $flavour"; } +$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0; + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or @@ -68,7 +70,7 @@ $key="r5"; $Tbl0="r3"; $Tbl1="r6"; $Tbl2="r7"; -$Tbl3="r2"; +$Tbl3=$out; # stay away from "r2"; $out is offloaded to stack $s0="r8"; $s1="r9"; @@ -76,7 +78,7 @@ $s2="r10"; $s3="r11"; $t0="r12"; -$t1="r13"; +$t1="r0"; # stay away from "r13"; $t2="r14"; $t3="r15"; @@ -100,9 +102,6 @@ $acc13="r29"; $acc14="r30"; $acc15="r31"; -# stay away from TLS pointer -if ($SIZE_T==8) { die if ($t1 ne "r13"); $t1="r0"; } -else { die if ($Tbl3 ne "r2"); $Tbl3=$t0; $t0="r0"; } $mask80=$Tbl2; $mask1b=$Tbl3; @@ -337,8 +336,7 @@ $code.=<<___; $STU $sp,-$FRAME($sp) mflr r0 - $PUSH $toc,`$FRAME-$SIZE_T*20`($sp) - $PUSH r13,`$FRAME-$SIZE_T*19`($sp) + $PUSH $out,`$FRAME-$SIZE_T*19`($sp) $PUSH r14,`$FRAME-$SIZE_T*18`($sp) $PUSH r15,`$FRAME-$SIZE_T*17`($sp) $PUSH r16,`$FRAME-$SIZE_T*16`($sp) @@ -365,16 +363,61 @@ $code.=<<___; bne Lenc_unaligned Lenc_unaligned_ok: +___ +$code.=<<___ if (!$LITTLE_ENDIAN); lwz $s0,0($inp) lwz $s1,4($inp) lwz $s2,8($inp) lwz $s3,12($inp) +___ +$code.=<<___ if ($LITTLE_ENDIAN); + lwz $t0,0($inp) + lwz $t1,4($inp) + lwz $t2,8($inp) + lwz $t3,12($inp) + rotlwi $s0,$t0,8 + rotlwi $s1,$t1,8 + rotlwi $s2,$t2,8 + rotlwi $s3,$t3,8 + rlwimi $s0,$t0,24,0,7 + rlwimi $s1,$t1,24,0,7 + rlwimi $s2,$t2,24,0,7 + rlwimi $s3,$t3,24,0,7 + rlwimi $s0,$t0,24,16,23 + rlwimi $s1,$t1,24,16,23 + rlwimi $s2,$t2,24,16,23 + rlwimi $s3,$t3,24,16,23 +___ +$code.=<<___; bl LAES_Te bl Lppc_AES_encrypt_compact + $POP $out,`$FRAME-$SIZE_T*19`($sp) +___ +$code.=<<___ if ($LITTLE_ENDIAN); + rotlwi $t0,$s0,8 + rotlwi $t1,$s1,8 + rotlwi $t2,$s2,8 + rotlwi $t3,$s3,8 + rlwimi $t0,$s0,24,0,7 + rlwimi $t1,$s1,24,0,7 + rlwimi $t2,$s2,24,0,7 + rlwimi $t3,$s3,24,0,7 + rlwimi $t0,$s0,24,16,23 + rlwimi $t1,$s1,24,16,23 + rlwimi $t2,$s2,24,16,23 + rlwimi $t3,$s3,24,16,23 + stw $t0,0($out) + stw $t1,4($out) + stw $t2,8($out) + stw $t3,12($out) +___ +$code.=<<___ if (!$LITTLE_ENDIAN); stw $s0,0($out) stw $s1,4($out) stw $s2,8($out) stw $s3,12($out) +___ +$code.=<<___; b Lenc_done Lenc_unaligned: @@ -417,6 +460,7 @@ Lenc_xpage: bl LAES_Te bl Lppc_AES_encrypt_compact + $POP $out,`$FRAME-$SIZE_T*19`($sp) extrwi $acc00,$s0,8,0 extrwi $acc01,$s0,8,8 @@ -449,8 +493,6 @@ Lenc_xpage: Lenc_done: $POP r0,`$FRAME+$LRSAVE`($sp) - $POP $toc,`$FRAME-$SIZE_T*20`($sp) - $POP r13,`$FRAME-$SIZE_T*19`($sp) $POP r14,`$FRAME-$SIZE_T*18`($sp) $POP r15,`$FRAME-$SIZE_T*17`($sp) $POP r16,`$FRAME-$SIZE_T*16`($sp) @@ -764,6 +806,7 @@ Lenc_compact_done: blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size .AES_encrypt,.-.AES_encrypt .globl .AES_decrypt .align 7 @@ -771,8 +814,7 @@ Lenc_compact_done: $STU $sp,-$FRAME($sp) mflr r0 - $PUSH $toc,`$FRAME-$SIZE_T*20`($sp) - $PUSH r13,`$FRAME-$SIZE_T*19`($sp) + $PUSH $out,`$FRAME-$SIZE_T*19`($sp) $PUSH r14,`$FRAME-$SIZE_T*18`($sp) $PUSH r15,`$FRAME-$SIZE_T*17`($sp) $PUSH r16,`$FRAME-$SIZE_T*16`($sp) @@ -799,16 +841,61 @@ Lenc_compact_done: bne Ldec_unaligned Ldec_unaligned_ok: +___ +$code.=<<___ if (!$LITTLE_ENDIAN); lwz $s0,0($inp) lwz $s1,4($inp) lwz $s2,8($inp) lwz $s3,12($inp) +___ +$code.=<<___ if ($LITTLE_ENDIAN); + lwz $t0,0($inp) + lwz $t1,4($inp) + lwz $t2,8($inp) + lwz $t3,12($inp) + rotlwi $s0,$t0,8 + rotlwi $s1,$t1,8 + rotlwi $s2,$t2,8 + rotlwi $s3,$t3,8 + rlwimi $s0,$t0,24,0,7 + rlwimi $s1,$t1,24,0,7 + rlwimi $s2,$t2,24,0,7 + rlwimi $s3,$t3,24,0,7 + rlwimi $s0,$t0,24,16,23 + rlwimi $s1,$t1,24,16,23 + rlwimi $s2,$t2,24,16,23 + rlwimi $s3,$t3,24,16,23 +___ +$code.=<<___; bl LAES_Td bl Lppc_AES_decrypt_compact + $POP $out,`$FRAME-$SIZE_T*19`($sp) +___ +$code.=<<___ if ($LITTLE_ENDIAN); + rotlwi $t0,$s0,8 + rotlwi $t1,$s1,8 + rotlwi $t2,$s2,8 + rotlwi $t3,$s3,8 + rlwimi $t0,$s0,24,0,7 + rlwimi $t1,$s1,24,0,7 + rlwimi $t2,$s2,24,0,7 + rlwimi $t3,$s3,24,0,7 + rlwimi $t0,$s0,24,16,23 + rlwimi $t1,$s1,24,16,23 + rlwimi $t2,$s2,24,16,23 + rlwimi $t3,$s3,24,16,23 + stw $t0,0($out) + stw $t1,4($out) + stw $t2,8($out) + stw $t3,12($out) +___ +$code.=<<___ if (!$LITTLE_ENDIAN); stw $s0,0($out) stw $s1,4($out) stw $s2,8($out) stw $s3,12($out) +___ +$code.=<<___; b Ldec_done Ldec_unaligned: @@ -851,6 +938,7 @@ Ldec_xpage: bl LAES_Td bl Lppc_AES_decrypt_compact + $POP $out,`$FRAME-$SIZE_T*19`($sp) extrwi $acc00,$s0,8,0 extrwi $acc01,$s0,8,8 @@ -883,8 +971,6 @@ Ldec_xpage: Ldec_done: $POP r0,`$FRAME+$LRSAVE`($sp) - $POP $toc,`$FRAME-$SIZE_T*20`($sp) - $POP r13,`$FRAME-$SIZE_T*19`($sp) $POP r14,`$FRAME-$SIZE_T*18`($sp) $POP r15,`$FRAME-$SIZE_T*17`($sp) $POP r16,`$FRAME-$SIZE_T*16`($sp) @@ -1355,6 +1441,7 @@ Ldec_compact_done: blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size .AES_decrypt,.-.AES_decrypt .asciz "AES for PPC, CRYPTOGAMS by " .align 7 diff --git a/crypto/aes/asm/aes-x86_64.pl b/crypto/aes/asm/aes-x86_64.pl index 34cbb5d84429..47f416375d1e 100755 --- a/crypto/aes/asm/aes-x86_64.pl +++ b/crypto/aes/asm/aes-x86_64.pl @@ -19,9 +19,10 @@ # Performance in number of cycles per processed byte for 128-bit key: # # ECB encrypt ECB decrypt CBC large chunk -# AMD64 33 41 13.0 -# EM64T 38 59 18.6(*) -# Core 2 30 43 14.5(*) +# AMD64 33 43 13.0 +# EM64T 38 56 18.6(*) +# Core 2 30 42 14.5(*) +# Atom 65 86 32.1(*) # # (*) with hyper-threading off @@ -366,68 +367,66 @@ $code.=<<___; movzb `&lo("$s0")`,$t0 movzb `&lo("$s1")`,$t1 movzb `&lo("$s2")`,$t2 - movzb ($sbox,$t0,1),$t0 - movzb ($sbox,$t1,1),$t1 - movzb ($sbox,$t2,1),$t2 - movzb `&lo("$s3")`,$t3 movzb `&hi("$s1")`,$acc0 movzb `&hi("$s2")`,$acc1 + shr \$16,$s2 + movzb `&hi("$s3")`,$acc2 + movzb ($sbox,$t0,1),$t0 + movzb ($sbox,$t1,1),$t1 + movzb ($sbox,$t2,1),$t2 movzb ($sbox,$t3,1),$t3 - movzb ($sbox,$acc0,1),$t4 #$t0 - movzb ($sbox,$acc1,1),$t5 #$t1 - movzb `&hi("$s3")`,$acc2 + movzb ($sbox,$acc0,1),$t4 #$t0 movzb `&hi("$s0")`,$acc0 - shr \$16,$s2 + movzb ($sbox,$acc1,1),$t5 #$t1 + movzb `&lo("$s2")`,$acc1 movzb ($sbox,$acc2,1),$acc2 #$t2 movzb ($sbox,$acc0,1),$acc0 #$t3 - shr \$16,$s3 - movzb `&lo("$s2")`,$acc1 shl \$8,$t4 + shr \$16,$s3 shl \$8,$t5 - movzb ($sbox,$acc1,1),$acc1 #$t0 xor $t4,$t0 - xor $t5,$t1 - - movzb `&lo("$s3")`,$t4 shr \$16,$s0 + movzb `&lo("$s3")`,$t4 shr \$16,$s1 - movzb `&lo("$s0")`,$t5 + xor $t5,$t1 shl \$8,$acc2 - shl \$8,$acc0 - movzb ($sbox,$t4,1),$t4 #$t1 - movzb ($sbox,$t5,1),$t5 #$t2 + movzb `&lo("$s0")`,$t5 + movzb ($sbox,$acc1,1),$acc1 #$t0 xor $acc2,$t2 - xor $acc0,$t3 + shl \$8,$acc0 movzb `&lo("$s1")`,$acc2 - movzb `&hi("$s3")`,$acc0 shl \$16,$acc1 - movzb ($sbox,$acc2,1),$acc2 #$t3 - movzb ($sbox,$acc0,1),$acc0 #$t0 + xor $acc0,$t3 + movzb ($sbox,$t4,1),$t4 #$t1 + movzb `&hi("$s3")`,$acc0 + movzb ($sbox,$t5,1),$t5 #$t2 xor $acc1,$t0 - movzb `&hi("$s0")`,$acc1 shr \$8,$s2 + movzb `&hi("$s0")`,$acc1 + shl \$16,$t4 shr \$8,$s1 + shl \$16,$t5 + xor $t4,$t1 + movzb ($sbox,$acc2,1),$acc2 #$t3 + movzb ($sbox,$acc0,1),$acc0 #$t0 movzb ($sbox,$acc1,1),$acc1 #$t1 movzb ($sbox,$s2,1),$s3 #$t3 movzb ($sbox,$s1,1),$s2 #$t2 - shl \$16,$t4 - shl \$16,$t5 + shl \$16,$acc2 - xor $t4,$t1 xor $t5,$t2 - xor $acc2,$t3 - shl \$24,$acc0 + xor $acc2,$t3 shl \$24,$acc1 - shl \$24,$s3 xor $acc0,$t0 - shl \$24,$s2 + shl \$24,$s3 xor $acc1,$t1 + shl \$24,$s2 mov $t0,$s0 mov $t1,$s1 xor $t2,$s2 @@ -466,12 +465,12 @@ sub enctransform() { my ($t3,$r20,$r21)=($acc2,"%r8d","%r9d"); $code.=<<___; - mov $s0,$acc0 - mov $s1,$acc1 - and \$0x80808080,$acc0 - and \$0x80808080,$acc1 - mov $acc0,$t0 - mov $acc1,$t1 + mov \$0x80808080,$t0 + mov \$0x80808080,$t1 + and $s0,$t0 + and $s1,$t1 + mov $t0,$acc0 + mov $t1,$acc1 shr \$7,$t0 lea ($s0,$s0),$r20 shr \$7,$t1 @@ -489,25 +488,25 @@ $code.=<<___; xor $r20,$s0 xor $r21,$s1 - mov $s2,$acc0 - mov $s3,$acc1 + mov \$0x80808080,$t2 rol \$24,$s0 + mov \$0x80808080,$t3 rol \$24,$s1 - and \$0x80808080,$acc0 - and \$0x80808080,$acc1 + and $s2,$t2 + and $s3,$t3 xor $r20,$s0 xor $r21,$s1 - mov $acc0,$t2 - mov $acc1,$t3 + mov $t2,$acc0 ror \$16,$t0 + mov $t3,$acc1 ror \$16,$t1 - shr \$7,$t2 lea ($s2,$s2),$r20 + shr \$7,$t2 xor $t0,$s0 - xor $t1,$s1 shr \$7,$t3 - lea ($s3,$s3),$r21 + xor $t1,$s1 ror \$8,$t0 + lea ($s3,$s3),$r21 ror \$8,$t1 sub $t2,$acc0 sub $t3,$acc1 @@ -523,23 +522,23 @@ $code.=<<___; xor $acc0,$r20 xor $acc1,$r21 + ror \$16,$t2 xor $r20,$s2 + ror \$16,$t3 xor $r21,$s3 rol \$24,$s2 + mov 0($sbox),$acc0 # prefetch Te4 rol \$24,$s3 xor $r20,$s2 - xor $r21,$s3 - mov 0($sbox),$acc0 # prefetch Te4 - ror \$16,$t2 - ror \$16,$t3 mov 64($sbox),$acc1 - xor $t2,$s2 - xor $t3,$s3 + xor $r21,$s3 mov 128($sbox),$r20 + xor $t2,$s2 ror \$8,$t2 + xor $t3,$s3 ror \$8,$t3 - mov 192($sbox),$r21 xor $t2,$s2 + mov 192($sbox),$r21 xor $t3,$s3 ___ } @@ -936,70 +935,69 @@ $code.=<<___; movzb `&lo("$s0")`,$t0 movzb `&lo("$s1")`,$t1 movzb `&lo("$s2")`,$t2 - movzb ($sbox,$t0,1),$t0 - movzb ($sbox,$t1,1),$t1 - movzb ($sbox,$t2,1),$t2 - movzb `&lo("$s3")`,$t3 movzb `&hi("$s3")`,$acc0 movzb `&hi("$s0")`,$acc1 + shr \$16,$s3 + movzb `&hi("$s1")`,$acc2 + movzb ($sbox,$t0,1),$t0 + movzb ($sbox,$t1,1),$t1 + movzb ($sbox,$t2,1),$t2 movzb ($sbox,$t3,1),$t3 - movzb ($sbox,$acc0,1),$t4 #$t0 - movzb ($sbox,$acc1,1),$t5 #$t1 - movzb `&hi("$s1")`,$acc2 + movzb ($sbox,$acc0,1),$t4 #$t0 movzb `&hi("$s2")`,$acc0 - shr \$16,$s2 + movzb ($sbox,$acc1,1),$t5 #$t1 movzb ($sbox,$acc2,1),$acc2 #$t2 movzb ($sbox,$acc0,1),$acc0 #$t3 - shr \$16,$s3 - movzb `&lo("$s2")`,$acc1 - shl \$8,$t4 + shr \$16,$s2 shl \$8,$t5 - movzb ($sbox,$acc1,1),$acc1 #$t0 - xor $t4,$t0 - xor $t5,$t1 - - movzb `&lo("$s3")`,$t4 + shl \$8,$t4 + movzb `&lo("$s2")`,$acc1 shr \$16,$s0 + xor $t4,$t0 shr \$16,$s1 - movzb `&lo("$s0")`,$t5 + movzb `&lo("$s3")`,$t4 + shl \$8,$acc2 + xor $t5,$t1 shl \$8,$acc0 - movzb ($sbox,$t4,1),$t4 #$t1 - movzb ($sbox,$t5,1),$t5 #$t2 + movzb `&lo("$s0")`,$t5 + movzb ($sbox,$acc1,1),$acc1 #$t0 xor $acc2,$t2 - xor $acc0,$t3 - movzb `&lo("$s1")`,$acc2 - movzb `&hi("$s1")`,$acc0 + shl \$16,$acc1 + xor $acc0,$t3 + movzb ($sbox,$t4,1),$t4 #$t1 + movzb `&hi("$s1")`,$acc0 movzb ($sbox,$acc2,1),$acc2 #$t3 - movzb ($sbox,$acc0,1),$acc0 #$t0 xor $acc1,$t0 - + movzb ($sbox,$t5,1),$t5 #$t2 movzb `&hi("$s2")`,$acc1 + + shl \$16,$acc2 shl \$16,$t4 shl \$16,$t5 - movzb ($sbox,$acc1,1),$s1 #$t1 + xor $acc2,$t3 + movzb `&hi("$s3")`,$acc2 xor $t4,$t1 + shr \$8,$s0 xor $t5,$t2 - movzb `&hi("$s3")`,$acc1 - shr \$8,$s0 - shl \$16,$acc2 - movzb ($sbox,$acc1,1),$s2 #$t2 + movzb ($sbox,$acc0,1),$acc0 #$t0 + movzb ($sbox,$acc1,1),$s1 #$t1 + movzb ($sbox,$acc2,1),$s2 #$t2 movzb ($sbox,$s0,1),$s3 #$t3 - xor $acc2,$t3 + mov $t0,$s0 shl \$24,$acc0 shl \$24,$s1 shl \$24,$s2 - xor $acc0,$t0 + xor $acc0,$s0 shl \$24,$s3 xor $t1,$s1 - mov $t0,$s0 xor $t2,$s2 xor $t3,$s3 ___ @@ -1014,12 +1012,12 @@ sub dectransform() my $prefetch = shift; $code.=<<___; - mov $tp10,$acc0 - mov $tp18,$acc8 - and $mask80,$acc0 - and $mask80,$acc8 - mov $acc0,$tp40 - mov $acc8,$tp48 + mov $mask80,$tp40 + mov $mask80,$tp48 + and $tp10,$tp40 + and $tp18,$tp48 + mov $tp40,$acc0 + mov $tp48,$acc8 shr \$7,$tp40 lea ($tp10,$tp10),$tp20 shr \$7,$tp48 @@ -1030,15 +1028,15 @@ $code.=<<___; and $maskfe,$tp28 and $mask1b,$acc0 and $mask1b,$acc8 - xor $tp20,$acc0 - xor $tp28,$acc8 - mov $acc0,$tp20 - mov $acc8,$tp28 - - and $mask80,$acc0 - and $mask80,$acc8 - mov $acc0,$tp80 - mov $acc8,$tp88 + xor $acc0,$tp20 + xor $acc8,$tp28 + mov $mask80,$tp80 + mov $mask80,$tp88 + + and $tp20,$tp80 + and $tp28,$tp88 + mov $tp80,$acc0 + mov $tp88,$acc8 shr \$7,$tp80 lea ($tp20,$tp20),$tp40 shr \$7,$tp88 @@ -1049,15 +1047,15 @@ $code.=<<___; and $maskfe,$tp48 and $mask1b,$acc0 and $mask1b,$acc8 - xor $tp40,$acc0 - xor $tp48,$acc8 - mov $acc0,$tp40 - mov $acc8,$tp48 - - and $mask80,$acc0 - and $mask80,$acc8 - mov $acc0,$tp80 - mov $acc8,$tp88 + xor $acc0,$tp40 + xor $acc8,$tp48 + mov $mask80,$tp80 + mov $mask80,$tp88 + + and $tp40,$tp80 + and $tp48,$tp88 + mov $tp80,$acc0 + mov $tp88,$acc8 shr \$7,$tp80 xor $tp10,$tp20 # tp2^=tp1 shr \$7,$tp88 @@ -1082,51 +1080,51 @@ $code.=<<___; mov $tp10,$acc0 mov $tp18,$acc8 xor $tp80,$tp40 # tp4^tp1^=tp8 - xor $tp88,$tp48 # tp4^tp1^=tp8 shr \$32,$acc0 + xor $tp88,$tp48 # tp4^tp1^=tp8 shr \$32,$acc8 xor $tp20,$tp80 # tp8^=tp8^tp2^tp1=tp2^tp1 - xor $tp28,$tp88 # tp8^=tp8^tp2^tp1=tp2^tp1 rol \$8,`&LO("$tp10")` # ROTATE(tp1^tp8,8) + xor $tp28,$tp88 # tp8^=tp8^tp2^tp1=tp2^tp1 rol \$8,`&LO("$tp18")` # ROTATE(tp1^tp8,8) xor $tp40,$tp80 # tp2^tp1^=tp8^tp4^tp1=tp8^tp4^tp2 + rol \$8,`&LO("$acc0")` # ROTATE(tp1^tp8,8) xor $tp48,$tp88 # tp2^tp1^=tp8^tp4^tp1=tp8^tp4^tp2 - rol \$8,`&LO("$acc0")` # ROTATE(tp1^tp8,8) rol \$8,`&LO("$acc8")` # ROTATE(tp1^tp8,8) xor `&LO("$tp80")`,`&LO("$tp10")` - xor `&LO("$tp88")`,`&LO("$tp18")` shr \$32,$tp80 + xor `&LO("$tp88")`,`&LO("$tp18")` shr \$32,$tp88 xor `&LO("$tp80")`,`&LO("$acc0")` xor `&LO("$tp88")`,`&LO("$acc8")` mov $tp20,$tp80 - mov $tp28,$tp88 - shr \$32,$tp80 - shr \$32,$tp88 rol \$24,`&LO("$tp20")` # ROTATE(tp2^tp1^tp8,24) + mov $tp28,$tp88 rol \$24,`&LO("$tp28")` # ROTATE(tp2^tp1^tp8,24) - rol \$24,`&LO("$tp80")` # ROTATE(tp2^tp1^tp8,24) - rol \$24,`&LO("$tp88")` # ROTATE(tp2^tp1^tp8,24) + shr \$32,$tp80 xor `&LO("$tp20")`,`&LO("$tp10")` + shr \$32,$tp88 xor `&LO("$tp28")`,`&LO("$tp18")` + rol \$24,`&LO("$tp80")` # ROTATE(tp2^tp1^tp8,24) mov $tp40,$tp20 + rol \$24,`&LO("$tp88")` # ROTATE(tp2^tp1^tp8,24) mov $tp48,$tp28 + shr \$32,$tp20 xor `&LO("$tp80")`,`&LO("$acc0")` + shr \$32,$tp28 xor `&LO("$tp88")`,`&LO("$acc8")` `"mov 0($sbox),$mask80" if ($prefetch)` - shr \$32,$tp20 - shr \$32,$tp28 - `"mov 64($sbox),$maskfe" if ($prefetch)` rol \$16,`&LO("$tp40")` # ROTATE(tp4^tp1^tp8,16) + `"mov 64($sbox),$maskfe" if ($prefetch)` rol \$16,`&LO("$tp48")` # ROTATE(tp4^tp1^tp8,16) `"mov 128($sbox),$mask1b" if ($prefetch)` rol \$16,`&LO("$tp20")` # ROTATE(tp4^tp1^tp8,16) - rol \$16,`&LO("$tp28")` # ROTATE(tp4^tp1^tp8,16) `"mov 192($sbox),$tp80" if ($prefetch)` xor `&LO("$tp40")`,`&LO("$tp10")` + rol \$16,`&LO("$tp28")` # ROTATE(tp4^tp1^tp8,16) xor `&LO("$tp48")`,`&LO("$tp18")` `"mov 256($sbox),$tp88" if ($prefetch)` xor `&LO("$tp20")`,`&LO("$acc0")` @@ -1302,10 +1300,6 @@ private_AES_set_encrypt_key: call _x86_64_AES_set_encrypt_key - mov 8(%rsp),%r15 - mov 16(%rsp),%r14 - mov 24(%rsp),%r13 - mov 32(%rsp),%r12 mov 40(%rsp),%rbp mov 48(%rsp),%rbx add \$56,%rsp diff --git a/crypto/aes/asm/aesni-mb-x86_64.pl b/crypto/aes/asm/aesni-mb-x86_64.pl new file mode 100755 index 000000000000..33b1aed3c0b4 --- /dev/null +++ b/crypto/aes/asm/aesni-mb-x86_64.pl @@ -0,0 +1,1395 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# Multi-buffer AES-NI procedures process several independent buffers +# in parallel by interleaving independent instructions. +# +# Cycles per byte for interleave factor 4: +# +# asymptotic measured +# --------------------------- +# Westmere 5.00/4=1.25 5.13/4=1.28 +# Atom 15.0/4=3.75 ?15.7/4=3.93 +# Sandy Bridge 5.06/4=1.27 5.18/4=1.29 +# Ivy Bridge 5.06/4=1.27 5.14/4=1.29 +# Haswell 4.44/4=1.11 4.44/4=1.11 +# Bulldozer 5.75/4=1.44 5.76/4=1.44 +# +# Cycles per byte for interleave factor 8 (not implemented for +# pre-AVX processors, where higher interleave factor incidentally +# doesn't result in improvement): +# +# asymptotic measured +# --------------------------- +# Sandy Bridge 5.06/8=0.64 7.10/8=0.89(*) +# Ivy Bridge 5.06/8=0.64 7.14/8=0.89(*) +# Haswell 5.00/8=0.63 5.00/8=0.63 +# Bulldozer 5.75/8=0.72 5.77/8=0.72 +# +# (*) Sandy/Ivy Bridge are known to handle high interleave factors +# suboptimally; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +$avx=0; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +# void aesni_multi_cbc_encrypt ( +# struct { void *inp,*out; int blocks; double iv[2]; } inp[8]; +# const AES_KEY *key, +# int num); /* 1 or 2 */ +# +$inp="%rdi"; # 1st arg +$key="%rsi"; # 2nd arg +$num="%edx"; + +@inptr=map("%r$_",(8..11)); +@outptr=map("%r$_",(12..15)); + +($rndkey0,$rndkey1)=("%xmm0","%xmm1"); +@out=map("%xmm$_",(2..5)); +@inp=map("%xmm$_",(6..9)); +($counters,$mask,$zero)=map("%xmm$_",(10..12)); + +($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx"); + +$code.=<<___; +.text + +.extern OPENSSL_ia32cap_P + +.globl aesni_multi_cbc_encrypt +.type aesni_multi_cbc_encrypt,\@function,3 +.align 32 +aesni_multi_cbc_encrypt: +___ +$code.=<<___ if ($avx); + cmp \$2,$num + jb .Lenc_non_avx + mov OPENSSL_ia32cap_P+4(%rip),%ecx + test \$`1<<28`,%ecx # AVX bit + jnz _avx_cbc_enc_shortcut + jmp .Lenc_non_avx +.align 16 +.Lenc_non_avx: +___ +$code.=<<___; + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,0x40(%rsp) + movaps %xmm11,0x50(%rsp) + movaps %xmm12,0x60(%rsp) + movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +___ +$code.=<<___; + # stack layout + # + # +0 output sink + # +16 input sink [original %rsp and $num] + # +32 counters + + sub \$48,%rsp + and \$-64,%rsp + mov %rax,16(%rsp) # original %rsp + +.Lenc4x_body: + movdqu ($key),$zero # 0-round key + lea 0x78($key),$key # size optimization + lea 40*2($inp),$inp + +.Lenc4x_loop_grande: + mov $num,24(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + mov `40*$i+16-40*2`($inp),$one # borrow $one for number of blocks + mov `40*$i+0-40*2`($inp),@inptr[$i] + cmp $num,$one + mov `40*$i+8-40*2`($inp),@outptr[$i] + cmovg $one,$num # find maximum + test $one,$one + movdqu `40*$i+24-40*2`($inp),@out[$i] # load IV + mov $one,`32+4*$i`(%rsp) # initialize counters + cmovle %rsp,@inptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Lenc4x_done + + movups 0x10-0x78($key),$rndkey1 + pxor $zero,@out[0] + movups 0x20-0x78($key),$rndkey0 + pxor $zero,@out[1] + mov 0xf0-0x78($key),$rounds + pxor $zero,@out[2] + movdqu (@inptr[0]),@inp[0] # load inputs + pxor $zero,@out[3] + movdqu (@inptr[1]),@inp[1] + pxor @inp[0],@out[0] + movdqu (@inptr[2]),@inp[2] + pxor @inp[1],@out[1] + movdqu (@inptr[3]),@inp[3] + pxor @inp[2],@out[2] + pxor @inp[3],@out[3] + movdqa 32(%rsp),$counters # load counters + xor $offset,$offset + jmp .Loop_enc4x + +.align 32 +.Loop_enc4x: + add \$16,$offset + lea 16(%rsp),$sink # sink pointer + mov \$1,$one # constant of 1 + sub $offset,$sink + + aesenc $rndkey1,@out[0] + prefetcht0 31(@inptr[0],$offset) # prefetch input + prefetcht0 31(@inptr[1],$offset) + aesenc $rndkey1,@out[1] + prefetcht0 31(@inptr[2],$offset) + prefetcht0 31(@inptr[2],$offset) + aesenc $rndkey1,@out[2] + aesenc $rndkey1,@out[3] + movups 0x30-0x78($key),$rndkey1 +___ +for($i=0;$i<4;$i++) { +my $rndkey = ($i&1) ? $rndkey1 : $rndkey0; +$code.=<<___; + cmp `32+4*$i`(%rsp),$one + aesenc $rndkey,@out[0] + aesenc $rndkey,@out[1] + aesenc $rndkey,@out[2] + cmovge $sink,@inptr[$i] # cancel input + cmovg $sink,@outptr[$i] # sink output + aesenc $rndkey,@out[3] + movups `0x40+16*$i-0x78`($key),$rndkey +___ +} +$code.=<<___; + movdqa $counters,$mask + aesenc $rndkey0,@out[0] + prefetcht0 15(@outptr[0],$offset) # prefetch output + prefetcht0 15(@outptr[1],$offset) + aesenc $rndkey0,@out[1] + prefetcht0 15(@outptr[2],$offset) + prefetcht0 15(@outptr[3],$offset) + aesenc $rndkey0,@out[2] + aesenc $rndkey0,@out[3] + movups 0x80-0x78($key),$rndkey0 + pxor $zero,$zero + + aesenc $rndkey1,@out[0] + pcmpgtd $zero,$mask + movdqu -0x78($key),$zero # reload 0-round key + aesenc $rndkey1,@out[1] + paddd $mask,$counters # decrement counters + movdqa $counters,32(%rsp) # update counters + aesenc $rndkey1,@out[2] + aesenc $rndkey1,@out[3] + movups 0x90-0x78($key),$rndkey1 + + cmp \$11,$rounds + + aesenc $rndkey0,@out[0] + aesenc $rndkey0,@out[1] + aesenc $rndkey0,@out[2] + aesenc $rndkey0,@out[3] + movups 0xa0-0x78($key),$rndkey0 + + jb .Lenc4x_tail + + aesenc $rndkey1,@out[0] + aesenc $rndkey1,@out[1] + aesenc $rndkey1,@out[2] + aesenc $rndkey1,@out[3] + movups 0xb0-0x78($key),$rndkey1 + + aesenc $rndkey0,@out[0] + aesenc $rndkey0,@out[1] + aesenc $rndkey0,@out[2] + aesenc $rndkey0,@out[3] + movups 0xc0-0x78($key),$rndkey0 + + je .Lenc4x_tail + + aesenc $rndkey1,@out[0] + aesenc $rndkey1,@out[1] + aesenc $rndkey1,@out[2] + aesenc $rndkey1,@out[3] + movups 0xd0-0x78($key),$rndkey1 + + aesenc $rndkey0,@out[0] + aesenc $rndkey0,@out[1] + aesenc $rndkey0,@out[2] + aesenc $rndkey0,@out[3] + movups 0xe0-0x78($key),$rndkey0 + jmp .Lenc4x_tail + +.align 32 +.Lenc4x_tail: + aesenc $rndkey1,@out[0] + aesenc $rndkey1,@out[1] + aesenc $rndkey1,@out[2] + aesenc $rndkey1,@out[3] + movdqu (@inptr[0],$offset),@inp[0] + movdqu 0x10-0x78($key),$rndkey1 + + aesenclast $rndkey0,@out[0] + movdqu (@inptr[1],$offset),@inp[1] + pxor $zero,@inp[0] + aesenclast $rndkey0,@out[1] + movdqu (@inptr[2],$offset),@inp[2] + pxor $zero,@inp[1] + aesenclast $rndkey0,@out[2] + movdqu (@inptr[3],$offset),@inp[3] + pxor $zero,@inp[2] + aesenclast $rndkey0,@out[3] + movdqu 0x20-0x78($key),$rndkey0 + pxor $zero,@inp[3] + + movups @out[0],-16(@outptr[0],$offset) + pxor @inp[0],@out[0] + movups @out[1],-16(@outptr[1],$offset) + pxor @inp[1],@out[1] + movups @out[2],-16(@outptr[2],$offset) + pxor @inp[2],@out[2] + movups @out[3],-16(@outptr[3],$offset) + pxor @inp[3],@out[3] + + dec $num + jnz .Loop_enc4x + + mov 16(%rsp),%rax # original %rsp + mov 24(%rsp),$num + + #pxor @inp[0],@out[0] + #pxor @inp[1],@out[1] + #movdqu @out[0],`40*0+24-40*2`($inp) # output iv FIX ME! + #pxor @inp[2],@out[2] + #movdqu @out[1],`40*1+24-40*2`($inp) + #pxor @inp[3],@out[3] + #movdqu @out[2],`40*2+24-40*2`($inp) # won't fix, let caller + #movdqu @out[3],`40*3+24-40*2`($inp) # figure this out... + + lea `40*4`($inp),$inp + dec $num + jnz .Lenc4x_loop_grande + +.Lenc4x_done: +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + #movaps -0x68(%rax),%xmm13 + #movaps -0x58(%rax),%xmm14 + #movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lenc4x_epilogue: + ret +.size aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt + +.globl aesni_multi_cbc_decrypt +.type aesni_multi_cbc_decrypt,\@function,3 +.align 32 +aesni_multi_cbc_decrypt: +___ +$code.=<<___ if ($avx); + cmp \$2,$num + jb .Ldec_non_avx + mov OPENSSL_ia32cap_P+4(%rip),%ecx + test \$`1<<28`,%ecx # AVX bit + jnz _avx_cbc_dec_shortcut + jmp .Ldec_non_avx +.align 16 +.Ldec_non_avx: +___ +$code.=<<___; + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,0x40(%rsp) + movaps %xmm11,0x50(%rsp) + movaps %xmm12,0x60(%rsp) + movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +___ +$code.=<<___; + # stack layout + # + # +0 output sink + # +16 input sink [original %rsp and $num] + # +32 counters + + sub \$48,%rsp + and \$-64,%rsp + mov %rax,16(%rsp) # original %rsp + +.Ldec4x_body: + movdqu ($key),$zero # 0-round key + lea 0x78($key),$key # size optimization + lea 40*2($inp),$inp + +.Ldec4x_loop_grande: + mov $num,24(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + mov `40*$i+16-40*2`($inp),$one # borrow $one for number of blocks + mov `40*$i+0-40*2`($inp),@inptr[$i] + cmp $num,$one + mov `40*$i+8-40*2`($inp),@outptr[$i] + cmovg $one,$num # find maximum + test $one,$one + movdqu `40*$i+24-40*2`($inp),@inp[$i] # load IV + mov $one,`32+4*$i`(%rsp) # initialize counters + cmovle %rsp,@inptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldec4x_done + + movups 0x10-0x78($key),$rndkey1 + movups 0x20-0x78($key),$rndkey0 + mov 0xf0-0x78($key),$rounds + movdqu (@inptr[0]),@out[0] # load inputs + movdqu (@inptr[1]),@out[1] + pxor $zero,@out[0] + movdqu (@inptr[2]),@out[2] + pxor $zero,@out[1] + movdqu (@inptr[3]),@out[3] + pxor $zero,@out[2] + pxor $zero,@out[3] + movdqa 32(%rsp),$counters # load counters + xor $offset,$offset + jmp .Loop_dec4x + +.align 32 +.Loop_dec4x: + add \$16,$offset + lea 16(%rsp),$sink # sink pointer + mov \$1,$one # constant of 1 + sub $offset,$sink + + aesdec $rndkey1,@out[0] + prefetcht0 31(@inptr[0],$offset) # prefetch input + prefetcht0 31(@inptr[1],$offset) + aesdec $rndkey1,@out[1] + prefetcht0 31(@inptr[2],$offset) + prefetcht0 31(@inptr[3],$offset) + aesdec $rndkey1,@out[2] + aesdec $rndkey1,@out[3] + movups 0x30-0x78($key),$rndkey1 +___ +for($i=0;$i<4;$i++) { +my $rndkey = ($i&1) ? $rndkey1 : $rndkey0; +$code.=<<___; + cmp `32+4*$i`(%rsp),$one + aesdec $rndkey,@out[0] + aesdec $rndkey,@out[1] + aesdec $rndkey,@out[2] + cmovge $sink,@inptr[$i] # cancel input + cmovg $sink,@outptr[$i] # sink output + aesdec $rndkey,@out[3] + movups `0x40+16*$i-0x78`($key),$rndkey +___ +} +$code.=<<___; + movdqa $counters,$mask + aesdec $rndkey0,@out[0] + prefetcht0 15(@outptr[0],$offset) # prefetch output + prefetcht0 15(@outptr[1],$offset) + aesdec $rndkey0,@out[1] + prefetcht0 15(@outptr[2],$offset) + prefetcht0 15(@outptr[3],$offset) + aesdec $rndkey0,@out[2] + aesdec $rndkey0,@out[3] + movups 0x80-0x78($key),$rndkey0 + pxor $zero,$zero + + aesdec $rndkey1,@out[0] + pcmpgtd $zero,$mask + movdqu -0x78($key),$zero # reload 0-round key + aesdec $rndkey1,@out[1] + paddd $mask,$counters # decrement counters + movdqa $counters,32(%rsp) # update counters + aesdec $rndkey1,@out[2] + aesdec $rndkey1,@out[3] + movups 0x90-0x78($key),$rndkey1 + + cmp \$11,$rounds + + aesdec $rndkey0,@out[0] + aesdec $rndkey0,@out[1] + aesdec $rndkey0,@out[2] + aesdec $rndkey0,@out[3] + movups 0xa0-0x78($key),$rndkey0 + + jb .Ldec4x_tail + + aesdec $rndkey1,@out[0] + aesdec $rndkey1,@out[1] + aesdec $rndkey1,@out[2] + aesdec $rndkey1,@out[3] + movups 0xb0-0x78($key),$rndkey1 + + aesdec $rndkey0,@out[0] + aesdec $rndkey0,@out[1] + aesdec $rndkey0,@out[2] + aesdec $rndkey0,@out[3] + movups 0xc0-0x78($key),$rndkey0 + + je .Ldec4x_tail + + aesdec $rndkey1,@out[0] + aesdec $rndkey1,@out[1] + aesdec $rndkey1,@out[2] + aesdec $rndkey1,@out[3] + movups 0xd0-0x78($key),$rndkey1 + + aesdec $rndkey0,@out[0] + aesdec $rndkey0,@out[1] + aesdec $rndkey0,@out[2] + aesdec $rndkey0,@out[3] + movups 0xe0-0x78($key),$rndkey0 + jmp .Ldec4x_tail + +.align 32 +.Ldec4x_tail: + aesdec $rndkey1,@out[0] + aesdec $rndkey1,@out[1] + aesdec $rndkey1,@out[2] + pxor $rndkey0,@inp[0] + pxor $rndkey0,@inp[1] + aesdec $rndkey1,@out[3] + movdqu 0x10-0x78($key),$rndkey1 + pxor $rndkey0,@inp[2] + pxor $rndkey0,@inp[3] + movdqu 0x20-0x78($key),$rndkey0 + + aesdeclast @inp[0],@out[0] + aesdeclast @inp[1],@out[1] + movdqu -16(@inptr[0],$offset),@inp[0] # load next IV + movdqu -16(@inptr[1],$offset),@inp[1] + aesdeclast @inp[2],@out[2] + aesdeclast @inp[3],@out[3] + movdqu -16(@inptr[2],$offset),@inp[2] + movdqu -16(@inptr[3],$offset),@inp[3] + + movups @out[0],-16(@outptr[0],$offset) + movdqu (@inptr[0],$offset),@out[0] + movups @out[1],-16(@outptr[1],$offset) + movdqu (@inptr[1],$offset),@out[1] + pxor $zero,@out[0] + movups @out[2],-16(@outptr[2],$offset) + movdqu (@inptr[2],$offset),@out[2] + pxor $zero,@out[1] + movups @out[3],-16(@outptr[3],$offset) + movdqu (@inptr[3],$offset),@out[3] + pxor $zero,@out[2] + pxor $zero,@out[3] + + dec $num + jnz .Loop_dec4x + + mov 16(%rsp),%rax # original %rsp + mov 24(%rsp),$num + + lea `40*4`($inp),$inp + dec $num + jnz .Ldec4x_loop_grande + +.Ldec4x_done: +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + #movaps -0x68(%rax),%xmm13 + #movaps -0x58(%rax),%xmm14 + #movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Ldec4x_epilogue: + ret +.size aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt +___ + + if ($avx) {{{ +my @ptr=map("%r$_",(8..15)); +my $offload=$sink; + +my @out=map("%xmm$_",(2..9)); +my @inp=map("%xmm$_",(10..13)); +my ($counters,$zero)=("%xmm14","%xmm15"); + +$code.=<<___; +.type aesni_multi_cbc_encrypt_avx,\@function,3 +.align 32 +aesni_multi_cbc_encrypt_avx: +_avx_cbc_enc_shortcut: + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,0x40(%rsp) + movaps %xmm11,0x50(%rsp) + movaps %xmm12,-0x78(%rax) + movaps %xmm13,-0x68(%rax) + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +___ +$code.=<<___; + # stack layout + # + # +0 output sink + # +16 input sink [original %rsp and $num] + # +32 counters + # +64 distances between inputs and outputs + # +128 off-load area for @inp[0..3] + + sub \$192,%rsp + and \$-128,%rsp + mov %rax,16(%rsp) # original %rsp + +.Lenc8x_body: + vzeroupper + vmovdqu ($key),$zero # 0-round key + lea 0x78($key),$key # size optimization + lea 40*4($inp),$inp + shr \$1,$num + +.Lenc8x_loop_grande: + #mov $num,24(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<8;$i++) { + my $temp = $i ? $offload : $offset; + $code.=<<___; + mov `40*$i+16-40*4`($inp),$one # borrow $one for number of blocks + mov `40*$i+0-40*4`($inp),@ptr[$i] # input pointer + cmp $num,$one + mov `40*$i+8-40*4`($inp),$temp # output pointer + cmovg $one,$num # find maximum + test $one,$one + vmovdqu `40*$i+24-40*4`($inp),@out[$i] # load IV + mov $one,`32+4*$i`(%rsp) # initialize counters + cmovle %rsp,@ptr[$i] # cancel input + sub @ptr[$i],$temp # distance between input and output + mov $temp,`64+8*$i`(%rsp) # initialize distances +___ +} +$code.=<<___; + test $num,$num + jz .Lenc8x_done + + vmovups 0x10-0x78($key),$rndkey1 + vmovups 0x20-0x78($key),$rndkey0 + mov 0xf0-0x78($key),$rounds + + vpxor (@ptr[0]),$zero,@inp[0] # load inputs and xor with 0-round + lea 128(%rsp),$offload # offload area + vpxor (@ptr[1]),$zero,@inp[1] + vpxor (@ptr[2]),$zero,@inp[2] + vpxor (@ptr[3]),$zero,@inp[3] + vpxor @inp[0],@out[0],@out[0] + vpxor (@ptr[4]),$zero,@inp[0] + vpxor @inp[1],@out[1],@out[1] + vpxor (@ptr[5]),$zero,@inp[1] + vpxor @inp[2],@out[2],@out[2] + vpxor (@ptr[6]),$zero,@inp[2] + vpxor @inp[3],@out[3],@out[3] + vpxor (@ptr[7]),$zero,@inp[3] + vpxor @inp[0],@out[4],@out[4] + mov \$1,$one # constant of 1 + vpxor @inp[1],@out[5],@out[5] + vpxor @inp[2],@out[6],@out[6] + vpxor @inp[3],@out[7],@out[7] + jmp .Loop_enc8x + +.align 32 +.Loop_enc8x: +___ +for($i=0;$i<8;$i++) { +my $rndkey=($i&1)?$rndkey0:$rndkey1; +$code.=<<___; + vaesenc $rndkey,@out[0],@out[0] + cmp 32+4*$i(%rsp),$one +___ +$code.=<<___ if ($i); + mov 64+8*$i(%rsp),$offset +___ +$code.=<<___; + vaesenc $rndkey,@out[1],@out[1] + prefetcht0 31(@ptr[$i]) # prefetch input + vaesenc $rndkey,@out[2],@out[2] +___ +$code.=<<___ if ($i>1); + prefetcht0 15(@ptr[$i-2]) # prefetch output +___ +$code.=<<___; + vaesenc $rndkey,@out[3],@out[3] + lea (@ptr[$i],$offset),$offset + cmovge %rsp,@ptr[$i] # cancel input + vaesenc $rndkey,@out[4],@out[4] + cmovg %rsp,$offset # sink output + vaesenc $rndkey,@out[5],@out[5] + sub @ptr[$i],$offset + vaesenc $rndkey,@out[6],@out[6] + vpxor 16(@ptr[$i]),$zero,@inp[$i%4] # load input and xor with 0-round + mov $offset,64+8*$i(%rsp) + vaesenc $rndkey,@out[7],@out[7] + vmovups `16*(3+$i)-0x78`($key),$rndkey + lea 16(@ptr[$i],$offset),@ptr[$i] # switch to output +___ +$code.=<<___ if ($i<4) + vmovdqu @inp[$i%4],`16*$i`($offload) # off-load +___ +} +$code.=<<___; + vmovdqu 32(%rsp),$counters + prefetcht0 15(@ptr[$i-2]) # prefetch output + prefetcht0 15(@ptr[$i-1]) + cmp \$11,$rounds + jb .Lenc8x_tail + + vaesenc $rndkey1,@out[0],@out[0] + vaesenc $rndkey1,@out[1],@out[1] + vaesenc $rndkey1,@out[2],@out[2] + vaesenc $rndkey1,@out[3],@out[3] + vaesenc $rndkey1,@out[4],@out[4] + vaesenc $rndkey1,@out[5],@out[5] + vaesenc $rndkey1,@out[6],@out[6] + vaesenc $rndkey1,@out[7],@out[7] + vmovups 0xb0-0x78($key),$rndkey1 + + vaesenc $rndkey0,@out[0],@out[0] + vaesenc $rndkey0,@out[1],@out[1] + vaesenc $rndkey0,@out[2],@out[2] + vaesenc $rndkey0,@out[3],@out[3] + vaesenc $rndkey0,@out[4],@out[4] + vaesenc $rndkey0,@out[5],@out[5] + vaesenc $rndkey0,@out[6],@out[6] + vaesenc $rndkey0,@out[7],@out[7] + vmovups 0xc0-0x78($key),$rndkey0 + je .Lenc8x_tail + + vaesenc $rndkey1,@out[0],@out[0] + vaesenc $rndkey1,@out[1],@out[1] + vaesenc $rndkey1,@out[2],@out[2] + vaesenc $rndkey1,@out[3],@out[3] + vaesenc $rndkey1,@out[4],@out[4] + vaesenc $rndkey1,@out[5],@out[5] + vaesenc $rndkey1,@out[6],@out[6] + vaesenc $rndkey1,@out[7],@out[7] + vmovups 0xd0-0x78($key),$rndkey1 + + vaesenc $rndkey0,@out[0],@out[0] + vaesenc $rndkey0,@out[1],@out[1] + vaesenc $rndkey0,@out[2],@out[2] + vaesenc $rndkey0,@out[3],@out[3] + vaesenc $rndkey0,@out[4],@out[4] + vaesenc $rndkey0,@out[5],@out[5] + vaesenc $rndkey0,@out[6],@out[6] + vaesenc $rndkey0,@out[7],@out[7] + vmovups 0xe0-0x78($key),$rndkey0 + +.Lenc8x_tail: + vaesenc $rndkey1,@out[0],@out[0] + vpxor $zero,$zero,$zero + vaesenc $rndkey1,@out[1],@out[1] + vaesenc $rndkey1,@out[2],@out[2] + vpcmpgtd $zero,$counters,$zero + vaesenc $rndkey1,@out[3],@out[3] + vaesenc $rndkey1,@out[4],@out[4] + vpaddd $counters,$zero,$zero # decrement counters + vmovdqu 48(%rsp),$counters + vaesenc $rndkey1,@out[5],@out[5] + mov 64(%rsp),$offset # pre-load 1st offset + vaesenc $rndkey1,@out[6],@out[6] + vaesenc $rndkey1,@out[7],@out[7] + vmovups 0x10-0x78($key),$rndkey1 + + vaesenclast $rndkey0,@out[0],@out[0] + vmovdqa $zero,32(%rsp) # update counters + vpxor $zero,$zero,$zero + vaesenclast $rndkey0,@out[1],@out[1] + vaesenclast $rndkey0,@out[2],@out[2] + vpcmpgtd $zero,$counters,$zero + vaesenclast $rndkey0,@out[3],@out[3] + vaesenclast $rndkey0,@out[4],@out[4] + vpaddd $zero,$counters,$counters # decrement counters + vmovdqu -0x78($key),$zero # 0-round + vaesenclast $rndkey0,@out[5],@out[5] + vaesenclast $rndkey0,@out[6],@out[6] + vmovdqa $counters,48(%rsp) # update counters + vaesenclast $rndkey0,@out[7],@out[7] + vmovups 0x20-0x78($key),$rndkey0 + + vmovups @out[0],-16(@ptr[0]) # write output + sub $offset,@ptr[0] # switch to input + vpxor 0x00($offload),@out[0],@out[0] + vmovups @out[1],-16(@ptr[1]) + sub `64+1*8`(%rsp),@ptr[1] + vpxor 0x10($offload),@out[1],@out[1] + vmovups @out[2],-16(@ptr[2]) + sub `64+2*8`(%rsp),@ptr[2] + vpxor 0x20($offload),@out[2],@out[2] + vmovups @out[3],-16(@ptr[3]) + sub `64+3*8`(%rsp),@ptr[3] + vpxor 0x30($offload),@out[3],@out[3] + vmovups @out[4],-16(@ptr[4]) + sub `64+4*8`(%rsp),@ptr[4] + vpxor @inp[0],@out[4],@out[4] + vmovups @out[5],-16(@ptr[5]) + sub `64+5*8`(%rsp),@ptr[5] + vpxor @inp[1],@out[5],@out[5] + vmovups @out[6],-16(@ptr[6]) + sub `64+6*8`(%rsp),@ptr[6] + vpxor @inp[2],@out[6],@out[6] + vmovups @out[7],-16(@ptr[7]) + sub `64+7*8`(%rsp),@ptr[7] + vpxor @inp[3],@out[7],@out[7] + + dec $num + jnz .Loop_enc8x + + mov 16(%rsp),%rax # original %rsp + #mov 24(%rsp),$num + #lea `40*8`($inp),$inp + #dec $num + #jnz .Lenc8x_loop_grande + +.Lenc8x_done: + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lenc8x_epilogue: + ret +.size aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx + +.type aesni_multi_cbc_decrypt_avx,\@function,3 +.align 32 +aesni_multi_cbc_decrypt_avx: +_avx_cbc_dec_shortcut: + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,0x40(%rsp) + movaps %xmm11,0x50(%rsp) + movaps %xmm12,-0x78(%rax) + movaps %xmm13,-0x68(%rax) + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +___ +$code.=<<___; + # stack layout + # + # +0 output sink + # +16 input sink [original %rsp and $num] + # +32 counters + # +64 distances between inputs and outputs + # +128 off-load area for @inp[0..3] + # +192 IV/input offload + + sub \$256,%rsp + and \$-256,%rsp + sub \$192,%rsp + mov %rax,16(%rsp) # original %rsp + +.Ldec8x_body: + vzeroupper + vmovdqu ($key),$zero # 0-round key + lea 0x78($key),$key # size optimization + lea 40*4($inp),$inp + shr \$1,$num + +.Ldec8x_loop_grande: + #mov $num,24(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<8;$i++) { + my $temp = $i ? $offload : $offset; + $code.=<<___; + mov `40*$i+16-40*4`($inp),$one # borrow $one for number of blocks + mov `40*$i+0-40*4`($inp),@ptr[$i] # input pointer + cmp $num,$one + mov `40*$i+8-40*4`($inp),$temp # output pointer + cmovg $one,$num # find maximum + test $one,$one + vmovdqu `40*$i+24-40*4`($inp),@out[$i] # load IV + mov $one,`32+4*$i`(%rsp) # initialize counters + cmovle %rsp,@ptr[$i] # cancel input + sub @ptr[$i],$temp # distance between input and output + mov $temp,`64+8*$i`(%rsp) # initialize distances + vmovdqu @out[$i],`192+16*$i`(%rsp) # offload IV +___ +} +$code.=<<___; + test $num,$num + jz .Ldec8x_done + + vmovups 0x10-0x78($key),$rndkey1 + vmovups 0x20-0x78($key),$rndkey0 + mov 0xf0-0x78($key),$rounds + lea 192+128(%rsp),$offload # offload area + + vmovdqu (@ptr[0]),@out[0] # load inputs + vmovdqu (@ptr[1]),@out[1] + vmovdqu (@ptr[2]),@out[2] + vmovdqu (@ptr[3]),@out[3] + vmovdqu (@ptr[4]),@out[4] + vmovdqu (@ptr[5]),@out[5] + vmovdqu (@ptr[6]),@out[6] + vmovdqu (@ptr[7]),@out[7] + vmovdqu @out[0],0x00($offload) # offload inputs + vpxor $zero,@out[0],@out[0] # xor inputs with 0-round + vmovdqu @out[1],0x10($offload) + vpxor $zero,@out[1],@out[1] + vmovdqu @out[2],0x20($offload) + vpxor $zero,@out[2],@out[2] + vmovdqu @out[3],0x30($offload) + vpxor $zero,@out[3],@out[3] + vmovdqu @out[4],0x40($offload) + vpxor $zero,@out[4],@out[4] + vmovdqu @out[5],0x50($offload) + vpxor $zero,@out[5],@out[5] + vmovdqu @out[6],0x60($offload) + vpxor $zero,@out[6],@out[6] + vmovdqu @out[7],0x70($offload) + vpxor $zero,@out[7],@out[7] + xor \$0x80,$offload + mov \$1,$one # constant of 1 + jmp .Loop_dec8x + +.align 32 +.Loop_dec8x: +___ +for($i=0;$i<8;$i++) { +my $rndkey=($i&1)?$rndkey0:$rndkey1; +$code.=<<___; + vaesdec $rndkey,@out[0],@out[0] + cmp 32+4*$i(%rsp),$one +___ +$code.=<<___ if ($i); + mov 64+8*$i(%rsp),$offset +___ +$code.=<<___; + vaesdec $rndkey,@out[1],@out[1] + prefetcht0 31(@ptr[$i]) # prefetch input + vaesdec $rndkey,@out[2],@out[2] +___ +$code.=<<___ if ($i>1); + prefetcht0 15(@ptr[$i-2]) # prefetch output +___ +$code.=<<___; + vaesdec $rndkey,@out[3],@out[3] + lea (@ptr[$i],$offset),$offset + cmovge %rsp,@ptr[$i] # cancel input + vaesdec $rndkey,@out[4],@out[4] + cmovg %rsp,$offset # sink output + vaesdec $rndkey,@out[5],@out[5] + sub @ptr[$i],$offset + vaesdec $rndkey,@out[6],@out[6] + vmovdqu 16(@ptr[$i]),@inp[$i%4] # load input + mov $offset,64+8*$i(%rsp) + vaesdec $rndkey,@out[7],@out[7] + vmovups `16*(3+$i)-0x78`($key),$rndkey + lea 16(@ptr[$i],$offset),@ptr[$i] # switch to output +___ +$code.=<<___ if ($i<4); + vmovdqu @inp[$i%4],`128+16*$i`(%rsp) # off-load +___ +} +$code.=<<___; + vmovdqu 32(%rsp),$counters + prefetcht0 15(@ptr[$i-2]) # prefetch output + prefetcht0 15(@ptr[$i-1]) + cmp \$11,$rounds + jb .Ldec8x_tail + + vaesdec $rndkey1,@out[0],@out[0] + vaesdec $rndkey1,@out[1],@out[1] + vaesdec $rndkey1,@out[2],@out[2] + vaesdec $rndkey1,@out[3],@out[3] + vaesdec $rndkey1,@out[4],@out[4] + vaesdec $rndkey1,@out[5],@out[5] + vaesdec $rndkey1,@out[6],@out[6] + vaesdec $rndkey1,@out[7],@out[7] + vmovups 0xb0-0x78($key),$rndkey1 + + vaesdec $rndkey0,@out[0],@out[0] + vaesdec $rndkey0,@out[1],@out[1] + vaesdec $rndkey0,@out[2],@out[2] + vaesdec $rndkey0,@out[3],@out[3] + vaesdec $rndkey0,@out[4],@out[4] + vaesdec $rndkey0,@out[5],@out[5] + vaesdec $rndkey0,@out[6],@out[6] + vaesdec $rndkey0,@out[7],@out[7] + vmovups 0xc0-0x78($key),$rndkey0 + je .Ldec8x_tail + + vaesdec $rndkey1,@out[0],@out[0] + vaesdec $rndkey1,@out[1],@out[1] + vaesdec $rndkey1,@out[2],@out[2] + vaesdec $rndkey1,@out[3],@out[3] + vaesdec $rndkey1,@out[4],@out[4] + vaesdec $rndkey1,@out[5],@out[5] + vaesdec $rndkey1,@out[6],@out[6] + vaesdec $rndkey1,@out[7],@out[7] + vmovups 0xd0-0x78($key),$rndkey1 + + vaesdec $rndkey0,@out[0],@out[0] + vaesdec $rndkey0,@out[1],@out[1] + vaesdec $rndkey0,@out[2],@out[2] + vaesdec $rndkey0,@out[3],@out[3] + vaesdec $rndkey0,@out[4],@out[4] + vaesdec $rndkey0,@out[5],@out[5] + vaesdec $rndkey0,@out[6],@out[6] + vaesdec $rndkey0,@out[7],@out[7] + vmovups 0xe0-0x78($key),$rndkey0 + +.Ldec8x_tail: + vaesdec $rndkey1,@out[0],@out[0] + vpxor $zero,$zero,$zero + vaesdec $rndkey1,@out[1],@out[1] + vaesdec $rndkey1,@out[2],@out[2] + vpcmpgtd $zero,$counters,$zero + vaesdec $rndkey1,@out[3],@out[3] + vaesdec $rndkey1,@out[4],@out[4] + vpaddd $counters,$zero,$zero # decrement counters + vmovdqu 48(%rsp),$counters + vaesdec $rndkey1,@out[5],@out[5] + mov 64(%rsp),$offset # pre-load 1st offset + vaesdec $rndkey1,@out[6],@out[6] + vaesdec $rndkey1,@out[7],@out[7] + vmovups 0x10-0x78($key),$rndkey1 + + vaesdeclast $rndkey0,@out[0],@out[0] + vmovdqa $zero,32(%rsp) # update counters + vpxor $zero,$zero,$zero + vaesdeclast $rndkey0,@out[1],@out[1] + vpxor 0x00($offload),@out[0],@out[0] # xor with IV + vaesdeclast $rndkey0,@out[2],@out[2] + vpxor 0x10($offload),@out[1],@out[1] + vpcmpgtd $zero,$counters,$zero + vaesdeclast $rndkey0,@out[3],@out[3] + vpxor 0x20($offload),@out[2],@out[2] + vaesdeclast $rndkey0,@out[4],@out[4] + vpxor 0x30($offload),@out[3],@out[3] + vpaddd $zero,$counters,$counters # decrement counters + vmovdqu -0x78($key),$zero # 0-round + vaesdeclast $rndkey0,@out[5],@out[5] + vpxor 0x40($offload),@out[4],@out[4] + vaesdeclast $rndkey0,@out[6],@out[6] + vpxor 0x50($offload),@out[5],@out[5] + vmovdqa $counters,48(%rsp) # update counters + vaesdeclast $rndkey0,@out[7],@out[7] + vpxor 0x60($offload),@out[6],@out[6] + vmovups 0x20-0x78($key),$rndkey0 + + vmovups @out[0],-16(@ptr[0]) # write output + sub $offset,@ptr[0] # switch to input + vmovdqu 128+0(%rsp),@out[0] + vpxor 0x70($offload),@out[7],@out[7] + vmovups @out[1],-16(@ptr[1]) + sub `64+1*8`(%rsp),@ptr[1] + vmovdqu @out[0],0x00($offload) + vpxor $zero,@out[0],@out[0] + vmovdqu 128+16(%rsp),@out[1] + vmovups @out[2],-16(@ptr[2]) + sub `64+2*8`(%rsp),@ptr[2] + vmovdqu @out[1],0x10($offload) + vpxor $zero,@out[1],@out[1] + vmovdqu 128+32(%rsp),@out[2] + vmovups @out[3],-16(@ptr[3]) + sub `64+3*8`(%rsp),@ptr[3] + vmovdqu @out[2],0x20($offload) + vpxor $zero,@out[2],@out[2] + vmovdqu 128+48(%rsp),@out[3] + vmovups @out[4],-16(@ptr[4]) + sub `64+4*8`(%rsp),@ptr[4] + vmovdqu @out[3],0x30($offload) + vpxor $zero,@out[3],@out[3] + vmovdqu @inp[0],0x40($offload) + vpxor @inp[0],$zero,@out[4] + vmovups @out[5],-16(@ptr[5]) + sub `64+5*8`(%rsp),@ptr[5] + vmovdqu @inp[1],0x50($offload) + vpxor @inp[1],$zero,@out[5] + vmovups @out[6],-16(@ptr[6]) + sub `64+6*8`(%rsp),@ptr[6] + vmovdqu @inp[2],0x60($offload) + vpxor @inp[2],$zero,@out[6] + vmovups @out[7],-16(@ptr[7]) + sub `64+7*8`(%rsp),@ptr[7] + vmovdqu @inp[3],0x70($offload) + vpxor @inp[3],$zero,@out[7] + + xor \$128,$offload + dec $num + jnz .Loop_dec8x + + mov 16(%rsp),%rax # original %rsp + #mov 24(%rsp),$num + #lea `40*8`($inp),$inp + #dec $num + #jnz .Ldec8x_loop_grande + +.Ldec8x_done: + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Ldec8x_epilogue: + ret +.size aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx +___ + }}} + +if ($win64) { +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lin_prologue + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lin_prologue + + mov 16(%rax),%rax # pull saved stack pointer + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore cotnext->R12 + mov %r13,224($context) # restore cotnext->R13 + mov %r14,232($context) # restore cotnext->R14 + mov %r15,240($context) # restore cotnext->R15 + + lea -56-10*16(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lin_prologue: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_aesni_multi_cbc_encrypt + .rva .LSEH_end_aesni_multi_cbc_encrypt + .rva .LSEH_info_aesni_multi_cbc_encrypt + .rva .LSEH_begin_aesni_multi_cbc_decrypt + .rva .LSEH_end_aesni_multi_cbc_decrypt + .rva .LSEH_info_aesni_multi_cbc_decrypt +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_aesni_multi_cbc_encrypt_avx + .rva .LSEH_end_aesni_multi_cbc_encrypt_avx + .rva .LSEH_info_aesni_multi_cbc_encrypt_avx + .rva .LSEH_begin_aesni_multi_cbc_decrypt_avx + .rva .LSEH_end_aesni_multi_cbc_decrypt_avx + .rva .LSEH_info_aesni_multi_cbc_decrypt_avx +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_aesni_multi_cbc_encrypt: + .byte 9,0,0,0 + .rva se_handler + .rva .Lenc4x_body,.Lenc4x_epilogue # HandlerData[] +.LSEH_info_aesni_multi_cbc_decrypt: + .byte 9,0,0,0 + .rva se_handler + .rva .Ldec4x_body,.Ldec4x_epilogue # HandlerData[] +___ +$code.=<<___ if ($avx); +.LSEH_info_aesni_multi_cbc_encrypt_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .Lenc8x_body,.Lenc8x_epilogue # HandlerData[] +.LSEH_info_aesni_multi_cbc_decrypt_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .Ldec8x_body,.Ldec8x_epilogue # HandlerData[] +___ +} +#################################################################### + +sub rex { + local *opcode=shift; + my ($dst,$src)=@_; + my $rex=0; + + $rex|=0x04 if($dst>=8); + $rex|=0x01 if($src>=8); + push @opcode,$rex|0x40 if($rex); +} + +sub aesni { + my $line=shift; + my @opcode=(0x66); + + if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { + rex(\@opcode,$4,$3); + push @opcode,0x0f,0x3a,0xdf; + push @opcode,0xc0|($3&7)|(($4&7)<<3); # ModR/M + my $c=$2; + push @opcode,$c=~/^0/?oct($c):$c; + return ".byte\t".join(',',@opcode); + } + elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my %opcodelet = ( + "aesimc" => 0xdb, + "aesenc" => 0xdc, "aesenclast" => 0xdd, + "aesdec" => 0xde, "aesdeclast" => 0xdf + ); + return undef if (!defined($opcodelet{$1})); + rex(\@opcode,$3,$2); + push @opcode,0x0f,0x38,$opcodelet{$1}; + push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } + elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) { + my %opcodelet = ( + "aesenc" => 0xdc, "aesenclast" => 0xdd, + "aesdec" => 0xde, "aesdeclast" => 0xdf + ); + return undef if (!defined($opcodelet{$1})); + my $off = $2; + push @opcode,0x44 if ($3>=8); + push @opcode,0x0f,0x38,$opcodelet{$1}; + push @opcode,0x44|(($3&7)<<3),0x24; # ModR/M + push @opcode,($off=~/^0/?oct($off):$off)&0xff; + return ".byte\t".join(',',@opcode); + } + return $line; +} + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; +$code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; + +print $code; +close STDOUT; diff --git a/crypto/aes/asm/aesni-sha1-x86_64.pl b/crypto/aes/asm/aesni-sha1-x86_64.pl index 3c8f6c19e7bd..97992adca7c3 100755 --- a/crypto/aes/asm/aesni-sha1-x86_64.pl +++ b/crypto/aes/asm/aesni-sha1-x86_64.pl @@ -21,16 +21,25 @@ # subroutine: # # AES-128-CBC +SHA1 stitch gain -# Westmere 3.77[+5.6] 9.37 6.65 +41% -# Sandy Bridge 5.05[+5.2(6.3)] 10.25(11.35) 6.16(7.08) +67%(+60%) +# Westmere 3.77[+5.3] 9.07 6.55 +38% +# Sandy Bridge 5.05[+5.0(6.1)] 10.06(11.15) 5.98(7.05) +68%(+58%) +# Ivy Bridge 5.05[+4.6] 9.65 5.54 +74% +# Haswell 4.43[+3.6(4.2)] 8.00(8.58) 4.55(5.21) +75%(+65%) +# Bulldozer 5.77[+6.0] 11.72 6.37 +84% # # AES-192-CBC -# Westmere 4.51 10.11 6.97 +45% -# Sandy Bridge 6.05 11.25(12.35) 6.34(7.27) +77%(+70%) +# Westmere 4.51 9.81 6.80 +44% +# Sandy Bridge 6.05 11.06(12.15) 6.11(7.19) +81%(+69%) +# Ivy Bridge 6.05 10.65 6.07 +75% +# Haswell 5.29 8.86(9.44) 5.32(5.32) +67%(+77%) +# Bulldozer 6.89 12.84 6.96 +84% # # AES-256-CBC -# Westmere 5.25 10.85 7.25 +50% -# Sandy Bridge 7.05 12.25(13.35) 7.06(7.70) +74%(+73%) +# Westmere 5.25 10.55 7.21 +46% +# Sandy Bridge 7.05 12.06(13.15) 7.12(7.72) +69%(+70%) +# Ivy Bridge 7.05 11.65 7.12 +64% +# Haswell 6.19 9.76(10.34) 6.21(6.25) +57%(+65%) +# Bulldozer 8.00 13.95 8.25 +69% # # (*) There are two code paths: SSSE3 and AVX. See sha1-568.pl for # background information. Above numbers in parentheses are SSSE3 @@ -45,8 +54,25 @@ # standalone AESNI-CBC decrypt: # # AES-128-CBC AES-192-CBC AES-256-CBC -# Westmere 1.31 1.55 1.80 -# Sandy Bridge 0.93 1.06 1.22 +# Westmere 1.25 1.50 1.75 +# Sandy Bridge 0.74 0.91 1.09 +# Ivy Bridge 0.74 0.90 1.11 +# Haswell 0.63 0.76 0.88 +# Bulldozer 0.70 0.85 0.99 + +# And indeed: +# +# AES-256-CBC +SHA1 stitch gain +# Westmere 1.75 7.20 6.68 +7.8% +# Sandy Bridge 1.09 6.09(7.22) 5.82(6.95) +4.6%(+3.9%) +# Ivy Bridge 1.11 5.70 5.45 +4.6% +# Haswell 0.88 4.45(5.00) 4.39(4.69) +1.4%(*)(+6.6%) +# Bulldozer 0.99 6.95 5.95 +17%(**) +# +# (*) Tiny improvement coefficient on Haswell is because we compare +# AVX1 stitch to sum with AVX2 SHA1. +# (**) Execution is fully dominated by integer code sequence and +# SIMD still hardly shows [in single-process benchmark;-] $flavour = shift; $output = shift; @@ -68,6 +94,11 @@ $avx=1 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && $avx=1 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && `ml64 2>&1` =~ /Version ([0-9]+)\./ && $1>=10); +$avx=1 if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/ && $2>=3.0); + +$shaext=1; ### set to zero if compiling for 1.0.1 + +$stitched_decrypt=0; open OUT,"| \"$^X\" $xlate $flavour $output"; *STDOUT=*OUT; @@ -86,11 +117,15 @@ $code.=<<___; .globl aesni_cbc_sha1_enc .type aesni_cbc_sha1_enc,\@abi-omnipotent -.align 16 +.align 32 aesni_cbc_sha1_enc: # caller should check for SSSE3 and AES-NI bits mov OPENSSL_ia32cap_P+0(%rip),%r10d - mov OPENSSL_ia32cap_P+4(%rip),%r11d + mov OPENSSL_ia32cap_P+4(%rip),%r11 +___ +$code.=<<___ if ($shaext); + bt \$61,%r11 # check SHA bit + jc aesni_cbc_sha1_enc_shaext ___ $code.=<<___ if ($avx); and \$`1<<28`,%r11d # mask AVX bit @@ -112,10 +147,21 @@ my @X=map("%xmm$_",(4..7,0..3)); my @Tx=map("%xmm$_",(8..10)); my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization my @T=("%esi","%edi"); -my $j=0; my $jj=0; my $r=0; my $sn=0; +my $j=0; my $jj=0; my $r=0; my $sn=0; my $rx=0; my $K_XX_XX="%r11"; -my ($iv,$in,$rndkey0)=map("%xmm$_",(11..13)); -my @rndkey=("%xmm14","%xmm15"); +my ($rndkey0,$iv,$in)=map("%xmm$_",(11..13)); # for enc +my @rndkey=("%xmm14","%xmm15"); # for enc +my ($inout0,$inout1,$inout2,$inout3)=map("%xmm$_",(12..15)); # for dec + +if (1) { # reassign for Atom Silvermont + # The goal is to minimize amount of instructions with more than + # 3 prefix bytes. Or in more practical terms to keep AES-NI *and* + # SSSE3 instructions to upper half of the register bank. + @X=map("%xmm$_",(8..11,4..7)); + @Tx=map("%xmm$_",(12,13,3)); + ($iv,$in,$rndkey0)=map("%xmm$_",(2,14,15)); + @rndkey=("%xmm0","%xmm1"); +} sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; @@ -129,7 +175,7 @@ my $_ror=sub { &ror(@_) }; $code.=<<___; .type aesni_cbc_sha1_enc_ssse3,\@function,6 -.align 16 +.align 32 aesni_cbc_sha1_enc_ssse3: mov `($win64?56:8)`(%rsp),$inp # load 7th argument #shr \$6,$len # debugging artefact @@ -161,16 +207,16 @@ $code.=<<___; mov $in0,%r12 # reassign arguments mov $out,%r13 mov $len,%r14 - mov $key,%r15 + lea 112($key),%r15 # size optimization movdqu ($ivp),$iv # load IV mov $ivp,88(%rsp) # save $ivp ___ -my ($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments +($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments my $rounds="${ivp}d"; $code.=<<___; shl \$6,$len sub $in0,$out - mov 240($key),$rounds + mov 240-112($key),$rounds add $inp,$len # end of input lea K_XX_XX(%rip),$K_XX_XX @@ -180,19 +226,22 @@ $code.=<<___; mov 12($ctx),$D mov $B,@T[0] # magic seed mov 16($ctx),$E + mov $C,@T[1] + xor $D,@T[1] + and @T[1],@T[0] - movdqa 64($K_XX_XX),@X[2] # pbswap mask + movdqa 64($K_XX_XX),@Tx[2] # pbswap mask movdqa 0($K_XX_XX),@Tx[1] # K_00_19 movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] movdqu 16($inp),@X[-3&7] movdqu 32($inp),@X[-2&7] movdqu 48($inp),@X[-1&7] - pshufb @X[2],@X[-4&7] # byte swap + pshufb @Tx[2],@X[-4&7] # byte swap + pshufb @Tx[2],@X[-3&7] + pshufb @Tx[2],@X[-2&7] add \$64,$inp - pshufb @X[2],@X[-3&7] - pshufb @X[2],@X[-2&7] - pshufb @X[2],@X[-1&7] paddd @Tx[1],@X[-4&7] # add K_00_19 + pshufb @Tx[2],@X[-1&7] paddd @Tx[1],@X[-3&7] paddd @Tx[1],@X[-2&7] movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU @@ -201,8 +250,8 @@ $code.=<<___; psubd @Tx[1],@X[-3&7] movdqa @X[-2&7],32(%rsp) psubd @Tx[1],@X[-2&7] - movups ($key),$rndkey0 # $key[0] - movups 16($key),$rndkey[0] # forward reference + movups -112($key),$rndkey0 # $key[0] + movups 16-112($key),$rndkey[0] # forward reference jmp .Loop_ssse3 ___ @@ -219,31 +268,31 @@ ___ ___ $code.=<<___; xorps $in,$iv + movups `32+16*$k-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*$k`($key),$rndkey[1] ___ } elsif ($k==9) { $sn++; $code.=<<___; cmp \$11,$rounds jb .Laesenclast$sn - movups `32+16*($k+0)`($key),$rndkey[1] + movups `32+16*($k+0)-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*($k+1)`($key),$rndkey[0] + movups `32+16*($k+1)-112`($key),$rndkey[0] aesenc $rndkey[1],$iv je .Laesenclast$sn - movups `32+16*($k+2)`($key),$rndkey[1] + movups `32+16*($k+2)-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*($k+3)`($key),$rndkey[0] + movups `32+16*($k+3)-112`($key),$rndkey[0] aesenc $rndkey[1],$iv .Laesenclast$sn: aesenclast $rndkey[0],$iv - movups 16($key),$rndkey[1] # forward reference + movups 16-112($key),$rndkey[1] # forward reference ___ } else { $code.=<<___; + movups `32+16*$k-112`($key),$rndkey[1] aesenc $rndkey[0],$iv - movups `32+16*$k`($key),$rndkey[1] ___ } $r++; unshift(@rndkey,pop(@rndkey)); @@ -255,74 +304,75 @@ sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 my @insns = (&$body,&$body,&$body,&$body); # 40 instructions my ($a,$b,$c,$d,$e); - &movdqa (@X[0],@X[-3&7]); - eval(shift(@insns)); + eval(shift(@insns)); # ror + &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]); eval(shift(@insns)); &movdqa (@Tx[0],@X[-1&7]); - &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]" + &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); - &paddd (@Tx[1],@X[-1&7]); + &punpcklqdq(@X[0],@X[-3&7]); # compose "X[-14]" in "X[0]", was &palignr(@X[0],@X[-4&7],8); eval(shift(@insns)); + eval(shift(@insns)); # rol eval(shift(@insns)); &psrldq (@Tx[0],4); # "X[-3]", 3 dwords eval(shift(@insns)); eval(shift(@insns)); + &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]" eval(shift(@insns)); - eval(shift(@insns)); - + eval(shift(@insns)); # ror &pxor (@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]" eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - eval(shift(@insns)); &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]" eval(shift(@insns)); - eval(shift(@insns)); + eval(shift(@insns)); # rol &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); &movdqa (@Tx[2],@X[0]); - &movdqa (@Tx[0],@X[0]); - eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); # ror + &movdqa (@Tx[0],@X[0]); eval(shift(@insns)); &pslldq (@Tx[2],12); # "X[0]"<<96, extract one dword &paddd (@X[0],@X[0]); eval(shift(@insns)); eval(shift(@insns)); - eval(shift(@insns)); - eval(shift(@insns)); &psrld (@Tx[0],31); eval(shift(@insns)); + eval(shift(@insns)); # rol eval(shift(@insns)); &movdqa (@Tx[1],@Tx[2]); eval(shift(@insns)); eval(shift(@insns)); &psrld (@Tx[2],30); - &por (@X[0],@Tx[0]); # "X[0]"<<<=1 eval(shift(@insns)); + eval(shift(@insns)); # ror + &por (@X[0],@Tx[0]); # "X[0]"<<<=1 eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); &pslld (@Tx[1],2); &pxor (@X[0],@Tx[2]); - eval(shift(@insns)); eval(shift(@insns)); &movdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX + eval(shift(@insns)); # rol eval(shift(@insns)); eval(shift(@insns)); &pxor (@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2 + &pshufd (@Tx[1],@X[-1&7],0xee) if ($Xi==7); # was &movdqa (@Tx[0],@X[-1&7]) in Xupdate_ssse3_32_79 foreach (@insns) { eval; } # remaining instructions [if any] @@ -333,27 +383,30 @@ sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 sub Xupdate_ssse3_32_79() { use integer; my $body = shift; - my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions my ($a,$b,$c,$d,$e); - &movdqa (@Tx[0],@X[-1&7]) if ($Xi==8); - eval(shift(@insns)); # body_20_39 + eval(shift(@insns)) if ($Xi==8); &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" - &palignr(@Tx[0],@X[-2&7],8); # compose "X[-6]" + eval(shift(@insns)) if ($Xi==8); + eval(shift(@insns)); # body_20_39 eval(shift(@insns)); + eval(shift(@insns)) if (@insns[1] =~ /_ror/); + eval(shift(@insns)) if (@insns[0] =~ /_ror/); + &punpcklqdq(@Tx[0],@X[-1&7]); # compose "X[-6]", was &palignr(@Tx[0],@X[-2&7],8); eval(shift(@insns)); eval(shift(@insns)); # rol &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]" eval(shift(@insns)); - eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/); + eval(shift(@insns)); if ($Xi%5) { &movdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX... } else { # ... or load next one &movdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)"); } - &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); # ror + &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-6]" @@ -361,29 +414,31 @@ sub Xupdate_ssse3_32_79() eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); # rol + eval(shift(@insns)) if (@insns[0] =~ /_ror/); &movdqa (@Tx[0],@X[0]); - &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); + &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU eval(shift(@insns)); # ror eval(shift(@insns)); + eval(shift(@insns)); # body_20_39 &pslld (@X[0],2); - eval(shift(@insns)); # body_20_39 eval(shift(@insns)); - &psrld (@Tx[0],30); eval(shift(@insns)); - eval(shift(@insns)); # rol + &psrld (@Tx[0],30); + eval(shift(@insns)) if (@insns[0] =~ /_rol/);# rol eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); # ror - eval(shift(@insns)); &por (@X[0],@Tx[0]); # "X[0]"<<<=2 - eval(shift(@insns)); # body_20_39 eval(shift(@insns)); - &movdqa (@Tx[1],@X[0]) if ($Xi<19); + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)) if (@insns[1] =~ /_rol/); + eval(shift(@insns)) if (@insns[0] =~ /_rol/); + &pshufd(@Tx[1],@X[-1&7],0xee) if ($Xi<19); # was &movdqa (@Tx[1],@X[0]) eval(shift(@insns)); eval(shift(@insns)); # rol eval(shift(@insns)); @@ -404,9 +459,10 @@ sub Xuplast_ssse3_80() my ($a,$b,$c,$d,$e); eval(shift(@insns)); - &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); @@ -415,17 +471,17 @@ sub Xuplast_ssse3_80() foreach (@insns) { eval; } # remaining instructions &cmp ($inp,$len); - &je (".Ldone_ssse3"); + &je (shift); unshift(@Tx,pop(@Tx)); - &movdqa (@X[2],"64($K_XX_XX)"); # pbswap mask + &movdqa (@Tx[2],"64($K_XX_XX)"); # pbswap mask &movdqa (@Tx[1],"0($K_XX_XX)"); # K_00_19 &movdqu (@X[-4&7],"0($inp)"); # load input &movdqu (@X[-3&7],"16($inp)"); &movdqu (@X[-2&7],"32($inp)"); &movdqu (@X[-1&7],"48($inp)"); - &pshufb (@X[-4&7],@X[2]); # byte swap + &pshufb (@X[-4&7],@Tx[2]); # byte swap &add ($inp,64); $Xi=0; @@ -439,7 +495,10 @@ sub Xloop_ssse3() eval(shift(@insns)); eval(shift(@insns)); - &pshufb (@X[($Xi-3)&7],@X[2]); + eval(shift(@insns)); + &pshufb (@X[($Xi-3)&7],@Tx[2]); + eval(shift(@insns)); + eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); &paddd (@X[($Xi-4)&7],@Tx[1]); @@ -450,6 +509,8 @@ sub Xloop_ssse3() &movdqa (eval(16*$Xi)."(%rsp)",@X[($Xi-4)&7]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); &psubd (@X[($Xi-4)&7],@Tx[1]); foreach (@insns) { eval; } @@ -465,76 +526,106 @@ sub Xtail_ssse3() foreach (@insns) { eval; } } -sub body_00_19 () { - use integer; - my ($k,$n); - my @r=( +my @body_00_19 = ( '($a,$b,$c,$d,$e)=@V;'. - '&add ($e,eval(4*($j&15))."(%rsp)");', # X[]+K xfer - '&xor ($c,$d);', - '&mov (@T[1],$a);', # $b in next round - '&$_rol ($a,5);', - '&and (@T[0],$c);', # ($b&($c^$d)) - '&xor ($c,$d);', # restore $c - '&xor (@T[0],$d);', - '&add ($e,$a);', '&$_ror ($b,$j?7:2);', # $b>>>2 - '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&xor (@T[0],$d);', + '&mov (@T[1],$a);', # $b for next round + + '&add ($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer + '&xor ($b,$c);', # $c^$d for next round + + '&$_rol ($a,5);', + '&add ($e,@T[0]);', + '&and (@T[1],$b);', # ($b&($c^$d)) for next round + + '&xor ($b,$c);', # restore $b + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); + +sub body_00_19 () { # ((c^d)&b)^d + # on start @T[0]=(c^d)&b + return &body_20_39() if ($rx==19); $rx++; + + use integer; + my ($k,$n); + my @r=@body_00_19; + $n = scalar(@r); $k = (($jj+1)*12/20)*20*$n/12; # 12 aesencs per these 20 rounds @r[$k%$n].='&$aesenc();' if ($jj==$k/$n); $jj++; + return @r; } -sub body_20_39 () { - use integer; - my ($k,$n); - my @r=( +my @body_20_39 = ( '($a,$b,$c,$d,$e)=@V;'. - '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer - '&xor (@T[0],$d);', # ($b^$d) - '&mov (@T[1],$a);', # $b in next round + '&add ($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer + '&xor (@T[0],$d) if($j==19);'. + '&xor (@T[0],$c) if($j> 19);', # ($b^$d^$c) + '&mov (@T[1],$a);', # $b for next round + '&$_rol ($a,5);', - '&xor (@T[0],$c);', # ($b^$d^$c) - '&add ($e,$a);', + '&add ($e,@T[0]);', + '&xor (@T[1],$c) if ($j< 79);', # $b^$d for next round + '&$_ror ($b,7);', # $b>>>2 - '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); + +sub body_20_39 () { # b^d^c + # on entry @T[0]=b^d + return &body_40_59() if ($rx==39); $rx++; + + use integer; + my ($k,$n); + my @r=@body_20_39; + $n = scalar(@r); $k = (($jj+1)*8/20)*20*$n/8; # 8 aesencs per these 20 rounds - @r[$k%$n].='&$aesenc();' if ($jj==$k/$n); + @r[$k%$n].='&$aesenc();' if ($jj==$k/$n && $rx!=20); $jj++; + return @r; } -sub body_40_59 () { - use integer; - my ($k,$n); - my @r=( +my @body_40_59 = ( '($a,$b,$c,$d,$e)=@V;'. - '&mov (@T[1],$c);', - '&xor ($c,$d);', - '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer - '&and (@T[1],$d);', - '&and (@T[0],$c);', # ($b&($c^$d)) + '&add ($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer + '&and (@T[0],$c) if ($j>=40);', # (b^c)&(c^d) + '&xor ($c,$d) if ($j>=40);', # restore $c + '&$_ror ($b,7);', # $b>>>2 - '&add ($e,@T[1]);', - '&mov (@T[1],$a);', # $b in next round + '&mov (@T[1],$a);', # $b for next round + '&xor (@T[0],$c);', + '&$_rol ($a,5);', '&add ($e,@T[0]);', - '&xor ($c,$d);', # restore $c - '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&xor (@T[1],$c) if ($j==59);'. + '&xor (@T[1],$b) if ($j< 59);', # b^c for next round + + '&xor ($b,$c) if ($j< 59);', # c^d for next round + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); + +sub body_40_59 () { # ((b^c)&(c^d))^c + # on entry @T[0]=(b^c), (c^=d) + $rx++; + + use integer; + my ($k,$n); + my @r=@body_40_59; + $n = scalar(@r); $k=(($jj+1)*12/20)*20*$n/12; # 12 aesencs per these 20 rounds - @r[$k%$n].='&$aesenc();' if ($jj==$k/$n); + @r[$k%$n].='&$aesenc();' if ($jj==$k/$n && $rx!=40); $jj++; + return @r; } $code.=<<___; -.align 16 +.align 32 .Loop_ssse3: ___ &Xupdate_ssse3_16_31(\&body_00_19); @@ -553,7 +644,7 @@ ___ &Xupdate_ssse3_32_79(\&body_40_59); &Xupdate_ssse3_32_79(\&body_40_59); &Xupdate_ssse3_32_79(\&body_20_39); - &Xuplast_ssse3_80(\&body_20_39); # can jump to "done" + &Xuplast_ssse3_80(\&body_20_39,".Ldone_ssse3"); # can jump to "done" $saved_j=$j; @saved_V=@V; $saved_r=$r; @saved_rndkey=@rndkey; @@ -575,11 +666,13 @@ $code.=<<___; mov @T[0],4($ctx) mov @T[0],$B # magic seed mov $C,8($ctx) + mov $C,@T[1] mov $D,12($ctx) + xor $D,@T[1] mov $E,16($ctx) + and @T[1],@T[0] jmp .Loop_ssse3 -.align 16 .Ldone_ssse3: ___ $jj=$j=$saved_j; @V=@saved_V; @@ -631,7 +724,278 @@ $code.=<<___; .size aesni_cbc_sha1_enc_ssse3,.-aesni_cbc_sha1_enc_ssse3 ___ -$j=$jj=$r=$sn=0; + if ($stitched_decrypt) {{{ +# reset +($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); +$j=$jj=$r=$rx=0; +$Xi=4; + +# reassign for Atom Silvermont (see above) +($inout0,$inout1,$inout2,$inout3,$rndkey0)=map("%xmm$_",(0..4)); +@X=map("%xmm$_",(8..13,6,7)); +@Tx=map("%xmm$_",(14,15,5)); + +my @aes256_dec = ( + '&movdqu($inout0,"0x00($in0)");', + '&movdqu($inout1,"0x10($in0)"); &pxor ($inout0,$rndkey0);', + '&movdqu($inout2,"0x20($in0)"); &pxor ($inout1,$rndkey0);', + '&movdqu($inout3,"0x30($in0)"); &pxor ($inout2,$rndkey0);', + + '&pxor ($inout3,$rndkey0); &movups ($rndkey0,"16-112($key)");', + '&movaps("64(%rsp)",@X[2]);', # save IV, originally @X[3] + undef,undef + ); +for ($i=0;$i<13;$i++) { + push (@aes256_dec,( + '&aesdec ($inout0,$rndkey0);', + '&aesdec ($inout1,$rndkey0);', + '&aesdec ($inout2,$rndkey0);', + '&aesdec ($inout3,$rndkey0); &movups($rndkey0,"'.(16*($i+2)-112).'($key)");' + )); + push (@aes256_dec,(undef,undef)) if (($i>=3 && $i<=5) || $i>=11); + push (@aes256_dec,(undef,undef)) if ($i==5); +} +push(@aes256_dec,( + '&aesdeclast ($inout0,$rndkey0); &movups (@X[0],"0x00($in0)");', + '&aesdeclast ($inout1,$rndkey0); &movups (@X[1],"0x10($in0)");', + '&aesdeclast ($inout2,$rndkey0); &movups (@X[2],"0x20($in0)");', + '&aesdeclast ($inout3,$rndkey0); &movups (@X[3],"0x30($in0)");', + + '&xorps ($inout0,"64(%rsp)"); &movdqu ($rndkey0,"-112($key)");', + '&xorps ($inout1,@X[0]); &movups ("0x00($out,$in0)",$inout0);', + '&xorps ($inout2,@X[1]); &movups ("0x10($out,$in0)",$inout1);', + '&xorps ($inout3,@X[2]); &movups ("0x20($out,$in0)",$inout2);', + + '&movups ("0x30($out,$in0)",$inout3);' + )); + +sub body_00_19_dec () { # ((c^d)&b)^d + # on start @T[0]=(c^d)&b + return &body_20_39_dec() if ($rx==19); + + my @r=@body_00_19; + + unshift (@r,@aes256_dec[$rx]) if (@aes256_dec[$rx]); + $rx++; + + return @r; +} + +sub body_20_39_dec () { # b^d^c + # on entry @T[0]=b^d + return &body_40_59_dec() if ($rx==39); + + my @r=@body_20_39; + + unshift (@r,@aes256_dec[$rx]) if (@aes256_dec[$rx]); + $rx++; + + return @r; +} + +sub body_40_59_dec () { # ((b^c)&(c^d))^c + # on entry @T[0]=(b^c), (c^=d) + + my @r=@body_40_59; + + unshift (@r,@aes256_dec[$rx]) if (@aes256_dec[$rx]); + $rx++; + + return @r; +} + +$code.=<<___; +.globl aesni256_cbc_sha1_dec +.type aesni256_cbc_sha1_dec,\@abi-omnipotent +.align 32 +aesni256_cbc_sha1_dec: + # caller should check for SSSE3 and AES-NI bits + mov OPENSSL_ia32cap_P+0(%rip),%r10d + mov OPENSSL_ia32cap_P+4(%rip),%r11d +___ +$code.=<<___ if ($avx); + and \$`1<<28`,%r11d # mask AVX bit + and \$`1<<30`,%r10d # mask "Intel CPU" bit + or %r11d,%r10d + cmp \$`1<<28|1<<30`,%r10d + je aesni256_cbc_sha1_dec_avx +___ +$code.=<<___; + jmp aesni256_cbc_sha1_dec_ssse3 + ret +.size aesni256_cbc_sha1_dec,.-aesni256_cbc_sha1_dec + +.type aesni256_cbc_sha1_dec_ssse3,\@function,6 +.align 32 +aesni256_cbc_sha1_dec_ssse3: + mov `($win64?56:8)`(%rsp),$inp # load 7th argument + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + lea `-104-($win64?10*16:0)`(%rsp),%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,96+0(%rsp) + movaps %xmm7,96+16(%rsp) + movaps %xmm8,96+32(%rsp) + movaps %xmm9,96+48(%rsp) + movaps %xmm10,96+64(%rsp) + movaps %xmm11,96+80(%rsp) + movaps %xmm12,96+96(%rsp) + movaps %xmm13,96+112(%rsp) + movaps %xmm14,96+128(%rsp) + movaps %xmm15,96+144(%rsp) +.Lprologue_dec_ssse3: +___ +$code.=<<___; + mov $in0,%r12 # reassign arguments + mov $out,%r13 + mov $len,%r14 + lea 112($key),%r15 # size optimization + movdqu ($ivp),@X[3] # load IV + #mov $ivp,88(%rsp) # save $ivp +___ +($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments +$code.=<<___; + shl \$6,$len + sub $in0,$out + add $inp,$len # end of input + + lea K_XX_XX(%rip),$K_XX_XX + mov 0($ctx),$A # load context + mov 4($ctx),$B + mov 8($ctx),$C + mov 12($ctx),$D + mov $B,@T[0] # magic seed + mov 16($ctx),$E + mov $C,@T[1] + xor $D,@T[1] + and @T[1],@T[0] + + movdqa 64($K_XX_XX),@Tx[2] # pbswap mask + movdqa 0($K_XX_XX),@Tx[1] # K_00_19 + movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] + movdqu 16($inp),@X[-3&7] + movdqu 32($inp),@X[-2&7] + movdqu 48($inp),@X[-1&7] + pshufb @Tx[2],@X[-4&7] # byte swap + add \$64,$inp + pshufb @Tx[2],@X[-3&7] + pshufb @Tx[2],@X[-2&7] + pshufb @Tx[2],@X[-1&7] + paddd @Tx[1],@X[-4&7] # add K_00_19 + paddd @Tx[1],@X[-3&7] + paddd @Tx[1],@X[-2&7] + movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU + psubd @Tx[1],@X[-4&7] # restore X[] + movdqa @X[-3&7],16(%rsp) + psubd @Tx[1],@X[-3&7] + movdqa @X[-2&7],32(%rsp) + psubd @Tx[1],@X[-2&7] + movdqu -112($key),$rndkey0 # $key[0] + jmp .Loop_dec_ssse3 + +.align 32 +.Loop_dec_ssse3: +___ + &Xupdate_ssse3_16_31(\&body_00_19_dec); + &Xupdate_ssse3_16_31(\&body_00_19_dec); + &Xupdate_ssse3_16_31(\&body_00_19_dec); + &Xupdate_ssse3_16_31(\&body_00_19_dec); + &Xupdate_ssse3_32_79(\&body_00_19_dec); + &Xupdate_ssse3_32_79(\&body_20_39_dec); + &Xupdate_ssse3_32_79(\&body_20_39_dec); + &Xupdate_ssse3_32_79(\&body_20_39_dec); + &Xupdate_ssse3_32_79(\&body_20_39_dec); + &Xupdate_ssse3_32_79(\&body_20_39_dec); + &Xupdate_ssse3_32_79(\&body_40_59_dec); + &Xupdate_ssse3_32_79(\&body_40_59_dec); + &Xupdate_ssse3_32_79(\&body_40_59_dec); + &Xupdate_ssse3_32_79(\&body_40_59_dec); + &Xupdate_ssse3_32_79(\&body_40_59_dec); + &Xupdate_ssse3_32_79(\&body_20_39_dec); + &Xuplast_ssse3_80(\&body_20_39_dec,".Ldone_dec_ssse3"); # can jump to "done" + + $saved_j=$j; @saved_V=@V; + $saved_rx=$rx; + + &Xloop_ssse3(\&body_20_39_dec); + &Xloop_ssse3(\&body_20_39_dec); + &Xloop_ssse3(\&body_20_39_dec); + + eval(@aes256_dec[-1]); # last store +$code.=<<___; + lea 64($in0),$in0 + + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + add 12($ctx),$D + mov $A,0($ctx) + add 16($ctx),$E + mov @T[0],4($ctx) + mov @T[0],$B # magic seed + mov $C,8($ctx) + mov $C,@T[1] + mov $D,12($ctx) + xor $D,@T[1] + mov $E,16($ctx) + and @T[1],@T[0] + jmp .Loop_dec_ssse3 + +.Ldone_dec_ssse3: +___ + $jj=$j=$saved_j; @V=@saved_V; + $rx=$saved_rx; + + &Xtail_ssse3(\&body_20_39_dec); + &Xtail_ssse3(\&body_20_39_dec); + &Xtail_ssse3(\&body_20_39_dec); + + eval(@aes256_dec[-1]); # last store +$code.=<<___; + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + mov $A,0($ctx) + add 12($ctx),$D + mov @T[0],4($ctx) + add 16($ctx),$E + mov $C,8($ctx) + mov $D,12($ctx) + mov $E,16($ctx) + movups @X[3],($ivp) # write IV +___ +$code.=<<___ if ($win64); + movaps 96+0(%rsp),%xmm6 + movaps 96+16(%rsp),%xmm7 + movaps 96+32(%rsp),%xmm8 + movaps 96+48(%rsp),%xmm9 + movaps 96+64(%rsp),%xmm10 + movaps 96+80(%rsp),%xmm11 + movaps 96+96(%rsp),%xmm12 + movaps 96+112(%rsp),%xmm13 + movaps 96+128(%rsp),%xmm14 + movaps 96+144(%rsp),%xmm15 +___ +$code.=<<___; + lea `104+($win64?10*16:0)`(%rsp),%rsi + mov 0(%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_dec_ssse3: + ret +.size aesni256_cbc_sha1_dec_ssse3,.-aesni256_cbc_sha1_dec_ssse3 +___ + }}} +$j=$jj=$r=$rx=0; if ($avx) { my ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); @@ -641,13 +1005,17 @@ my @X=map("%xmm$_",(4..7,0..3)); my @Tx=map("%xmm$_",(8..10)); my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization my @T=("%esi","%edi"); +my ($rndkey0,$iv,$in)=map("%xmm$_",(11..13)); +my @rndkey=("%xmm14","%xmm15"); +my ($inout0,$inout1,$inout2,$inout3)=map("%xmm$_",(12..15)); # for dec +my $Kx=@Tx[2]; my $_rol=sub { &shld(@_[0],@_) }; my $_ror=sub { &shrd(@_[0],@_) }; $code.=<<___; .type aesni_cbc_sha1_enc_avx,\@function,6 -.align 16 +.align 32 aesni_cbc_sha1_enc_avx: mov `($win64?56:8)`(%rsp),$inp # load 7th argument #shr \$6,$len # debugging artefact @@ -680,17 +1048,16 @@ $code.=<<___; mov $in0,%r12 # reassign arguments mov $out,%r13 mov $len,%r14 - mov $key,%r15 + lea 112($key),%r15 # size optimization vmovdqu ($ivp),$iv # load IV mov $ivp,88(%rsp) # save $ivp ___ -my ($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments +($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments my $rounds="${ivp}d"; $code.=<<___; shl \$6,$len sub $in0,$out - mov 240($key),$rounds - add \$112,$key # size optimization + mov 240-112($key),$rounds add $inp,$len # end of input lea K_XX_XX(%rip),$K_XX_XX @@ -700,9 +1067,12 @@ $code.=<<___; mov 12($ctx),$D mov $B,@T[0] # magic seed mov 16($ctx),$E + mov $C,@T[1] + xor $D,@T[1] + and @T[1],@T[0] vmovdqa 64($K_XX_XX),@X[2] # pbswap mask - vmovdqa 0($K_XX_XX),@Tx[1] # K_00_19 + vmovdqa 0($K_XX_XX),$Kx # K_00_19 vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] vmovdqu 16($inp),@X[-3&7] vmovdqu 32($inp),@X[-2&7] @@ -712,13 +1082,13 @@ $code.=<<___; vpshufb @X[2],@X[-3&7],@X[-3&7] vpshufb @X[2],@X[-2&7],@X[-2&7] vpshufb @X[2],@X[-1&7],@X[-1&7] - vpaddd @Tx[1],@X[-4&7],@X[0] # add K_00_19 - vpaddd @Tx[1],@X[-3&7],@X[1] - vpaddd @Tx[1],@X[-2&7],@X[2] + vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19 + vpaddd $Kx,@X[-3&7],@X[1] + vpaddd $Kx,@X[-2&7],@X[2] vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU vmovdqa @X[1],16(%rsp) vmovdqa @X[2],32(%rsp) - vmovups -112($key),$rndkey0 # $key[0] + vmovups -112($key),$rndkey[1] # $key[0] vmovups 16-112($key),$rndkey[0] # forward reference jmp .Loop_avx ___ @@ -728,14 +1098,14 @@ my $aesenc=sub { my ($n,$k)=($r/10,$r%10); if ($k==0) { $code.=<<___; - vmovups `16*$n`($in0),$in # load input - vxorps $rndkey0,$in,$in + vmovdqu `16*$n`($in0),$in # load input + vpxor $rndkey[1],$in,$in ___ $code.=<<___ if ($n); vmovups $iv,`16*($n-1)`($out,$in0) # write output ___ $code.=<<___; - vxorps $in,$iv,$iv + vpxor $in,$iv,$iv vaesenc $rndkey[0],$iv,$iv vmovups `32+16*$k-112`($key),$rndkey[1] ___ @@ -755,6 +1125,7 @@ ___ vmovups `32+16*($k+3)-112`($key),$rndkey[0] .Lvaesenclast$sn: vaesenclast $rndkey[0],$iv,$iv + vmovups -112($key),$rndkey[0] vmovups 16-112($key),$rndkey[1] # forward reference ___ } else { @@ -778,10 +1149,10 @@ sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 eval(shift(@insns)); eval(shift(@insns)); - &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + &vpaddd (@Tx[1],$Kx,@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); - &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords + &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords eval(shift(@insns)); eval(shift(@insns)); &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" @@ -807,31 +1178,31 @@ sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 eval(shift(@insns)); eval(shift(@insns)); - &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword + &vpslldq(@Tx[1],@X[0],12); # "X[0]"<<96, extract one dword &vpaddd (@X[0],@X[0],@X[0]); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - &vpsrld (@Tx[1],@Tx[2],30); &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1 + &vpsrld (@Tx[0],@Tx[1],30); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - &vpslld (@Tx[2],@Tx[2],2); - &vpxor (@X[0],@X[0],@Tx[1]); + &vpslld (@Tx[1],@Tx[1],2); + &vpxor (@X[0],@X[0],@Tx[0]); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2 + &vpxor (@X[0],@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2 eval(shift(@insns)); eval(shift(@insns)); - &vmovdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX + &vmovdqa ($Kx,eval(16*(($Xi)/5))."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX eval(shift(@insns)); eval(shift(@insns)); @@ -839,7 +1210,6 @@ sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 foreach (@insns) { eval; } # remaining instructions [if any] $Xi++; push(@X,shift(@X)); # "rotate" X[] - push(@Tx,shift(@Tx)); } sub Xupdate_avx_32_79() @@ -858,12 +1228,8 @@ sub Xupdate_avx_32_79() &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" eval(shift(@insns)); eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/); - if ($Xi%5) { - &vmovdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX... - } else { # ... or load next one - &vmovdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)"); - } - &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + &vpaddd (@Tx[1],$Kx,@X[-1&7]); + &vmovdqa ($Kx,eval(16*($Xi/5))."($K_XX_XX)") if ($Xi%5==0); eval(shift(@insns)); # ror eval(shift(@insns)); @@ -893,7 +1259,6 @@ sub Xupdate_avx_32_79() &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2 eval(shift(@insns)); # body_20_39 eval(shift(@insns)); - &vmovdqa (@Tx[1],@X[0]) if ($Xi<19); eval(shift(@insns)); eval(shift(@insns)); # rol eval(shift(@insns)); @@ -904,7 +1269,6 @@ sub Xupdate_avx_32_79() foreach (@insns) { eval; } # remaining instructions $Xi++; push(@X,shift(@X)); # "rotate" X[] - push(@Tx,shift(@Tx)); } sub Xuplast_avx_80() @@ -914,28 +1278,26 @@ sub Xuplast_avx_80() my ($a,$b,$c,$d,$e); eval(shift(@insns)); - &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + &vpaddd (@Tx[1],$Kx,@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU + &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU foreach (@insns) { eval; } # remaining instructions &cmp ($inp,$len); - &je (".Ldone_avx"); + &je (shift); - unshift(@Tx,pop(@Tx)); - - &vmovdqa(@X[2],"64($K_XX_XX)"); # pbswap mask - &vmovdqa(@Tx[1],"0($K_XX_XX)"); # K_00_19 + &vmovdqa(@Tx[1],"64($K_XX_XX)"); # pbswap mask + &vmovdqa($Kx,"0($K_XX_XX)"); # K_00_19 &vmovdqu(@X[-4&7],"0($inp)"); # load input &vmovdqu(@X[-3&7],"16($inp)"); &vmovdqu(@X[-2&7],"32($inp)"); &vmovdqu(@X[-1&7],"48($inp)"); - &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap + &vpshufb(@X[-4&7],@X[-4&7],@Tx[1]); # byte swap &add ($inp,64); $Xi=0; @@ -949,15 +1311,15 @@ sub Xloop_avx() eval(shift(@insns)); eval(shift(@insns)); - &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]); + &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@Tx[1]); eval(shift(@insns)); eval(shift(@insns)); - &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@Tx[1]); + &vpaddd (@Tx[0],@X[($Xi-4)&7],$Kx); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - &vmovdqa(eval(16*$Xi)."(%rsp)",@X[$Xi&7]); # X[]+K xfer to IALU + &vmovdqa(eval(16*$Xi)."(%rsp)",@Tx[0]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); @@ -975,7 +1337,7 @@ sub Xtail_avx() } $code.=<<___; -.align 16 +.align 32 .Loop_avx: ___ &Xupdate_avx_16_31(\&body_00_19); @@ -994,7 +1356,7 @@ ___ &Xupdate_avx_32_79(\&body_40_59); &Xupdate_avx_32_79(\&body_40_59); &Xupdate_avx_32_79(\&body_20_39); - &Xuplast_avx_80(\&body_20_39); # can jump to "done" + &Xuplast_avx_80(\&body_20_39,".Ldone_avx"); # can jump to "done" $saved_j=$j; @saved_V=@V; $saved_r=$r; @saved_rndkey=@rndkey; @@ -1016,11 +1378,13 @@ $code.=<<___; mov @T[0],4($ctx) mov @T[0],$B # magic seed mov $C,8($ctx) + mov $C,@T[1] mov $D,12($ctx) + xor $D,@T[1] mov $E,16($ctx) + and @T[1],@T[0] jmp .Loop_avx -.align 16 .Ldone_avx: ___ $jj=$j=$saved_j; @V=@saved_V; @@ -1072,6 +1436,218 @@ $code.=<<___; ret .size aesni_cbc_sha1_enc_avx,.-aesni_cbc_sha1_enc_avx ___ + + if ($stitched_decrypt) {{{ +# reset +($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); + +$j=$jj=$r=$rx=0; +$Xi=4; + +@aes256_dec = ( + '&vpxor ($inout0,$rndkey0,"0x00($in0)");', + '&vpxor ($inout1,$rndkey0,"0x10($in0)");', + '&vpxor ($inout2,$rndkey0,"0x20($in0)");', + '&vpxor ($inout3,$rndkey0,"0x30($in0)");', + + '&vmovups($rndkey0,"16-112($key)");', + '&vmovups("64(%rsp)",@X[2]);', # save IV, originally @X[3] + undef,undef + ); +for ($i=0;$i<13;$i++) { + push (@aes256_dec,( + '&vaesdec ($inout0,$inout0,$rndkey0);', + '&vaesdec ($inout1,$inout1,$rndkey0);', + '&vaesdec ($inout2,$inout2,$rndkey0);', + '&vaesdec ($inout3,$inout3,$rndkey0); &vmovups($rndkey0,"'.(16*($i+2)-112).'($key)");' + )); + push (@aes256_dec,(undef,undef)) if (($i>=3 && $i<=5) || $i>=11); + push (@aes256_dec,(undef,undef)) if ($i==5); +} +push(@aes256_dec,( + '&vaesdeclast ($inout0,$inout0,$rndkey0); &vmovups(@X[0],"0x00($in0)");', + '&vaesdeclast ($inout1,$inout1,$rndkey0); &vmovups(@X[1],"0x10($in0)");', + '&vaesdeclast ($inout2,$inout2,$rndkey0); &vmovups(@X[2],"0x20($in0)");', + '&vaesdeclast ($inout3,$inout3,$rndkey0); &vmovups(@X[3],"0x30($in0)");', + + '&vxorps ($inout0,$inout0,"64(%rsp)"); &vmovdqu($rndkey0,"-112($key)");', + '&vxorps ($inout1,$inout1,@X[0]); &vmovups("0x00($out,$in0)",$inout0);', + '&vxorps ($inout2,$inout2,@X[1]); &vmovups("0x10($out,$in0)",$inout1);', + '&vxorps ($inout3,$inout3,@X[2]); &vmovups("0x20($out,$in0)",$inout2);', + + '&vmovups ("0x30($out,$in0)",$inout3);' + )); + +$code.=<<___; +.type aesni256_cbc_sha1_dec_avx,\@function,6 +.align 32 +aesni256_cbc_sha1_dec_avx: + mov `($win64?56:8)`(%rsp),$inp # load 7th argument + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + lea `-104-($win64?10*16:0)`(%rsp),%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,96+0(%rsp) + movaps %xmm7,96+16(%rsp) + movaps %xmm8,96+32(%rsp) + movaps %xmm9,96+48(%rsp) + movaps %xmm10,96+64(%rsp) + movaps %xmm11,96+80(%rsp) + movaps %xmm12,96+96(%rsp) + movaps %xmm13,96+112(%rsp) + movaps %xmm14,96+128(%rsp) + movaps %xmm15,96+144(%rsp) +.Lprologue_dec_avx: +___ +$code.=<<___; + vzeroall + mov $in0,%r12 # reassign arguments + mov $out,%r13 + mov $len,%r14 + lea 112($key),%r15 # size optimization + vmovdqu ($ivp),@X[3] # load IV +___ +($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments +$code.=<<___; + shl \$6,$len + sub $in0,$out + add $inp,$len # end of input + + lea K_XX_XX(%rip),$K_XX_XX + mov 0($ctx),$A # load context + mov 4($ctx),$B + mov 8($ctx),$C + mov 12($ctx),$D + mov $B,@T[0] # magic seed + mov 16($ctx),$E + mov $C,@T[1] + xor $D,@T[1] + and @T[1],@T[0] + + vmovdqa 64($K_XX_XX),@X[2] # pbswap mask + vmovdqa 0($K_XX_XX),$Kx # K_00_19 + vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] + vmovdqu 16($inp),@X[-3&7] + vmovdqu 32($inp),@X[-2&7] + vmovdqu 48($inp),@X[-1&7] + vpshufb @X[2],@X[-4&7],@X[-4&7] # byte swap + add \$64,$inp + vpshufb @X[2],@X[-3&7],@X[-3&7] + vpshufb @X[2],@X[-2&7],@X[-2&7] + vpshufb @X[2],@X[-1&7],@X[-1&7] + vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19 + vpaddd $Kx,@X[-3&7],@X[1] + vpaddd $Kx,@X[-2&7],@X[2] + vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU + vmovdqa @X[1],16(%rsp) + vmovdqa @X[2],32(%rsp) + vmovups -112($key),$rndkey0 # $key[0] + jmp .Loop_dec_avx + +.align 32 +.Loop_dec_avx: +___ + &Xupdate_avx_16_31(\&body_00_19_dec); + &Xupdate_avx_16_31(\&body_00_19_dec); + &Xupdate_avx_16_31(\&body_00_19_dec); + &Xupdate_avx_16_31(\&body_00_19_dec); + &Xupdate_avx_32_79(\&body_00_19_dec); + &Xupdate_avx_32_79(\&body_20_39_dec); + &Xupdate_avx_32_79(\&body_20_39_dec); + &Xupdate_avx_32_79(\&body_20_39_dec); + &Xupdate_avx_32_79(\&body_20_39_dec); + &Xupdate_avx_32_79(\&body_20_39_dec); + &Xupdate_avx_32_79(\&body_40_59_dec); + &Xupdate_avx_32_79(\&body_40_59_dec); + &Xupdate_avx_32_79(\&body_40_59_dec); + &Xupdate_avx_32_79(\&body_40_59_dec); + &Xupdate_avx_32_79(\&body_40_59_dec); + &Xupdate_avx_32_79(\&body_20_39_dec); + &Xuplast_avx_80(\&body_20_39_dec,".Ldone_dec_avx"); # can jump to "done" + + $saved_j=$j; @saved_V=@V; + $saved_rx=$rx; + + &Xloop_avx(\&body_20_39_dec); + &Xloop_avx(\&body_20_39_dec); + &Xloop_avx(\&body_20_39_dec); + + eval(@aes256_dec[-1]); # last store +$code.=<<___; + lea 64($in0),$in0 + + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + add 12($ctx),$D + mov $A,0($ctx) + add 16($ctx),$E + mov @T[0],4($ctx) + mov @T[0],$B # magic seed + mov $C,8($ctx) + mov $C,@T[1] + mov $D,12($ctx) + xor $D,@T[1] + mov $E,16($ctx) + and @T[1],@T[0] + jmp .Loop_dec_avx + +.Ldone_dec_avx: +___ + $jj=$j=$saved_j; @V=@saved_V; + $rx=$saved_rx; + + &Xtail_avx(\&body_20_39_dec); + &Xtail_avx(\&body_20_39_dec); + &Xtail_avx(\&body_20_39_dec); + + eval(@aes256_dec[-1]); # last store +$code.=<<___; + + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + mov $A,0($ctx) + add 12($ctx),$D + mov @T[0],4($ctx) + add 16($ctx),$E + mov $C,8($ctx) + mov $D,12($ctx) + mov $E,16($ctx) + vmovups @X[3],($ivp) # write IV + vzeroall +___ +$code.=<<___ if ($win64); + movaps 96+0(%rsp),%xmm6 + movaps 96+16(%rsp),%xmm7 + movaps 96+32(%rsp),%xmm8 + movaps 96+48(%rsp),%xmm9 + movaps 96+64(%rsp),%xmm10 + movaps 96+80(%rsp),%xmm11 + movaps 96+96(%rsp),%xmm12 + movaps 96+112(%rsp),%xmm13 + movaps 96+128(%rsp),%xmm14 + movaps 96+144(%rsp),%xmm15 +___ +$code.=<<___; + lea `104+($win64?10*16:0)`(%rsp),%rsi + mov 0(%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_dec_avx: + ret +.size aesni256_cbc_sha1_dec_avx,.-aesni256_cbc_sha1_dec_avx +___ + }}} } $code.=<<___; .align 64 @@ -1081,11 +1657,180 @@ K_XX_XX: .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask +.byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 .asciz "AESNI-CBC+SHA1 stitch for x86_64, CRYPTOGAMS by " .align 64 ___ + if ($shaext) {{{ +($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); + +$rounds="%r11d"; + +($iv,$in,$rndkey0)=map("%xmm$_",(2,14,15)); +@rndkey=("%xmm0","%xmm1"); +$r=0; + +my ($BSWAP,$ABCD,$E,$E_,$ABCD_SAVE,$E_SAVE)=map("%xmm$_",(7..12)); +my @MSG=map("%xmm$_",(3..6)); + +$code.=<<___; +.type aesni_cbc_sha1_enc_shaext,\@function,6 +.align 32 +aesni_cbc_sha1_enc_shaext: + mov `($win64?56:8)`(%rsp),$inp # load 7th argument +___ +$code.=<<___ if ($win64); + lea `-8-10*16`(%rsp),%rsp + movaps %xmm6,-8-10*16(%rax) + movaps %xmm7,-8-9*16(%rax) + movaps %xmm8,-8-8*16(%rax) + movaps %xmm9,-8-7*16(%rax) + movaps %xmm10,-8-6*16(%rax) + movaps %xmm11,-8-5*16(%rax) + movaps %xmm12,-8-4*16(%rax) + movaps %xmm13,-8-3*16(%rax) + movaps %xmm14,-8-2*16(%rax) + movaps %xmm15,-8-1*16(%rax) +.Lprologue_shaext: +___ +$code.=<<___; + movdqu ($ctx),$ABCD + movd 16($ctx),$E + movdqa K_XX_XX+0x50(%rip),$BSWAP # byte-n-word swap + + mov 240($key),$rounds + sub $in0,$out + movups ($key),$rndkey0 # $key[0] + movups 16($key),$rndkey[0] # forward reference + lea 112($key),$key # size optimization + + pshufd \$0b00011011,$ABCD,$ABCD # flip word order + pshufd \$0b00011011,$E,$E # flip word order + jmp .Loop_shaext +.align 16 +.Loop_shaext: +___ + &$aesenc(); +$code.=<<___; + movdqu ($inp),@MSG[0] + movdqa $E,$E_SAVE # offload $E + pshufb $BSWAP,@MSG[0] + movdqu 0x10($inp),@MSG[1] + movdqa $ABCD,$ABCD_SAVE # offload $ABCD +___ + &$aesenc(); +$code.=<<___; + pshufb $BSWAP,@MSG[1] + + paddd @MSG[0],$E + movdqu 0x20($inp),@MSG[2] + lea 0x40($inp),$inp + pxor $E_SAVE,@MSG[0] # black magic +___ + &$aesenc(); +$code.=<<___; + pxor $E_SAVE,@MSG[0] # black magic + movdqa $ABCD,$E_ + pshufb $BSWAP,@MSG[2] + sha1rnds4 \$0,$E,$ABCD # 0-3 + sha1nexte @MSG[1],$E_ +___ + &$aesenc(); +$code.=<<___; + sha1msg1 @MSG[1],@MSG[0] + movdqu -0x10($inp),@MSG[3] + movdqa $ABCD,$E + pshufb $BSWAP,@MSG[3] +___ + &$aesenc(); +$code.=<<___; + sha1rnds4 \$0,$E_,$ABCD # 4-7 + sha1nexte @MSG[2],$E + pxor @MSG[2],@MSG[0] + sha1msg1 @MSG[2],@MSG[1] +___ + &$aesenc(); + +for($i=2;$i<20-4;$i++) { +$code.=<<___; + movdqa $ABCD,$E_ + sha1rnds4 \$`int($i/5)`,$E,$ABCD # 8-11 + sha1nexte @MSG[3],$E_ +___ + &$aesenc(); +$code.=<<___; + sha1msg2 @MSG[3],@MSG[0] + pxor @MSG[3],@MSG[1] + sha1msg1 @MSG[3],@MSG[2] +___ + ($E,$E_)=($E_,$E); + push(@MSG,shift(@MSG)); + + &$aesenc(); +} +$code.=<<___; + movdqa $ABCD,$E_ + sha1rnds4 \$3,$E,$ABCD # 64-67 + sha1nexte @MSG[3],$E_ + sha1msg2 @MSG[3],@MSG[0] + pxor @MSG[3],@MSG[1] +___ + &$aesenc(); +$code.=<<___; + movdqa $ABCD,$E + sha1rnds4 \$3,$E_,$ABCD # 68-71 + sha1nexte @MSG[0],$E + sha1msg2 @MSG[0],@MSG[1] +___ + &$aesenc(); +$code.=<<___; + movdqa $E_SAVE,@MSG[0] + movdqa $ABCD,$E_ + sha1rnds4 \$3,$E,$ABCD # 72-75 + sha1nexte @MSG[1],$E_ +___ + &$aesenc(); +$code.=<<___; + movdqa $ABCD,$E + sha1rnds4 \$3,$E_,$ABCD # 76-79 + sha1nexte $MSG[0],$E +___ + while($r<40) { &$aesenc(); } # remaining aesenc's +$code.=<<___; + dec $len + + paddd $ABCD_SAVE,$ABCD + movups $iv,48($out,$in0) # write output + lea 64($in0),$in0 + jnz .Loop_shaext + + pshufd \$0b00011011,$ABCD,$ABCD + pshufd \$0b00011011,$E,$E + movups $iv,($ivp) # write IV + movdqu $ABCD,($ctx) + movd $E,16($ctx) +___ +$code.=<<___ if ($win64); + movaps -8-10*16(%rax),%xmm6 + movaps -8-9*16(%rax),%xmm7 + movaps -8-8*16(%rax),%xmm8 + movaps -8-7*16(%rax),%xmm9 + movaps -8-6*16(%rax),%xmm10 + movaps -8-5*16(%rax),%xmm11 + movaps -8-4*16(%rax),%xmm12 + movaps -8-3*16(%rax),%xmm13 + movaps -8-2*16(%rax),%xmm14 + movaps -8-1*16(%rax),%xmm15 + mov %rax,%rsp +.Lepilogue_shaext: +___ +$code.=<<___; + ret +.size aesni_cbc_sha1_enc_shaext,.-aesni_cbc_sha1_enc_shaext +___ + }}} # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, # CONTEXT *context,DISPATCHER_CONTEXT *disp) if ($win64) { @@ -1127,7 +1872,21 @@ ssse3_handler: lea (%rsi,%r10),%r10 # epilogue label cmp %r10,%rbx # context->Rip>=epilogue label jae .Lcommon_seh_tail +___ +$code.=<<___ if ($shaext); + lea aesni_cbc_sha1_enc_shaext(%rip),%r10 + cmp %r10,%rbx + jb .Lseh_no_shaext + lea (%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + lea 168(%rax),%rax # adjust stack pointer + jmp .Lcommon_seh_tail +.Lseh_no_shaext: +___ +$code.=<<___; lea 96(%rax),%rsi lea 512($context),%rdi # &context.Xmm6 mov \$20,%ecx @@ -1199,6 +1958,11 @@ $code.=<<___ if ($avx); .rva .LSEH_end_aesni_cbc_sha1_enc_avx .rva .LSEH_info_aesni_cbc_sha1_enc_avx ___ +$code.=<<___ if ($shaext); + .rva .LSEH_begin_aesni_cbc_sha1_enc_shaext + .rva .LSEH_end_aesni_cbc_sha1_enc_shaext + .rva .LSEH_info_aesni_cbc_sha1_enc_shaext +___ $code.=<<___; .section .xdata .align 8 @@ -1213,6 +1977,12 @@ $code.=<<___ if ($avx); .rva ssse3_handler .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[] ___ +$code.=<<___ if ($shaext); +.LSEH_info_aesni_cbc_sha1_enc_shaext: + .byte 9,0,0,0 + .rva ssse3_handler + .rva .Lprologue_shaext,.Lepilogue_shaext # HandlerData[] +___ } #################################################################### @@ -1223,28 +1993,65 @@ sub rex { $rex|=0x04 if($dst>=8); $rex|=0x01 if($src>=8); - push @opcode,$rex|0x40 if($rex); + unshift @opcode,$rex|0x40 if($rex); +} + +sub sha1rnds4 { + if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x3a,0xcc); + rex(\@opcode,$3,$2); + push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + my $c=$1; + push @opcode,$c=~/^0/?oct($c):$c; + return ".byte\t".join(',',@opcode); + } else { + return "sha1rnds4\t".@_[0]; + } +} + +sub sha1op38 { + my $instr = shift; + my %opcodelet = ( + "sha1nexte" => 0xc8, + "sha1msg1" => 0xc9, + "sha1msg2" => 0xca ); + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x38); + rex(\@opcode,$2,$1); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } } sub aesni { my $line=shift; - my @opcode=(0x66); + my @opcode=(0x0f,0x38); if ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { my %opcodelet = ( - "aesenc" => 0xdc, "aesenclast" => 0xdd + "aesenc" => 0xdc, "aesenclast" => 0xdd, + "aesdec" => 0xde, "aesdeclast" => 0xdf ); return undef if (!defined($opcodelet{$1})); rex(\@opcode,$3,$2); - push @opcode,0x0f,0x38,$opcodelet{$1}; - push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + push @opcode,$opcodelet{$1},0xc0|($2&7)|(($3&7)<<3); # ModR/M + unshift @opcode,0x66; return ".byte\t".join(',',@opcode); } return $line; } -$code =~ s/\`([^\`]*)\`/eval($1)/gem; -$code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or + s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo or + s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/geo; -print $code; + print $_,"\n"; +} close STDOUT; diff --git a/crypto/aes/asm/aesni-sha256-x86_64.pl b/crypto/aes/asm/aesni-sha256-x86_64.pl new file mode 100755 index 000000000000..19b0433b3b1b --- /dev/null +++ b/crypto/aes/asm/aesni-sha256-x86_64.pl @@ -0,0 +1,1708 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# January 2013 +# +# This is AESNI-CBC+SHA256 stitch implementation. The idea, as spelled +# in http://download.intel.com/design/intarch/papers/323686.pdf, is +# that since AESNI-CBC encrypt exhibit *very* low instruction-level +# parallelism, interleaving it with another algorithm would allow to +# utilize processor resources better and achieve better performance. +# SHA256 instruction sequences(*) are taken from sha512-x86_64.pl and +# AESNI code is weaved into it. As SHA256 dominates execution time, +# stitch performance does not depend on AES key length. Below are +# performance numbers in cycles per processed byte, less is better, +# for standalone AESNI-CBC encrypt, standalone SHA256, and stitched +# subroutine: +# +# AES-128/-192/-256+SHA256 this(**)gain +# Sandy Bridge 5.05/6.05/7.05+11.6 13.0 +28%/36%/43% +# Ivy Bridge 5.05/6.05/7.05+10.3 11.6 +32%/41%/50% +# Haswell 4.43/5.29/6.19+7.80 8.79 +39%/49%/59% +# Bulldozer 5.77/6.89/8.00+13.7 13.7 +42%/50%/58% +# +# (*) there are XOP, AVX1 and AVX2 code pathes, meaning that +# Westmere is omitted from loop, this is because gain was not +# estimated high enough to justify the effort; +# (**) these are EVP-free results, results obtained with 'speed +# -evp aes-256-cbc-hmac-sha256' will vary by percent or two; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=12); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +$shaext=$avx; ### set to zero if compiling for 1.0.1 +$avx=1 if (!$shaext && $avx); + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +$func="aesni_cbc_sha256_enc"; +$TABLE="K256"; +$SZ=4; +@ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%eax","%ebx","%ecx","%edx", + "%r8d","%r9d","%r10d","%r11d"); +($T1,$a0,$a1,$a2,$a3)=("%r12d","%r13d","%r14d","%r15d","%esi"); +@Sigma0=( 2,13,22); +@Sigma1=( 6,11,25); +@sigma0=( 7,18, 3); +@sigma1=(17,19,10); +$rounds=64; + +######################################################################## +# void aesni_cbc_sha256_enc(const void *inp, +# void *out, +# size_t length, +# const AES_KEY *key, +# unsigned char *iv, +# SHA256_CTX *ctx, +# const void *in0); +($inp, $out, $len, $key, $ivp, $ctx, $in0) = +("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); + +$Tbl="%rbp"; + +$_inp="16*$SZ+0*8(%rsp)"; +$_out="16*$SZ+1*8(%rsp)"; +$_end="16*$SZ+2*8(%rsp)"; +$_key="16*$SZ+3*8(%rsp)"; +$_ivp="16*$SZ+4*8(%rsp)"; +$_ctx="16*$SZ+5*8(%rsp)"; +$_in0="16*$SZ+6*8(%rsp)"; +$_rsp="16*$SZ+7*8(%rsp)"; +$framesz=16*$SZ+8*8; + +$code=<<___; +.text + +.extern OPENSSL_ia32cap_P +.globl $func +.type $func,\@abi-omnipotent +.align 16 +$func: +___ + if ($avx) { +$code.=<<___; + lea OPENSSL_ia32cap_P(%rip),%r11 + mov \$1,%eax + cmp \$0,`$win64?"%rcx":"%rdi"` + je .Lprobe + mov 0(%r11),%eax + mov 4(%r11),%r10 +___ +$code.=<<___ if ($shaext); + bt \$61,%r10 # check for SHA + jc ${func}_shaext +___ +$code.=<<___; + mov %r10,%r11 + shr \$32,%r11 + + test \$`1<<11`,%r10d # check for XOP + jnz ${func}_xop +___ +$code.=<<___ if ($avx>1); + and \$`1<<8|1<<5|1<<3`,%r11d # check for BMI2+AVX2+BMI1 + cmp \$`1<<8|1<<5|1<<3`,%r11d + je ${func}_avx2 +___ +$code.=<<___; + and \$`1<<30`,%eax # mask "Intel CPU" bit + and \$`1<<28|1<<9`,%r10d # mask AVX+SSSE3 bits + or %eax,%r10d + cmp \$`1<<28|1<<9|1<<30`,%r10d + je ${func}_avx + ud2 +___ + } +$code.=<<___; + xor %eax,%eax + cmp \$0,`$win64?"%rcx":"%rdi"` + je .Lprobe + ud2 +.Lprobe: + ret +.size $func,.-$func + +.align 64 +.type $TABLE,\@object +$TABLE: + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f + .long 0,0,0,0, 0,0,0,0, -1,-1,-1,-1 + .long 0,0,0,0, 0,0,0,0 + .asciz "AESNI-CBC+SHA256 stitch for x86_64, CRYPTOGAMS by " +.align 64 +___ + +###################################################################### +# SIMD code paths +# +{{{ +($iv,$inout,$roundkey,$temp, + $mask10,$mask12,$mask14,$offload)=map("%xmm$_",(8..15)); + +$aesni_cbc_idx=0; +@aesni_cbc_block = ( +## &vmovdqu ($roundkey,"0x00-0x80($inp)");' +## &vmovdqu ($inout,($inp)); +## &mov ($_inp,$inp); + + '&vpxor ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x10-0x80($inp)");', + + '&vpxor ($inout,$inout,$iv);', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x20-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x30-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x40-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x50-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x60-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x70-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x80-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x90-0x80($inp)");', + + '&vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0xa0-0x80($inp)");', + + '&vaesenclast ($temp,$inout,$roundkey);'. + ' &vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0xb0-0x80($inp)");', + + '&vpand ($iv,$temp,$mask10);'. + ' &vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0xc0-0x80($inp)");', + + '&vaesenclast ($temp,$inout,$roundkey);'. + ' &vaesenc ($inout,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0xd0-0x80($inp)");', + + '&vpand ($temp,$temp,$mask12);'. + ' &vaesenc ($inout,$inout,$roundkey);'. + '&vmovdqu ($roundkey,"0xe0-0x80($inp)");', + + '&vpor ($iv,$iv,$temp);'. + ' &vaesenclast ($temp,$inout,$roundkey);'. + ' &vmovdqu ($roundkey,"0x00-0x80($inp)");' + +## &mov ($inp,$_inp); +## &mov ($out,$_out); +## &vpand ($temp,$temp,$mask14); +## &vpor ($iv,$iv,$temp); +## &vmovdqu ($iv,($out,$inp); +## &lea (inp,16($inp)); +); + +my $a4=$T1; +my ($a,$b,$c,$d,$e,$f,$g,$h); + +sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; + my $arg = pop; + $arg = "\$$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; +} + +sub body_00_15 () { + ( + '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'. + + '&ror ($a0,$Sigma1[2]-$Sigma1[1])', + '&mov ($a,$a1)', + '&mov ($a4,$f)', + + '&xor ($a0,$e)', + '&ror ($a1,$Sigma0[2]-$Sigma0[1])', + '&xor ($a4,$g)', # f^g + + '&ror ($a0,$Sigma1[1]-$Sigma1[0])', + '&xor ($a1,$a)', + '&and ($a4,$e)', # (f^g)&e + + @aesni_cbc_block[$aesni_cbc_idx++]. + '&xor ($a0,$e)', + '&add ($h,$SZ*($i&15)."(%rsp)")', # h+=X[i]+K[i] + '&mov ($a2,$a)', + + '&ror ($a1,$Sigma0[1]-$Sigma0[0])', + '&xor ($a4,$g)', # Ch(e,f,g)=((f^g)&e)^g + '&xor ($a2,$b)', # a^b, b^c in next round + + '&ror ($a0,$Sigma1[0])', # Sigma1(e) + '&add ($h,$a4)', # h+=Ch(e,f,g) + '&and ($a3,$a2)', # (b^c)&(a^b) + + '&xor ($a1,$a)', + '&add ($h,$a0)', # h+=Sigma1(e) + '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b) + + '&add ($d,$h)', # d+=h + '&ror ($a1,$Sigma0[0])', # Sigma0(a) + '&add ($h,$a3)', # h+=Maj(a,b,c) + + '&mov ($a0,$d)', + '&add ($a1,$h);'. # h+=Sigma0(a) + '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;' + ); +} + +if ($avx) {{ +###################################################################### +# XOP code path +# +$code.=<<___; +.type ${func}_xop,\@function,6 +.align 64 +${func}_xop: +.Lxop_shortcut: + mov `($win64?56:8)`(%rsp),$in0 # load 7th parameter + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + mov %rsp,%r11 # copy %rsp + sub \$`$framesz+$win64*16*10`,%rsp + and \$-64,%rsp # align stack frame + + shl \$6,$len + sub $inp,$out # re-bias + sub $inp,$in0 + add $inp,$len # end of input + + #mov $inp,$_inp # saved later + mov $out,$_out + mov $len,$_end + #mov $key,$_key # remains resident in $inp register + mov $ivp,$_ivp + mov $ctx,$_ctx + mov $in0,$_in0 + mov %r11,$_rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,`$framesz+16*0`(%rsp) + movaps %xmm7,`$framesz+16*1`(%rsp) + movaps %xmm8,`$framesz+16*2`(%rsp) + movaps %xmm9,`$framesz+16*3`(%rsp) + movaps %xmm10,`$framesz+16*4`(%rsp) + movaps %xmm11,`$framesz+16*5`(%rsp) + movaps %xmm12,`$framesz+16*6`(%rsp) + movaps %xmm13,`$framesz+16*7`(%rsp) + movaps %xmm14,`$framesz+16*8`(%rsp) + movaps %xmm15,`$framesz+16*9`(%rsp) +___ +$code.=<<___; +.Lprologue_xop: + vzeroall + + mov $inp,%r12 # borrow $a4 + lea 0x80($key),$inp # size optimization, reassign + lea $TABLE+`$SZ*2*$rounds+32`(%rip),%r13 # borrow $a0 + mov 0xf0-0x80($inp),%r14d # rounds, borrow $a1 + mov $ctx,%r15 # borrow $a2 + mov $in0,%rsi # borrow $a3 + vmovdqu ($ivp),$iv # load IV + sub \$9,%r14 + + mov $SZ*0(%r15),$A + mov $SZ*1(%r15),$B + mov $SZ*2(%r15),$C + mov $SZ*3(%r15),$D + mov $SZ*4(%r15),$E + mov $SZ*5(%r15),$F + mov $SZ*6(%r15),$G + mov $SZ*7(%r15),$H + + vmovdqa 0x00(%r13,%r14,8),$mask14 + vmovdqa 0x10(%r13,%r14,8),$mask12 + vmovdqa 0x20(%r13,%r14,8),$mask10 + vmovdqu 0x00-0x80($inp),$roundkey + jmp .Lloop_xop +___ + if ($SZ==4) { # SHA256 + my @X = map("%xmm$_",(0..3)); + my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7)); + +$code.=<<___; +.align 16 +.Lloop_xop: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu 0x00(%rsi,%r12),@X[0] + vmovdqu 0x10(%rsi,%r12),@X[1] + vmovdqu 0x20(%rsi,%r12),@X[2] + vmovdqu 0x30(%rsi,%r12),@X[3] + vpshufb $t3,@X[0],@X[0] + lea $TABLE(%rip),$Tbl + vpshufb $t3,@X[1],@X[1] + vpshufb $t3,@X[2],@X[2] + vpaddd 0x00($Tbl),@X[0],$t0 + vpshufb $t3,@X[3],@X[3] + vpaddd 0x20($Tbl),@X[1],$t1 + vpaddd 0x40($Tbl),@X[2],$t2 + vpaddd 0x60($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + mov $A,$a1 + vmovdqa $t1,0x10(%rsp) + mov $B,$a3 + vmovdqa $t2,0x20(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x30(%rsp) + mov $E,$a0 + jmp .Lxop_00_47 + +.align 16 +.Lxop_00_47: + sub \$-16*2*$SZ,$Tbl # size optimization + vmovdqu (%r12),$inout # $a4 + mov %r12,$_inp # $a4 +___ +sub XOP_256_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 104 instructions + + &vpalignr ($t0,@X[1],@X[0],$SZ); # X[1..4] + eval(shift(@insns)); + eval(shift(@insns)); + &vpalignr ($t3,@X[3],@X[2],$SZ); # X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t1,$t0,8*$SZ-$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrld ($t0,$t0,$sigma0[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t3); # X[0..3] += X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t2,$t1,$sigma0[1]-$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t0,$t0,$t1); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t3,@X[3],8*$SZ-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t0,$t0,$t2); # sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrld ($t2,@X[3],$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t0); # X[0..3] += sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t1); # sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrldq ($t3,$t3,8); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t3,@X[0],8*$SZ-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrld ($t2,@X[0],$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t1); # sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpslldq ($t3,$t3,8); # 22 instructions + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t3); # X[2..3] += sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd ($t2,@X[0],16*2*$j."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa (16*$j."(%rsp)",$t2); +} + + $aesni_cbc_idx=0; + for ($i=0,$j=0; $j<4; $j++) { + &XOP_256_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &mov ("%r12",$_inp); # borrow $a4 + &vpand ($temp,$temp,$mask14); + &mov ("%r15",$_out); # borrow $a2 + &vpor ($iv,$iv,$temp); + &vmovdqu ("(%r15,%r12)",$iv); # write output + &lea ("%r12","16(%r12)"); # inp++ + + &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0); + &jne (".Lxop_00_47"); + + &vmovdqu ($inout,"(%r12)"); + &mov ($_inp,"%r12"); + + $aesni_cbc_idx=0; + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } + } +$code.=<<___; + mov $_inp,%r12 # borrow $a4 + mov $_out,%r13 # borrow $a0 + mov $_ctx,%r15 # borrow $a2 + mov $_in0,%rsi # borrow $a3 + + vpand $mask14,$temp,$temp + mov $a1,$A + vpor $temp,$iv,$iv + vmovdqu $iv,(%r13,%r12) # write output + lea 16(%r12),%r12 # inp++ + + add $SZ*0(%r15),$A + add $SZ*1(%r15),$B + add $SZ*2(%r15),$C + add $SZ*3(%r15),$D + add $SZ*4(%r15),$E + add $SZ*5(%r15),$F + add $SZ*6(%r15),$G + add $SZ*7(%r15),$H + + cmp $_end,%r12 + + mov $A,$SZ*0(%r15) + mov $B,$SZ*1(%r15) + mov $C,$SZ*2(%r15) + mov $D,$SZ*3(%r15) + mov $E,$SZ*4(%r15) + mov $F,$SZ*5(%r15) + mov $G,$SZ*6(%r15) + mov $H,$SZ*7(%r15) + + jb .Lloop_xop + + mov $_ivp,$ivp + mov $_rsp,%rsi + vmovdqu $iv,($ivp) # output IV + vzeroall +___ +$code.=<<___ if ($win64); + movaps `$framesz+16*0`(%rsp),%xmm6 + movaps `$framesz+16*1`(%rsp),%xmm7 + movaps `$framesz+16*2`(%rsp),%xmm8 + movaps `$framesz+16*3`(%rsp),%xmm9 + movaps `$framesz+16*4`(%rsp),%xmm10 + movaps `$framesz+16*5`(%rsp),%xmm11 + movaps `$framesz+16*6`(%rsp),%xmm12 + movaps `$framesz+16*7`(%rsp),%xmm13 + movaps `$framesz+16*8`(%rsp),%xmm14 + movaps `$framesz+16*9`(%rsp),%xmm15 +___ +$code.=<<___; + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_xop: + ret +.size ${func}_xop,.-${func}_xop +___ +###################################################################### +# AVX+shrd code path +# +local *ror = sub { &shrd(@_[0],@_) }; + +$code.=<<___; +.type ${func}_avx,\@function,6 +.align 64 +${func}_avx: +.Lavx_shortcut: + mov `($win64?56:8)`(%rsp),$in0 # load 7th parameter + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + mov %rsp,%r11 # copy %rsp + sub \$`$framesz+$win64*16*10`,%rsp + and \$-64,%rsp # align stack frame + + shl \$6,$len + sub $inp,$out # re-bias + sub $inp,$in0 + add $inp,$len # end of input + + #mov $inp,$_inp # saved later + mov $out,$_out + mov $len,$_end + #mov $key,$_key # remains resident in $inp register + mov $ivp,$_ivp + mov $ctx,$_ctx + mov $in0,$_in0 + mov %r11,$_rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,`$framesz+16*0`(%rsp) + movaps %xmm7,`$framesz+16*1`(%rsp) + movaps %xmm8,`$framesz+16*2`(%rsp) + movaps %xmm9,`$framesz+16*3`(%rsp) + movaps %xmm10,`$framesz+16*4`(%rsp) + movaps %xmm11,`$framesz+16*5`(%rsp) + movaps %xmm12,`$framesz+16*6`(%rsp) + movaps %xmm13,`$framesz+16*7`(%rsp) + movaps %xmm14,`$framesz+16*8`(%rsp) + movaps %xmm15,`$framesz+16*9`(%rsp) +___ +$code.=<<___; +.Lprologue_avx: + vzeroall + + mov $inp,%r12 # borrow $a4 + lea 0x80($key),$inp # size optimization, reassign + lea $TABLE+`$SZ*2*$rounds+32`(%rip),%r13 # borrow $a0 + mov 0xf0-0x80($inp),%r14d # rounds, borrow $a1 + mov $ctx,%r15 # borrow $a2 + mov $in0,%rsi # borrow $a3 + vmovdqu ($ivp),$iv # load IV + sub \$9,%r14 + + mov $SZ*0(%r15),$A + mov $SZ*1(%r15),$B + mov $SZ*2(%r15),$C + mov $SZ*3(%r15),$D + mov $SZ*4(%r15),$E + mov $SZ*5(%r15),$F + mov $SZ*6(%r15),$G + mov $SZ*7(%r15),$H + + vmovdqa 0x00(%r13,%r14,8),$mask14 + vmovdqa 0x10(%r13,%r14,8),$mask12 + vmovdqa 0x20(%r13,%r14,8),$mask10 + vmovdqu 0x00-0x80($inp),$roundkey +___ + if ($SZ==4) { # SHA256 + my @X = map("%xmm$_",(0..3)); + my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7)); + +$code.=<<___; + jmp .Lloop_avx +.align 16 +.Lloop_avx: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu 0x00(%rsi,%r12),@X[0] + vmovdqu 0x10(%rsi,%r12),@X[1] + vmovdqu 0x20(%rsi,%r12),@X[2] + vmovdqu 0x30(%rsi,%r12),@X[3] + vpshufb $t3,@X[0],@X[0] + lea $TABLE(%rip),$Tbl + vpshufb $t3,@X[1],@X[1] + vpshufb $t3,@X[2],@X[2] + vpaddd 0x00($Tbl),@X[0],$t0 + vpshufb $t3,@X[3],@X[3] + vpaddd 0x20($Tbl),@X[1],$t1 + vpaddd 0x40($Tbl),@X[2],$t2 + vpaddd 0x60($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + mov $A,$a1 + vmovdqa $t1,0x10(%rsp) + mov $B,$a3 + vmovdqa $t2,0x20(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x30(%rsp) + mov $E,$a0 + jmp .Lavx_00_47 + +.align 16 +.Lavx_00_47: + sub \$-16*2*$SZ,$Tbl # size optimization + vmovdqu (%r12),$inout # $a4 + mov %r12,$_inp # $a4 +___ +sub Xupdate_256_AVX () { + ( + '&vpalignr ($t0,@X[1],@X[0],$SZ)', # X[1..4] + '&vpalignr ($t3,@X[3],@X[2],$SZ)', # X[9..12] + '&vpsrld ($t2,$t0,$sigma0[0]);', + '&vpaddd (@X[0],@X[0],$t3)', # X[0..3] += X[9..12] + '&vpsrld ($t3,$t0,$sigma0[2])', + '&vpslld ($t1,$t0,8*$SZ-$sigma0[1]);', + '&vpxor ($t0,$t3,$t2)', + '&vpshufd ($t3,@X[3],0b11111010)',# X[14..15] + '&vpsrld ($t2,$t2,$sigma0[1]-$sigma0[0]);', + '&vpxor ($t0,$t0,$t1)', + '&vpslld ($t1,$t1,$sigma0[1]-$sigma0[0]);', + '&vpxor ($t0,$t0,$t2)', + '&vpsrld ($t2,$t3,$sigma1[2]);', + '&vpxor ($t0,$t0,$t1)', # sigma0(X[1..4]) + '&vpsrlq ($t3,$t3,$sigma1[0]);', + '&vpaddd (@X[0],@X[0],$t0)', # X[0..3] += sigma0(X[1..4]) + '&vpxor ($t2,$t2,$t3);', + '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])', + '&vpxor ($t2,$t2,$t3)', # sigma1(X[14..15]) + '&vpshufd ($t2,$t2,0b10000100)', + '&vpsrldq ($t2,$t2,8)', + '&vpaddd (@X[0],@X[0],$t2)', # X[0..1] += sigma1(X[14..15]) + '&vpshufd ($t3,@X[0],0b01010000)',# X[16..17] + '&vpsrld ($t2,$t3,$sigma1[2])', + '&vpsrlq ($t3,$t3,$sigma1[0])', + '&vpxor ($t2,$t2,$t3);', + '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])', + '&vpxor ($t2,$t2,$t3)', + '&vpshufd ($t2,$t2,0b11101000)', + '&vpslldq ($t2,$t2,8)', + '&vpaddd (@X[0],@X[0],$t2)' # X[2..3] += sigma1(X[16..17]) + ); +} + +sub AVX_256_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 104 instructions + + foreach (Xupdate_256_AVX()) { # 29 instructions + eval; + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + } + &vpaddd ($t2,@X[0],16*2*$j."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa (16*$j."(%rsp)",$t2); +} + + $aesni_cbc_idx=0; + for ($i=0,$j=0; $j<4; $j++) { + &AVX_256_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &mov ("%r12",$_inp); # borrow $a4 + &vpand ($temp,$temp,$mask14); + &mov ("%r15",$_out); # borrow $a2 + &vpor ($iv,$iv,$temp); + &vmovdqu ("(%r15,%r12)",$iv); # write output + &lea ("%r12","16(%r12)"); # inp++ + + &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0); + &jne (".Lavx_00_47"); + + &vmovdqu ($inout,"(%r12)"); + &mov ($_inp,"%r12"); + + $aesni_cbc_idx=0; + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } + + } +$code.=<<___; + mov $_inp,%r12 # borrow $a4 + mov $_out,%r13 # borrow $a0 + mov $_ctx,%r15 # borrow $a2 + mov $_in0,%rsi # borrow $a3 + + vpand $mask14,$temp,$temp + mov $a1,$A + vpor $temp,$iv,$iv + vmovdqu $iv,(%r13,%r12) # write output + lea 16(%r12),%r12 # inp++ + + add $SZ*0(%r15),$A + add $SZ*1(%r15),$B + add $SZ*2(%r15),$C + add $SZ*3(%r15),$D + add $SZ*4(%r15),$E + add $SZ*5(%r15),$F + add $SZ*6(%r15),$G + add $SZ*7(%r15),$H + + cmp $_end,%r12 + + mov $A,$SZ*0(%r15) + mov $B,$SZ*1(%r15) + mov $C,$SZ*2(%r15) + mov $D,$SZ*3(%r15) + mov $E,$SZ*4(%r15) + mov $F,$SZ*5(%r15) + mov $G,$SZ*6(%r15) + mov $H,$SZ*7(%r15) + jb .Lloop_avx + + mov $_ivp,$ivp + mov $_rsp,%rsi + vmovdqu $iv,($ivp) # output IV + vzeroall +___ +$code.=<<___ if ($win64); + movaps `$framesz+16*0`(%rsp),%xmm6 + movaps `$framesz+16*1`(%rsp),%xmm7 + movaps `$framesz+16*2`(%rsp),%xmm8 + movaps `$framesz+16*3`(%rsp),%xmm9 + movaps `$framesz+16*4`(%rsp),%xmm10 + movaps `$framesz+16*5`(%rsp),%xmm11 + movaps `$framesz+16*6`(%rsp),%xmm12 + movaps `$framesz+16*7`(%rsp),%xmm13 + movaps `$framesz+16*8`(%rsp),%xmm14 + movaps `$framesz+16*9`(%rsp),%xmm15 +___ +$code.=<<___; + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_avx: + ret +.size ${func}_avx,.-${func}_avx +___ + +if ($avx>1) {{ +###################################################################### +# AVX2+BMI code path +# +my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp +my $PUSH8=8*2*$SZ; +use integer; + +sub bodyx_00_15 () { + # at start $a1 should be zero, $a3 - $b^$c and $a4 copy of $f + ( + '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'. + + '&add ($h,(32*($i/(16/$SZ))+$SZ*($i%(16/$SZ)))%$PUSH8.$base)', # h+=X[i]+K[i] + '&and ($a4,$e)', # f&e + '&rorx ($a0,$e,$Sigma1[2])', + '&rorx ($a2,$e,$Sigma1[1])', + + '&lea ($a,"($a,$a1)")', # h+=Sigma0(a) from the past + '&lea ($h,"($h,$a4)")', + '&andn ($a4,$e,$g)', # ~e&g + '&xor ($a0,$a2)', + + '&rorx ($a1,$e,$Sigma1[0])', + '&lea ($h,"($h,$a4)")', # h+=Ch(e,f,g)=(e&f)+(~e&g) + '&xor ($a0,$a1)', # Sigma1(e) + '&mov ($a2,$a)', + + '&rorx ($a4,$a,$Sigma0[2])', + '&lea ($h,"($h,$a0)")', # h+=Sigma1(e) + '&xor ($a2,$b)', # a^b, b^c in next round + '&rorx ($a1,$a,$Sigma0[1])', + + '&rorx ($a0,$a,$Sigma0[0])', + '&lea ($d,"($d,$h)")', # d+=h + '&and ($a3,$a2)', # (b^c)&(a^b) + @aesni_cbc_block[$aesni_cbc_idx++]. + '&xor ($a1,$a4)', + + '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b) + '&xor ($a1,$a0)', # Sigma0(a) + '&lea ($h,"($h,$a3)");'. # h+=Maj(a,b,c) + '&mov ($a4,$e)', # copy of f in future + + '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;' + ); + # and at the finish one has to $a+=$a1 +} + +$code.=<<___; +.type ${func}_avx2,\@function,6 +.align 64 +${func}_avx2: +.Lavx2_shortcut: + mov `($win64?56:8)`(%rsp),$in0 # load 7th parameter + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + mov %rsp,%r11 # copy %rsp + sub \$`2*$SZ*$rounds+8*8+$win64*16*10`,%rsp + and \$-256*$SZ,%rsp # align stack frame + add \$`2*$SZ*($rounds-8)`,%rsp + + shl \$6,$len + sub $inp,$out # re-bias + sub $inp,$in0 + add $inp,$len # end of input + + #mov $inp,$_inp # saved later + #mov $out,$_out # kept in $offload + mov $len,$_end + #mov $key,$_key # remains resident in $inp register + mov $ivp,$_ivp + mov $ctx,$_ctx + mov $in0,$_in0 + mov %r11,$_rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,`$framesz+16*0`(%rsp) + movaps %xmm7,`$framesz+16*1`(%rsp) + movaps %xmm8,`$framesz+16*2`(%rsp) + movaps %xmm9,`$framesz+16*3`(%rsp) + movaps %xmm10,`$framesz+16*4`(%rsp) + movaps %xmm11,`$framesz+16*5`(%rsp) + movaps %xmm12,`$framesz+16*6`(%rsp) + movaps %xmm13,`$framesz+16*7`(%rsp) + movaps %xmm14,`$framesz+16*8`(%rsp) + movaps %xmm15,`$framesz+16*9`(%rsp) +___ +$code.=<<___; +.Lprologue_avx2: + vzeroall + + mov $inp,%r13 # borrow $a0 + vpinsrq \$1,$out,$offload,$offload + lea 0x80($key),$inp # size optimization, reassign + lea $TABLE+`$SZ*2*$rounds+32`(%rip),%r12 # borrow $a4 + mov 0xf0-0x80($inp),%r14d # rounds, borrow $a1 + mov $ctx,%r15 # borrow $a2 + mov $in0,%rsi # borrow $a3 + vmovdqu ($ivp),$iv # load IV + lea -9(%r14),%r14 + + vmovdqa 0x00(%r12,%r14,8),$mask14 + vmovdqa 0x10(%r12,%r14,8),$mask12 + vmovdqa 0x20(%r12,%r14,8),$mask10 + + sub \$-16*$SZ,%r13 # inp++, size optimization + mov $SZ*0(%r15),$A + lea (%rsi,%r13),%r12 # borrow $a0 + mov $SZ*1(%r15),$B + cmp $len,%r13 # $_end + mov $SZ*2(%r15),$C + cmove %rsp,%r12 # next block or random data + mov $SZ*3(%r15),$D + mov $SZ*4(%r15),$E + mov $SZ*5(%r15),$F + mov $SZ*6(%r15),$G + mov $SZ*7(%r15),$H + vmovdqu 0x00-0x80($inp),$roundkey +___ + if ($SZ==4) { # SHA256 + my @X = map("%ymm$_",(0..3)); + my ($t0,$t1,$t2,$t3) = map("%ymm$_",(4..7)); + +$code.=<<___; + jmp .Loop_avx2 +.align 16 +.Loop_avx2: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu -16*$SZ+0(%rsi,%r13),%xmm0 + vmovdqu -16*$SZ+16(%rsi,%r13),%xmm1 + vmovdqu -16*$SZ+32(%rsi,%r13),%xmm2 + vmovdqu -16*$SZ+48(%rsi,%r13),%xmm3 + + vinserti128 \$1,(%r12),@X[0],@X[0] + vinserti128 \$1,16(%r12),@X[1],@X[1] + vpshufb $t3,@X[0],@X[0] + vinserti128 \$1,32(%r12),@X[2],@X[2] + vpshufb $t3,@X[1],@X[1] + vinserti128 \$1,48(%r12),@X[3],@X[3] + + lea $TABLE(%rip),$Tbl + vpshufb $t3,@X[2],@X[2] + lea -16*$SZ(%r13),%r13 + vpaddd 0x00($Tbl),@X[0],$t0 + vpshufb $t3,@X[3],@X[3] + vpaddd 0x20($Tbl),@X[1],$t1 + vpaddd 0x40($Tbl),@X[2],$t2 + vpaddd 0x60($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + xor $a1,$a1 + vmovdqa $t1,0x20(%rsp) + lea -$PUSH8(%rsp),%rsp + mov $B,$a3 + vmovdqa $t2,0x00(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x20(%rsp) + mov $F,$a4 + sub \$-16*2*$SZ,$Tbl # size optimization + jmp .Lavx2_00_47 + +.align 16 +.Lavx2_00_47: + vmovdqu (%r13),$inout + vpinsrq \$0,%r13,$offload,$offload +___ + +sub AVX2_256_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 96 instructions +my $base = "+2*$PUSH8(%rsp)"; + + &lea ("%rsp","-$PUSH8(%rsp)") if (($j%2)==0); + foreach (Xupdate_256_AVX()) { # 29 instructions + eval; + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + } + &vpaddd ($t2,@X[0],16*2*$j."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa ((32*$j)%$PUSH8."(%rsp)",$t2); +} + $aesni_cbc_idx=0; + for ($i=0,$j=0; $j<4; $j++) { + &AVX2_256_00_47($j,\&bodyx_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &vmovq ("%r13",$offload); # borrow $a0 + &vpextrq ("%r15",$offload,1); # borrow $a2 + &vpand ($temp,$temp,$mask14); + &vpor ($iv,$iv,$temp); + &vmovdqu ("(%r15,%r13)",$iv); # write output + &lea ("%r13","16(%r13)"); # inp++ + + &lea ($Tbl,16*2*$SZ."($Tbl)"); + &cmpb (($SZ-1)."($Tbl)",0); + &jne (".Lavx2_00_47"); + + &vmovdqu ($inout,"(%r13)"); + &vpinsrq ($offload,$offload,"%r13",0); + + $aesni_cbc_idx=0; + for ($i=0; $i<16; ) { + my $base=$i<8?"+$PUSH8(%rsp)":"(%rsp)"; + foreach(bodyx_00_15()) { eval; } + } + } +$code.=<<___; + vpextrq \$1,$offload,%r12 # $_out, borrow $a4 + vmovq $offload,%r13 # $_inp, borrow $a0 + mov `2*$SZ*$rounds+5*8`(%rsp),%r15 # $_ctx, borrow $a2 + add $a1,$A + lea `2*$SZ*($rounds-8)`(%rsp),$Tbl + + vpand $mask14,$temp,$temp + vpor $temp,$iv,$iv + vmovdqu $iv,(%r12,%r13) # write output + lea 16(%r13),%r13 + + add $SZ*0(%r15),$A + add $SZ*1(%r15),$B + add $SZ*2(%r15),$C + add $SZ*3(%r15),$D + add $SZ*4(%r15),$E + add $SZ*5(%r15),$F + add $SZ*6(%r15),$G + add $SZ*7(%r15),$H + + mov $A,$SZ*0(%r15) + mov $B,$SZ*1(%r15) + mov $C,$SZ*2(%r15) + mov $D,$SZ*3(%r15) + mov $E,$SZ*4(%r15) + mov $F,$SZ*5(%r15) + mov $G,$SZ*6(%r15) + mov $H,$SZ*7(%r15) + + cmp `$PUSH8+2*8`($Tbl),%r13 # $_end + je .Ldone_avx2 + + xor $a1,$a1 + mov $B,$a3 + mov $F,$a4 + xor $C,$a3 # magic + jmp .Lower_avx2 +.align 16 +.Lower_avx2: + vmovdqu (%r13),$inout + vpinsrq \$0,%r13,$offload,$offload +___ + $aesni_cbc_idx=0; + for ($i=0; $i<16; ) { + my $base="+16($Tbl)"; + foreach(bodyx_00_15()) { eval; } + &lea ($Tbl,"-$PUSH8($Tbl)") if ($i==8); + } +$code.=<<___; + vmovq $offload,%r13 # borrow $a0 + vpextrq \$1,$offload,%r15 # borrow $a2 + vpand $mask14,$temp,$temp + vpor $temp,$iv,$iv + lea -$PUSH8($Tbl),$Tbl + vmovdqu $iv,(%r15,%r13) # write output + lea 16(%r13),%r13 # inp++ + cmp %rsp,$Tbl + jae .Lower_avx2 + + mov `2*$SZ*$rounds+5*8`(%rsp),%r15 # $_ctx, borrow $a2 + lea 16*$SZ(%r13),%r13 + mov `2*$SZ*$rounds+6*8`(%rsp),%rsi # $_in0, borrow $a3 + add $a1,$A + lea `2*$SZ*($rounds-8)`(%rsp),%rsp + + add $SZ*0(%r15),$A + add $SZ*1(%r15),$B + add $SZ*2(%r15),$C + add $SZ*3(%r15),$D + add $SZ*4(%r15),$E + add $SZ*5(%r15),$F + add $SZ*6(%r15),$G + lea (%rsi,%r13),%r12 + add $SZ*7(%r15),$H + + cmp $_end,%r13 + + mov $A,$SZ*0(%r15) + cmove %rsp,%r12 # next block or stale data + mov $B,$SZ*1(%r15) + mov $C,$SZ*2(%r15) + mov $D,$SZ*3(%r15) + mov $E,$SZ*4(%r15) + mov $F,$SZ*5(%r15) + mov $G,$SZ*6(%r15) + mov $H,$SZ*7(%r15) + + jbe .Loop_avx2 + lea (%rsp),$Tbl + +.Ldone_avx2: + lea ($Tbl),%rsp + mov $_ivp,$ivp + mov $_rsp,%rsi + vmovdqu $iv,($ivp) # output IV + vzeroall +___ +$code.=<<___ if ($win64); + movaps `$framesz+16*0`(%rsp),%xmm6 + movaps `$framesz+16*1`(%rsp),%xmm7 + movaps `$framesz+16*2`(%rsp),%xmm8 + movaps `$framesz+16*3`(%rsp),%xmm9 + movaps `$framesz+16*4`(%rsp),%xmm10 + movaps `$framesz+16*5`(%rsp),%xmm11 + movaps `$framesz+16*6`(%rsp),%xmm12 + movaps `$framesz+16*7`(%rsp),%xmm13 + movaps `$framesz+16*8`(%rsp),%xmm14 + movaps `$framesz+16*9`(%rsp),%xmm15 +___ +$code.=<<___; + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_avx2: + ret +.size ${func}_avx2,.-${func}_avx2 +___ +}} +}} +{{ +my ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); + +my ($rounds,$Tbl)=("%r11d","%rbx"); + +my ($iv,$in,$rndkey0)=map("%xmm$_",(6,14,15)); +my @rndkey=("%xmm4","%xmm5"); +my $r=0; +my $sn=0; + +my ($Wi,$ABEF,$CDGH,$TMP,$BSWAP,$ABEF_SAVE,$CDGH_SAVE)=map("%xmm$_",(0..3,7..9)); +my @MSG=map("%xmm$_",(10..13)); + +my $aesenc=sub { + use integer; + my ($n,$k)=($r/10,$r%10); + if ($k==0) { + $code.=<<___; + movups `16*$n`($in0),$in # load input + xorps $rndkey0,$in +___ + $code.=<<___ if ($n); + movups $iv,`16*($n-1)`($out,$in0) # write output +___ + $code.=<<___; + xorps $in,$iv + movups `32+16*$k-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv +___ + } elsif ($k==9) { + $sn++; + $code.=<<___; + cmp \$11,$rounds + jb .Laesenclast$sn + movups `32+16*($k+0)-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv + movups `32+16*($k+1)-112`($key),$rndkey[0] + aesenc $rndkey[1],$iv + je .Laesenclast$sn + movups `32+16*($k+2)-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv + movups `32+16*($k+3)-112`($key),$rndkey[0] + aesenc $rndkey[1],$iv +.Laesenclast$sn: + aesenclast $rndkey[0],$iv + movups 16-112($key),$rndkey[1] # forward reference + nop +___ + } else { + $code.=<<___; + movups `32+16*$k-112`($key),$rndkey[1] + aesenc $rndkey[0],$iv +___ + } + $r++; unshift(@rndkey,pop(@rndkey)); +}; + +if ($shaext) { +my $Tbl="%rax"; + +$code.=<<___; +.type ${func}_shaext,\@function,6 +.align 32 +${func}_shaext: + mov `($win64?56:8)`(%rsp),$inp # load 7th argument +___ +$code.=<<___ if ($win64); + lea `-8-10*16`(%rsp),%rsp + movaps %xmm6,-8-10*16(%rax) + movaps %xmm7,-8-9*16(%rax) + movaps %xmm8,-8-8*16(%rax) + movaps %xmm9,-8-7*16(%rax) + movaps %xmm10,-8-6*16(%rax) + movaps %xmm11,-8-5*16(%rax) + movaps %xmm12,-8-4*16(%rax) + movaps %xmm13,-8-3*16(%rax) + movaps %xmm14,-8-2*16(%rax) + movaps %xmm15,-8-1*16(%rax) +.Lprologue_shaext: +___ +$code.=<<___; + lea K256+0x80(%rip),$Tbl + movdqu ($ctx),$ABEF # DCBA + movdqu 16($ctx),$CDGH # HGFE + movdqa 0x200-0x80($Tbl),$TMP # byte swap mask + + mov 240($key),$rounds + sub $in0,$out + movups ($key),$rndkey0 # $key[0] + movups 16($key),$rndkey[0] # forward reference + lea 112($key),$key # size optimization + + pshufd \$0x1b,$ABEF,$Wi # ABCD + pshufd \$0xb1,$ABEF,$ABEF # CDAB + pshufd \$0x1b,$CDGH,$CDGH # EFGH + movdqa $TMP,$BSWAP # offload + palignr \$8,$CDGH,$ABEF # ABEF + punpcklqdq $Wi,$CDGH # CDGH + + jmp .Loop_shaext + +.align 16 +.Loop_shaext: + movdqu ($inp),@MSG[0] + movdqu 0x10($inp),@MSG[1] + movdqu 0x20($inp),@MSG[2] + pshufb $TMP,@MSG[0] + movdqu 0x30($inp),@MSG[3] + + movdqa 0*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + pshufb $TMP,@MSG[1] + movdqa $CDGH,$CDGH_SAVE # offload + movdqa $ABEF,$ABEF_SAVE # offload +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 0-3 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 1*32-0x80($Tbl),$Wi + paddd @MSG[1],$Wi + pshufb $TMP,@MSG[2] + lea 0x40($inp),$inp +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 4-7 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 2*32-0x80($Tbl),$Wi + paddd @MSG[2],$Wi + pshufb $TMP,@MSG[3] + sha256msg1 @MSG[1],@MSG[0] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 8-11 + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[3],$TMP + palignr \$4,@MSG[2],$TMP + paddd $TMP,@MSG[0] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 3*32-0x80($Tbl),$Wi + paddd @MSG[3],$Wi + sha256msg2 @MSG[3],@MSG[0] + sha256msg1 @MSG[2],@MSG[1] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 12-15 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + movdqa @MSG[0],$TMP + palignr \$4,@MSG[3],$TMP + paddd $TMP,@MSG[1] + sha256rnds2 $CDGH,$ABEF +___ +for($i=4;$i<16-3;$i++) { + &$aesenc() if (($r%10)==0); +$code.=<<___; + movdqa $i*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + sha256msg2 @MSG[0],@MSG[1] + sha256msg1 @MSG[3],@MSG[2] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 16-19... + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[1],$TMP + palignr \$4,@MSG[0],$TMP + paddd $TMP,@MSG[2] +___ + &$aesenc(); + &$aesenc() if ($r==19); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF +___ + push(@MSG,shift(@MSG)); +} +$code.=<<___; + movdqa 13*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + sha256msg2 @MSG[0],@MSG[1] + sha256msg1 @MSG[3],@MSG[2] +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 52-55 + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[1],$TMP + palignr \$4,@MSG[0],$TMP + paddd $TMP,@MSG[2] +___ + &$aesenc(); + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 14*32-0x80($Tbl),$Wi + paddd @MSG[1],$Wi + sha256msg2 @MSG[1],@MSG[2] + movdqa $BSWAP,$TMP +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 56-59 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + + movdqa 15*32-0x80($Tbl),$Wi + paddd @MSG[2],$Wi +___ + &$aesenc(); + &$aesenc(); +$code.=<<___; + sha256rnds2 $ABEF,$CDGH # 60-63 + pshufd \$0x0e,$Wi,$Wi +___ + &$aesenc(); +$code.=<<___; + sha256rnds2 $CDGH,$ABEF + #pxor $CDGH,$rndkey0 # black magic +___ + while ($r<40) { &$aesenc(); } # remaining aesenc's +$code.=<<___; + #xorps $CDGH,$rndkey0 # black magic + paddd $CDGH_SAVE,$CDGH + paddd $ABEF_SAVE,$ABEF + + dec $len + movups $iv,48($out,$in0) # write output + lea 64($in0),$in0 + jnz .Loop_shaext + + pshufd \$0xb1,$CDGH,$CDGH # DCHG + pshufd \$0x1b,$ABEF,$TMP # FEBA + pshufd \$0xb1,$ABEF,$ABEF # BAFE + punpckhqdq $CDGH,$ABEF # DCBA + palignr \$8,$TMP,$CDGH # HGFE + + movups $iv,($ivp) # write IV + movdqu $ABEF,($ctx) + movdqu $CDGH,16($ctx) +___ +$code.=<<___ if ($win64); + movaps 0*16(%rsp),%xmm6 + movaps 1*16(%rsp),%xmm7 + movaps 2*16(%rsp),%xmm8 + movaps 3*16(%rsp),%xmm9 + movaps 4*16(%rsp),%xmm10 + movaps 5*16(%rsp),%xmm11 + movaps 6*16(%rsp),%xmm12 + movaps 7*16(%rsp),%xmm13 + movaps 8*16(%rsp),%xmm14 + movaps 9*16(%rsp),%xmm15 + lea 8+10*16(%rsp),%rsp +.Lepilogue_shaext: +___ +$code.=<<___; + ret +.size ${func}_shaext,.-${func}_shaext +___ +} +}}}}} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64 && $avx) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HanderlData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lin_prologue +___ +$code.=<<___ if ($shaext); + lea aesni_cbc_sha256_enc_shaext(%rip),%r10 + cmp %r10,%rbx + jb .Lnot_in_shaext + + lea (%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + lea 168(%rax),%rax # adjust stack pointer + jmp .Lin_prologue +.Lnot_in_shaext: +___ +$code.=<<___ if ($avx>1); + lea .Lavx2_shortcut(%rip),%r10 + cmp %r10,%rbx # context->RipRbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R15 + + lea 16*$SZ+8*8(%rsi),%rsi # Xmm6- save area + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lin_prologue: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler + +.section .pdata + .rva .LSEH_begin_${func}_xop + .rva .LSEH_end_${func}_xop + .rva .LSEH_info_${func}_xop + + .rva .LSEH_begin_${func}_avx + .rva .LSEH_end_${func}_avx + .rva .LSEH_info_${func}_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_${func}_avx2 + .rva .LSEH_end_${func}_avx2 + .rva .LSEH_info_${func}_avx2 +___ +$code.=<<___ if ($shaext); + .rva .LSEH_begin_${func}_shaext + .rva .LSEH_end_${func}_shaext + .rva .LSEH_info_${func}_shaext +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_${func}_xop: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_xop,.Lepilogue_xop # HandlerData[] + +.LSEH_info_${func}_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[] +___ +$code.=<<___ if ($avx>1); +.LSEH_info_${func}_avx2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_avx2,.Lepilogue_avx2 # HandlerData[] +___ +$code.=<<___ if ($shaext); +.LSEH_info_${func}_shaext: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_shaext,.Lepilogue_shaext # HandlerData[] +___ +} + +#################################################################### +sub rex { + local *opcode=shift; + my ($dst,$src)=@_; + my $rex=0; + + $rex|=0x04 if($dst>=8); + $rex|=0x01 if($src>=8); + unshift @opcode,$rex|0x40 if($rex); +} + +{ + my %opcodelet = ( + "sha256rnds2" => 0xcb, + "sha256msg1" => 0xcc, + "sha256msg2" => 0xcd ); + + sub sha256op38 { + my $instr = shift; + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x38); + rex(\@opcode,$2,$1); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } + } +} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +$code =~ s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/gem; +print $code; +close STDOUT; diff --git a/crypto/aes/asm/aesni-x86.pl b/crypto/aes/asm/aesni-x86.pl index 3dc345b585f6..f67df8cf13da 100755 --- a/crypto/aes/asm/aesni-x86.pl +++ b/crypto/aes/asm/aesni-x86.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -43,6 +43,17 @@ # Add aesni_xts_[en|de]crypt. Westmere spends 1.50 cycles processing # one byte out of 8KB with 128-bit key, Sandy Bridge - 1.09. +###################################################################### +# Current large-block performance in cycles per byte processed with +# 128-bit key (less is better). +# +# CBC en-/decrypt CTR XTS ECB +# Westmere 3.77/1.37 1.37 1.52 1.27 +# * Bridge 5.07/0.98 0.99 1.09 0.91 +# Haswell 4.44/0.80 0.97 1.03 0.72 +# Silvermont 5.77/3.56 3.67 4.03 3.46 +# Bulldozer 5.80/0.98 1.05 1.24 0.93 + $PREFIX="aesni"; # if $PREFIX is set to "AES", the script # generates drop-in replacement for # crypto/aes/asm/aes-586.pl:-) @@ -54,8 +65,11 @@ require "x86asm.pl"; &asm_init($ARGV[0],$0); -if ($PREFIX eq "aesni") { $movekey=*movups; } -else { $movekey=*movups; } +&external_label("OPENSSL_ia32cap_P"); +&static_label("key_const"); + +if ($PREFIX eq "aesni") { $movekey=\&movups; } +else { $movekey=\&movups; } $len="eax"; $rounds="ecx"; @@ -170,7 +184,10 @@ sub aesni_generate1 # fully unrolled loop { &aesni_inline_generate1("enc"); } else { &call ("_aesni_encrypt1"); } + &pxor ($rndkey0,$rndkey0); # clear register bank + &pxor ($rndkey1,$rndkey1); &movups (&QWP(0,"eax"),$inout0); + &pxor ($inout0,$inout0); &ret (); &function_end_B("${PREFIX}_encrypt"); @@ -186,7 +203,10 @@ sub aesni_generate1 # fully unrolled loop { &aesni_inline_generate1("dec"); } else { &call ("_aesni_decrypt1"); } + &pxor ($rndkey0,$rndkey0); # clear register bank + &pxor ($rndkey1,$rndkey1); &movups (&QWP(0,"eax"),$inout0); + &pxor ($inout0,$inout0); &ret (); &function_end_B("${PREFIX}_decrypt"); @@ -196,37 +216,71 @@ sub aesni_generate1 # fully unrolled loop # every *2nd* cycle. Thus 3x interleave was the one providing optimal # utilization, i.e. when subroutine's throughput is virtually same as # of non-interleaved subroutine [for number of input blocks up to 3]. -# This is why it makes no sense to implement 2x subroutine. -# aes[enc|dec] latency in next processor generation is 8, but the -# instructions can be scheduled every cycle. Optimal interleave for -# new processor is therefore 8x, but it's unfeasible to accommodate it -# in XMM registers addreassable in 32-bit mode and therefore 6x is -# used instead... +# This is why it originally made no sense to implement 2x subroutine. +# But times change and it became appropriate to spend extra 192 bytes +# on 2x subroutine on Atom Silvermont account. For processors that +# can schedule aes[enc|dec] every cycle optimal interleave factor +# equals to corresponding instructions latency. 8x is optimal for +# * Bridge, but it's unfeasible to accommodate such implementation +# in XMM registers addreassable in 32-bit mode and therefore maximum +# of 6x is used instead... + +sub aesni_generate2 +{ my $p=shift; + + &function_begin_B("_aesni_${p}rypt2"); + &$movekey ($rndkey0,&QWP(0,$key)); + &shl ($rounds,4); + &$movekey ($rndkey1,&QWP(16,$key)); + &xorps ($inout0,$rndkey0); + &pxor ($inout1,$rndkey0); + &$movekey ($rndkey0,&QWP(32,$key)); + &lea ($key,&DWP(32,$key,$rounds)); + &neg ($rounds); + &add ($rounds,16); + + &set_label("${p}2_loop"); + eval"&aes${p} ($inout0,$rndkey1)"; + eval"&aes${p} ($inout1,$rndkey1)"; + &$movekey ($rndkey1,&QWP(0,$key,$rounds)); + &add ($rounds,32); + eval"&aes${p} ($inout0,$rndkey0)"; + eval"&aes${p} ($inout1,$rndkey0)"; + &$movekey ($rndkey0,&QWP(-16,$key,$rounds)); + &jnz (&label("${p}2_loop")); + eval"&aes${p} ($inout0,$rndkey1)"; + eval"&aes${p} ($inout1,$rndkey1)"; + eval"&aes${p}last ($inout0,$rndkey0)"; + eval"&aes${p}last ($inout1,$rndkey0)"; + &ret(); + &function_end_B("_aesni_${p}rypt2"); +} sub aesni_generate3 { my $p=shift; &function_begin_B("_aesni_${p}rypt3"); &$movekey ($rndkey0,&QWP(0,$key)); - &shr ($rounds,1); + &shl ($rounds,4); &$movekey ($rndkey1,&QWP(16,$key)); - &lea ($key,&DWP(32,$key)); &xorps ($inout0,$rndkey0); &pxor ($inout1,$rndkey0); &pxor ($inout2,$rndkey0); - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(32,$key)); + &lea ($key,&DWP(32,$key,$rounds)); + &neg ($rounds); + &add ($rounds,16); &set_label("${p}3_loop"); eval"&aes${p} ($inout0,$rndkey1)"; eval"&aes${p} ($inout1,$rndkey1)"; - &dec ($rounds); eval"&aes${p} ($inout2,$rndkey1)"; - &$movekey ($rndkey1,&QWP(16,$key)); + &$movekey ($rndkey1,&QWP(0,$key,$rounds)); + &add ($rounds,32); eval"&aes${p} ($inout0,$rndkey0)"; eval"&aes${p} ($inout1,$rndkey0)"; - &lea ($key,&DWP(32,$key)); eval"&aes${p} ($inout2,$rndkey0)"; - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(-16,$key,$rounds)); &jnz (&label("${p}3_loop")); eval"&aes${p} ($inout0,$rndkey1)"; eval"&aes${p} ($inout1,$rndkey1)"; @@ -248,27 +302,29 @@ sub aesni_generate4 &function_begin_B("_aesni_${p}rypt4"); &$movekey ($rndkey0,&QWP(0,$key)); &$movekey ($rndkey1,&QWP(16,$key)); - &shr ($rounds,1); - &lea ($key,&DWP(32,$key)); + &shl ($rounds,4); &xorps ($inout0,$rndkey0); &pxor ($inout1,$rndkey0); &pxor ($inout2,$rndkey0); &pxor ($inout3,$rndkey0); - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(32,$key)); + &lea ($key,&DWP(32,$key,$rounds)); + &neg ($rounds); + &data_byte (0x0f,0x1f,0x40,0x00); + &add ($rounds,16); &set_label("${p}4_loop"); eval"&aes${p} ($inout0,$rndkey1)"; eval"&aes${p} ($inout1,$rndkey1)"; - &dec ($rounds); eval"&aes${p} ($inout2,$rndkey1)"; eval"&aes${p} ($inout3,$rndkey1)"; - &$movekey ($rndkey1,&QWP(16,$key)); + &$movekey ($rndkey1,&QWP(0,$key,$rounds)); + &add ($rounds,32); eval"&aes${p} ($inout0,$rndkey0)"; eval"&aes${p} ($inout1,$rndkey0)"; - &lea ($key,&DWP(32,$key)); eval"&aes${p} ($inout2,$rndkey0)"; eval"&aes${p} ($inout3,$rndkey0)"; - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(-16,$key,$rounds)); &jnz (&label("${p}4_loop")); eval"&aes${p} ($inout0,$rndkey1)"; @@ -289,43 +345,41 @@ sub aesni_generate6 &function_begin_B("_aesni_${p}rypt6"); &static_label("_aesni_${p}rypt6_enter"); &$movekey ($rndkey0,&QWP(0,$key)); - &shr ($rounds,1); + &shl ($rounds,4); &$movekey ($rndkey1,&QWP(16,$key)); - &lea ($key,&DWP(32,$key)); &xorps ($inout0,$rndkey0); &pxor ($inout1,$rndkey0); # pxor does better here - eval"&aes${p} ($inout0,$rndkey1)"; &pxor ($inout2,$rndkey0); - eval"&aes${p} ($inout1,$rndkey1)"; + eval"&aes${p} ($inout0,$rndkey1)"; &pxor ($inout3,$rndkey0); - &dec ($rounds); - eval"&aes${p} ($inout2,$rndkey1)"; &pxor ($inout4,$rndkey0); - eval"&aes${p} ($inout3,$rndkey1)"; + eval"&aes${p} ($inout1,$rndkey1)"; + &lea ($key,&DWP(32,$key,$rounds)); + &neg ($rounds); + eval"&aes${p} ($inout2,$rndkey1)"; &pxor ($inout5,$rndkey0); - eval"&aes${p} ($inout4,$rndkey1)"; - &$movekey ($rndkey0,&QWP(0,$key)); - eval"&aes${p} ($inout5,$rndkey1)"; - &jmp (&label("_aesni_${p}rypt6_enter")); + &$movekey ($rndkey0,&QWP(0,$key,$rounds)); + &add ($rounds,16); + &jmp (&label("_aesni_${p}rypt6_inner")); &set_label("${p}6_loop",16); eval"&aes${p} ($inout0,$rndkey1)"; eval"&aes${p} ($inout1,$rndkey1)"; - &dec ($rounds); eval"&aes${p} ($inout2,$rndkey1)"; + &set_label("_aesni_${p}rypt6_inner"); eval"&aes${p} ($inout3,$rndkey1)"; eval"&aes${p} ($inout4,$rndkey1)"; eval"&aes${p} ($inout5,$rndkey1)"; - &set_label("_aesni_${p}rypt6_enter",16); - &$movekey ($rndkey1,&QWP(16,$key)); + &set_label("_aesni_${p}rypt6_enter"); + &$movekey ($rndkey1,&QWP(0,$key,$rounds)); + &add ($rounds,32); eval"&aes${p} ($inout0,$rndkey0)"; eval"&aes${p} ($inout1,$rndkey0)"; - &lea ($key,&DWP(32,$key)); eval"&aes${p} ($inout2,$rndkey0)"; eval"&aes${p} ($inout3,$rndkey0)"; eval"&aes${p} ($inout4,$rndkey0)"; eval"&aes${p} ($inout5,$rndkey0)"; - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(-16,$key,$rounds)); &jnz (&label("${p}6_loop")); eval"&aes${p} ($inout0,$rndkey1)"; @@ -343,6 +397,8 @@ sub aesni_generate6 &ret(); &function_end_B("_aesni_${p}rypt6"); } +&aesni_generate2("enc") if ($PREFIX eq "aesni"); +&aesni_generate2("dec"); &aesni_generate3("enc") if ($PREFIX eq "aesni"); &aesni_generate3("dec"); &aesni_generate4("enc") if ($PREFIX eq "aesni"); @@ -446,8 +502,7 @@ if ($PREFIX eq "aesni") { &jmp (&label("ecb_ret")); &set_label("ecb_enc_two",16); - &xorps ($inout2,$inout2); - &call ("_aesni_encrypt3"); + &call ("_aesni_encrypt2"); &movups (&QWP(0,$out),$inout0); &movups (&QWP(0x10,$out),$inout1); &jmp (&label("ecb_ret")); @@ -547,8 +602,7 @@ if ($PREFIX eq "aesni") { &jmp (&label("ecb_ret")); &set_label("ecb_dec_two",16); - &xorps ($inout2,$inout2); - &call ("_aesni_decrypt3"); + &call ("_aesni_decrypt2"); &movups (&QWP(0,$out),$inout0); &movups (&QWP(0x10,$out),$inout1); &jmp (&label("ecb_ret")); @@ -568,6 +622,14 @@ if ($PREFIX eq "aesni") { &movups (&QWP(0x30,$out),$inout3); &set_label("ecb_ret"); + &pxor ("xmm0","xmm0"); # clear register bank + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &pxor ("xmm3","xmm3"); + &pxor ("xmm4","xmm4"); + &pxor ("xmm5","xmm5"); + &pxor ("xmm6","xmm6"); + &pxor ("xmm7","xmm7"); &function_end("aesni_ecb_encrypt"); ###################################################################### @@ -610,11 +672,13 @@ if ($PREFIX eq "aesni") { &mov (&DWP(24,"esp"),$key_); &mov (&DWP(28,"esp"),$key_); - &shr ($rounds,1); + &shl ($rounds,4); + &mov ($rounds_,16); &lea ($key_,&DWP(0,$key)); &movdqa ($inout3,&QWP(0,"esp")); &movdqa ($inout0,$ivec); - &mov ($rounds_,$rounds); + &lea ($key,&DWP(32,$key,$rounds)); + &sub ($rounds_,$rounds); &pshufb ($ivec,$inout3); &set_label("ccm64_enc_outer"); @@ -625,38 +689,45 @@ if ($PREFIX eq "aesni") { &xorps ($inout0,$rndkey0); &$movekey ($rndkey1,&QWP(16,$key_)); &xorps ($rndkey0,$in0); - &lea ($key,&DWP(32,$key_)); &xorps ($cmac,$rndkey0); # cmac^=inp - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(32,$key_)); &set_label("ccm64_enc2_loop"); &aesenc ($inout0,$rndkey1); - &dec ($rounds); &aesenc ($cmac,$rndkey1); - &$movekey ($rndkey1,&QWP(16,$key)); + &$movekey ($rndkey1,&QWP(0,$key,$rounds)); + &add ($rounds,32); &aesenc ($inout0,$rndkey0); - &lea ($key,&DWP(32,$key)); &aesenc ($cmac,$rndkey0); - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(-16,$key,$rounds)); &jnz (&label("ccm64_enc2_loop")); &aesenc ($inout0,$rndkey1); &aesenc ($cmac,$rndkey1); &paddq ($ivec,&QWP(16,"esp")); + &dec ($len); &aesenclast ($inout0,$rndkey0); &aesenclast ($cmac,$rndkey0); - &dec ($len); &lea ($inp,&DWP(16,$inp)); &xorps ($in0,$inout0); # inp^=E(ivec) &movdqa ($inout0,$ivec); &movups (&QWP(0,$out),$in0); # save output - &lea ($out,&DWP(16,$out)); &pshufb ($inout0,$inout3); + &lea ($out,&DWP(16,$out)); &jnz (&label("ccm64_enc_outer")); &mov ("esp",&DWP(48,"esp")); &mov ($out,&wparam(5)); &movups (&QWP(0,$out),$cmac); + + &pxor ("xmm0","xmm0"); # clear register bank + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &pxor ("xmm3","xmm3"); + &pxor ("xmm4","xmm4"); + &pxor ("xmm5","xmm5"); + &pxor ("xmm6","xmm6"); + &pxor ("xmm7","xmm7"); &function_end("aesni_ccm64_encrypt_blocks"); &function_begin("aesni_ccm64_decrypt_blocks"); @@ -700,15 +771,19 @@ if ($PREFIX eq "aesni") { { &aesni_inline_generate1("enc"); } else { &call ("_aesni_encrypt1"); } + &shl ($rounds_,4); + &mov ($rounds,16); &movups ($in0,&QWP(0,$inp)); # load inp &paddq ($ivec,&QWP(16,"esp")); &lea ($inp,&QWP(16,$inp)); + &sub ($rounds,$rounds_); + &lea ($key,&DWP(32,$key_,$rounds_)); + &mov ($rounds_,$rounds); &jmp (&label("ccm64_dec_outer")); &set_label("ccm64_dec_outer",16); &xorps ($in0,$inout0); # inp ^= E(ivec) &movdqa ($inout0,$ivec); - &mov ($rounds,$rounds_); &movups (&QWP(0,$out),$in0); # save output &lea ($out,&DWP(16,$out)); &pshufb ($inout0,$inout3); @@ -717,34 +792,33 @@ if ($PREFIX eq "aesni") { &jz (&label("ccm64_dec_break")); &$movekey ($rndkey0,&QWP(0,$key_)); - &shr ($rounds,1); + &mov ($rounds,$rounds_); &$movekey ($rndkey1,&QWP(16,$key_)); &xorps ($in0,$rndkey0); - &lea ($key,&DWP(32,$key_)); &xorps ($inout0,$rndkey0); &xorps ($cmac,$in0); # cmac^=out - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(32,$key_)); &set_label("ccm64_dec2_loop"); &aesenc ($inout0,$rndkey1); - &dec ($rounds); &aesenc ($cmac,$rndkey1); - &$movekey ($rndkey1,&QWP(16,$key)); + &$movekey ($rndkey1,&QWP(0,$key,$rounds)); + &add ($rounds,32); &aesenc ($inout0,$rndkey0); - &lea ($key,&DWP(32,$key)); &aesenc ($cmac,$rndkey0); - &$movekey ($rndkey0,&QWP(0,$key)); + &$movekey ($rndkey0,&QWP(-16,$key,$rounds)); &jnz (&label("ccm64_dec2_loop")); &movups ($in0,&QWP(0,$inp)); # load inp &paddq ($ivec,&QWP(16,"esp")); &aesenc ($inout0,$rndkey1); &aesenc ($cmac,$rndkey1); - &lea ($inp,&QWP(16,$inp)); &aesenclast ($inout0,$rndkey0); &aesenclast ($cmac,$rndkey0); + &lea ($inp,&QWP(16,$inp)); &jmp (&label("ccm64_dec_outer")); &set_label("ccm64_dec_break",16); + &mov ($rounds,&DWP(240,$key_)); &mov ($key,$key_); if ($inline) { &aesni_inline_generate1("enc",$cmac,$in0); } @@ -754,6 +828,15 @@ if ($PREFIX eq "aesni") { &mov ("esp",&DWP(48,"esp")); &mov ($out,&wparam(5)); &movups (&QWP(0,$out),$cmac); + + &pxor ("xmm0","xmm0"); # clear register bank + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &pxor ("xmm3","xmm3"); + &pxor ("xmm4","xmm4"); + &pxor ("xmm5","xmm5"); + &pxor ("xmm6","xmm6"); + &pxor ("xmm7","xmm7"); &function_end("aesni_ccm64_decrypt_blocks"); } @@ -763,7 +846,7 @@ if ($PREFIX eq "aesni") { # const char *ivec); # # Handles only complete blocks, operates on 32-bit counter and -# does not update *ivec! (see engine/eng_aesni.c for details) +# does not update *ivec! (see crypto/modes/ctr128.c for details) # # stack layout: # 0 pshufb mask @@ -810,66 +893,61 @@ if ($PREFIX eq "aesni") { # compose 2 vectors of 3x32-bit counters &bswap ($rounds_); - &pxor ($rndkey1,$rndkey1); &pxor ($rndkey0,$rndkey0); + &pxor ($rndkey1,$rndkey1); &movdqa ($inout0,&QWP(0,"esp")); # load byte-swap mask - &pinsrd ($rndkey1,$rounds_,0); + &pinsrd ($rndkey0,$rounds_,0); &lea ($key_,&DWP(3,$rounds_)); - &pinsrd ($rndkey0,$key_,0); + &pinsrd ($rndkey1,$key_,0); &inc ($rounds_); - &pinsrd ($rndkey1,$rounds_,1); + &pinsrd ($rndkey0,$rounds_,1); &inc ($key_); - &pinsrd ($rndkey0,$key_,1); + &pinsrd ($rndkey1,$key_,1); &inc ($rounds_); - &pinsrd ($rndkey1,$rounds_,2); + &pinsrd ($rndkey0,$rounds_,2); &inc ($key_); - &pinsrd ($rndkey0,$key_,2); - &movdqa (&QWP(48,"esp"),$rndkey1); # save 1st triplet - &pshufb ($rndkey1,$inout0); # byte swap - &movdqa (&QWP(64,"esp"),$rndkey0); # save 2nd triplet + &pinsrd ($rndkey1,$key_,2); + &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet &pshufb ($rndkey0,$inout0); # byte swap + &movdqu ($inout4,&QWP(0,$key)); # key[0] + &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet + &pshufb ($rndkey1,$inout0); # byte swap - &pshufd ($inout0,$rndkey1,3<<6); # place counter to upper dword - &pshufd ($inout1,$rndkey1,2<<6); + &pshufd ($inout0,$rndkey0,3<<6); # place counter to upper dword + &pshufd ($inout1,$rndkey0,2<<6); &cmp ($len,6); &jb (&label("ctr32_tail")); - &movdqa (&QWP(32,"esp"),$inout5); # save counter-less ivec - &shr ($rounds,1); + &pxor ($inout5,$inout4); # counter-less ivec^key[0] + &shl ($rounds,4); + &mov ($rounds_,16); + &movdqa (&QWP(32,"esp"),$inout5); # save counter-less ivec^key[0] &mov ($key_,$key); # backup $key - &mov ($rounds_,$rounds); # backup $rounds + &sub ($rounds_,$rounds); # backup twisted $rounds + &lea ($key,&DWP(32,$key,$rounds)); &sub ($len,6); &jmp (&label("ctr32_loop6")); &set_label("ctr32_loop6",16); - &pshufd ($inout2,$rndkey1,1<<6); - &movdqa ($rndkey1,&QWP(32,"esp")); # pull counter-less ivec - &pshufd ($inout3,$rndkey0,3<<6); - &por ($inout0,$rndkey1); # merge counter-less ivec - &pshufd ($inout4,$rndkey0,2<<6); - &por ($inout1,$rndkey1); - &pshufd ($inout5,$rndkey0,1<<6); - &por ($inout2,$rndkey1); - &por ($inout3,$rndkey1); - &por ($inout4,$rndkey1); - &por ($inout5,$rndkey1); - - # inlining _aesni_encrypt6's prologue gives ~4% improvement... - &$movekey ($rndkey0,&QWP(0,$key_)); - &$movekey ($rndkey1,&QWP(16,$key_)); - &lea ($key,&DWP(32,$key_)); - &dec ($rounds); - &pxor ($inout0,$rndkey0); + # inlining _aesni_encrypt6's prologue gives ~6% improvement... + &pshufd ($inout2,$rndkey0,1<<6); + &movdqa ($rndkey0,&QWP(32,"esp")); # pull counter-less ivec + &pshufd ($inout3,$rndkey1,3<<6); + &pxor ($inout0,$rndkey0); # merge counter-less ivec + &pshufd ($inout4,$rndkey1,2<<6); &pxor ($inout1,$rndkey0); - &aesenc ($inout0,$rndkey1); + &pshufd ($inout5,$rndkey1,1<<6); + &$movekey ($rndkey1,&QWP(16,$key_)); &pxor ($inout2,$rndkey0); - &aesenc ($inout1,$rndkey1); &pxor ($inout3,$rndkey0); - &aesenc ($inout2,$rndkey1); + &aesenc ($inout0,$rndkey1); &pxor ($inout4,$rndkey0); - &aesenc ($inout3,$rndkey1); &pxor ($inout5,$rndkey0); + &aesenc ($inout1,$rndkey1); + &$movekey ($rndkey0,&QWP(32,$key_)); + &mov ($rounds,$rounds_); + &aesenc ($inout2,$rndkey1); + &aesenc ($inout3,$rndkey1); &aesenc ($inout4,$rndkey1); - &$movekey ($rndkey0,&QWP(0,$key)); &aesenc ($inout5,$rndkey1); &call (&label("_aesni_encrypt6_enter")); @@ -882,12 +960,12 @@ if ($PREFIX eq "aesni") { &movups (&QWP(0,$out),$inout0); &movdqa ($rndkey0,&QWP(16,"esp")); # load increment &xorps ($inout2,$rndkey1); - &movdqa ($rndkey1,&QWP(48,"esp")); # load 1st triplet + &movdqa ($rndkey1,&QWP(64,"esp")); # load 2nd triplet &movups (&QWP(0x10,$out),$inout1); &movups (&QWP(0x20,$out),$inout2); - &paddd ($rndkey1,$rndkey0); # 1st triplet increment - &paddd ($rndkey0,&QWP(64,"esp")); # 2nd triplet increment + &paddd ($rndkey1,$rndkey0); # 2nd triplet increment + &paddd ($rndkey0,&QWP(48,"esp")); # 1st triplet increment &movdqa ($inout0,&QWP(0,"esp")); # load byte swap mask &movups ($inout1,&QWP(0x30,$inp)); @@ -895,44 +973,44 @@ if ($PREFIX eq "aesni") { &xorps ($inout3,$inout1); &movups ($inout1,&QWP(0x50,$inp)); &lea ($inp,&DWP(0x60,$inp)); - &movdqa (&QWP(48,"esp"),$rndkey1); # save 1st triplet - &pshufb ($rndkey1,$inout0); # byte swap + &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet + &pshufb ($rndkey0,$inout0); # byte swap &xorps ($inout4,$inout2); &movups (&QWP(0x30,$out),$inout3); &xorps ($inout5,$inout1); - &movdqa (&QWP(64,"esp"),$rndkey0); # save 2nd triplet - &pshufb ($rndkey0,$inout0); # byte swap + &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet + &pshufb ($rndkey1,$inout0); # byte swap &movups (&QWP(0x40,$out),$inout4); - &pshufd ($inout0,$rndkey1,3<<6); + &pshufd ($inout0,$rndkey0,3<<6); &movups (&QWP(0x50,$out),$inout5); &lea ($out,&DWP(0x60,$out)); - &mov ($rounds,$rounds_); - &pshufd ($inout1,$rndkey1,2<<6); + &pshufd ($inout1,$rndkey0,2<<6); &sub ($len,6); &jnc (&label("ctr32_loop6")); &add ($len,6); &jz (&label("ctr32_ret")); + &movdqu ($inout5,&QWP(0,$key_)); &mov ($key,$key_); - &lea ($rounds,&DWP(1,"",$rounds,2)); # restore $rounds - &movdqa ($inout5,&QWP(32,"esp")); # pull count-less ivec + &pxor ($inout5,&QWP(32,"esp")); # restore count-less ivec + &mov ($rounds,&DWP(240,$key_)); # restore $rounds &set_label("ctr32_tail"); &por ($inout0,$inout5); &cmp ($len,2); &jb (&label("ctr32_one")); - &pshufd ($inout2,$rndkey1,1<<6); + &pshufd ($inout2,$rndkey0,1<<6); &por ($inout1,$inout5); &je (&label("ctr32_two")); - &pshufd ($inout3,$rndkey0,3<<6); + &pshufd ($inout3,$rndkey1,3<<6); &por ($inout2,$inout5); &cmp ($len,4); &jb (&label("ctr32_three")); - &pshufd ($inout4,$rndkey0,2<<6); + &pshufd ($inout4,$rndkey1,2<<6); &por ($inout3,$inout5); &je (&label("ctr32_four")); @@ -970,7 +1048,7 @@ if ($PREFIX eq "aesni") { &jmp (&label("ctr32_ret")); &set_label("ctr32_two",16); - &call ("_aesni_encrypt3"); + &call ("_aesni_encrypt2"); &movups ($inout3,&QWP(0,$inp)); &movups ($inout4,&QWP(0x10,$inp)); &xorps ($inout0,$inout3); @@ -1008,6 +1086,17 @@ if ($PREFIX eq "aesni") { &movups (&QWP(0x30,$out),$inout3); &set_label("ctr32_ret"); + &pxor ("xmm0","xmm0"); # clear register bank + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &pxor ("xmm3","xmm3"); + &pxor ("xmm4","xmm4"); + &movdqa (&QWP(32,"esp"),"xmm0"); # clear stack + &pxor ("xmm5","xmm5"); + &movdqa (&QWP(48,"esp"),"xmm0"); + &pxor ("xmm6","xmm6"); + &movdqa (&QWP(64,"esp"),"xmm0"); + &pxor ("xmm7","xmm7"); &mov ("esp",&DWP(80,"esp")); &function_end("aesni_ctr32_encrypt_blocks"); @@ -1057,8 +1146,10 @@ if ($PREFIX eq "aesni") { &sub ($len,16*6); &jc (&label("xts_enc_short")); - &shr ($rounds,1); - &mov ($rounds_,$rounds); + &shl ($rounds,4); + &mov ($rounds_,16); + &sub ($rounds_,$rounds); + &lea ($key,&DWP(32,$key,$rounds)); &jmp (&label("xts_enc_loop6")); &set_label("xts_enc_loop6",16); @@ -1080,6 +1171,7 @@ if ($PREFIX eq "aesni") { &pxor ($inout5,$tweak); # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0] + &mov ($rounds,$rounds_); # restore $rounds &movdqu ($inout1,&QWP(16*1,$inp)); &xorps ($inout0,$rndkey0); # input^=rndkey[0] &movdqu ($inout2,&QWP(16*2,$inp)); @@ -1096,19 +1188,17 @@ if ($PREFIX eq "aesni") { &pxor ($inout5,$rndkey1); &$movekey ($rndkey1,&QWP(16,$key_)); - &lea ($key,&DWP(32,$key_)); &pxor ($inout1,&QWP(16*1,"esp")); - &aesenc ($inout0,$rndkey1); &pxor ($inout2,&QWP(16*2,"esp")); - &aesenc ($inout1,$rndkey1); + &aesenc ($inout0,$rndkey1); &pxor ($inout3,&QWP(16*3,"esp")); - &dec ($rounds); - &aesenc ($inout2,$rndkey1); &pxor ($inout4,&QWP(16*4,"esp")); - &aesenc ($inout3,$rndkey1); + &aesenc ($inout1,$rndkey1); &pxor ($inout5,$rndkey0); + &$movekey ($rndkey0,&QWP(32,$key_)); + &aesenc ($inout2,$rndkey1); + &aesenc ($inout3,$rndkey1); &aesenc ($inout4,$rndkey1); - &$movekey ($rndkey0,&QWP(0,$key)); &aesenc ($inout5,$rndkey1); &call (&label("_aesni_encrypt6_enter")); @@ -1135,13 +1225,12 @@ if ($PREFIX eq "aesni") { &paddq ($tweak,$tweak); # &psllq($tweak,1); &pand ($twres,$twmask); # isolate carry and residue &pcmpgtd($twtmp,$tweak); # broadcast upper bits - &mov ($rounds,$rounds_); # restore $rounds &pxor ($tweak,$twres); &sub ($len,16*6); &jnc (&label("xts_enc_loop6")); - &lea ($rounds,&DWP(1,"",$rounds,2)); # restore $rounds + &mov ($rounds,&DWP(240,$key_)); # restore $rounds &mov ($key,$key_); # restore $key &mov ($rounds_,$rounds); @@ -1241,9 +1330,8 @@ if ($PREFIX eq "aesni") { &lea ($inp,&DWP(16*2,$inp)); &xorps ($inout0,$inout3); # input^=tweak &xorps ($inout1,$inout4); - &xorps ($inout2,$inout2); - &call ("_aesni_encrypt3"); + &call ("_aesni_encrypt2"); &xorps ($inout0,$inout3); # output^=tweak &xorps ($inout1,$inout4); @@ -1350,6 +1438,20 @@ if ($PREFIX eq "aesni") { &movups (&QWP(-16,$out),$inout0); # write output &set_label("xts_enc_ret"); + &pxor ("xmm0","xmm0"); # clear register bank + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack + &pxor ("xmm3","xmm3"); + &movdqa (&QWP(16*1,"esp"),"xmm0"); + &pxor ("xmm4","xmm4"); + &movdqa (&QWP(16*2,"esp"),"xmm0"); + &pxor ("xmm5","xmm5"); + &movdqa (&QWP(16*3,"esp"),"xmm0"); + &pxor ("xmm6","xmm6"); + &movdqa (&QWP(16*4,"esp"),"xmm0"); + &pxor ("xmm7","xmm7"); + &movdqa (&QWP(16*5,"esp"),"xmm0"); &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp &function_end("aesni_xts_encrypt"); @@ -1399,8 +1501,10 @@ if ($PREFIX eq "aesni") { &sub ($len,16*6); &jc (&label("xts_dec_short")); - &shr ($rounds,1); - &mov ($rounds_,$rounds); + &shl ($rounds,4); + &mov ($rounds_,16); + &sub ($rounds_,$rounds); + &lea ($key,&DWP(32,$key,$rounds)); &jmp (&label("xts_dec_loop6")); &set_label("xts_dec_loop6",16); @@ -1422,6 +1526,7 @@ if ($PREFIX eq "aesni") { &pxor ($inout5,$tweak); # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0] + &mov ($rounds,$rounds_); &movdqu ($inout1,&QWP(16*1,$inp)); &xorps ($inout0,$rndkey0); # input^=rndkey[0] &movdqu ($inout2,&QWP(16*2,$inp)); @@ -1438,19 +1543,17 @@ if ($PREFIX eq "aesni") { &pxor ($inout5,$rndkey1); &$movekey ($rndkey1,&QWP(16,$key_)); - &lea ($key,&DWP(32,$key_)); &pxor ($inout1,&QWP(16*1,"esp")); - &aesdec ($inout0,$rndkey1); &pxor ($inout2,&QWP(16*2,"esp")); - &aesdec ($inout1,$rndkey1); + &aesdec ($inout0,$rndkey1); &pxor ($inout3,&QWP(16*3,"esp")); - &dec ($rounds); - &aesdec ($inout2,$rndkey1); &pxor ($inout4,&QWP(16*4,"esp")); - &aesdec ($inout3,$rndkey1); + &aesdec ($inout1,$rndkey1); &pxor ($inout5,$rndkey0); + &$movekey ($rndkey0,&QWP(32,$key_)); + &aesdec ($inout2,$rndkey1); + &aesdec ($inout3,$rndkey1); &aesdec ($inout4,$rndkey1); - &$movekey ($rndkey0,&QWP(0,$key)); &aesdec ($inout5,$rndkey1); &call (&label("_aesni_decrypt6_enter")); @@ -1477,13 +1580,12 @@ if ($PREFIX eq "aesni") { &paddq ($tweak,$tweak); # &psllq($tweak,1); &pand ($twres,$twmask); # isolate carry and residue &pcmpgtd($twtmp,$tweak); # broadcast upper bits - &mov ($rounds,$rounds_); # restore $rounds &pxor ($tweak,$twres); &sub ($len,16*6); &jnc (&label("xts_dec_loop6")); - &lea ($rounds,&DWP(1,"",$rounds,2)); # restore $rounds + &mov ($rounds,&DWP(240,$key_)); # restore $rounds &mov ($key,$key_); # restore $key &mov ($rounds_,$rounds); @@ -1584,7 +1686,7 @@ if ($PREFIX eq "aesni") { &xorps ($inout0,$inout3); # input^=tweak &xorps ($inout1,$inout4); - &call ("_aesni_decrypt3"); + &call ("_aesni_decrypt2"); &xorps ($inout0,$inout3); # output^=tweak &xorps ($inout1,$inout4); @@ -1712,6 +1814,20 @@ if ($PREFIX eq "aesni") { &movups (&QWP(0,$out),$inout0); # write output &set_label("xts_dec_ret"); + &pxor ("xmm0","xmm0"); # clear register bank + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack + &pxor ("xmm3","xmm3"); + &movdqa (&QWP(16*1,"esp"),"xmm0"); + &pxor ("xmm4","xmm4"); + &movdqa (&QWP(16*2,"esp"),"xmm0"); + &pxor ("xmm5","xmm5"); + &movdqa (&QWP(16*3,"esp"),"xmm0"); + &pxor ("xmm6","xmm6"); + &movdqa (&QWP(16*4,"esp"),"xmm0"); + &pxor ("xmm7","xmm7"); + &movdqa (&QWP(16*5,"esp"),"xmm0"); &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp &function_end("aesni_xts_decrypt"); } @@ -1764,6 +1880,7 @@ if ($PREFIX eq "aesni") { &add ($len,16); &jnz (&label("cbc_enc_tail")); &movaps ($ivec,$inout0); + &pxor ($inout0,$inout0); &jmp (&label("cbc_ret")); &set_label("cbc_enc_tail"); @@ -1816,7 +1933,7 @@ if ($PREFIX eq "aesni") { &movups (&QWP(0x10,$out),$inout1); &lea ($inp,&DWP(0x60,$inp)); &movups (&QWP(0x20,$out),$inout2); - &mov ($rounds,$rounds_) # restore $rounds + &mov ($rounds,$rounds_); # restore $rounds &movups (&QWP(0x30,$out),$inout3); &mov ($key,$key_); # restore $key &movups (&QWP(0x40,$out),$inout4); @@ -1827,7 +1944,7 @@ if ($PREFIX eq "aesni") { &movaps ($inout0,$inout5); &movaps ($ivec,$rndkey0); &add ($len,0x50); - &jle (&label("cbc_dec_tail_collected")); + &jle (&label("cbc_dec_clear_tail_collected")); &movups (&QWP(0,$out),$inout0); &lea ($out,&DWP(0x10,$out)); &set_label("cbc_dec_tail"); @@ -1866,10 +1983,14 @@ if ($PREFIX eq "aesni") { &xorps ($inout4,$rndkey0); &movups (&QWP(0,$out),$inout0); &movups (&QWP(0x10,$out),$inout1); + &pxor ($inout1,$inout1); &movups (&QWP(0x20,$out),$inout2); + &pxor ($inout2,$inout2); &movups (&QWP(0x30,$out),$inout3); + &pxor ($inout3,$inout3); &lea ($out,&DWP(0x40,$out)); &movaps ($inout0,$inout4); + &pxor ($inout4,$inout4); &sub ($len,0x50); &jmp (&label("cbc_dec_tail_collected")); @@ -1884,12 +2005,12 @@ if ($PREFIX eq "aesni") { &jmp (&label("cbc_dec_tail_collected")); &set_label("cbc_dec_two",16); - &xorps ($inout2,$inout2); - &call ("_aesni_decrypt3"); + &call ("_aesni_decrypt2"); &xorps ($inout0,$ivec); &xorps ($inout1,$in0); &movups (&QWP(0,$out),$inout0); &movaps ($inout0,$inout1); + &pxor ($inout1,$inout1); &lea ($out,&DWP(0x10,$out)); &movaps ($ivec,$in1); &sub ($len,0x20); @@ -1902,7 +2023,9 @@ if ($PREFIX eq "aesni") { &xorps ($inout2,$in1); &movups (&QWP(0,$out),$inout0); &movaps ($inout0,$inout2); + &pxor ($inout2,$inout2); &movups (&QWP(0x10,$out),$inout1); + &pxor ($inout1,$inout1); &lea ($out,&DWP(0x20,$out)); &movups ($ivec,&QWP(0x20,$inp)); &sub ($len,0x30); @@ -1918,29 +2041,44 @@ if ($PREFIX eq "aesni") { &movups (&QWP(0,$out),$inout0); &xorps ($inout2,$rndkey1); &movups (&QWP(0x10,$out),$inout1); + &pxor ($inout1,$inout1); &xorps ($inout3,$rndkey0); &movups (&QWP(0x20,$out),$inout2); + &pxor ($inout2,$inout2); &lea ($out,&DWP(0x30,$out)); &movaps ($inout0,$inout3); + &pxor ($inout3,$inout3); &sub ($len,0x40); + &jmp (&label("cbc_dec_tail_collected")); +&set_label("cbc_dec_clear_tail_collected",16); + &pxor ($inout1,$inout1); + &pxor ($inout2,$inout2); + &pxor ($inout3,$inout3); + &pxor ($inout4,$inout4); &set_label("cbc_dec_tail_collected"); &and ($len,15); &jnz (&label("cbc_dec_tail_partial")); &movups (&QWP(0,$out),$inout0); + &pxor ($rndkey0,$rndkey0); &jmp (&label("cbc_ret")); &set_label("cbc_dec_tail_partial",16); &movaps (&QWP(0,"esp"),$inout0); + &pxor ($rndkey0,$rndkey0); &mov ("ecx",16); &mov ($inp,"esp"); &sub ("ecx",$len); &data_word(0xA4F3F689); # rep movsb + &movdqa (&QWP(0,"esp"),$inout0); &set_label("cbc_ret"); &mov ("esp",&DWP(16,"esp")); # pull original %esp &mov ($key_,&wparam(4)); + &pxor ($inout0,$inout0); + &pxor ($rndkey1,$rndkey1); &movups (&QWP(0,$key_),$ivec); # output IV + &pxor ($ivec,$ivec); &set_label("cbc_abort"); &function_end("${PREFIX}_cbc_encrypt"); @@ -1957,14 +2095,24 @@ if ($PREFIX eq "aesni") { # $round rounds &function_begin_B("_aesni_set_encrypt_key"); + &push ("ebp"); + &push ("ebx"); &test ("eax","eax"); &jz (&label("bad_pointer")); &test ($key,$key); &jz (&label("bad_pointer")); + &call (&label("pic")); +&set_label("pic"); + &blindpop("ebx"); + &lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx")); + + &picmeup("ebp","OPENSSL_ia32cap_P","ebx",&label("key_const")); &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey &xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0 + &mov ("ebp",&DWP(4,"ebp")); &lea ($key,&DWP(16,$key)); + &and ("ebp",1<<28|1<<11); # AVX and XOP bits &cmp ($rounds,256); &je (&label("14rounds")); &cmp ($rounds,192); @@ -1973,6 +2121,9 @@ if ($PREFIX eq "aesni") { &jne (&label("bad_keybits")); &set_label("10rounds",16); + &cmp ("ebp",1<<28); + &je (&label("10rounds_alt")); + &mov ($rounds,9); &$movekey (&QWP(-16,$key),"xmm0"); # round 0 &aeskeygenassist("xmm1","xmm0",0x01); # round 1 @@ -1997,8 +2148,8 @@ if ($PREFIX eq "aesni") { &call (&label("key_128")); &$movekey (&QWP(0,$key),"xmm0"); &mov (&DWP(80,$key),$rounds); - &xor ("eax","eax"); - &ret(); + + &jmp (&label("good_key")); &set_label("key_128",16); &$movekey (&QWP(0,$key),"xmm0"); @@ -2012,10 +2163,78 @@ if ($PREFIX eq "aesni") { &xorps ("xmm0","xmm1"); &ret(); +&set_label("10rounds_alt",16); + &movdqa ("xmm5",&QWP(0x00,"ebx")); + &mov ($rounds,8); + &movdqa ("xmm4",&QWP(0x20,"ebx")); + &movdqa ("xmm2","xmm0"); + &movdqu (&QWP(-16,$key),"xmm0"); + +&set_label("loop_key128"); + &pshufb ("xmm0","xmm5"); + &aesenclast ("xmm0","xmm4"); + &pslld ("xmm4",1); + &lea ($key,&DWP(16,$key)); + + &movdqa ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm2","xmm3"); + + &pxor ("xmm0","xmm2"); + &movdqu (&QWP(-16,$key),"xmm0"); + &movdqa ("xmm2","xmm0"); + + &dec ($rounds); + &jnz (&label("loop_key128")); + + &movdqa ("xmm4",&QWP(0x30,"ebx")); + + &pshufb ("xmm0","xmm5"); + &aesenclast ("xmm0","xmm4"); + &pslld ("xmm4",1); + + &movdqa ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm2","xmm3"); + + &pxor ("xmm0","xmm2"); + &movdqu (&QWP(0,$key),"xmm0"); + + &movdqa ("xmm2","xmm0"); + &pshufb ("xmm0","xmm5"); + &aesenclast ("xmm0","xmm4"); + + &movdqa ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm3","xmm2"); + &pslldq ("xmm2",4); + &pxor ("xmm2","xmm3"); + + &pxor ("xmm0","xmm2"); + &movdqu (&QWP(16,$key),"xmm0"); + + &mov ($rounds,9); + &mov (&DWP(96,$key),$rounds); + + &jmp (&label("good_key")); + &set_label("12rounds",16); &movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey + &cmp ("ebp",1<<28); + &je (&label("12rounds_alt")); + &mov ($rounds,11); - &$movekey (&QWP(-16,$key),"xmm0") # round 0 + &$movekey (&QWP(-16,$key),"xmm0"); # round 0 &aeskeygenassist("xmm1","xmm2",0x01); # round 1,2 &call (&label("key_192a_cold")); &aeskeygenassist("xmm1","xmm2",0x02); # round 2,3 @@ -2034,8 +2253,8 @@ if ($PREFIX eq "aesni") { &call (&label("key_192b")); &$movekey (&QWP(0,$key),"xmm0"); &mov (&DWP(48,$key),$rounds); - &xor ("eax","eax"); - &ret(); + + &jmp (&label("good_key")); &set_label("key_192a",16); &$movekey (&QWP(0,$key),"xmm0"); @@ -2065,10 +2284,52 @@ if ($PREFIX eq "aesni") { &lea ($key,&DWP(32,$key)); &jmp (&label("key_192b_warm")); +&set_label("12rounds_alt",16); + &movdqa ("xmm5",&QWP(0x10,"ebx")); + &movdqa ("xmm4",&QWP(0x20,"ebx")); + &mov ($rounds,8); + &movdqu (&QWP(-16,$key),"xmm0"); + +&set_label("loop_key192"); + &movq (&QWP(0,$key),"xmm2"); + &movdqa ("xmm1","xmm2"); + &pshufb ("xmm2","xmm5"); + &aesenclast ("xmm2","xmm4"); + &pslld ("xmm4",1); + &lea ($key,&DWP(24,$key)); + + &movdqa ("xmm3","xmm0"); + &pslldq ("xmm0",4); + &pxor ("xmm3","xmm0"); + &pslldq ("xmm0",4); + &pxor ("xmm3","xmm0"); + &pslldq ("xmm0",4); + &pxor ("xmm0","xmm3"); + + &pshufd ("xmm3","xmm0",0xff); + &pxor ("xmm3","xmm1"); + &pslldq ("xmm1",4); + &pxor ("xmm3","xmm1"); + + &pxor ("xmm0","xmm2"); + &pxor ("xmm2","xmm3"); + &movdqu (&QWP(-16,$key),"xmm0"); + + &dec ($rounds); + &jnz (&label("loop_key192")); + + &mov ($rounds,11); + &mov (&DWP(32,$key),$rounds); + + &jmp (&label("good_key")); + &set_label("14rounds",16); &movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey - &mov ($rounds,13); &lea ($key,&DWP(16,$key)); + &cmp ("ebp",1<<28); + &je (&label("14rounds_alt")); + + &mov ($rounds,13); &$movekey (&QWP(-32,$key),"xmm0"); # round 0 &$movekey (&QWP(-16,$key),"xmm2"); # round 1 &aeskeygenassist("xmm1","xmm2",0x01); # round 2 @@ -2100,7 +2361,8 @@ if ($PREFIX eq "aesni") { &$movekey (&QWP(0,$key),"xmm0"); &mov (&DWP(16,$key),$rounds); &xor ("eax","eax"); - &ret(); + + &jmp (&label("good_key")); &set_label("key_256a",16); &$movekey (&QWP(0,$key),"xmm2"); @@ -2126,11 +2388,77 @@ if ($PREFIX eq "aesni") { &xorps ("xmm2","xmm1"); &ret(); +&set_label("14rounds_alt",16); + &movdqa ("xmm5",&QWP(0x00,"ebx")); + &movdqa ("xmm4",&QWP(0x20,"ebx")); + &mov ($rounds,7); + &movdqu (&QWP(-32,$key),"xmm0"); + &movdqa ("xmm1","xmm2"); + &movdqu (&QWP(-16,$key),"xmm2"); + +&set_label("loop_key256"); + &pshufb ("xmm2","xmm5"); + &aesenclast ("xmm2","xmm4"); + + &movdqa ("xmm3","xmm0"); + &pslldq ("xmm0",4); + &pxor ("xmm3","xmm0"); + &pslldq ("xmm0",4); + &pxor ("xmm3","xmm0"); + &pslldq ("xmm0",4); + &pxor ("xmm0","xmm3"); + &pslld ("xmm4",1); + + &pxor ("xmm0","xmm2"); + &movdqu (&QWP(0,$key),"xmm0"); + + &dec ($rounds); + &jz (&label("done_key256")); + + &pshufd ("xmm2","xmm0",0xff); + &pxor ("xmm3","xmm3"); + &aesenclast ("xmm2","xmm3"); + + &movdqa ("xmm3","xmm1") + &pslldq ("xmm1",4); + &pxor ("xmm3","xmm1"); + &pslldq ("xmm1",4); + &pxor ("xmm3","xmm1"); + &pslldq ("xmm1",4); + &pxor ("xmm1","xmm3"); + + &pxor ("xmm2","xmm1"); + &movdqu (&QWP(16,$key),"xmm2"); + &lea ($key,&DWP(32,$key)); + &movdqa ("xmm1","xmm2"); + &jmp (&label("loop_key256")); + +&set_label("done_key256"); + &mov ($rounds,13); + &mov (&DWP(16,$key),$rounds); + +&set_label("good_key"); + &pxor ("xmm0","xmm0"); + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &pxor ("xmm3","xmm3"); + &pxor ("xmm4","xmm4"); + &pxor ("xmm5","xmm5"); + &xor ("eax","eax"); + &pop ("ebx"); + &pop ("ebp"); + &ret (); + &set_label("bad_pointer",4); &mov ("eax",-1); + &pop ("ebx"); + &pop ("ebp"); &ret (); &set_label("bad_keybits",4); + &pxor ("xmm0","xmm0"); &mov ("eax",-2); + &pop ("ebx"); + &pop ("ebp"); &ret (); &function_end_B("_aesni_set_encrypt_key"); @@ -2152,7 +2480,7 @@ if ($PREFIX eq "aesni") { &mov ($key,&wparam(2)); &call ("_aesni_set_encrypt_key"); &mov ($key,&wparam(2)); - &shl ($rounds,4) # rounds-1 after _aesni_set_encrypt_key + &shl ($rounds,4); # rounds-1 after _aesni_set_encrypt_key &test ("eax","eax"); &jnz (&label("dec_key_ret")); &lea ("eax",&DWP(16,$key,$rounds)); # end of key schedule @@ -2180,10 +2508,18 @@ if ($PREFIX eq "aesni") { &aesimc ("xmm0","xmm0"); &$movekey (&QWP(0,$key),"xmm0"); + &pxor ("xmm0","xmm0"); + &pxor ("xmm1","xmm1"); &xor ("eax","eax"); # return success &set_label("dec_key_ret"); &ret (); &function_end_B("${PREFIX}_set_decrypt_key"); + +&set_label("key_const",64); +&data_word(0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d); +&data_word(0x04070605,0x04070605,0x04070605,0x04070605); +&data_word(1,1,1,1); +&data_word(0x1b,0x1b,0x1b,0x1b); &asciz("AES for Intel AES-NI, CRYPTOGAMS by "); &asm_finish(); diff --git a/crypto/aes/asm/aesni-x86_64.pl b/crypto/aes/asm/aesni-x86_64.pl index c9270dfddc18..25ca574f6a2f 100755 --- a/crypto/aes/asm/aesni-x86_64.pl +++ b/crypto/aes/asm/aesni-x86_64.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -129,8 +129,8 @@ # # Further data for other parallelizable modes: # -# CBC decrypt 1.16 0.93 0.93 -# CTR 1.14 0.91 n/a +# CBC decrypt 1.16 0.93 0.74 +# CTR 1.14 0.91 0.74 # # Well, given 3x column it's probably inappropriate to call the limit # asymptotic, if it can be surpassed, isn't it? What happens there? @@ -153,10 +153,25 @@ # April 2011 # -# Add aesni_xts_[en|de]crypt. Westmere spends 1.33 cycles processing -# one byte out of 8KB with 128-bit key, Sandy Bridge - 0.97. Just like +# Add aesni_xts_[en|de]crypt. Westmere spends 1.25 cycles processing +# one byte out of 8KB with 128-bit key, Sandy Bridge - 0.90. Just like # in CTR mode AES instruction interleave factor was chosen to be 6x. +###################################################################### +# Current large-block performance in cycles per byte processed with +# 128-bit key (less is better). +# +# CBC en-/decrypt CTR XTS ECB +# Westmere 3.77/1.25 1.25 1.25 1.26 +# * Bridge 5.07/0.74 0.75 0.90 0.85 +# Haswell 4.44/0.63 0.63 0.73 0.63 +# Silvermont 5.75/3.54 3.56 4.12 3.87(*) +# Bulldozer 5.77/0.70 0.72 0.90 0.70 +# +# (*) Atom Silvermont ECB result is suboptimal because of penalties +# incurred by operations on %xmm8-15. As ECB is not considered +# critical, nothing was done to mitigate the problem. + $PREFIX="aesni"; # if $PREFIX is set to "AES", the script # generates drop-in replacement for # crypto/aes/asm/aes-x86_64.pl:-) @@ -180,6 +195,7 @@ $movkey = $PREFIX eq "aesni" ? "movups" : "movups"; ("%rdi","%rsi","%rdx","%rcx"); # Unix order $code=".text\n"; +$code.=".extern OPENSSL_ia32cap_P\n"; $rounds="%eax"; # input to and changed by aesni_[en|de]cryptN !!! # this is natural Unix argument order for public $PREFIX_[ecb|cbc]_encrypt ... @@ -247,7 +263,10 @@ ${PREFIX}_encrypt: ___ &aesni_generate1("enc",$key,$rounds); $code.=<<___; + pxor $rndkey0,$rndkey0 # clear register bank + pxor $rndkey1,$rndkey1 movups $inout0,($out) # output + pxor $inout0,$inout0 ret .size ${PREFIX}_encrypt,.-${PREFIX}_encrypt @@ -260,7 +279,10 @@ ${PREFIX}_decrypt: ___ &aesni_generate1("dec",$key,$rounds); $code.=<<___; + pxor $rndkey0,$rndkey0 # clear register bank + pxor $rndkey1,$rndkey1 movups $inout0,($out) # output + pxor $inout0,$inout0 ret .size ${PREFIX}_decrypt, .-${PREFIX}_decrypt ___ @@ -272,10 +294,49 @@ ___ # every *2nd* cycle. Thus 3x interleave was the one providing optimal # utilization, i.e. when subroutine's throughput is virtually same as # of non-interleaved subroutine [for number of input blocks up to 3]. -# This is why it makes no sense to implement 2x subroutine. -# aes[enc|dec] latency in next processor generation is 8, but the -# instructions can be scheduled every cycle. Optimal interleave for -# new processor is therefore 8x... +# This is why it originally made no sense to implement 2x subroutine. +# But times change and it became appropriate to spend extra 192 bytes +# on 2x subroutine on Atom Silvermont account. For processors that +# can schedule aes[enc|dec] every cycle optimal interleave factor +# equals to corresponding instructions latency. 8x is optimal for +# * Bridge and "super-optimal" for other Intel CPUs... + +sub aesni_generate2 { +my $dir=shift; +# As already mentioned it takes in $key and $rounds, which are *not* +# preserved. $inout[0-1] is cipher/clear text... +$code.=<<___; +.type _aesni_${dir}rypt2,\@abi-omnipotent +.align 16 +_aesni_${dir}rypt2: + $movkey ($key),$rndkey0 + shl \$4,$rounds + $movkey 16($key),$rndkey1 + xorps $rndkey0,$inout0 + xorps $rndkey0,$inout1 + $movkey 32($key),$rndkey0 + lea 32($key,$rounds),$key + neg %rax # $rounds + add \$16,%rax + +.L${dir}_loop2: + aes${dir} $rndkey1,$inout0 + aes${dir} $rndkey1,$inout1 + $movkey ($key,%rax),$rndkey1 + add \$32,%rax + aes${dir} $rndkey0,$inout0 + aes${dir} $rndkey0,$inout1 + $movkey -16($key,%rax),$rndkey0 + jnz .L${dir}_loop2 + + aes${dir} $rndkey1,$inout0 + aes${dir} $rndkey1,$inout1 + aes${dir}last $rndkey0,$inout0 + aes${dir}last $rndkey0,$inout1 + ret +.size _aesni_${dir}rypt2,.-_aesni_${dir}rypt2 +___ +} sub aesni_generate3 { my $dir=shift; # As already mentioned it takes in $key and $rounds, which are *not* @@ -285,25 +346,26 @@ $code.=<<___; .align 16 _aesni_${dir}rypt3: $movkey ($key),$rndkey0 - shr \$1,$rounds + shl \$4,$rounds $movkey 16($key),$rndkey1 - lea 32($key),$key xorps $rndkey0,$inout0 xorps $rndkey0,$inout1 xorps $rndkey0,$inout2 - $movkey ($key),$rndkey0 + $movkey 32($key),$rndkey0 + lea 32($key,$rounds),$key + neg %rax # $rounds + add \$16,%rax .L${dir}_loop3: aes${dir} $rndkey1,$inout0 aes${dir} $rndkey1,$inout1 - dec $rounds aes${dir} $rndkey1,$inout2 - $movkey 16($key),$rndkey1 + $movkey ($key,%rax),$rndkey1 + add \$32,%rax aes${dir} $rndkey0,$inout0 aes${dir} $rndkey0,$inout1 - lea 32($key),$key aes${dir} $rndkey0,$inout2 - $movkey ($key),$rndkey0 + $movkey -16($key,%rax),$rndkey0 jnz .L${dir}_loop3 aes${dir} $rndkey1,$inout0 @@ -329,28 +391,30 @@ $code.=<<___; .align 16 _aesni_${dir}rypt4: $movkey ($key),$rndkey0 - shr \$1,$rounds + shl \$4,$rounds $movkey 16($key),$rndkey1 - lea 32($key),$key xorps $rndkey0,$inout0 xorps $rndkey0,$inout1 xorps $rndkey0,$inout2 xorps $rndkey0,$inout3 - $movkey ($key),$rndkey0 + $movkey 32($key),$rndkey0 + lea 32($key,$rounds),$key + neg %rax # $rounds + .byte 0x0f,0x1f,0x00 + add \$16,%rax .L${dir}_loop4: aes${dir} $rndkey1,$inout0 aes${dir} $rndkey1,$inout1 - dec $rounds aes${dir} $rndkey1,$inout2 aes${dir} $rndkey1,$inout3 - $movkey 16($key),$rndkey1 + $movkey ($key,%rax),$rndkey1 + add \$32,%rax aes${dir} $rndkey0,$inout0 aes${dir} $rndkey0,$inout1 - lea 32($key),$key aes${dir} $rndkey0,$inout2 aes${dir} $rndkey0,$inout3 - $movkey ($key),$rndkey0 + $movkey -16($key,%rax),$rndkey0 jnz .L${dir}_loop4 aes${dir} $rndkey1,$inout0 @@ -374,43 +438,40 @@ $code.=<<___; .align 16 _aesni_${dir}rypt6: $movkey ($key),$rndkey0 - shr \$1,$rounds + shl \$4,$rounds $movkey 16($key),$rndkey1 - lea 32($key),$key xorps $rndkey0,$inout0 pxor $rndkey0,$inout1 - aes${dir} $rndkey1,$inout0 pxor $rndkey0,$inout2 + aes${dir} $rndkey1,$inout0 + lea 32($key,$rounds),$key + neg %rax # $rounds aes${dir} $rndkey1,$inout1 pxor $rndkey0,$inout3 - aes${dir} $rndkey1,$inout2 pxor $rndkey0,$inout4 - aes${dir} $rndkey1,$inout3 + aes${dir} $rndkey1,$inout2 pxor $rndkey0,$inout5 - dec $rounds - aes${dir} $rndkey1,$inout4 - $movkey ($key),$rndkey0 - aes${dir} $rndkey1,$inout5 + $movkey ($key,%rax),$rndkey0 + add \$16,%rax jmp .L${dir}_loop6_enter .align 16 .L${dir}_loop6: aes${dir} $rndkey1,$inout0 aes${dir} $rndkey1,$inout1 - dec $rounds aes${dir} $rndkey1,$inout2 +.L${dir}_loop6_enter: aes${dir} $rndkey1,$inout3 aes${dir} $rndkey1,$inout4 aes${dir} $rndkey1,$inout5 -.L${dir}_loop6_enter: # happens to be 16-byte aligned - $movkey 16($key),$rndkey1 + $movkey ($key,%rax),$rndkey1 + add \$32,%rax aes${dir} $rndkey0,$inout0 aes${dir} $rndkey0,$inout1 - lea 32($key),$key aes${dir} $rndkey0,$inout2 aes${dir} $rndkey0,$inout3 aes${dir} $rndkey0,$inout4 aes${dir} $rndkey0,$inout5 - $movkey ($key),$rndkey0 + $movkey -16($key,%rax),$rndkey0 jnz .L${dir}_loop6 aes${dir} $rndkey1,$inout0 @@ -438,52 +499,46 @@ $code.=<<___; .align 16 _aesni_${dir}rypt8: $movkey ($key),$rndkey0 - shr \$1,$rounds + shl \$4,$rounds $movkey 16($key),$rndkey1 - lea 32($key),$key xorps $rndkey0,$inout0 xorps $rndkey0,$inout1 - aes${dir} $rndkey1,$inout0 pxor $rndkey0,$inout2 - aes${dir} $rndkey1,$inout1 pxor $rndkey0,$inout3 - aes${dir} $rndkey1,$inout2 pxor $rndkey0,$inout4 - aes${dir} $rndkey1,$inout3 + lea 32($key,$rounds),$key + neg %rax # $rounds + aes${dir} $rndkey1,$inout0 pxor $rndkey0,$inout5 - dec $rounds - aes${dir} $rndkey1,$inout4 pxor $rndkey0,$inout6 - aes${dir} $rndkey1,$inout5 + aes${dir} $rndkey1,$inout1 pxor $rndkey0,$inout7 - $movkey ($key),$rndkey0 - aes${dir} $rndkey1,$inout6 - aes${dir} $rndkey1,$inout7 - $movkey 16($key),$rndkey1 - jmp .L${dir}_loop8_enter + $movkey ($key,%rax),$rndkey0 + add \$16,%rax + jmp .L${dir}_loop8_inner .align 16 .L${dir}_loop8: aes${dir} $rndkey1,$inout0 aes${dir} $rndkey1,$inout1 - dec $rounds +.L${dir}_loop8_inner: aes${dir} $rndkey1,$inout2 aes${dir} $rndkey1,$inout3 aes${dir} $rndkey1,$inout4 aes${dir} $rndkey1,$inout5 aes${dir} $rndkey1,$inout6 aes${dir} $rndkey1,$inout7 - $movkey 16($key),$rndkey1 -.L${dir}_loop8_enter: # happens to be 16-byte aligned +.L${dir}_loop8_enter: + $movkey ($key,%rax),$rndkey1 + add \$32,%rax aes${dir} $rndkey0,$inout0 aes${dir} $rndkey0,$inout1 - lea 32($key),$key aes${dir} $rndkey0,$inout2 aes${dir} $rndkey0,$inout3 aes${dir} $rndkey0,$inout4 aes${dir} $rndkey0,$inout5 aes${dir} $rndkey0,$inout6 aes${dir} $rndkey0,$inout7 - $movkey ($key),$rndkey0 + $movkey -16($key,%rax),$rndkey0 jnz .L${dir}_loop8 aes${dir} $rndkey1,$inout0 @@ -506,6 +561,8 @@ _aesni_${dir}rypt8: .size _aesni_${dir}rypt8,.-_aesni_${dir}rypt8 ___ } +&aesni_generate2("enc") if ($PREFIX eq "aesni"); +&aesni_generate2("dec"); &aesni_generate3("enc") if ($PREFIX eq "aesni"); &aesni_generate3("dec"); &aesni_generate4("enc") if ($PREFIX eq "aesni"); @@ -528,15 +585,15 @@ aesni_ecb_encrypt: ___ $code.=<<___ if ($win64); lea -0x58(%rsp),%rsp - movaps %xmm6,(%rsp) + movaps %xmm6,(%rsp) # offload $inout4..7 movaps %xmm7,0x10(%rsp) movaps %xmm8,0x20(%rsp) movaps %xmm9,0x30(%rsp) .Lecb_enc_body: ___ $code.=<<___; - and \$-16,$len - jz .Lecb_ret + and \$-16,$len # if ($len<16) + jz .Lecb_ret # return mov 240($key),$rounds # key->rounds $movkey ($key),$rndkey0 @@ -545,10 +602,10 @@ $code.=<<___; test %r8d,%r8d # 5th argument jz .Lecb_decrypt #--------------------------- ECB ENCRYPT ------------------------------# - cmp \$0x80,$len - jb .Lecb_enc_tail + cmp \$0x80,$len # if ($len<8*16) + jb .Lecb_enc_tail # short input - movdqu ($inp),$inout0 + movdqu ($inp),$inout0 # load 8 input blocks movdqu 0x10($inp),$inout1 movdqu 0x20($inp),$inout2 movdqu 0x30($inp),$inout3 @@ -556,14 +613,14 @@ $code.=<<___; movdqu 0x50($inp),$inout5 movdqu 0x60($inp),$inout6 movdqu 0x70($inp),$inout7 - lea 0x80($inp),$inp - sub \$0x80,$len + lea 0x80($inp),$inp # $inp+=8*16 + sub \$0x80,$len # $len-=8*16 (can be zero) jmp .Lecb_enc_loop8_enter .align 16 .Lecb_enc_loop8: - movups $inout0,($out) + movups $inout0,($out) # store 8 output blocks mov $key_,$key # restore $key - movdqu ($inp),$inout0 + movdqu ($inp),$inout0 # load 8 input blocks mov $rnds_,$rounds # restore $rounds movups $inout1,0x10($out) movdqu 0x10($inp),$inout1 @@ -578,17 +635,17 @@ $code.=<<___; movups $inout6,0x60($out) movdqu 0x60($inp),$inout6 movups $inout7,0x70($out) - lea 0x80($out),$out + lea 0x80($out),$out # $out+=8*16 movdqu 0x70($inp),$inout7 - lea 0x80($inp),$inp + lea 0x80($inp),$inp # $inp+=8*16 .Lecb_enc_loop8_enter: call _aesni_encrypt8 sub \$0x80,$len - jnc .Lecb_enc_loop8 + jnc .Lecb_enc_loop8 # loop if $len-=8*16 didn't borrow - movups $inout0,($out) + movups $inout0,($out) # store 8 output blocks mov $key_,$key # restore $key movups $inout1,0x10($out) mov $rnds_,$rounds # restore $rounds @@ -598,11 +655,11 @@ $code.=<<___; movups $inout5,0x50($out) movups $inout6,0x60($out) movups $inout7,0x70($out) - lea 0x80($out),$out - add \$0x80,$len - jz .Lecb_ret + lea 0x80($out),$out # $out+=8*16 + add \$0x80,$len # restore real remaining $len + jz .Lecb_ret # done if ($len==0) -.Lecb_enc_tail: +.Lecb_enc_tail: # $len is less than 8*16 movups ($inp),$inout0 cmp \$0x20,$len jb .Lecb_enc_one @@ -619,8 +676,9 @@ $code.=<<___; movups 0x50($inp),$inout5 je .Lecb_enc_six movdqu 0x60($inp),$inout6 + xorps $inout7,$inout7 call _aesni_encrypt8 - movups $inout0,($out) + movups $inout0,($out) # store 7 output blocks movups $inout1,0x10($out) movups $inout2,0x20($out) movups $inout3,0x30($out) @@ -633,26 +691,25 @@ $code.=<<___; ___ &aesni_generate1("enc",$key,$rounds); $code.=<<___; - movups $inout0,($out) + movups $inout0,($out) # store one output block jmp .Lecb_ret .align 16 .Lecb_enc_two: - xorps $inout2,$inout2 - call _aesni_encrypt3 - movups $inout0,($out) + call _aesni_encrypt2 + movups $inout0,($out) # store 2 output blocks movups $inout1,0x10($out) jmp .Lecb_ret .align 16 .Lecb_enc_three: call _aesni_encrypt3 - movups $inout0,($out) + movups $inout0,($out) # store 3 output blocks movups $inout1,0x10($out) movups $inout2,0x20($out) jmp .Lecb_ret .align 16 .Lecb_enc_four: call _aesni_encrypt4 - movups $inout0,($out) + movups $inout0,($out) # store 4 output blocks movups $inout1,0x10($out) movups $inout2,0x20($out) movups $inout3,0x30($out) @@ -661,7 +718,7 @@ $code.=<<___; .Lecb_enc_five: xorps $inout5,$inout5 call _aesni_encrypt6 - movups $inout0,($out) + movups $inout0,($out) # store 5 output blocks movups $inout1,0x10($out) movups $inout2,0x20($out) movups $inout3,0x30($out) @@ -670,7 +727,7 @@ $code.=<<___; .align 16 .Lecb_enc_six: call _aesni_encrypt6 - movups $inout0,($out) + movups $inout0,($out) # store 6 output blocks movups $inout1,0x10($out) movups $inout2,0x20($out) movups $inout3,0x30($out) @@ -680,10 +737,10 @@ $code.=<<___; #--------------------------- ECB DECRYPT ------------------------------# .align 16 .Lecb_decrypt: - cmp \$0x80,$len - jb .Lecb_dec_tail + cmp \$0x80,$len # if ($len<8*16) + jb .Lecb_dec_tail # short input - movdqu ($inp),$inout0 + movdqu ($inp),$inout0 # load 8 input blocks movdqu 0x10($inp),$inout1 movdqu 0x20($inp),$inout2 movdqu 0x30($inp),$inout3 @@ -691,14 +748,14 @@ $code.=<<___; movdqu 0x50($inp),$inout5 movdqu 0x60($inp),$inout6 movdqu 0x70($inp),$inout7 - lea 0x80($inp),$inp - sub \$0x80,$len + lea 0x80($inp),$inp # $inp+=8*16 + sub \$0x80,$len # $len-=8*16 (can be zero) jmp .Lecb_dec_loop8_enter .align 16 .Lecb_dec_loop8: - movups $inout0,($out) + movups $inout0,($out) # store 8 output blocks mov $key_,$key # restore $key - movdqu ($inp),$inout0 + movdqu ($inp),$inout0 # load 8 input blocks mov $rnds_,$rounds # restore $rounds movups $inout1,0x10($out) movdqu 0x10($inp),$inout1 @@ -713,30 +770,38 @@ $code.=<<___; movups $inout6,0x60($out) movdqu 0x60($inp),$inout6 movups $inout7,0x70($out) - lea 0x80($out),$out + lea 0x80($out),$out # $out+=8*16 movdqu 0x70($inp),$inout7 - lea 0x80($inp),$inp + lea 0x80($inp),$inp # $inp+=8*16 .Lecb_dec_loop8_enter: call _aesni_decrypt8 $movkey ($key_),$rndkey0 sub \$0x80,$len - jnc .Lecb_dec_loop8 + jnc .Lecb_dec_loop8 # loop if $len-=8*16 didn't borrow - movups $inout0,($out) + movups $inout0,($out) # store 8 output blocks + pxor $inout0,$inout0 # clear register bank mov $key_,$key # restore $key movups $inout1,0x10($out) + pxor $inout1,$inout1 mov $rnds_,$rounds # restore $rounds movups $inout2,0x20($out) + pxor $inout2,$inout2 movups $inout3,0x30($out) + pxor $inout3,$inout3 movups $inout4,0x40($out) + pxor $inout4,$inout4 movups $inout5,0x50($out) + pxor $inout5,$inout5 movups $inout6,0x60($out) + pxor $inout6,$inout6 movups $inout7,0x70($out) - lea 0x80($out),$out - add \$0x80,$len - jz .Lecb_ret + pxor $inout7,$inout7 + lea 0x80($out),$out # $out+=8*16 + add \$0x80,$len # restore real remaining $len + jz .Lecb_ret # done if ($len==0) .Lecb_dec_tail: movups ($inp),$inout0 @@ -756,71 +821,107 @@ $code.=<<___; je .Lecb_dec_six movups 0x60($inp),$inout6 $movkey ($key),$rndkey0 + xorps $inout7,$inout7 call _aesni_decrypt8 - movups $inout0,($out) + movups $inout0,($out) # store 7 output blocks + pxor $inout0,$inout0 # clear register bank movups $inout1,0x10($out) + pxor $inout1,$inout1 movups $inout2,0x20($out) + pxor $inout2,$inout2 movups $inout3,0x30($out) + pxor $inout3,$inout3 movups $inout4,0x40($out) + pxor $inout4,$inout4 movups $inout5,0x50($out) + pxor $inout5,$inout5 movups $inout6,0x60($out) + pxor $inout6,$inout6 + pxor $inout7,$inout7 jmp .Lecb_ret .align 16 .Lecb_dec_one: ___ &aesni_generate1("dec",$key,$rounds); $code.=<<___; - movups $inout0,($out) + movups $inout0,($out) # store one output block + pxor $inout0,$inout0 # clear register bank jmp .Lecb_ret .align 16 .Lecb_dec_two: - xorps $inout2,$inout2 - call _aesni_decrypt3 - movups $inout0,($out) + call _aesni_decrypt2 + movups $inout0,($out) # store 2 output blocks + pxor $inout0,$inout0 # clear register bank movups $inout1,0x10($out) + pxor $inout1,$inout1 jmp .Lecb_ret .align 16 .Lecb_dec_three: call _aesni_decrypt3 - movups $inout0,($out) + movups $inout0,($out) # store 3 output blocks + pxor $inout0,$inout0 # clear register bank movups $inout1,0x10($out) + pxor $inout1,$inout1 movups $inout2,0x20($out) + pxor $inout2,$inout2 jmp .Lecb_ret .align 16 .Lecb_dec_four: call _aesni_decrypt4 - movups $inout0,($out) + movups $inout0,($out) # store 4 output blocks + pxor $inout0,$inout0 # clear register bank movups $inout1,0x10($out) + pxor $inout1,$inout1 movups $inout2,0x20($out) + pxor $inout2,$inout2 movups $inout3,0x30($out) + pxor $inout3,$inout3 jmp .Lecb_ret .align 16 .Lecb_dec_five: xorps $inout5,$inout5 call _aesni_decrypt6 - movups $inout0,($out) + movups $inout0,($out) # store 5 output blocks + pxor $inout0,$inout0 # clear register bank movups $inout1,0x10($out) + pxor $inout1,$inout1 movups $inout2,0x20($out) + pxor $inout2,$inout2 movups $inout3,0x30($out) + pxor $inout3,$inout3 movups $inout4,0x40($out) + pxor $inout4,$inout4 + pxor $inout5,$inout5 jmp .Lecb_ret .align 16 .Lecb_dec_six: call _aesni_decrypt6 - movups $inout0,($out) + movups $inout0,($out) # store 6 output blocks + pxor $inout0,$inout0 # clear register bank movups $inout1,0x10($out) + pxor $inout1,$inout1 movups $inout2,0x20($out) + pxor $inout2,$inout2 movups $inout3,0x30($out) + pxor $inout3,$inout3 movups $inout4,0x40($out) + pxor $inout4,$inout4 movups $inout5,0x50($out) + pxor $inout5,$inout5 .Lecb_ret: + xorps $rndkey0,$rndkey0 # %xmm0 + pxor $rndkey1,$rndkey1 ___ $code.=<<___ if ($win64); movaps (%rsp),%xmm6 + movaps %xmm0,(%rsp) # clear stack movaps 0x10(%rsp),%xmm7 + movaps %xmm0,0x10(%rsp) movaps 0x20(%rsp),%xmm8 + movaps %xmm0,0x20(%rsp) movaps 0x30(%rsp),%xmm9 + movaps %xmm0,0x30(%rsp) lea 0x58(%rsp),%rsp .Lecb_enc_ret: ___ @@ -842,7 +943,8 @@ ___ { my $cmac="%r9"; # 6th argument -my $increment="%xmm6"; +my $increment="%xmm9"; +my $iv="%xmm6"; my $bswap_mask="%xmm7"; $code.=<<___; @@ -853,10 +955,10 @@ aesni_ccm64_encrypt_blocks: ___ $code.=<<___ if ($win64); lea -0x58(%rsp),%rsp - movaps %xmm6,(%rsp) - movaps %xmm7,0x10(%rsp) - movaps %xmm8,0x20(%rsp) - movaps %xmm9,0x30(%rsp) + movaps %xmm6,(%rsp) # $iv + movaps %xmm7,0x10(%rsp) # $bswap_mask + movaps %xmm8,0x20(%rsp) # $in0 + movaps %xmm9,0x30(%rsp) # $increment .Lccm64_enc_body: ___ $code.=<<___; @@ -865,58 +967,68 @@ $code.=<<___; movdqa .Lincrement64(%rip),$increment movdqa .Lbswap_mask(%rip),$bswap_mask - shr \$1,$rounds + shl \$4,$rounds + mov \$16,$rnds_ lea 0($key),$key_ movdqu ($cmac),$inout1 movdqa $iv,$inout0 - mov $rounds,$rnds_ + lea 32($key,$rounds),$key # end of key schedule pshufb $bswap_mask,$iv + sub %rax,%r10 # twisted $rounds jmp .Lccm64_enc_outer .align 16 .Lccm64_enc_outer: $movkey ($key_),$rndkey0 - mov $rnds_,$rounds + mov %r10,%rax movups ($inp),$in0 # load inp xorps $rndkey0,$inout0 # counter $movkey 16($key_),$rndkey1 xorps $in0,$rndkey0 - lea 32($key_),$key xorps $rndkey0,$inout1 # cmac^=inp - $movkey ($key),$rndkey0 + $movkey 32($key_),$rndkey0 .Lccm64_enc2_loop: aesenc $rndkey1,$inout0 - dec $rounds aesenc $rndkey1,$inout1 - $movkey 16($key),$rndkey1 + $movkey ($key,%rax),$rndkey1 + add \$32,%rax aesenc $rndkey0,$inout0 - lea 32($key),$key aesenc $rndkey0,$inout1 - $movkey 0($key),$rndkey0 + $movkey -16($key,%rax),$rndkey0 jnz .Lccm64_enc2_loop aesenc $rndkey1,$inout0 aesenc $rndkey1,$inout1 paddq $increment,$iv + dec $len # $len-- ($len is in blocks) aesenclast $rndkey0,$inout0 aesenclast $rndkey0,$inout1 - dec $len lea 16($inp),$inp xorps $inout0,$in0 # inp ^= E(iv) movdqa $iv,$inout0 movups $in0,($out) # save output - lea 16($out),$out pshufb $bswap_mask,$inout0 - jnz .Lccm64_enc_outer + lea 16($out),$out # $out+=16 + jnz .Lccm64_enc_outer # loop if ($len!=0) - movups $inout1,($cmac) + pxor $rndkey0,$rndkey0 # clear register bank + pxor $rndkey1,$rndkey1 + pxor $inout0,$inout0 + movups $inout1,($cmac) # store resulting mac + pxor $inout1,$inout1 + pxor $in0,$in0 + pxor $iv,$iv ___ $code.=<<___ if ($win64); movaps (%rsp),%xmm6 + movaps %xmm0,(%rsp) # clear stack movaps 0x10(%rsp),%xmm7 + movaps %xmm0,0x10(%rsp) movaps 0x20(%rsp),%xmm8 + movaps %xmm0,0x20(%rsp) movaps 0x30(%rsp),%xmm9 + movaps %xmm0,0x30(%rsp) lea 0x58(%rsp),%rsp .Lccm64_enc_ret: ___ @@ -933,10 +1045,10 @@ aesni_ccm64_decrypt_blocks: ___ $code.=<<___ if ($win64); lea -0x58(%rsp),%rsp - movaps %xmm6,(%rsp) - movaps %xmm7,0x10(%rsp) - movaps %xmm8,0x20(%rsp) - movaps %xmm9,0x30(%rsp) + movaps %xmm6,(%rsp) # $iv + movaps %xmm7,0x10(%rsp) # $bswap_mask + movaps %xmm8,0x20(%rsp) # $in8 + movaps %xmm9,0x30(%rsp) # $increment .Lccm64_dec_body: ___ $code.=<<___; @@ -953,63 +1065,77 @@ $code.=<<___; ___ &aesni_generate1("enc",$key,$rounds); $code.=<<___; + shl \$4,$rnds_ + mov \$16,$rounds movups ($inp),$in0 # load inp paddq $increment,$iv - lea 16($inp),$inp + lea 16($inp),$inp # $inp+=16 + sub %r10,%rax # twisted $rounds + lea 32($key_,$rnds_),$key # end of key schedule + mov %rax,%r10 jmp .Lccm64_dec_outer .align 16 .Lccm64_dec_outer: xorps $inout0,$in0 # inp ^= E(iv) movdqa $iv,$inout0 - mov $rnds_,$rounds movups $in0,($out) # save output - lea 16($out),$out + lea 16($out),$out # $out+=16 pshufb $bswap_mask,$inout0 - sub \$1,$len - jz .Lccm64_dec_break + sub \$1,$len # $len-- ($len is in blocks) + jz .Lccm64_dec_break # if ($len==0) break $movkey ($key_),$rndkey0 - shr \$1,$rounds + mov %r10,%rax $movkey 16($key_),$rndkey1 xorps $rndkey0,$in0 - lea 32($key_),$key xorps $rndkey0,$inout0 xorps $in0,$inout1 # cmac^=out - $movkey ($key),$rndkey0 - + $movkey 32($key_),$rndkey0 + jmp .Lccm64_dec2_loop +.align 16 .Lccm64_dec2_loop: aesenc $rndkey1,$inout0 - dec $rounds aesenc $rndkey1,$inout1 - $movkey 16($key),$rndkey1 + $movkey ($key,%rax),$rndkey1 + add \$32,%rax aesenc $rndkey0,$inout0 - lea 32($key),$key aesenc $rndkey0,$inout1 - $movkey 0($key),$rndkey0 + $movkey -16($key,%rax),$rndkey0 jnz .Lccm64_dec2_loop - movups ($inp),$in0 # load inp + movups ($inp),$in0 # load input paddq $increment,$iv aesenc $rndkey1,$inout0 aesenc $rndkey1,$inout1 - lea 16($inp),$inp aesenclast $rndkey0,$inout0 aesenclast $rndkey0,$inout1 + lea 16($inp),$inp # $inp+=16 jmp .Lccm64_dec_outer .align 16 .Lccm64_dec_break: #xorps $in0,$inout1 # cmac^=out + mov 240($key_),$rounds ___ &aesni_generate1("enc",$key_,$rounds,$inout1,$in0); $code.=<<___; - movups $inout1,($cmac) + pxor $rndkey0,$rndkey0 # clear register bank + pxor $rndkey1,$rndkey1 + pxor $inout0,$inout0 + movups $inout1,($cmac) # store resulting mac + pxor $inout1,$inout1 + pxor $in0,$in0 + pxor $iv,$iv ___ $code.=<<___ if ($win64); movaps (%rsp),%xmm6 + movaps %xmm0,(%rsp) # clear stack movaps 0x10(%rsp),%xmm7 + movaps %xmm0,0x10(%rsp) movaps 0x20(%rsp),%xmm8 + movaps %xmm0,0x20(%rsp) movaps 0x30(%rsp),%xmm9 + movaps %xmm0,0x30(%rsp) lea 0x58(%rsp),%rsp .Lccm64_dec_ret: ___ @@ -1024,285 +1150,567 @@ ___ # const char *ivec); # # Handles only complete blocks, operates on 32-bit counter and -# does not update *ivec! (see engine/eng_aesni.c for details) +# does not update *ivec! (see crypto/modes/ctr128.c for details) # +# Overhaul based on suggestions from Shay Gueron and Vlad Krasnov, +# http://rt.openssl.org/Ticket/Display.html?id=3021&user=guest&pass=guest. +# Keywords are full unroll and modulo-schedule counter calculations +# with zero-round key xor. { -my $reserved = $win64?0:-0x28; -my ($in0,$in1,$in2,$in3)=map("%xmm$_",(8..11)); -my ($iv0,$iv1,$ivec)=("%xmm12","%xmm13","%xmm14"); -my $bswap_mask="%xmm15"; +my ($in0,$in1,$in2,$in3,$in4,$in5)=map("%xmm$_",(10..15)); +my ($key0,$ctr)=("${key_}d","${ivp}d"); +my $frame_size = 0x80 + ($win64?160:0); $code.=<<___; .globl aesni_ctr32_encrypt_blocks .type aesni_ctr32_encrypt_blocks,\@function,5 .align 16 aesni_ctr32_encrypt_blocks: + cmp \$1,$len + jne .Lctr32_bulk + + # handle single block without allocating stack frame, + # useful when handling edges + movups ($ivp),$inout0 + movups ($inp),$inout1 + mov 240($key),%edx # key->rounds +___ + &aesni_generate1("enc",$key,"%edx"); +$code.=<<___; + pxor $rndkey0,$rndkey0 # clear register bank + pxor $rndkey1,$rndkey1 + xorps $inout1,$inout0 + pxor $inout1,$inout1 + movups $inout0,($out) + xorps $inout0,$inout0 + jmp .Lctr32_epilogue + +.align 16 +.Lctr32_bulk: + lea (%rsp),%rax + push %rbp + sub \$$frame_size,%rsp + and \$-16,%rsp # Linux kernel stack can be incorrectly seeded ___ $code.=<<___ if ($win64); - lea -0xc8(%rsp),%rsp - movaps %xmm6,0x20(%rsp) - movaps %xmm7,0x30(%rsp) - movaps %xmm8,0x40(%rsp) - movaps %xmm9,0x50(%rsp) - movaps %xmm10,0x60(%rsp) - movaps %xmm11,0x70(%rsp) - movaps %xmm12,0x80(%rsp) - movaps %xmm13,0x90(%rsp) - movaps %xmm14,0xa0(%rsp) - movaps %xmm15,0xb0(%rsp) + movaps %xmm6,-0xa8(%rax) # offload everything + movaps %xmm7,-0x98(%rax) + movaps %xmm8,-0x88(%rax) + movaps %xmm9,-0x78(%rax) + movaps %xmm10,-0x68(%rax) + movaps %xmm11,-0x58(%rax) + movaps %xmm12,-0x48(%rax) + movaps %xmm13,-0x38(%rax) + movaps %xmm14,-0x28(%rax) + movaps %xmm15,-0x18(%rax) .Lctr32_body: ___ $code.=<<___; - cmp \$1,$len - je .Lctr32_one_shortcut - - movdqu ($ivp),$ivec - movdqa .Lbswap_mask(%rip),$bswap_mask - xor $rounds,$rounds - pextrd \$3,$ivec,$rnds_ # pull 32-bit counter - pinsrd \$3,$rounds,$ivec # wipe 32-bit counter - + lea -8(%rax),%rbp + + # 8 16-byte words on top of stack are counter values + # xor-ed with zero-round key + + movdqu ($ivp),$inout0 + movdqu ($key),$rndkey0 + mov 12($ivp),$ctr # counter LSB + pxor $rndkey0,$inout0 + mov 12($key),$key0 # 0-round key LSB + movdqa $inout0,0x00(%rsp) # populate counter block + bswap $ctr + movdqa $inout0,$inout1 + movdqa $inout0,$inout2 + movdqa $inout0,$inout3 + movdqa $inout0,0x40(%rsp) + movdqa $inout0,0x50(%rsp) + movdqa $inout0,0x60(%rsp) + mov %rdx,%r10 # about to borrow %rdx + movdqa $inout0,0x70(%rsp) + + lea 1($ctr),%rax + lea 2($ctr),%rdx + bswap %eax + bswap %edx + xor $key0,%eax + xor $key0,%edx + pinsrd \$3,%eax,$inout1 + lea 3($ctr),%rax + movdqa $inout1,0x10(%rsp) + pinsrd \$3,%edx,$inout2 + bswap %eax + mov %r10,%rdx # restore %rdx + lea 4($ctr),%r10 + movdqa $inout2,0x20(%rsp) + xor $key0,%eax + bswap %r10d + pinsrd \$3,%eax,$inout3 + xor $key0,%r10d + movdqa $inout3,0x30(%rsp) + lea 5($ctr),%r9 + mov %r10d,0x40+12(%rsp) + bswap %r9d + lea 6($ctr),%r10 mov 240($key),$rounds # key->rounds - bswap $rnds_ - pxor $iv0,$iv0 # vector of 3 32-bit counters - pxor $iv1,$iv1 # vector of 3 32-bit counters - pinsrd \$0,$rnds_,$iv0 - lea 3($rnds_),$key_ - pinsrd \$0,$key_,$iv1 - inc $rnds_ - pinsrd \$1,$rnds_,$iv0 - inc $key_ - pinsrd \$1,$key_,$iv1 - inc $rnds_ - pinsrd \$2,$rnds_,$iv0 - inc $key_ - pinsrd \$2,$key_,$iv1 - movdqa $iv0,$reserved(%rsp) - pshufb $bswap_mask,$iv0 - movdqa $iv1,`$reserved+0x10`(%rsp) - pshufb $bswap_mask,$iv1 - - pshufd \$`3<<6`,$iv0,$inout0 # place counter to upper dword - pshufd \$`2<<6`,$iv0,$inout1 - pshufd \$`1<<6`,$iv0,$inout2 - cmp \$6,$len - jb .Lctr32_tail - shr \$1,$rounds - mov $key,$key_ # backup $key - mov $rounds,$rnds_ # backup $rounds - sub \$6,$len + xor $key0,%r9d + bswap %r10d + mov %r9d,0x50+12(%rsp) + xor $key0,%r10d + lea 7($ctr),%r9 + mov %r10d,0x60+12(%rsp) + bswap %r9d + mov OPENSSL_ia32cap_P+4(%rip),%r10d + xor $key0,%r9d + and \$`1<<26|1<<22`,%r10d # isolate XSAVE+MOVBE + mov %r9d,0x70+12(%rsp) + + $movkey 0x10($key),$rndkey1 + + movdqa 0x40(%rsp),$inout4 + movdqa 0x50(%rsp),$inout5 + + cmp \$8,$len # $len is in blocks + jb .Lctr32_tail # short input if ($len<8) + + sub \$6,$len # $len is biased by -6 + cmp \$`1<<22`,%r10d # check for MOVBE without XSAVE + je .Lctr32_6x # [which denotes Atom Silvermont] + + lea 0x80($key),$key # size optimization + sub \$2,$len # $len is biased by -8 + jmp .Lctr32_loop8 + +.align 16 +.Lctr32_6x: + shl \$4,$rounds + mov \$48,$rnds_ + bswap $key0 + lea 32($key,$rounds),$key # end of key schedule + sub %rax,%r10 # twisted $rounds jmp .Lctr32_loop6 .align 16 .Lctr32_loop6: - pshufd \$`3<<6`,$iv1,$inout3 - por $ivec,$inout0 # merge counter-less ivec - $movkey ($key_),$rndkey0 - pshufd \$`2<<6`,$iv1,$inout4 - por $ivec,$inout1 - $movkey 16($key_),$rndkey1 - pshufd \$`1<<6`,$iv1,$inout5 - por $ivec,$inout2 - por $ivec,$inout3 - xorps $rndkey0,$inout0 - por $ivec,$inout4 - por $ivec,$inout5 - - # inline _aesni_encrypt6 and interleave last rounds - # with own code... + add \$6,$ctr # next counter value + $movkey -48($key,$rnds_),$rndkey0 + aesenc $rndkey1,$inout0 + mov $ctr,%eax + xor $key0,%eax + aesenc $rndkey1,$inout1 + movbe %eax,`0x00+12`(%rsp) # store next counter value + lea 1($ctr),%eax + aesenc $rndkey1,$inout2 + xor $key0,%eax + movbe %eax,`0x10+12`(%rsp) + aesenc $rndkey1,$inout3 + lea 2($ctr),%eax + xor $key0,%eax + aesenc $rndkey1,$inout4 + movbe %eax,`0x20+12`(%rsp) + lea 3($ctr),%eax + aesenc $rndkey1,$inout5 + $movkey -32($key,$rnds_),$rndkey1 + xor $key0,%eax - pxor $rndkey0,$inout1 + aesenc $rndkey0,$inout0 + movbe %eax,`0x30+12`(%rsp) + lea 4($ctr),%eax + aesenc $rndkey0,$inout1 + xor $key0,%eax + movbe %eax,`0x40+12`(%rsp) + aesenc $rndkey0,$inout2 + lea 5($ctr),%eax + xor $key0,%eax + aesenc $rndkey0,$inout3 + movbe %eax,`0x50+12`(%rsp) + mov %r10,%rax # mov $rnds_,$rounds + aesenc $rndkey0,$inout4 + aesenc $rndkey0,$inout5 + $movkey -16($key,$rnds_),$rndkey0 + + call .Lenc_loop6 + + movdqu ($inp),$inout6 # load 6 input blocks + movdqu 0x10($inp),$inout7 + movdqu 0x20($inp),$in0 + movdqu 0x30($inp),$in1 + movdqu 0x40($inp),$in2 + movdqu 0x50($inp),$in3 + lea 0x60($inp),$inp # $inp+=6*16 + $movkey -64($key,$rnds_),$rndkey1 + pxor $inout0,$inout6 # inp^=E(ctr) + movaps 0x00(%rsp),$inout0 # load next counter [xor-ed with 0 round] + pxor $inout1,$inout7 + movaps 0x10(%rsp),$inout1 + pxor $inout2,$in0 + movaps 0x20(%rsp),$inout2 + pxor $inout3,$in1 + movaps 0x30(%rsp),$inout3 + pxor $inout4,$in2 + movaps 0x40(%rsp),$inout4 + pxor $inout5,$in3 + movaps 0x50(%rsp),$inout5 + movdqu $inout6,($out) # store 6 output blocks + movdqu $inout7,0x10($out) + movdqu $in0,0x20($out) + movdqu $in1,0x30($out) + movdqu $in2,0x40($out) + movdqu $in3,0x50($out) + lea 0x60($out),$out # $out+=6*16 + + sub \$6,$len + jnc .Lctr32_loop6 # loop if $len-=6 didn't borrow + + add \$6,$len # restore real remaining $len + jz .Lctr32_done # done if ($len==0) + + lea -48($rnds_),$rounds + lea -80($key,$rnds_),$key # restore $key + neg $rounds + shr \$4,$rounds # restore $rounds + jmp .Lctr32_tail + +.align 32 +.Lctr32_loop8: + add \$8,$ctr # next counter value + movdqa 0x60(%rsp),$inout6 aesenc $rndkey1,$inout0 - lea 32($key_),$key - pxor $rndkey0,$inout2 + mov $ctr,%r9d + movdqa 0x70(%rsp),$inout7 aesenc $rndkey1,$inout1 - movdqa .Lincrement32(%rip),$iv1 - pxor $rndkey0,$inout3 + bswap %r9d + $movkey 0x20-0x80($key),$rndkey0 aesenc $rndkey1,$inout2 - movdqa $reserved(%rsp),$iv0 - pxor $rndkey0,$inout4 + xor $key0,%r9d + nop aesenc $rndkey1,$inout3 - pxor $rndkey0,$inout5 - $movkey ($key),$rndkey0 - dec $rounds + mov %r9d,0x00+12(%rsp) # store next counter value + lea 1($ctr),%r9 aesenc $rndkey1,$inout4 aesenc $rndkey1,$inout5 - jmp .Lctr32_enc_loop6_enter -.align 16 -.Lctr32_enc_loop6: + aesenc $rndkey1,$inout6 + aesenc $rndkey1,$inout7 + $movkey 0x30-0x80($key),$rndkey1 +___ +for($i=2;$i<8;$i++) { +my $rndkeyx = ($i&1)?$rndkey1:$rndkey0; +$code.=<<___; + bswap %r9d + aesenc $rndkeyx,$inout0 + aesenc $rndkeyx,$inout1 + xor $key0,%r9d + .byte 0x66,0x90 + aesenc $rndkeyx,$inout2 + aesenc $rndkeyx,$inout3 + mov %r9d,`0x10*($i-1)`+12(%rsp) + lea $i($ctr),%r9 + aesenc $rndkeyx,$inout4 + aesenc $rndkeyx,$inout5 + aesenc $rndkeyx,$inout6 + aesenc $rndkeyx,$inout7 + $movkey `0x20+0x10*$i`-0x80($key),$rndkeyx +___ +} +$code.=<<___; + bswap %r9d + aesenc $rndkey0,$inout0 + aesenc $rndkey0,$inout1 + aesenc $rndkey0,$inout2 + xor $key0,%r9d + movdqu 0x00($inp),$in0 # start loading input + aesenc $rndkey0,$inout3 + mov %r9d,0x70+12(%rsp) + cmp \$11,$rounds + aesenc $rndkey0,$inout4 + aesenc $rndkey0,$inout5 + aesenc $rndkey0,$inout6 + aesenc $rndkey0,$inout7 + $movkey 0xa0-0x80($key),$rndkey0 + + jb .Lctr32_enc_done + aesenc $rndkey1,$inout0 aesenc $rndkey1,$inout1 - dec $rounds aesenc $rndkey1,$inout2 aesenc $rndkey1,$inout3 aesenc $rndkey1,$inout4 aesenc $rndkey1,$inout5 -.Lctr32_enc_loop6_enter: - $movkey 16($key),$rndkey1 + aesenc $rndkey1,$inout6 + aesenc $rndkey1,$inout7 + $movkey 0xb0-0x80($key),$rndkey1 + aesenc $rndkey0,$inout0 aesenc $rndkey0,$inout1 - lea 32($key),$key aesenc $rndkey0,$inout2 aesenc $rndkey0,$inout3 aesenc $rndkey0,$inout4 aesenc $rndkey0,$inout5 - $movkey ($key),$rndkey0 - jnz .Lctr32_enc_loop6 + aesenc $rndkey0,$inout6 + aesenc $rndkey0,$inout7 + $movkey 0xc0-0x80($key),$rndkey0 + je .Lctr32_enc_done aesenc $rndkey1,$inout0 - paddd $iv1,$iv0 # increment counter vector aesenc $rndkey1,$inout1 - paddd `$reserved+0x10`(%rsp),$iv1 aesenc $rndkey1,$inout2 - movdqa $iv0,$reserved(%rsp) # save counter vector aesenc $rndkey1,$inout3 - movdqa $iv1,`$reserved+0x10`(%rsp) aesenc $rndkey1,$inout4 - pshufb $bswap_mask,$iv0 # byte swap aesenc $rndkey1,$inout5 - pshufb $bswap_mask,$iv1 - - aesenclast $rndkey0,$inout0 - movups ($inp),$in0 # load input - aesenclast $rndkey0,$inout1 - movups 0x10($inp),$in1 - aesenclast $rndkey0,$inout2 - movups 0x20($inp),$in2 - aesenclast $rndkey0,$inout3 - movups 0x30($inp),$in3 - aesenclast $rndkey0,$inout4 - movups 0x40($inp),$rndkey1 - aesenclast $rndkey0,$inout5 - movups 0x50($inp),$rndkey0 - lea 0x60($inp),$inp - - xorps $inout0,$in0 # xor - pshufd \$`3<<6`,$iv0,$inout0 - xorps $inout1,$in1 - pshufd \$`2<<6`,$iv0,$inout1 - movups $in0,($out) # store output - xorps $inout2,$in2 - pshufd \$`1<<6`,$iv0,$inout2 - movups $in1,0x10($out) - xorps $inout3,$in3 - movups $in2,0x20($out) - xorps $inout4,$rndkey1 - movups $in3,0x30($out) - xorps $inout5,$rndkey0 - movups $rndkey1,0x40($out) - movups $rndkey0,0x50($out) - lea 0x60($out),$out - mov $rnds_,$rounds - sub \$6,$len - jnc .Lctr32_loop6 + aesenc $rndkey1,$inout6 + aesenc $rndkey1,$inout7 + $movkey 0xd0-0x80($key),$rndkey1 - add \$6,$len - jz .Lctr32_done - mov $key_,$key # restore $key - lea 1($rounds,$rounds),$rounds # restore original value + aesenc $rndkey0,$inout0 + aesenc $rndkey0,$inout1 + aesenc $rndkey0,$inout2 + aesenc $rndkey0,$inout3 + aesenc $rndkey0,$inout4 + aesenc $rndkey0,$inout5 + aesenc $rndkey0,$inout6 + aesenc $rndkey0,$inout7 + $movkey 0xe0-0x80($key),$rndkey0 + jmp .Lctr32_enc_done + +.align 16 +.Lctr32_enc_done: + movdqu 0x10($inp),$in1 + pxor $rndkey0,$in0 # input^=round[last] + movdqu 0x20($inp),$in2 + pxor $rndkey0,$in1 + movdqu 0x30($inp),$in3 + pxor $rndkey0,$in2 + movdqu 0x40($inp),$in4 + pxor $rndkey0,$in3 + movdqu 0x50($inp),$in5 + pxor $rndkey0,$in4 + pxor $rndkey0,$in5 + aesenc $rndkey1,$inout0 + aesenc $rndkey1,$inout1 + aesenc $rndkey1,$inout2 + aesenc $rndkey1,$inout3 + aesenc $rndkey1,$inout4 + aesenc $rndkey1,$inout5 + aesenc $rndkey1,$inout6 + aesenc $rndkey1,$inout7 + movdqu 0x60($inp),$rndkey1 # borrow $rndkey1 for inp[6] + lea 0x80($inp),$inp # $inp+=8*16 + + aesenclast $in0,$inout0 # $inN is inp[N]^round[last] + pxor $rndkey0,$rndkey1 # borrowed $rndkey + movdqu 0x70-0x80($inp),$in0 + aesenclast $in1,$inout1 + pxor $rndkey0,$in0 + movdqa 0x00(%rsp),$in1 # load next counter block + aesenclast $in2,$inout2 + aesenclast $in3,$inout3 + movdqa 0x10(%rsp),$in2 + movdqa 0x20(%rsp),$in3 + aesenclast $in4,$inout4 + aesenclast $in5,$inout5 + movdqa 0x30(%rsp),$in4 + movdqa 0x40(%rsp),$in5 + aesenclast $rndkey1,$inout6 + movdqa 0x50(%rsp),$rndkey0 + $movkey 0x10-0x80($key),$rndkey1#real 1st-round key + aesenclast $in0,$inout7 + + movups $inout0,($out) # store 8 output blocks + movdqa $in1,$inout0 + movups $inout1,0x10($out) + movdqa $in2,$inout1 + movups $inout2,0x20($out) + movdqa $in3,$inout2 + movups $inout3,0x30($out) + movdqa $in4,$inout3 + movups $inout4,0x40($out) + movdqa $in5,$inout4 + movups $inout5,0x50($out) + movdqa $rndkey0,$inout5 + movups $inout6,0x60($out) + movups $inout7,0x70($out) + lea 0x80($out),$out # $out+=8*16 + + sub \$8,$len + jnc .Lctr32_loop8 # loop if $len-=8 didn't borrow + + add \$8,$len # restore real remainig $len + jz .Lctr32_done # done if ($len==0) + lea -0x80($key),$key .Lctr32_tail: - por $ivec,$inout0 - movups ($inp),$in0 - cmp \$2,$len - jb .Lctr32_one - - por $ivec,$inout1 - movups 0x10($inp),$in1 - je .Lctr32_two - - pshufd \$`3<<6`,$iv1,$inout3 - por $ivec,$inout2 - movups 0x20($inp),$in2 + # note that at this point $inout0..5 are populated with + # counter values xor-ed with 0-round key + lea 16($key),$key cmp \$4,$len - jb .Lctr32_three + jb .Lctr32_loop3 + je .Lctr32_loop4 - pshufd \$`2<<6`,$iv1,$inout4 - por $ivec,$inout3 - movups 0x30($inp),$in3 - je .Lctr32_four + # if ($len>4) compute 7 E(counter) + shl \$4,$rounds + movdqa 0x60(%rsp),$inout6 + pxor $inout7,$inout7 - por $ivec,$inout4 - xorps $inout5,$inout5 + $movkey 16($key),$rndkey0 + aesenc $rndkey1,$inout0 + aesenc $rndkey1,$inout1 + lea 32-16($key,$rounds),$key# prepare for .Lenc_loop8_enter + neg %rax + aesenc $rndkey1,$inout2 + add \$16,%rax # prepare for .Lenc_loop8_enter + movups ($inp),$in0 + aesenc $rndkey1,$inout3 + aesenc $rndkey1,$inout4 + movups 0x10($inp),$in1 # pre-load input + movups 0x20($inp),$in2 + aesenc $rndkey1,$inout5 + aesenc $rndkey1,$inout6 + + call .Lenc_loop8_enter + + movdqu 0x30($inp),$in3 + pxor $in0,$inout0 + movdqu 0x40($inp),$in0 + pxor $in1,$inout1 + movdqu $inout0,($out) # store output + pxor $in2,$inout2 + movdqu $inout1,0x10($out) + pxor $in3,$inout3 + movdqu $inout2,0x20($out) + pxor $in0,$inout4 + movdqu $inout3,0x30($out) + movdqu $inout4,0x40($out) + cmp \$6,$len + jb .Lctr32_done # $len was 5, stop store - call _aesni_encrypt6 + movups 0x50($inp),$in1 + xorps $in1,$inout5 + movups $inout5,0x50($out) + je .Lctr32_done # $len was 6, stop store - movups 0x40($inp),$rndkey1 - xorps $inout0,$in0 - xorps $inout1,$in1 - movups $in0,($out) - xorps $inout2,$in2 - movups $in1,0x10($out) - xorps $inout3,$in3 - movups $in2,0x20($out) - xorps $inout4,$rndkey1 - movups $in3,0x30($out) - movups $rndkey1,0x40($out) - jmp .Lctr32_done - -.align 16 -.Lctr32_one_shortcut: - movups ($ivp),$inout0 - movups ($inp),$in0 - mov 240($key),$rounds # key->rounds -.Lctr32_one: -___ - &aesni_generate1("enc",$key,$rounds); -$code.=<<___; - xorps $inout0,$in0 - movups $in0,($out) - jmp .Lctr32_done + movups 0x60($inp),$in2 + xorps $in2,$inout6 + movups $inout6,0x60($out) + jmp .Lctr32_done # $len was 7, stop store -.align 16 -.Lctr32_two: - xorps $inout2,$inout2 - call _aesni_encrypt3 - xorps $inout0,$in0 - xorps $inout1,$in1 - movups $in0,($out) - movups $in1,0x10($out) - jmp .Lctr32_done +.align 32 +.Lctr32_loop4: + aesenc $rndkey1,$inout0 + lea 16($key),$key + dec $rounds + aesenc $rndkey1,$inout1 + aesenc $rndkey1,$inout2 + aesenc $rndkey1,$inout3 + $movkey ($key),$rndkey1 + jnz .Lctr32_loop4 + aesenclast $rndkey1,$inout0 + aesenclast $rndkey1,$inout1 + movups ($inp),$in0 # load input + movups 0x10($inp),$in1 + aesenclast $rndkey1,$inout2 + aesenclast $rndkey1,$inout3 + movups 0x20($inp),$in2 + movups 0x30($inp),$in3 -.align 16 -.Lctr32_three: - call _aesni_encrypt3 - xorps $inout0,$in0 - xorps $inout1,$in1 - movups $in0,($out) - xorps $inout2,$in2 - movups $in1,0x10($out) - movups $in2,0x20($out) - jmp .Lctr32_done + xorps $in0,$inout0 + movups $inout0,($out) # store output + xorps $in1,$inout1 + movups $inout1,0x10($out) + pxor $in2,$inout2 + movdqu $inout2,0x20($out) + pxor $in3,$inout3 + movdqu $inout3,0x30($out) + jmp .Lctr32_done # $len was 4, stop store + +.align 32 +.Lctr32_loop3: + aesenc $rndkey1,$inout0 + lea 16($key),$key + dec $rounds + aesenc $rndkey1,$inout1 + aesenc $rndkey1,$inout2 + $movkey ($key),$rndkey1 + jnz .Lctr32_loop3 + aesenclast $rndkey1,$inout0 + aesenclast $rndkey1,$inout1 + aesenclast $rndkey1,$inout2 + + movups ($inp),$in0 # load input + xorps $in0,$inout0 + movups $inout0,($out) # store output + cmp \$2,$len + jb .Lctr32_done # $len was 1, stop store -.align 16 -.Lctr32_four: - call _aesni_encrypt4 - xorps $inout0,$in0 - xorps $inout1,$in1 - movups $in0,($out) - xorps $inout2,$in2 - movups $in1,0x10($out) - xorps $inout3,$in3 - movups $in2,0x20($out) - movups $in3,0x30($out) + movups 0x10($inp),$in1 + xorps $in1,$inout1 + movups $inout1,0x10($out) + je .Lctr32_done # $len was 2, stop store + + movups 0x20($inp),$in2 + xorps $in2,$inout2 + movups $inout2,0x20($out) # $len was 3, stop store .Lctr32_done: + xorps %xmm0,%xmm0 # clear regiser bank + xor $key0,$key0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 +___ +$code.=<<___ if (!$win64); + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 + movaps %xmm0,0x00(%rsp) # clear stack + pxor %xmm8,%xmm8 + movaps %xmm0,0x10(%rsp) + pxor %xmm9,%xmm9 + movaps %xmm0,0x20(%rsp) + pxor %xmm10,%xmm10 + movaps %xmm0,0x30(%rsp) + pxor %xmm11,%xmm11 + movaps %xmm0,0x40(%rsp) + pxor %xmm12,%xmm12 + movaps %xmm0,0x50(%rsp) + pxor %xmm13,%xmm13 + movaps %xmm0,0x60(%rsp) + pxor %xmm14,%xmm14 + movaps %xmm0,0x70(%rsp) + pxor %xmm15,%xmm15 ___ $code.=<<___ if ($win64); - movaps 0x20(%rsp),%xmm6 - movaps 0x30(%rsp),%xmm7 - movaps 0x40(%rsp),%xmm8 - movaps 0x50(%rsp),%xmm9 - movaps 0x60(%rsp),%xmm10 - movaps 0x70(%rsp),%xmm11 - movaps 0x80(%rsp),%xmm12 - movaps 0x90(%rsp),%xmm13 - movaps 0xa0(%rsp),%xmm14 - movaps 0xb0(%rsp),%xmm15 - lea 0xc8(%rsp),%rsp -.Lctr32_ret: + movaps -0xa0(%rbp),%xmm6 + movaps %xmm0,-0xa0(%rbp) # clear stack + movaps -0x90(%rbp),%xmm7 + movaps %xmm0,-0x90(%rbp) + movaps -0x80(%rbp),%xmm8 + movaps %xmm0,-0x80(%rbp) + movaps -0x70(%rbp),%xmm9 + movaps %xmm0,-0x70(%rbp) + movaps -0x60(%rbp),%xmm10 + movaps %xmm0,-0x60(%rbp) + movaps -0x50(%rbp),%xmm11 + movaps %xmm0,-0x50(%rbp) + movaps -0x40(%rbp),%xmm12 + movaps %xmm0,-0x40(%rbp) + movaps -0x30(%rbp),%xmm13 + movaps %xmm0,-0x30(%rbp) + movaps -0x20(%rbp),%xmm14 + movaps %xmm0,-0x20(%rbp) + movaps -0x10(%rbp),%xmm15 + movaps %xmm0,-0x10(%rbp) + movaps %xmm0,0x00(%rsp) + movaps %xmm0,0x10(%rsp) + movaps %xmm0,0x20(%rsp) + movaps %xmm0,0x30(%rsp) + movaps %xmm0,0x40(%rsp) + movaps %xmm0,0x50(%rsp) + movaps %xmm0,0x60(%rsp) + movaps %xmm0,0x70(%rsp) ___ $code.=<<___; + lea (%rbp),%rsp + pop %rbp +.Lctr32_epilogue: ret .size aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks ___ @@ -1317,252 +1725,297 @@ ___ my @tweak=map("%xmm$_",(10..15)); my ($twmask,$twres,$twtmp)=("%xmm8","%xmm9",@tweak[4]); my ($key2,$ivp,$len_)=("%r8","%r9","%r9"); -my $frame_size = 0x68 + ($win64?160:0); +my $frame_size = 0x70 + ($win64?160:0); $code.=<<___; .globl aesni_xts_encrypt .type aesni_xts_encrypt,\@function,6 .align 16 aesni_xts_encrypt: - lea -$frame_size(%rsp),%rsp + lea (%rsp),%rax + push %rbp + sub \$$frame_size,%rsp + and \$-16,%rsp # Linux kernel stack can be incorrectly seeded ___ $code.=<<___ if ($win64); - movaps %xmm6,0x60(%rsp) - movaps %xmm7,0x70(%rsp) - movaps %xmm8,0x80(%rsp) - movaps %xmm9,0x90(%rsp) - movaps %xmm10,0xa0(%rsp) - movaps %xmm11,0xb0(%rsp) - movaps %xmm12,0xc0(%rsp) - movaps %xmm13,0xd0(%rsp) - movaps %xmm14,0xe0(%rsp) - movaps %xmm15,0xf0(%rsp) + movaps %xmm6,-0xa8(%rax) # offload everything + movaps %xmm7,-0x98(%rax) + movaps %xmm8,-0x88(%rax) + movaps %xmm9,-0x78(%rax) + movaps %xmm10,-0x68(%rax) + movaps %xmm11,-0x58(%rax) + movaps %xmm12,-0x48(%rax) + movaps %xmm13,-0x38(%rax) + movaps %xmm14,-0x28(%rax) + movaps %xmm15,-0x18(%rax) .Lxts_enc_body: ___ $code.=<<___; - movups ($ivp),@tweak[5] # load clear-text tweak + lea -8(%rax),%rbp + movups ($ivp),$inout0 # load clear-text tweak mov 240(%r8),$rounds # key2->rounds mov 240($key),$rnds_ # key1->rounds ___ # generate the tweak - &aesni_generate1("enc",$key2,$rounds,@tweak[5]); + &aesni_generate1("enc",$key2,$rounds,$inout0); $code.=<<___; + $movkey ($key),$rndkey0 # zero round key mov $key,$key_ # backup $key mov $rnds_,$rounds # backup $rounds + shl \$4,$rnds_ mov $len,$len_ # backup $len and \$-16,$len + $movkey 16($key,$rnds_),$rndkey1 # last round key + movdqa .Lxts_magic(%rip),$twmask - pxor $twtmp,$twtmp - pcmpgtd @tweak[5],$twtmp # broadcast upper bits + movdqa $inout0,@tweak[5] + pshufd \$0x5f,$inout0,$twres + pxor $rndkey0,$rndkey1 ___ + # alternative tweak calculation algorithm is based on suggestions + # by Shay Gueron. psrad doesn't conflict with AES-NI instructions + # and should help in the future... for ($i=0;$i<4;$i++) { $code.=<<___; - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp + movdqa $twres,$twtmp + paddd $twres,$twres movdqa @tweak[5],@tweak[$i] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - pand $twmask,$twres # isolate carry and residue - pcmpgtd @tweak[5],$twtmp # broadcat upper bits - pxor $twres,@tweak[5] + psrad \$31,$twtmp # broadcast upper bits + paddq @tweak[5],@tweak[5] + pand $twmask,$twtmp + pxor $rndkey0,@tweak[$i] + pxor $twtmp,@tweak[5] ___ } $code.=<<___; + movdqa @tweak[5],@tweak[4] + psrad \$31,$twres + paddq @tweak[5],@tweak[5] + pand $twmask,$twres + pxor $rndkey0,@tweak[4] + pxor $twres,@tweak[5] + movaps $rndkey1,0x60(%rsp) # save round[0]^round[last] + sub \$16*6,$len - jc .Lxts_enc_short + jc .Lxts_enc_short # if $len-=6*16 borrowed - shr \$1,$rounds - sub \$1,$rounds - mov $rounds,$rnds_ + mov \$16+96,$rounds + lea 32($key_,$rnds_),$key # end of key schedule + sub %r10,%rax # twisted $rounds + $movkey 16($key_),$rndkey1 + mov %rax,%r10 # backup twisted $rounds + lea .Lxts_magic(%rip),%r8 jmp .Lxts_enc_grandloop -.align 16 +.align 32 .Lxts_enc_grandloop: - pshufd \$0x13,$twtmp,$twres - movdqa @tweak[5],@tweak[4] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak movdqu `16*0`($inp),$inout0 # load input - pand $twmask,$twres # isolate carry and residue + movdqa $rndkey0,$twmask movdqu `16*1`($inp),$inout1 - pxor $twres,@tweak[5] - + pxor @tweak[0],$inout0 # input^=tweak^round[0] movdqu `16*2`($inp),$inout2 - pxor @tweak[0],$inout0 # input^=tweak - movdqu `16*3`($inp),$inout3 pxor @tweak[1],$inout1 - movdqu `16*4`($inp),$inout4 + aesenc $rndkey1,$inout0 + movdqu `16*3`($inp),$inout3 pxor @tweak[2],$inout2 - movdqu `16*5`($inp),$inout5 - lea `16*6`($inp),$inp + aesenc $rndkey1,$inout1 + movdqu `16*4`($inp),$inout4 pxor @tweak[3],$inout3 - $movkey ($key_),$rndkey0 + aesenc $rndkey1,$inout2 + movdqu `16*5`($inp),$inout5 + pxor @tweak[5],$twmask # round[0]^=tweak[5] + movdqa 0x60(%rsp),$twres # load round[0]^round[last] pxor @tweak[4],$inout4 - pxor @tweak[5],$inout5 + aesenc $rndkey1,$inout3 + $movkey 32($key_),$rndkey0 + lea `16*6`($inp),$inp + pxor $twmask,$inout5 - # inline _aesni_encrypt6 and interleave first and last rounds - # with own code... - $movkey 16($key_),$rndkey1 - pxor $rndkey0,$inout0 - pxor $rndkey0,$inout1 - movdqa @tweak[0],`16*0`(%rsp) # put aside tweaks - aesenc $rndkey1,$inout0 - lea 32($key_),$key - pxor $rndkey0,$inout2 - movdqa @tweak[1],`16*1`(%rsp) - aesenc $rndkey1,$inout1 - pxor $rndkey0,$inout3 - movdqa @tweak[2],`16*2`(%rsp) - aesenc $rndkey1,$inout2 - pxor $rndkey0,$inout4 - movdqa @tweak[3],`16*3`(%rsp) - aesenc $rndkey1,$inout3 - pxor $rndkey0,$inout5 - $movkey ($key),$rndkey0 - dec $rounds - movdqa @tweak[4],`16*4`(%rsp) + pxor $twres,@tweak[0] # calclulate tweaks^round[last] aesenc $rndkey1,$inout4 - movdqa @tweak[5],`16*5`(%rsp) + pxor $twres,@tweak[1] + movdqa @tweak[0],`16*0`(%rsp) # put aside tweaks^round[last] aesenc $rndkey1,$inout5 - pxor $twtmp,$twtmp - pcmpgtd @tweak[5],$twtmp - jmp .Lxts_enc_loop6_enter + $movkey 48($key_),$rndkey1 + pxor $twres,@tweak[2] -.align 16 + aesenc $rndkey0,$inout0 + pxor $twres,@tweak[3] + movdqa @tweak[1],`16*1`(%rsp) + aesenc $rndkey0,$inout1 + pxor $twres,@tweak[4] + movdqa @tweak[2],`16*2`(%rsp) + aesenc $rndkey0,$inout2 + aesenc $rndkey0,$inout3 + pxor $twres,$twmask + movdqa @tweak[4],`16*4`(%rsp) + aesenc $rndkey0,$inout4 + aesenc $rndkey0,$inout5 + $movkey 64($key_),$rndkey0 + movdqa $twmask,`16*5`(%rsp) + pshufd \$0x5f,@tweak[5],$twres + jmp .Lxts_enc_loop6 +.align 32 .Lxts_enc_loop6: aesenc $rndkey1,$inout0 aesenc $rndkey1,$inout1 - dec $rounds aesenc $rndkey1,$inout2 aesenc $rndkey1,$inout3 aesenc $rndkey1,$inout4 aesenc $rndkey1,$inout5 -.Lxts_enc_loop6_enter: - $movkey 16($key),$rndkey1 + $movkey -64($key,%rax),$rndkey1 + add \$32,%rax + aesenc $rndkey0,$inout0 aesenc $rndkey0,$inout1 - lea 32($key),$key aesenc $rndkey0,$inout2 aesenc $rndkey0,$inout3 aesenc $rndkey0,$inout4 aesenc $rndkey0,$inout5 - $movkey ($key),$rndkey0 + $movkey -80($key,%rax),$rndkey0 jnz .Lxts_enc_loop6 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - paddq @tweak[5],@tweak[5] # psllq 1,$tweak + movdqa (%r8),$twmask # start calculating next tweak + movdqa $twres,$twtmp + paddd $twres,$twres aesenc $rndkey1,$inout0 - pand $twmask,$twres # isolate carry and residue + paddq @tweak[5],@tweak[5] + psrad \$31,$twtmp aesenc $rndkey1,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcast upper bits + pand $twmask,$twtmp + $movkey ($key_),@tweak[0] # load round[0] aesenc $rndkey1,$inout2 - pxor $twres,@tweak[5] aesenc $rndkey1,$inout3 aesenc $rndkey1,$inout4 + pxor $twtmp,@tweak[5] + movaps @tweak[0],@tweak[1] # copy round[0] aesenc $rndkey1,$inout5 - $movkey 16($key),$rndkey1 + $movkey -64($key),$rndkey1 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[0] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak + movdqa $twres,$twtmp aesenc $rndkey0,$inout0 - pand $twmask,$twres # isolate carry and residue + paddd $twres,$twres + pxor @tweak[5],@tweak[0] aesenc $rndkey0,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits + psrad \$31,$twtmp + paddq @tweak[5],@tweak[5] aesenc $rndkey0,$inout2 - pxor $twres,@tweak[5] aesenc $rndkey0,$inout3 + pand $twmask,$twtmp + movaps @tweak[1],@tweak[2] aesenc $rndkey0,$inout4 + pxor $twtmp,@tweak[5] + movdqa $twres,$twtmp aesenc $rndkey0,$inout5 - $movkey 32($key),$rndkey0 + $movkey -48($key),$rndkey0 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[1] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak + paddd $twres,$twres aesenc $rndkey1,$inout0 - pand $twmask,$twres # isolate carry and residue + pxor @tweak[5],@tweak[1] + psrad \$31,$twtmp aesenc $rndkey1,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits + paddq @tweak[5],@tweak[5] + pand $twmask,$twtmp aesenc $rndkey1,$inout2 - pxor $twres,@tweak[5] aesenc $rndkey1,$inout3 + movdqa @tweak[3],`16*3`(%rsp) + pxor $twtmp,@tweak[5] aesenc $rndkey1,$inout4 + movaps @tweak[2],@tweak[3] + movdqa $twres,$twtmp aesenc $rndkey1,$inout5 + $movkey -32($key),$rndkey1 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[2] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - aesenclast $rndkey0,$inout0 - pand $twmask,$twres # isolate carry and residue - aesenclast $rndkey0,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits - aesenclast $rndkey0,$inout2 - pxor $twres,@tweak[5] - aesenclast $rndkey0,$inout3 - aesenclast $rndkey0,$inout4 - aesenclast $rndkey0,$inout5 - - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[3] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - xorps `16*0`(%rsp),$inout0 # output^=tweak - pand $twmask,$twres # isolate carry and residue - xorps `16*1`(%rsp),$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits + paddd $twres,$twres + aesenc $rndkey0,$inout0 + pxor @tweak[5],@tweak[2] + psrad \$31,$twtmp + aesenc $rndkey0,$inout1 + paddq @tweak[5],@tweak[5] + pand $twmask,$twtmp + aesenc $rndkey0,$inout2 + aesenc $rndkey0,$inout3 + aesenc $rndkey0,$inout4 + pxor $twtmp,@tweak[5] + movaps @tweak[3],@tweak[4] + aesenc $rndkey0,$inout5 + + movdqa $twres,$rndkey0 + paddd $twres,$twres + aesenc $rndkey1,$inout0 + pxor @tweak[5],@tweak[3] + psrad \$31,$rndkey0 + aesenc $rndkey1,$inout1 + paddq @tweak[5],@tweak[5] + pand $twmask,$rndkey0 + aesenc $rndkey1,$inout2 + aesenc $rndkey1,$inout3 + pxor $rndkey0,@tweak[5] + $movkey ($key_),$rndkey0 + aesenc $rndkey1,$inout4 + aesenc $rndkey1,$inout5 + $movkey 16($key_),$rndkey1 + + pxor @tweak[5],@tweak[4] + aesenclast `16*0`(%rsp),$inout0 + psrad \$31,$twres + paddq @tweak[5],@tweak[5] + aesenclast `16*1`(%rsp),$inout1 + aesenclast `16*2`(%rsp),$inout2 + pand $twmask,$twres + mov %r10,%rax # restore $rounds + aesenclast `16*3`(%rsp),$inout3 + aesenclast `16*4`(%rsp),$inout4 + aesenclast `16*5`(%rsp),$inout5 pxor $twres,@tweak[5] - xorps `16*2`(%rsp),$inout2 - movups $inout0,`16*0`($out) # write output - xorps `16*3`(%rsp),$inout3 - movups $inout1,`16*1`($out) - xorps `16*4`(%rsp),$inout4 - movups $inout2,`16*2`($out) - xorps `16*5`(%rsp),$inout5 - movups $inout3,`16*3`($out) - mov $rnds_,$rounds # restore $rounds - movups $inout4,`16*4`($out) - movups $inout5,`16*5`($out) - lea `16*6`($out),$out + lea `16*6`($out),$out # $out+=6*16 + movups $inout0,`-16*6`($out) # store 6 output blocks + movups $inout1,`-16*5`($out) + movups $inout2,`-16*4`($out) + movups $inout3,`-16*3`($out) + movups $inout4,`-16*2`($out) + movups $inout5,`-16*1`($out) sub \$16*6,$len - jnc .Lxts_enc_grandloop + jnc .Lxts_enc_grandloop # loop if $len-=6*16 didn't borrow - lea 3($rounds,$rounds),$rounds # restore original value + mov \$16+96,$rounds + sub $rnds_,$rounds mov $key_,$key # restore $key - mov $rounds,$rnds_ # backup $rounds + shr \$4,$rounds # restore original value .Lxts_enc_short: - add \$16*6,$len - jz .Lxts_enc_done + # at the point @tweak[0..5] are populated with tweak values + mov $rounds,$rnds_ # backup $rounds + pxor $rndkey0,@tweak[0] + add \$16*6,$len # restore real remaining $len + jz .Lxts_enc_done # done if ($len==0) + pxor $rndkey0,@tweak[1] cmp \$0x20,$len - jb .Lxts_enc_one - je .Lxts_enc_two + jb .Lxts_enc_one # $len is 1*16 + pxor $rndkey0,@tweak[2] + je .Lxts_enc_two # $len is 2*16 + pxor $rndkey0,@tweak[3] cmp \$0x40,$len - jb .Lxts_enc_three - je .Lxts_enc_four - - pshufd \$0x13,$twtmp,$twres - movdqa @tweak[5],@tweak[4] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - movdqu ($inp),$inout0 - pand $twmask,$twres # isolate carry and residue - movdqu 16*1($inp),$inout1 - pxor $twres,@tweak[5] + jb .Lxts_enc_three # $len is 3*16 + pxor $rndkey0,@tweak[4] + je .Lxts_enc_four # $len is 4*16 + movdqu ($inp),$inout0 # $len is 5*16 + movdqu 16*1($inp),$inout1 movdqu 16*2($inp),$inout2 pxor @tweak[0],$inout0 movdqu 16*3($inp),$inout3 pxor @tweak[1],$inout1 movdqu 16*4($inp),$inout4 - lea 16*5($inp),$inp + lea 16*5($inp),$inp # $inp+=5*16 pxor @tweak[2],$inout2 pxor @tweak[3],$inout3 pxor @tweak[4],$inout4 + pxor $inout5,$inout5 call _aesni_encrypt6 @@ -1570,46 +2023,46 @@ $code.=<<___; movdqa @tweak[5],@tweak[0] xorps @tweak[1],$inout1 xorps @tweak[2],$inout2 - movdqu $inout0,($out) + movdqu $inout0,($out) # store 5 output blocks xorps @tweak[3],$inout3 movdqu $inout1,16*1($out) xorps @tweak[4],$inout4 movdqu $inout2,16*2($out) movdqu $inout3,16*3($out) movdqu $inout4,16*4($out) - lea 16*5($out),$out + lea 16*5($out),$out # $out+=5*16 jmp .Lxts_enc_done .align 16 .Lxts_enc_one: movups ($inp),$inout0 - lea 16*1($inp),$inp + lea 16*1($inp),$inp # inp+=1*16 xorps @tweak[0],$inout0 ___ &aesni_generate1("enc",$key,$rounds); $code.=<<___; xorps @tweak[0],$inout0 movdqa @tweak[1],@tweak[0] - movups $inout0,($out) - lea 16*1($out),$out + movups $inout0,($out) # store one output block + lea 16*1($out),$out # $out+=1*16 jmp .Lxts_enc_done .align 16 .Lxts_enc_two: movups ($inp),$inout0 movups 16($inp),$inout1 - lea 32($inp),$inp + lea 32($inp),$inp # $inp+=2*16 xorps @tweak[0],$inout0 xorps @tweak[1],$inout1 - call _aesni_encrypt3 + call _aesni_encrypt2 xorps @tweak[0],$inout0 movdqa @tweak[2],@tweak[0] xorps @tweak[1],$inout1 - movups $inout0,($out) + movups $inout0,($out) # store 2 output blocks movups $inout1,16*1($out) - lea 16*2($out),$out + lea 16*2($out),$out # $out+=2*16 jmp .Lxts_enc_done .align 16 @@ -1617,7 +2070,7 @@ $code.=<<___; movups ($inp),$inout0 movups 16*1($inp),$inout1 movups 16*2($inp),$inout2 - lea 16*3($inp),$inp + lea 16*3($inp),$inp # $inp+=3*16 xorps @tweak[0],$inout0 xorps @tweak[1],$inout1 xorps @tweak[2],$inout2 @@ -1628,10 +2081,10 @@ $code.=<<___; movdqa @tweak[3],@tweak[0] xorps @tweak[1],$inout1 xorps @tweak[2],$inout2 - movups $inout0,($out) + movups $inout0,($out) # store 3 output blocks movups $inout1,16*1($out) movups $inout2,16*2($out) - lea 16*3($out),$out + lea 16*3($out),$out # $out+=3*16 jmp .Lxts_enc_done .align 16 @@ -1641,28 +2094,28 @@ $code.=<<___; movups 16*2($inp),$inout2 xorps @tweak[0],$inout0 movups 16*3($inp),$inout3 - lea 16*4($inp),$inp + lea 16*4($inp),$inp # $inp+=4*16 xorps @tweak[1],$inout1 xorps @tweak[2],$inout2 xorps @tweak[3],$inout3 call _aesni_encrypt4 - xorps @tweak[0],$inout0 - movdqa @tweak[5],@tweak[0] - xorps @tweak[1],$inout1 - xorps @tweak[2],$inout2 - movups $inout0,($out) - xorps @tweak[3],$inout3 - movups $inout1,16*1($out) - movups $inout2,16*2($out) - movups $inout3,16*3($out) - lea 16*4($out),$out + pxor @tweak[0],$inout0 + movdqa @tweak[4],@tweak[0] + pxor @tweak[1],$inout1 + pxor @tweak[2],$inout2 + movdqu $inout0,($out) # store 4 output blocks + pxor @tweak[3],$inout3 + movdqu $inout1,16*1($out) + movdqu $inout2,16*2($out) + movdqu $inout3,16*3($out) + lea 16*4($out),$out # $out+=4*16 jmp .Lxts_enc_done .align 16 .Lxts_enc_done: - and \$15,$len_ + and \$15,$len_ # see if $len%16 is 0 jz .Lxts_enc_ret mov $len_,$len @@ -1689,21 +2142,64 @@ $code.=<<___; movups $inout0,-16($out) .Lxts_enc_ret: + xorps %xmm0,%xmm0 # clear register bank + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 +___ +$code.=<<___ if (!$win64); + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 + movaps %xmm0,0x00(%rsp) # clear stack + pxor %xmm8,%xmm8 + movaps %xmm0,0x10(%rsp) + pxor %xmm9,%xmm9 + movaps %xmm0,0x20(%rsp) + pxor %xmm10,%xmm10 + movaps %xmm0,0x30(%rsp) + pxor %xmm11,%xmm11 + movaps %xmm0,0x40(%rsp) + pxor %xmm12,%xmm12 + movaps %xmm0,0x50(%rsp) + pxor %xmm13,%xmm13 + movaps %xmm0,0x60(%rsp) + pxor %xmm14,%xmm14 + pxor %xmm15,%xmm15 ___ $code.=<<___ if ($win64); - movaps 0x60(%rsp),%xmm6 - movaps 0x70(%rsp),%xmm7 - movaps 0x80(%rsp),%xmm8 - movaps 0x90(%rsp),%xmm9 - movaps 0xa0(%rsp),%xmm10 - movaps 0xb0(%rsp),%xmm11 - movaps 0xc0(%rsp),%xmm12 - movaps 0xd0(%rsp),%xmm13 - movaps 0xe0(%rsp),%xmm14 - movaps 0xf0(%rsp),%xmm15 + movaps -0xa0(%rbp),%xmm6 + movaps %xmm0,-0xa0(%rbp) # clear stack + movaps -0x90(%rbp),%xmm7 + movaps %xmm0,-0x90(%rbp) + movaps -0x80(%rbp),%xmm8 + movaps %xmm0,-0x80(%rbp) + movaps -0x70(%rbp),%xmm9 + movaps %xmm0,-0x70(%rbp) + movaps -0x60(%rbp),%xmm10 + movaps %xmm0,-0x60(%rbp) + movaps -0x50(%rbp),%xmm11 + movaps %xmm0,-0x50(%rbp) + movaps -0x40(%rbp),%xmm12 + movaps %xmm0,-0x40(%rbp) + movaps -0x30(%rbp),%xmm13 + movaps %xmm0,-0x30(%rbp) + movaps -0x20(%rbp),%xmm14 + movaps %xmm0,-0x20(%rbp) + movaps -0x10(%rbp),%xmm15 + movaps %xmm0,-0x10(%rbp) + movaps %xmm0,0x00(%rsp) + movaps %xmm0,0x10(%rsp) + movaps %xmm0,0x20(%rsp) + movaps %xmm0,0x30(%rsp) + movaps %xmm0,0x40(%rsp) + movaps %xmm0,0x50(%rsp) + movaps %xmm0,0x60(%rsp) ___ $code.=<<___; - lea $frame_size(%rsp),%rsp + lea (%rbp),%rsp + pop %rbp .Lxts_enc_epilogue: ret .size aesni_xts_encrypt,.-aesni_xts_encrypt @@ -1714,28 +2210,32 @@ $code.=<<___; .type aesni_xts_decrypt,\@function,6 .align 16 aesni_xts_decrypt: - lea -$frame_size(%rsp),%rsp + lea (%rsp),%rax + push %rbp + sub \$$frame_size,%rsp + and \$-16,%rsp # Linux kernel stack can be incorrectly seeded ___ $code.=<<___ if ($win64); - movaps %xmm6,0x60(%rsp) - movaps %xmm7,0x70(%rsp) - movaps %xmm8,0x80(%rsp) - movaps %xmm9,0x90(%rsp) - movaps %xmm10,0xa0(%rsp) - movaps %xmm11,0xb0(%rsp) - movaps %xmm12,0xc0(%rsp) - movaps %xmm13,0xd0(%rsp) - movaps %xmm14,0xe0(%rsp) - movaps %xmm15,0xf0(%rsp) + movaps %xmm6,-0xa8(%rax) # offload everything + movaps %xmm7,-0x98(%rax) + movaps %xmm8,-0x88(%rax) + movaps %xmm9,-0x78(%rax) + movaps %xmm10,-0x68(%rax) + movaps %xmm11,-0x58(%rax) + movaps %xmm12,-0x48(%rax) + movaps %xmm13,-0x38(%rax) + movaps %xmm14,-0x28(%rax) + movaps %xmm15,-0x18(%rax) .Lxts_dec_body: ___ $code.=<<___; - movups ($ivp),@tweak[5] # load clear-text tweak + lea -8(%rax),%rbp + movups ($ivp),$inout0 # load clear-text tweak mov 240($key2),$rounds # key2->rounds mov 240($key),$rnds_ # key1->rounds ___ # generate the tweak - &aesni_generate1("enc",$key2,$rounds,@tweak[5]); + &aesni_generate1("enc",$key2,$rounds,$inout0); $code.=<<___; xor %eax,%eax # if ($len%16) len-=16; test \$15,$len @@ -1743,219 +2243,256 @@ $code.=<<___; shl \$4,%rax sub %rax,$len + $movkey ($key),$rndkey0 # zero round key mov $key,$key_ # backup $key mov $rnds_,$rounds # backup $rounds + shl \$4,$rnds_ mov $len,$len_ # backup $len and \$-16,$len + $movkey 16($key,$rnds_),$rndkey1 # last round key + movdqa .Lxts_magic(%rip),$twmask - pxor $twtmp,$twtmp - pcmpgtd @tweak[5],$twtmp # broadcast upper bits + movdqa $inout0,@tweak[5] + pshufd \$0x5f,$inout0,$twres + pxor $rndkey0,$rndkey1 ___ for ($i=0;$i<4;$i++) { $code.=<<___; - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp + movdqa $twres,$twtmp + paddd $twres,$twres movdqa @tweak[5],@tweak[$i] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - pand $twmask,$twres # isolate carry and residue - pcmpgtd @tweak[5],$twtmp # broadcat upper bits - pxor $twres,@tweak[5] + psrad \$31,$twtmp # broadcast upper bits + paddq @tweak[5],@tweak[5] + pand $twmask,$twtmp + pxor $rndkey0,@tweak[$i] + pxor $twtmp,@tweak[5] ___ } $code.=<<___; + movdqa @tweak[5],@tweak[4] + psrad \$31,$twres + paddq @tweak[5],@tweak[5] + pand $twmask,$twres + pxor $rndkey0,@tweak[4] + pxor $twres,@tweak[5] + movaps $rndkey1,0x60(%rsp) # save round[0]^round[last] + sub \$16*6,$len - jc .Lxts_dec_short + jc .Lxts_dec_short # if $len-=6*16 borrowed - shr \$1,$rounds - sub \$1,$rounds - mov $rounds,$rnds_ + mov \$16+96,$rounds + lea 32($key_,$rnds_),$key # end of key schedule + sub %r10,%rax # twisted $rounds + $movkey 16($key_),$rndkey1 + mov %rax,%r10 # backup twisted $rounds + lea .Lxts_magic(%rip),%r8 jmp .Lxts_dec_grandloop -.align 16 +.align 32 .Lxts_dec_grandloop: - pshufd \$0x13,$twtmp,$twres - movdqa @tweak[5],@tweak[4] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak movdqu `16*0`($inp),$inout0 # load input - pand $twmask,$twres # isolate carry and residue + movdqa $rndkey0,$twmask movdqu `16*1`($inp),$inout1 - pxor $twres,@tweak[5] - + pxor @tweak[0],$inout0 # intput^=tweak^round[0] movdqu `16*2`($inp),$inout2 - pxor @tweak[0],$inout0 # input^=tweak - movdqu `16*3`($inp),$inout3 pxor @tweak[1],$inout1 - movdqu `16*4`($inp),$inout4 + aesdec $rndkey1,$inout0 + movdqu `16*3`($inp),$inout3 pxor @tweak[2],$inout2 - movdqu `16*5`($inp),$inout5 - lea `16*6`($inp),$inp + aesdec $rndkey1,$inout1 + movdqu `16*4`($inp),$inout4 pxor @tweak[3],$inout3 - $movkey ($key_),$rndkey0 + aesdec $rndkey1,$inout2 + movdqu `16*5`($inp),$inout5 + pxor @tweak[5],$twmask # round[0]^=tweak[5] + movdqa 0x60(%rsp),$twres # load round[0]^round[last] pxor @tweak[4],$inout4 - pxor @tweak[5],$inout5 + aesdec $rndkey1,$inout3 + $movkey 32($key_),$rndkey0 + lea `16*6`($inp),$inp + pxor $twmask,$inout5 - # inline _aesni_decrypt6 and interleave first and last rounds - # with own code... - $movkey 16($key_),$rndkey1 - pxor $rndkey0,$inout0 - pxor $rndkey0,$inout1 - movdqa @tweak[0],`16*0`(%rsp) # put aside tweaks - aesdec $rndkey1,$inout0 - lea 32($key_),$key - pxor $rndkey0,$inout2 - movdqa @tweak[1],`16*1`(%rsp) - aesdec $rndkey1,$inout1 - pxor $rndkey0,$inout3 - movdqa @tweak[2],`16*2`(%rsp) - aesdec $rndkey1,$inout2 - pxor $rndkey0,$inout4 - movdqa @tweak[3],`16*3`(%rsp) - aesdec $rndkey1,$inout3 - pxor $rndkey0,$inout5 - $movkey ($key),$rndkey0 - dec $rounds - movdqa @tweak[4],`16*4`(%rsp) + pxor $twres,@tweak[0] # calclulate tweaks^round[last] aesdec $rndkey1,$inout4 - movdqa @tweak[5],`16*5`(%rsp) + pxor $twres,@tweak[1] + movdqa @tweak[0],`16*0`(%rsp) # put aside tweaks^last round key aesdec $rndkey1,$inout5 - pxor $twtmp,$twtmp - pcmpgtd @tweak[5],$twtmp - jmp .Lxts_dec_loop6_enter + $movkey 48($key_),$rndkey1 + pxor $twres,@tweak[2] -.align 16 + aesdec $rndkey0,$inout0 + pxor $twres,@tweak[3] + movdqa @tweak[1],`16*1`(%rsp) + aesdec $rndkey0,$inout1 + pxor $twres,@tweak[4] + movdqa @tweak[2],`16*2`(%rsp) + aesdec $rndkey0,$inout2 + aesdec $rndkey0,$inout3 + pxor $twres,$twmask + movdqa @tweak[4],`16*4`(%rsp) + aesdec $rndkey0,$inout4 + aesdec $rndkey0,$inout5 + $movkey 64($key_),$rndkey0 + movdqa $twmask,`16*5`(%rsp) + pshufd \$0x5f,@tweak[5],$twres + jmp .Lxts_dec_loop6 +.align 32 .Lxts_dec_loop6: aesdec $rndkey1,$inout0 aesdec $rndkey1,$inout1 - dec $rounds aesdec $rndkey1,$inout2 aesdec $rndkey1,$inout3 aesdec $rndkey1,$inout4 aesdec $rndkey1,$inout5 -.Lxts_dec_loop6_enter: - $movkey 16($key),$rndkey1 + $movkey -64($key,%rax),$rndkey1 + add \$32,%rax + aesdec $rndkey0,$inout0 aesdec $rndkey0,$inout1 - lea 32($key),$key aesdec $rndkey0,$inout2 aesdec $rndkey0,$inout3 aesdec $rndkey0,$inout4 aesdec $rndkey0,$inout5 - $movkey ($key),$rndkey0 + $movkey -80($key,%rax),$rndkey0 jnz .Lxts_dec_loop6 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - paddq @tweak[5],@tweak[5] # psllq 1,$tweak + movdqa (%r8),$twmask # start calculating next tweak + movdqa $twres,$twtmp + paddd $twres,$twres aesdec $rndkey1,$inout0 - pand $twmask,$twres # isolate carry and residue + paddq @tweak[5],@tweak[5] + psrad \$31,$twtmp aesdec $rndkey1,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcast upper bits + pand $twmask,$twtmp + $movkey ($key_),@tweak[0] # load round[0] aesdec $rndkey1,$inout2 - pxor $twres,@tweak[5] aesdec $rndkey1,$inout3 aesdec $rndkey1,$inout4 + pxor $twtmp,@tweak[5] + movaps @tweak[0],@tweak[1] # copy round[0] aesdec $rndkey1,$inout5 - $movkey 16($key),$rndkey1 + $movkey -64($key),$rndkey1 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[0] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak + movdqa $twres,$twtmp aesdec $rndkey0,$inout0 - pand $twmask,$twres # isolate carry and residue + paddd $twres,$twres + pxor @tweak[5],@tweak[0] aesdec $rndkey0,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits + psrad \$31,$twtmp + paddq @tweak[5],@tweak[5] aesdec $rndkey0,$inout2 - pxor $twres,@tweak[5] aesdec $rndkey0,$inout3 + pand $twmask,$twtmp + movaps @tweak[1],@tweak[2] aesdec $rndkey0,$inout4 + pxor $twtmp,@tweak[5] + movdqa $twres,$twtmp aesdec $rndkey0,$inout5 - $movkey 32($key),$rndkey0 + $movkey -48($key),$rndkey0 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[1] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak + paddd $twres,$twres aesdec $rndkey1,$inout0 - pand $twmask,$twres # isolate carry and residue + pxor @tweak[5],@tweak[1] + psrad \$31,$twtmp aesdec $rndkey1,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits + paddq @tweak[5],@tweak[5] + pand $twmask,$twtmp aesdec $rndkey1,$inout2 - pxor $twres,@tweak[5] aesdec $rndkey1,$inout3 + movdqa @tweak[3],`16*3`(%rsp) + pxor $twtmp,@tweak[5] aesdec $rndkey1,$inout4 + movaps @tweak[2],@tweak[3] + movdqa $twres,$twtmp aesdec $rndkey1,$inout5 + $movkey -32($key),$rndkey1 - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[2] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - aesdeclast $rndkey0,$inout0 - pand $twmask,$twres # isolate carry and residue - aesdeclast $rndkey0,$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits - aesdeclast $rndkey0,$inout2 - pxor $twres,@tweak[5] - aesdeclast $rndkey0,$inout3 - aesdeclast $rndkey0,$inout4 - aesdeclast $rndkey0,$inout5 - - pshufd \$0x13,$twtmp,$twres - pxor $twtmp,$twtmp - movdqa @tweak[5],@tweak[3] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - xorps `16*0`(%rsp),$inout0 # output^=tweak - pand $twmask,$twres # isolate carry and residue - xorps `16*1`(%rsp),$inout1 - pcmpgtd @tweak[5],$twtmp # broadcat upper bits + paddd $twres,$twres + aesdec $rndkey0,$inout0 + pxor @tweak[5],@tweak[2] + psrad \$31,$twtmp + aesdec $rndkey0,$inout1 + paddq @tweak[5],@tweak[5] + pand $twmask,$twtmp + aesdec $rndkey0,$inout2 + aesdec $rndkey0,$inout3 + aesdec $rndkey0,$inout4 + pxor $twtmp,@tweak[5] + movaps @tweak[3],@tweak[4] + aesdec $rndkey0,$inout5 + + movdqa $twres,$rndkey0 + paddd $twres,$twres + aesdec $rndkey1,$inout0 + pxor @tweak[5],@tweak[3] + psrad \$31,$rndkey0 + aesdec $rndkey1,$inout1 + paddq @tweak[5],@tweak[5] + pand $twmask,$rndkey0 + aesdec $rndkey1,$inout2 + aesdec $rndkey1,$inout3 + pxor $rndkey0,@tweak[5] + $movkey ($key_),$rndkey0 + aesdec $rndkey1,$inout4 + aesdec $rndkey1,$inout5 + $movkey 16($key_),$rndkey1 + + pxor @tweak[5],@tweak[4] + aesdeclast `16*0`(%rsp),$inout0 + psrad \$31,$twres + paddq @tweak[5],@tweak[5] + aesdeclast `16*1`(%rsp),$inout1 + aesdeclast `16*2`(%rsp),$inout2 + pand $twmask,$twres + mov %r10,%rax # restore $rounds + aesdeclast `16*3`(%rsp),$inout3 + aesdeclast `16*4`(%rsp),$inout4 + aesdeclast `16*5`(%rsp),$inout5 pxor $twres,@tweak[5] - xorps `16*2`(%rsp),$inout2 - movups $inout0,`16*0`($out) # write output - xorps `16*3`(%rsp),$inout3 - movups $inout1,`16*1`($out) - xorps `16*4`(%rsp),$inout4 - movups $inout2,`16*2`($out) - xorps `16*5`(%rsp),$inout5 - movups $inout3,`16*3`($out) - mov $rnds_,$rounds # restore $rounds - movups $inout4,`16*4`($out) - movups $inout5,`16*5`($out) - lea `16*6`($out),$out + lea `16*6`($out),$out # $out+=6*16 + movups $inout0,`-16*6`($out) # store 6 output blocks + movups $inout1,`-16*5`($out) + movups $inout2,`-16*4`($out) + movups $inout3,`-16*3`($out) + movups $inout4,`-16*2`($out) + movups $inout5,`-16*1`($out) sub \$16*6,$len - jnc .Lxts_dec_grandloop + jnc .Lxts_dec_grandloop # loop if $len-=6*16 didn't borrow - lea 3($rounds,$rounds),$rounds # restore original value + mov \$16+96,$rounds + sub $rnds_,$rounds mov $key_,$key # restore $key - mov $rounds,$rnds_ # backup $rounds + shr \$4,$rounds # restore original value .Lxts_dec_short: - add \$16*6,$len - jz .Lxts_dec_done + # at the point @tweak[0..5] are populated with tweak values + mov $rounds,$rnds_ # backup $rounds + pxor $rndkey0,@tweak[0] + pxor $rndkey0,@tweak[1] + add \$16*6,$len # restore real remaining $len + jz .Lxts_dec_done # done if ($len==0) + pxor $rndkey0,@tweak[2] cmp \$0x20,$len - jb .Lxts_dec_one - je .Lxts_dec_two + jb .Lxts_dec_one # $len is 1*16 + pxor $rndkey0,@tweak[3] + je .Lxts_dec_two # $len is 2*16 + pxor $rndkey0,@tweak[4] cmp \$0x40,$len - jb .Lxts_dec_three - je .Lxts_dec_four - - pshufd \$0x13,$twtmp,$twres - movdqa @tweak[5],@tweak[4] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - movdqu ($inp),$inout0 - pand $twmask,$twres # isolate carry and residue - movdqu 16*1($inp),$inout1 - pxor $twres,@tweak[5] + jb .Lxts_dec_three # $len is 3*16 + je .Lxts_dec_four # $len is 4*16 + movdqu ($inp),$inout0 # $len is 5*16 + movdqu 16*1($inp),$inout1 movdqu 16*2($inp),$inout2 pxor @tweak[0],$inout0 movdqu 16*3($inp),$inout3 pxor @tweak[1],$inout1 movdqu 16*4($inp),$inout4 - lea 16*5($inp),$inp + lea 16*5($inp),$inp # $inp+=5*16 pxor @tweak[2],$inout2 pxor @tweak[3],$inout3 pxor @tweak[4],$inout4 @@ -1965,7 +2502,7 @@ $code.=<<___; xorps @tweak[0],$inout0 xorps @tweak[1],$inout1 xorps @tweak[2],$inout2 - movdqu $inout0,($out) + movdqu $inout0,($out) # store 5 output blocks xorps @tweak[3],$inout3 movdqu $inout1,16*1($out) xorps @tweak[4],$inout4 @@ -1974,7 +2511,7 @@ $code.=<<___; movdqu $inout3,16*3($out) pcmpgtd @tweak[5],$twtmp movdqu $inout4,16*4($out) - lea 16*5($out),$out + lea 16*5($out),$out # $out+=5*16 pshufd \$0x13,$twtmp,@tweak[1] # $twres and \$15,$len_ jz .Lxts_dec_ret @@ -1988,35 +2525,35 @@ $code.=<<___; .align 16 .Lxts_dec_one: movups ($inp),$inout0 - lea 16*1($inp),$inp + lea 16*1($inp),$inp # $inp+=1*16 xorps @tweak[0],$inout0 ___ &aesni_generate1("dec",$key,$rounds); $code.=<<___; xorps @tweak[0],$inout0 movdqa @tweak[1],@tweak[0] - movups $inout0,($out) + movups $inout0,($out) # store one output block movdqa @tweak[2],@tweak[1] - lea 16*1($out),$out + lea 16*1($out),$out # $out+=1*16 jmp .Lxts_dec_done .align 16 .Lxts_dec_two: movups ($inp),$inout0 movups 16($inp),$inout1 - lea 32($inp),$inp + lea 32($inp),$inp # $inp+=2*16 xorps @tweak[0],$inout0 xorps @tweak[1],$inout1 - call _aesni_decrypt3 + call _aesni_decrypt2 xorps @tweak[0],$inout0 movdqa @tweak[2],@tweak[0] xorps @tweak[1],$inout1 movdqa @tweak[3],@tweak[1] - movups $inout0,($out) + movups $inout0,($out) # store 2 output blocks movups $inout1,16*1($out) - lea 16*2($out),$out + lea 16*2($out),$out # $out+=2*16 jmp .Lxts_dec_done .align 16 @@ -2024,7 +2561,7 @@ $code.=<<___; movups ($inp),$inout0 movups 16*1($inp),$inout1 movups 16*2($inp),$inout2 - lea 16*3($inp),$inp + lea 16*3($inp),$inp # $inp+=3*16 xorps @tweak[0],$inout0 xorps @tweak[1],$inout1 xorps @tweak[2],$inout2 @@ -2034,50 +2571,44 @@ $code.=<<___; xorps @tweak[0],$inout0 movdqa @tweak[3],@tweak[0] xorps @tweak[1],$inout1 - movdqa @tweak[5],@tweak[1] + movdqa @tweak[4],@tweak[1] xorps @tweak[2],$inout2 - movups $inout0,($out) + movups $inout0,($out) # store 3 output blocks movups $inout1,16*1($out) movups $inout2,16*2($out) - lea 16*3($out),$out + lea 16*3($out),$out # $out+=3*16 jmp .Lxts_dec_done .align 16 .Lxts_dec_four: - pshufd \$0x13,$twtmp,$twres - movdqa @tweak[5],@tweak[4] - paddq @tweak[5],@tweak[5] # psllq 1,$tweak - movups ($inp),$inout0 - pand $twmask,$twres # isolate carry and residue - movups 16*1($inp),$inout1 - pxor $twres,@tweak[5] - + movups ($inp),$inout0 + movups 16*1($inp),$inout1 movups 16*2($inp),$inout2 xorps @tweak[0],$inout0 movups 16*3($inp),$inout3 - lea 16*4($inp),$inp + lea 16*4($inp),$inp # $inp+=4*16 xorps @tweak[1],$inout1 xorps @tweak[2],$inout2 xorps @tweak[3],$inout3 call _aesni_decrypt4 - xorps @tweak[0],$inout0 + pxor @tweak[0],$inout0 movdqa @tweak[4],@tweak[0] - xorps @tweak[1],$inout1 + pxor @tweak[1],$inout1 movdqa @tweak[5],@tweak[1] - xorps @tweak[2],$inout2 - movups $inout0,($out) - xorps @tweak[3],$inout3 - movups $inout1,16*1($out) - movups $inout2,16*2($out) - movups $inout3,16*3($out) - lea 16*4($out),$out + pxor @tweak[2],$inout2 + movdqu $inout0,($out) # store 4 output blocks + pxor @tweak[3],$inout3 + movdqu $inout1,16*1($out) + movdqu $inout2,16*2($out) + movdqu $inout3,16*3($out) + lea 16*4($out),$out # $out+=4*16 jmp .Lxts_dec_done .align 16 .Lxts_dec_done: - and \$15,$len_ + and \$15,$len_ # see if $len%16 is 0 jz .Lxts_dec_ret .Lxts_dec_done2: mov $len_,$len @@ -2115,21 +2646,64 @@ $code.=<<___; movups $inout0,($out) .Lxts_dec_ret: + xorps %xmm0,%xmm0 # clear register bank + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 +___ +$code.=<<___ if (!$win64); + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 + movaps %xmm0,0x00(%rsp) # clear stack + pxor %xmm8,%xmm8 + movaps %xmm0,0x10(%rsp) + pxor %xmm9,%xmm9 + movaps %xmm0,0x20(%rsp) + pxor %xmm10,%xmm10 + movaps %xmm0,0x30(%rsp) + pxor %xmm11,%xmm11 + movaps %xmm0,0x40(%rsp) + pxor %xmm12,%xmm12 + movaps %xmm0,0x50(%rsp) + pxor %xmm13,%xmm13 + movaps %xmm0,0x60(%rsp) + pxor %xmm14,%xmm14 + pxor %xmm15,%xmm15 ___ $code.=<<___ if ($win64); - movaps 0x60(%rsp),%xmm6 - movaps 0x70(%rsp),%xmm7 - movaps 0x80(%rsp),%xmm8 - movaps 0x90(%rsp),%xmm9 - movaps 0xa0(%rsp),%xmm10 - movaps 0xb0(%rsp),%xmm11 - movaps 0xc0(%rsp),%xmm12 - movaps 0xd0(%rsp),%xmm13 - movaps 0xe0(%rsp),%xmm14 - movaps 0xf0(%rsp),%xmm15 + movaps -0xa0(%rbp),%xmm6 + movaps %xmm0,-0xa0(%rbp) # clear stack + movaps -0x90(%rbp),%xmm7 + movaps %xmm0,-0x90(%rbp) + movaps -0x80(%rbp),%xmm8 + movaps %xmm0,-0x80(%rbp) + movaps -0x70(%rbp),%xmm9 + movaps %xmm0,-0x70(%rbp) + movaps -0x60(%rbp),%xmm10 + movaps %xmm0,-0x60(%rbp) + movaps -0x50(%rbp),%xmm11 + movaps %xmm0,-0x50(%rbp) + movaps -0x40(%rbp),%xmm12 + movaps %xmm0,-0x40(%rbp) + movaps -0x30(%rbp),%xmm13 + movaps %xmm0,-0x30(%rbp) + movaps -0x20(%rbp),%xmm14 + movaps %xmm0,-0x20(%rbp) + movaps -0x10(%rbp),%xmm15 + movaps %xmm0,-0x10(%rbp) + movaps %xmm0,0x00(%rsp) + movaps %xmm0,0x10(%rsp) + movaps %xmm0,0x20(%rsp) + movaps %xmm0,0x30(%rsp) + movaps %xmm0,0x40(%rsp) + movaps %xmm0,0x50(%rsp) + movaps %xmm0,0x60(%rsp) ___ $code.=<<___; - lea $frame_size(%rsp),%rsp + lea (%rbp),%rsp + pop %rbp .Lxts_dec_epilogue: ret .size aesni_xts_decrypt,.-aesni_xts_decrypt @@ -2141,7 +2715,10 @@ ___ # size_t length, const AES_KEY *key, # unsigned char *ivp,const int enc); { -my $reserved = $win64?0x40:-0x18; # used in decrypt +my $frame_size = 0x10 + ($win64?0xa0:0); # used in decrypt +my ($iv,$in0,$in1,$in2,$in3,$in4)=map("%xmm$_",(10..15)); +my $inp_=$key_; + $code.=<<___; .globl ${PREFIX}_cbc_encrypt .type ${PREFIX}_cbc_encrypt,\@function,6 @@ -2177,7 +2754,11 @@ $code.=<<___; jnc .Lcbc_enc_loop add \$16,$len jnz .Lcbc_enc_tail + pxor $rndkey0,$rndkey0 # clear register bank + pxor $rndkey1,$rndkey1 movups $inout0,($ivp) + pxor $inout0,$inout0 + pxor $inout1,$inout1 jmp .Lcbc_ret .Lcbc_enc_tail: @@ -2197,283 +2778,483 @@ $code.=<<___; #--------------------------- CBC DECRYPT ------------------------------# .align 16 .Lcbc_decrypt: + cmp \$16,$len + jne .Lcbc_decrypt_bulk + + # handle single block without allocating stack frame, + # useful in ciphertext stealing mode + movdqu ($inp),$inout0 # load input + movdqu ($ivp),$inout1 # load iv + movdqa $inout0,$inout2 # future iv +___ + &aesni_generate1("dec",$key,$rnds_); +$code.=<<___; + pxor $rndkey0,$rndkey0 # clear register bank + pxor $rndkey1,$rndkey1 + movdqu $inout2,($ivp) # store iv + xorps $inout1,$inout0 # ^=iv + pxor $inout1,$inout1 + movups $inout0,($out) # store output + pxor $inout0,$inout0 + jmp .Lcbc_ret +.align 16 +.Lcbc_decrypt_bulk: + lea (%rsp),%rax + push %rbp + sub \$$frame_size,%rsp + and \$-16,%rsp # Linux kernel stack can be incorrectly seeded ___ $code.=<<___ if ($win64); - lea -0x58(%rsp),%rsp - movaps %xmm6,(%rsp) - movaps %xmm7,0x10(%rsp) - movaps %xmm8,0x20(%rsp) - movaps %xmm9,0x30(%rsp) + movaps %xmm6,0x10(%rsp) + movaps %xmm7,0x20(%rsp) + movaps %xmm8,0x30(%rsp) + movaps %xmm9,0x40(%rsp) + movaps %xmm10,0x50(%rsp) + movaps %xmm11,0x60(%rsp) + movaps %xmm12,0x70(%rsp) + movaps %xmm13,0x80(%rsp) + movaps %xmm14,0x90(%rsp) + movaps %xmm15,0xa0(%rsp) .Lcbc_decrypt_body: ___ $code.=<<___; + lea -8(%rax),%rbp movups ($ivp),$iv mov $rnds_,$rounds - cmp \$0x70,$len + cmp \$0x50,$len jbe .Lcbc_dec_tail - shr \$1,$rnds_ - sub \$0x70,$len - mov $rnds_,$rounds - movaps $iv,$reserved(%rsp) + + $movkey ($key),$rndkey0 + movdqu 0x00($inp),$inout0 # load input + movdqu 0x10($inp),$inout1 + movdqa $inout0,$in0 + movdqu 0x20($inp),$inout2 + movdqa $inout1,$in1 + movdqu 0x30($inp),$inout3 + movdqa $inout2,$in2 + movdqu 0x40($inp),$inout4 + movdqa $inout3,$in3 + movdqu 0x50($inp),$inout5 + movdqa $inout4,$in4 + mov OPENSSL_ia32cap_P+4(%rip),%r9d + cmp \$0x70,$len + jbe .Lcbc_dec_six_or_seven + + and \$`1<<26|1<<22`,%r9d # isolate XSAVE+MOVBE + sub \$0x50,$len # $len is biased by -5*16 + cmp \$`1<<22`,%r9d # check for MOVBE without XSAVE + je .Lcbc_dec_loop6_enter # [which denotes Atom Silvermont] + sub \$0x20,$len # $len is biased by -7*16 + lea 0x70($key),$key # size optimization jmp .Lcbc_dec_loop8_enter .align 16 .Lcbc_dec_loop8: - movaps $rndkey0,$reserved(%rsp) # save IV movups $inout7,($out) lea 0x10($out),$out .Lcbc_dec_loop8_enter: - $movkey ($key),$rndkey0 - movups ($inp),$inout0 # load input - movups 0x10($inp),$inout1 - $movkey 16($key),$rndkey1 + movdqu 0x60($inp),$inout6 + pxor $rndkey0,$inout0 + movdqu 0x70($inp),$inout7 + pxor $rndkey0,$inout1 + $movkey 0x10-0x70($key),$rndkey1 + pxor $rndkey0,$inout2 + xor $inp_,$inp_ + cmp \$0x70,$len # is there at least 0x60 bytes ahead? + pxor $rndkey0,$inout3 + pxor $rndkey0,$inout4 + pxor $rndkey0,$inout5 + pxor $rndkey0,$inout6 - lea 32($key),$key - movdqu 0x20($inp),$inout2 - xorps $rndkey0,$inout0 - movdqu 0x30($inp),$inout3 - xorps $rndkey0,$inout1 - movdqu 0x40($inp),$inout4 aesdec $rndkey1,$inout0 - pxor $rndkey0,$inout2 - movdqu 0x50($inp),$inout5 + pxor $rndkey0,$inout7 + $movkey 0x20-0x70($key),$rndkey0 aesdec $rndkey1,$inout1 - pxor $rndkey0,$inout3 - movdqu 0x60($inp),$inout6 aesdec $rndkey1,$inout2 - pxor $rndkey0,$inout4 - movdqu 0x70($inp),$inout7 aesdec $rndkey1,$inout3 - pxor $rndkey0,$inout5 - dec $rounds aesdec $rndkey1,$inout4 - pxor $rndkey0,$inout6 aesdec $rndkey1,$inout5 - pxor $rndkey0,$inout7 - $movkey ($key),$rndkey0 aesdec $rndkey1,$inout6 + setnc ${inp_}b + shl \$7,$inp_ aesdec $rndkey1,$inout7 - $movkey 16($key),$rndkey1 - - call .Ldec_loop8_enter + add $inp,$inp_ + $movkey 0x30-0x70($key),$rndkey1 +___ +for($i=1;$i<12;$i++) { +my $rndkeyx = ($i&1)?$rndkey0:$rndkey1; +$code.=<<___ if ($i==7); + cmp \$11,$rounds +___ +$code.=<<___; + aesdec $rndkeyx,$inout0 + aesdec $rndkeyx,$inout1 + aesdec $rndkeyx,$inout2 + aesdec $rndkeyx,$inout3 + aesdec $rndkeyx,$inout4 + aesdec $rndkeyx,$inout5 + aesdec $rndkeyx,$inout6 + aesdec $rndkeyx,$inout7 + $movkey `0x30+0x10*$i`-0x70($key),$rndkeyx +___ +$code.=<<___ if ($i<6 || (!($i&1) && $i>7)); + nop +___ +$code.=<<___ if ($i==7); + jb .Lcbc_dec_done +___ +$code.=<<___ if ($i==9); + je .Lcbc_dec_done +___ +$code.=<<___ if ($i==11); + jmp .Lcbc_dec_done +___ +} +$code.=<<___; +.align 16 +.Lcbc_dec_done: + aesdec $rndkey1,$inout0 + aesdec $rndkey1,$inout1 + pxor $rndkey0,$iv + pxor $rndkey0,$in0 + aesdec $rndkey1,$inout2 + aesdec $rndkey1,$inout3 + pxor $rndkey0,$in1 + pxor $rndkey0,$in2 + aesdec $rndkey1,$inout4 + aesdec $rndkey1,$inout5 + pxor $rndkey0,$in3 + pxor $rndkey0,$in4 + aesdec $rndkey1,$inout6 + aesdec $rndkey1,$inout7 + movdqu 0x50($inp),$rndkey1 + + aesdeclast $iv,$inout0 + movdqu 0x60($inp),$iv # borrow $iv + pxor $rndkey0,$rndkey1 + aesdeclast $in0,$inout1 + pxor $rndkey0,$iv + movdqu 0x70($inp),$rndkey0 # next IV + aesdeclast $in1,$inout2 + lea 0x80($inp),$inp + movdqu 0x00($inp_),$in0 + aesdeclast $in2,$inout3 + aesdeclast $in3,$inout4 + movdqu 0x10($inp_),$in1 + movdqu 0x20($inp_),$in2 + aesdeclast $in4,$inout5 + aesdeclast $rndkey1,$inout6 + movdqu 0x30($inp_),$in3 + movdqu 0x40($inp_),$in4 + aesdeclast $iv,$inout7 + movdqa $rndkey0,$iv # return $iv + movdqu 0x50($inp_),$rndkey1 + $movkey -0x70($key),$rndkey0 + + movups $inout0,($out) # store output + movdqa $in0,$inout0 + movups $inout1,0x10($out) + movdqa $in1,$inout1 + movups $inout2,0x20($out) + movdqa $in2,$inout2 + movups $inout3,0x30($out) + movdqa $in3,$inout3 + movups $inout4,0x40($out) + movdqa $in4,$inout4 + movups $inout5,0x50($out) + movdqa $rndkey1,$inout5 + movups $inout6,0x60($out) + lea 0x70($out),$out - movups ($inp),$rndkey1 # re-load input - movups 0x10($inp),$rndkey0 - xorps $reserved(%rsp),$inout0 # ^= IV - xorps $rndkey1,$inout1 - movups 0x20($inp),$rndkey1 - xorps $rndkey0,$inout2 - movups 0x30($inp),$rndkey0 - xorps $rndkey1,$inout3 - movups 0x40($inp),$rndkey1 - xorps $rndkey0,$inout4 - movups 0x50($inp),$rndkey0 - xorps $rndkey1,$inout5 - movups 0x60($inp),$rndkey1 - xorps $rndkey0,$inout6 - movups 0x70($inp),$rndkey0 # IV - xorps $rndkey1,$inout7 - movups $inout0,($out) - movups $inout1,0x10($out) - movups $inout2,0x20($out) - movups $inout3,0x30($out) - mov $rnds_,$rounds # restore $rounds - movups $inout4,0x40($out) - mov $key_,$key # restore $key - movups $inout5,0x50($out) - lea 0x80($inp),$inp - movups $inout6,0x60($out) - lea 0x70($out),$out sub \$0x80,$len ja .Lcbc_dec_loop8 movaps $inout7,$inout0 - movaps $rndkey0,$iv + lea -0x70($key),$key add \$0x70,$len - jle .Lcbc_dec_tail_collected - movups $inout0,($out) - lea 1($rnds_,$rnds_),$rounds + jle .Lcbc_dec_clear_tail_collected + movups $inout7,($out) + lea 0x10($out),$out + cmp \$0x50,$len + jbe .Lcbc_dec_tail + + movaps $in0,$inout0 +.Lcbc_dec_six_or_seven: + cmp \$0x60,$len + ja .Lcbc_dec_seven + + movaps $inout5,$inout6 + call _aesni_decrypt6 + pxor $iv,$inout0 # ^= IV + movaps $inout6,$iv + pxor $in0,$inout1 + movdqu $inout0,($out) + pxor $in1,$inout2 + movdqu $inout1,0x10($out) + pxor $inout1,$inout1 # clear register bank + pxor $in2,$inout3 + movdqu $inout2,0x20($out) + pxor $inout2,$inout2 + pxor $in3,$inout4 + movdqu $inout3,0x30($out) + pxor $inout3,$inout3 + pxor $in4,$inout5 + movdqu $inout4,0x40($out) + pxor $inout4,$inout4 + lea 0x50($out),$out + movdqa $inout5,$inout0 + pxor $inout5,$inout5 + jmp .Lcbc_dec_tail_collected + +.align 16 +.Lcbc_dec_seven: + movups 0x60($inp),$inout6 + xorps $inout7,$inout7 + call _aesni_decrypt8 + movups 0x50($inp),$inout7 + pxor $iv,$inout0 # ^= IV + movups 0x60($inp),$iv + pxor $in0,$inout1 + movdqu $inout0,($out) + pxor $in1,$inout2 + movdqu $inout1,0x10($out) + pxor $inout1,$inout1 # clear register bank + pxor $in2,$inout3 + movdqu $inout2,0x20($out) + pxor $inout2,$inout2 + pxor $in3,$inout4 + movdqu $inout3,0x30($out) + pxor $inout3,$inout3 + pxor $in4,$inout5 + movdqu $inout4,0x40($out) + pxor $inout4,$inout4 + pxor $inout7,$inout6 + movdqu $inout5,0x50($out) + pxor $inout5,$inout5 + lea 0x60($out),$out + movdqa $inout6,$inout0 + pxor $inout6,$inout6 + pxor $inout7,$inout7 + jmp .Lcbc_dec_tail_collected + +.align 16 +.Lcbc_dec_loop6: + movups $inout5,($out) + lea 0x10($out),$out + movdqu 0x00($inp),$inout0 # load input + movdqu 0x10($inp),$inout1 + movdqa $inout0,$in0 + movdqu 0x20($inp),$inout2 + movdqa $inout1,$in1 + movdqu 0x30($inp),$inout3 + movdqa $inout2,$in2 + movdqu 0x40($inp),$inout4 + movdqa $inout3,$in3 + movdqu 0x50($inp),$inout5 + movdqa $inout4,$in4 +.Lcbc_dec_loop6_enter: + lea 0x60($inp),$inp + movdqa $inout5,$inout6 + + call _aesni_decrypt6 + + pxor $iv,$inout0 # ^= IV + movdqa $inout6,$iv + pxor $in0,$inout1 + movdqu $inout0,($out) + pxor $in1,$inout2 + movdqu $inout1,0x10($out) + pxor $in2,$inout3 + movdqu $inout2,0x20($out) + pxor $in3,$inout4 + mov $key_,$key + movdqu $inout3,0x30($out) + pxor $in4,$inout5 + mov $rnds_,$rounds + movdqu $inout4,0x40($out) + lea 0x50($out),$out + sub \$0x60,$len + ja .Lcbc_dec_loop6 + + movdqa $inout5,$inout0 + add \$0x50,$len + jle .Lcbc_dec_clear_tail_collected + movups $inout5,($out) lea 0x10($out),$out + .Lcbc_dec_tail: movups ($inp),$inout0 - movaps $inout0,$in0 - cmp \$0x10,$len - jbe .Lcbc_dec_one + sub \$0x10,$len + jbe .Lcbc_dec_one # $len is 1*16 or less movups 0x10($inp),$inout1 - movaps $inout1,$in1 - cmp \$0x20,$len - jbe .Lcbc_dec_two + movaps $inout0,$in0 + sub \$0x10,$len + jbe .Lcbc_dec_two # $len is 2*16 or less movups 0x20($inp),$inout2 - movaps $inout2,$in2 - cmp \$0x30,$len - jbe .Lcbc_dec_three + movaps $inout1,$in1 + sub \$0x10,$len + jbe .Lcbc_dec_three # $len is 3*16 or less movups 0x30($inp),$inout3 - cmp \$0x40,$len - jbe .Lcbc_dec_four - - movups 0x40($inp),$inout4 - cmp \$0x50,$len - jbe .Lcbc_dec_five - - movups 0x50($inp),$inout5 - cmp \$0x60,$len - jbe .Lcbc_dec_six + movaps $inout2,$in2 + sub \$0x10,$len + jbe .Lcbc_dec_four # $len is 4*16 or less - movups 0x60($inp),$inout6 - movaps $iv,$reserved(%rsp) # save IV - call _aesni_decrypt8 - movups ($inp),$rndkey1 - movups 0x10($inp),$rndkey0 - xorps $reserved(%rsp),$inout0 # ^= IV - xorps $rndkey1,$inout1 - movups 0x20($inp),$rndkey1 - xorps $rndkey0,$inout2 - movups 0x30($inp),$rndkey0 - xorps $rndkey1,$inout3 - movups 0x40($inp),$rndkey1 - xorps $rndkey0,$inout4 - movups 0x50($inp),$rndkey0 - xorps $rndkey1,$inout5 - movups 0x60($inp),$iv # IV - xorps $rndkey0,$inout6 - movups $inout0,($out) - movups $inout1,0x10($out) - movups $inout2,0x20($out) - movups $inout3,0x30($out) - movups $inout4,0x40($out) - movups $inout5,0x50($out) - lea 0x60($out),$out - movaps $inout6,$inout0 - sub \$0x70,$len + movups 0x40($inp),$inout4 # $len is 5*16 or less + movaps $inout3,$in3 + movaps $inout4,$in4 + xorps $inout5,$inout5 + call _aesni_decrypt6 + pxor $iv,$inout0 + movaps $in4,$iv + pxor $in0,$inout1 + movdqu $inout0,($out) + pxor $in1,$inout2 + movdqu $inout1,0x10($out) + pxor $inout1,$inout1 # clear register bank + pxor $in2,$inout3 + movdqu $inout2,0x20($out) + pxor $inout2,$inout2 + pxor $in3,$inout4 + movdqu $inout3,0x30($out) + pxor $inout3,$inout3 + lea 0x40($out),$out + movdqa $inout4,$inout0 + pxor $inout4,$inout4 + pxor $inout5,$inout5 + sub \$0x10,$len jmp .Lcbc_dec_tail_collected + .align 16 .Lcbc_dec_one: + movaps $inout0,$in0 ___ &aesni_generate1("dec",$key,$rounds); $code.=<<___; xorps $iv,$inout0 movaps $in0,$iv - sub \$0x10,$len jmp .Lcbc_dec_tail_collected .align 16 .Lcbc_dec_two: - xorps $inout2,$inout2 - call _aesni_decrypt3 - xorps $iv,$inout0 - xorps $in0,$inout1 - movups $inout0,($out) + movaps $inout1,$in1 + call _aesni_decrypt2 + pxor $iv,$inout0 movaps $in1,$iv - movaps $inout1,$inout0 + pxor $in0,$inout1 + movdqu $inout0,($out) + movdqa $inout1,$inout0 + pxor $inout1,$inout1 # clear register bank lea 0x10($out),$out - sub \$0x20,$len jmp .Lcbc_dec_tail_collected .align 16 .Lcbc_dec_three: + movaps $inout2,$in2 call _aesni_decrypt3 - xorps $iv,$inout0 - xorps $in0,$inout1 - movups $inout0,($out) - xorps $in1,$inout2 - movups $inout1,0x10($out) + pxor $iv,$inout0 movaps $in2,$iv - movaps $inout2,$inout0 + pxor $in0,$inout1 + movdqu $inout0,($out) + pxor $in1,$inout2 + movdqu $inout1,0x10($out) + pxor $inout1,$inout1 # clear register bank + movdqa $inout2,$inout0 + pxor $inout2,$inout2 lea 0x20($out),$out - sub \$0x30,$len jmp .Lcbc_dec_tail_collected .align 16 .Lcbc_dec_four: + movaps $inout3,$in3 call _aesni_decrypt4 - xorps $iv,$inout0 - movups 0x30($inp),$iv - xorps $in0,$inout1 - movups $inout0,($out) - xorps $in1,$inout2 - movups $inout1,0x10($out) - xorps $in2,$inout3 - movups $inout2,0x20($out) - movaps $inout3,$inout0 + pxor $iv,$inout0 + movaps $in3,$iv + pxor $in0,$inout1 + movdqu $inout0,($out) + pxor $in1,$inout2 + movdqu $inout1,0x10($out) + pxor $inout1,$inout1 # clear register bank + pxor $in2,$inout3 + movdqu $inout2,0x20($out) + pxor $inout2,$inout2 + movdqa $inout3,$inout0 + pxor $inout3,$inout3 lea 0x30($out),$out - sub \$0x40,$len - jmp .Lcbc_dec_tail_collected -.align 16 -.Lcbc_dec_five: - xorps $inout5,$inout5 - call _aesni_decrypt6 - movups 0x10($inp),$rndkey1 - movups 0x20($inp),$rndkey0 - xorps $iv,$inout0 - xorps $in0,$inout1 - xorps $rndkey1,$inout2 - movups 0x30($inp),$rndkey1 - xorps $rndkey0,$inout3 - movups 0x40($inp),$iv - xorps $rndkey1,$inout4 - movups $inout0,($out) - movups $inout1,0x10($out) - movups $inout2,0x20($out) - movups $inout3,0x30($out) - lea 0x40($out),$out - movaps $inout4,$inout0 - sub \$0x50,$len - jmp .Lcbc_dec_tail_collected -.align 16 -.Lcbc_dec_six: - call _aesni_decrypt6 - movups 0x10($inp),$rndkey1 - movups 0x20($inp),$rndkey0 - xorps $iv,$inout0 - xorps $in0,$inout1 - xorps $rndkey1,$inout2 - movups 0x30($inp),$rndkey1 - xorps $rndkey0,$inout3 - movups 0x40($inp),$rndkey0 - xorps $rndkey1,$inout4 - movups 0x50($inp),$iv - xorps $rndkey0,$inout5 - movups $inout0,($out) - movups $inout1,0x10($out) - movups $inout2,0x20($out) - movups $inout3,0x30($out) - movups $inout4,0x40($out) - lea 0x50($out),$out - movaps $inout5,$inout0 - sub \$0x60,$len jmp .Lcbc_dec_tail_collected + .align 16 +.Lcbc_dec_clear_tail_collected: + pxor $inout1,$inout1 # clear register bank + pxor $inout2,$inout2 + pxor $inout3,$inout3 +___ +$code.=<<___ if (!$win64); + pxor $inout4,$inout4 # %xmm6..9 + pxor $inout5,$inout5 + pxor $inout6,$inout6 + pxor $inout7,$inout7 +___ +$code.=<<___; .Lcbc_dec_tail_collected: - and \$15,$len movups $iv,($ivp) + and \$15,$len jnz .Lcbc_dec_tail_partial movups $inout0,($out) + pxor $inout0,$inout0 jmp .Lcbc_dec_ret .align 16 .Lcbc_dec_tail_partial: - movaps $inout0,$reserved(%rsp) + movaps $inout0,(%rsp) + pxor $inout0,$inout0 mov \$16,%rcx mov $out,%rdi sub $len,%rcx - lea $reserved(%rsp),%rsi - .long 0x9066A4F3 # rep movsb + lea (%rsp),%rsi + .long 0x9066A4F3 # rep movsb + movdqa $inout0,(%rsp) .Lcbc_dec_ret: + xorps $rndkey0,$rndkey0 # %xmm0 + pxor $rndkey1,$rndkey1 ___ $code.=<<___ if ($win64); - movaps (%rsp),%xmm6 - movaps 0x10(%rsp),%xmm7 - movaps 0x20(%rsp),%xmm8 - movaps 0x30(%rsp),%xmm9 - lea 0x58(%rsp),%rsp + movaps 0x10(%rsp),%xmm6 + movaps %xmm0,0x10(%rsp) # clear stack + movaps 0x20(%rsp),%xmm7 + movaps %xmm0,0x20(%rsp) + movaps 0x30(%rsp),%xmm8 + movaps %xmm0,0x30(%rsp) + movaps 0x40(%rsp),%xmm9 + movaps %xmm0,0x40(%rsp) + movaps 0x50(%rsp),%xmm10 + movaps %xmm0,0x50(%rsp) + movaps 0x60(%rsp),%xmm11 + movaps %xmm0,0x60(%rsp) + movaps 0x70(%rsp),%xmm12 + movaps %xmm0,0x70(%rsp) + movaps 0x80(%rsp),%xmm13 + movaps %xmm0,0x80(%rsp) + movaps 0x90(%rsp),%xmm14 + movaps %xmm0,0x90(%rsp) + movaps 0xa0(%rsp),%xmm15 + movaps %xmm0,0xa0(%rsp) ___ $code.=<<___; + lea (%rbp),%rsp + pop %rbp .Lcbc_ret: ret .size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt ___ } -# int $PREFIX_set_[en|de]crypt_key (const unsigned char *userKey, +# int ${PREFIX}_set_decrypt_key(const unsigned char *inp, # int bits, AES_KEY *key) +# +# input: $inp user-supplied key +# $bits $inp length in bits +# $key pointer to key schedule +# output: %eax 0 denoting success, -1 or -2 - failure (see C) +# *$key key schedule +# { my ($inp,$bits,$key) = @_4args; $bits =~ s/%r/%e/; @@ -2510,7 +3291,9 @@ ${PREFIX}_set_decrypt_key: $movkey ($key),%xmm0 # inverse middle aesimc %xmm0,%xmm0 + pxor %xmm1,%xmm1 $movkey %xmm0,($inp) + pxor %xmm0,%xmm0 .Ldec_key_ret: add \$8,%rsp ret @@ -2527,6 +3310,22 @@ ___ # Agressively optimized in respect to aeskeygenassist's critical path # and is contained in %xmm0-5 to meet Win64 ABI requirement. # +# int ${PREFIX}_set_encrypt_key(const unsigned char *inp, +# int bits, AES_KEY * const key); +# +# input: $inp user-supplied key +# $bits $inp length in bits +# $key pointer to key schedule +# output: %eax 0 denoting success, -1 or -2 - failure (see C) +# $bits rounds-1 (used in aesni_set_decrypt_key) +# *$key key schedule +# $key pointer to key schedule (used in +# aesni_set_decrypt_key) +# +# Subroutine is frame-less, which means that only volatile registers +# are used. Note that it's declared "abi-omnipotent", which means that +# amount of volatile registers is smaller on Windows. +# $code.=<<___; .globl ${PREFIX}_set_encrypt_key .type ${PREFIX}_set_encrypt_key,\@abi-omnipotent @@ -2540,9 +3339,11 @@ __aesni_set_encrypt_key: test $key,$key jz .Lenc_key_ret + mov \$`1<<28|1<<11`,%r10d # AVX and XOP bits movups ($inp),%xmm0 # pull first 128 bits of *userKey xorps %xmm4,%xmm4 # low dword of xmm4 is assumed 0 - lea 16($key),%rax + and OPENSSL_ia32cap_P+4(%rip),%r10d + lea 16($key),%rax # %rax is used as modifiable copy of $key cmp \$256,$bits je .L14rounds cmp \$192,$bits @@ -2552,6 +3353,9 @@ __aesni_set_encrypt_key: .L10rounds: mov \$9,$bits # 10 rounds for 128-bit key + cmp \$`1<<28`,%r10d # AVX, bit no XOP + je .L10rounds_alt + $movkey %xmm0,($key) # round 0 aeskeygenassist \$0x1,%xmm0,%xmm1 # round 1 call .Lkey_expansion_128_cold @@ -2578,10 +3382,80 @@ __aesni_set_encrypt_key: xor %eax,%eax jmp .Lenc_key_ret +.align 16 +.L10rounds_alt: + movdqa .Lkey_rotate(%rip),%xmm5 + mov \$8,%r10d + movdqa .Lkey_rcon1(%rip),%xmm4 + movdqa %xmm0,%xmm2 + movdqu %xmm0,($key) + jmp .Loop_key128 + +.align 16 +.Loop_key128: + pshufb %xmm5,%xmm0 + aesenclast %xmm4,%xmm0 + pslld \$1,%xmm4 + lea 16(%rax),%rax + + movdqa %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,-16(%rax) + movdqa %xmm0,%xmm2 + + dec %r10d + jnz .Loop_key128 + + movdqa .Lkey_rcon1b(%rip),%xmm4 + + pshufb %xmm5,%xmm0 + aesenclast %xmm4,%xmm0 + pslld \$1,%xmm4 + + movdqa %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + movdqa %xmm0,%xmm2 + pshufb %xmm5,%xmm0 + aesenclast %xmm4,%xmm0 + + movdqa %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,16(%rax) + + mov $bits,96(%rax) # 240($key) + xor %eax,%eax + jmp .Lenc_key_ret + .align 16 .L12rounds: movq 16($inp),%xmm2 # remaining 1/3 of *userKey mov \$11,$bits # 12 rounds for 192 + cmp \$`1<<28`,%r10d # AVX, but no XOP + je .L12rounds_alt + $movkey %xmm0,($key) # round 0 aeskeygenassist \$0x1,%xmm2,%xmm1 # round 1,2 call .Lkey_expansion_192a_cold @@ -2604,11 +3478,55 @@ __aesni_set_encrypt_key: xor %rax, %rax jmp .Lenc_key_ret +.align 16 +.L12rounds_alt: + movdqa .Lkey_rotate192(%rip),%xmm5 + movdqa .Lkey_rcon1(%rip),%xmm4 + mov \$8,%r10d + movdqu %xmm0,($key) + jmp .Loop_key192 + +.align 16 +.Loop_key192: + movq %xmm2,0(%rax) + movdqa %xmm2,%xmm1 + pshufb %xmm5,%xmm2 + aesenclast %xmm4,%xmm2 + pslld \$1, %xmm4 + lea 24(%rax),%rax + + movdqa %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm3,%xmm0 + + pshufd \$0xff,%xmm0,%xmm3 + pxor %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm1,%xmm3 + + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm2 + movdqu %xmm0,-16(%rax) + + dec %r10d + jnz .Loop_key192 + + mov $bits,32(%rax) # 240($key) + xor %eax,%eax + jmp .Lenc_key_ret + .align 16 .L14rounds: movups 16($inp),%xmm2 # remaning half of *userKey mov \$13,$bits # 14 rounds for 256 lea 16(%rax),%rax + cmp \$`1<<28`,%r10d # AVX, but no XOP + je .L14rounds_alt + $movkey %xmm0,($key) # round 0 $movkey %xmm2,16($key) # round 1 aeskeygenassist \$0x1,%xmm2,%xmm1 # round 2 @@ -2642,10 +3560,70 @@ __aesni_set_encrypt_key: xor %rax,%rax jmp .Lenc_key_ret +.align 16 +.L14rounds_alt: + movdqa .Lkey_rotate(%rip),%xmm5 + movdqa .Lkey_rcon1(%rip),%xmm4 + mov \$7,%r10d + movdqu %xmm0,0($key) + movdqa %xmm2,%xmm1 + movdqu %xmm2,16($key) + jmp .Loop_key256 + +.align 16 +.Loop_key256: + pshufb %xmm5,%xmm2 + aesenclast %xmm4,%xmm2 + + movdqa %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm3,%xmm0 + pslld \$1,%xmm4 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + dec %r10d + jz .Ldone_key256 + + pshufd \$0xff,%xmm0,%xmm2 + pxor %xmm3,%xmm3 + aesenclast %xmm3,%xmm2 + + movdqa %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm3,%xmm1 + + pxor %xmm1,%xmm2 + movdqu %xmm2,16(%rax) + lea 32(%rax),%rax + movdqa %xmm2,%xmm1 + + jmp .Loop_key256 + +.Ldone_key256: + mov $bits,16(%rax) # 240($key) + xor %eax,%eax + jmp .Lenc_key_ret + .align 16 .Lbad_keybits: mov \$-2,%rax .Lenc_key_ret: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 add \$8,%rsp ret .LSEH_end_set_encrypt_key: @@ -2733,6 +3711,16 @@ $code.=<<___; .long 1,0,0,0 .Lxts_magic: .long 0x87,0,1,0 +.Lincrement1: + .byte 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +.Lkey_rotate: + .long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d +.Lkey_rotate192: + .long 0x04070605,0x04070605,0x04070605,0x04070605 +.Lkey_rcon1: + .long 1,1,1,1 +.Lkey_rcon1b: + .long 0x1b,0x1b,0x1b,0x1b .asciz "AES for Intel AES-NI, CRYPTOGAMS by " .align 64 @@ -2791,45 +3779,9 @@ ecb_ccm64_se_handler: jmp .Lcommon_seh_tail .size ecb_ccm64_se_handler,.-ecb_ccm64_se_handler -.type ctr32_se_handler,\@abi-omnipotent +.type ctr_xts_se_handler,\@abi-omnipotent .align 16 -ctr32_se_handler: - push %rsi - push %rdi - push %rbx - push %rbp - push %r12 - push %r13 - push %r14 - push %r15 - pushfq - sub \$64,%rsp - - mov 120($context),%rax # pull context->Rax - mov 248($context),%rbx # pull context->Rip - - lea .Lctr32_body(%rip),%r10 - cmp %r10,%rbx # context->Rip<"prologue" label - jb .Lcommon_seh_tail - - mov 152($context),%rax # pull context->Rsp - - lea .Lctr32_ret(%rip),%r10 - cmp %r10,%rbx - jae .Lcommon_seh_tail - - lea 0x20(%rax),%rsi # %xmm save area - lea 512($context),%rdi # &context.Xmm6 - mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) - .long 0xa548f3fc # cld; rep movsq - lea 0xc8(%rax),%rax # adjust stack pointer - - jmp .Lcommon_seh_tail -.size ctr32_se_handler,.-ctr32_se_handler - -.type xts_se_handler,\@abi-omnipotent -.align 16 -xts_se_handler: +ctr_xts_se_handler: push %rsi push %rdi push %rbx @@ -2859,14 +3811,14 @@ xts_se_handler: cmp %r10,%rbx # context->Rip>=epilogue label jae .Lcommon_seh_tail - lea 0x60(%rax),%rsi # %xmm save area + mov 160($context),%rax # pull context->Rbp + lea -0xa0(%rax),%rsi # %xmm save area lea 512($context),%rdi # & context.Xmm6 mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) .long 0xa548f3fc # cld; rep movsq - lea 0x68+160(%rax),%rax # adjust stack pointer - jmp .Lcommon_seh_tail -.size xts_se_handler,.-xts_se_handler + jmp .Lcommon_rbp_tail +.size ctr_xts_se_handler,.-ctr_xts_se_handler ___ $code.=<<___; .type cbc_se_handler,\@abi-omnipotent @@ -2886,7 +3838,7 @@ cbc_se_handler: mov 152($context),%rax # pull context->Rsp mov 248($context),%rbx # pull context->Rip - lea .Lcbc_decrypt(%rip),%r10 + lea .Lcbc_decrypt_bulk(%rip),%r10 cmp %r10,%rbx # context->Rip<"prologue" label jb .Lcommon_seh_tail @@ -2898,11 +3850,16 @@ cbc_se_handler: cmp %r10,%rbx # context->Rip>="epilogue" label jae .Lcommon_seh_tail - lea 0(%rax),%rsi # top of stack + lea 16(%rax),%rsi # %xmm save area lea 512($context),%rdi # &context.Xmm6 - mov \$8,%ecx # 4*sizeof(%xmm0)/sizeof(%rax) + mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) .long 0xa548f3fc # cld; rep movsq - lea 0x58(%rax),%rax # adjust stack pointer + +.Lcommon_rbp_tail: + mov 160($context),%rax # pull context->Rbp + mov (%rax),%rbp # restore saved %rbp + lea 8(%rax),%rax # adjust stack pointer + mov %rbp,160($context) # restore context->Rbp jmp .Lcommon_seh_tail .Lrestore_cbc_rax: @@ -3006,14 +3963,15 @@ $code.=<<___ if ($PREFIX eq "aesni"); .rva .Lccm64_dec_body,.Lccm64_dec_ret # HandlerData[] .LSEH_info_ctr32: .byte 9,0,0,0 - .rva ctr32_se_handler + .rva ctr_xts_se_handler + .rva .Lctr32_body,.Lctr32_epilogue # HandlerData[] .LSEH_info_xts_enc: .byte 9,0,0,0 - .rva xts_se_handler + .rva ctr_xts_se_handler .rva .Lxts_enc_body,.Lxts_enc_epilogue # HandlerData[] .LSEH_info_xts_dec: .byte 9,0,0,0 - .rva xts_se_handler + .rva ctr_xts_se_handler .rva .Lxts_dec_body,.Lxts_dec_epilogue # HandlerData[] ___ $code.=<<___; @@ -3060,11 +4018,30 @@ sub aesni { push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M return ".byte\t".join(',',@opcode); } + elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) { + my %opcodelet = ( + "aesenc" => 0xdc, "aesenclast" => 0xdd, + "aesdec" => 0xde, "aesdeclast" => 0xdf + ); + return undef if (!defined($opcodelet{$1})); + my $off = $2; + push @opcode,0x44 if ($3>=8); + push @opcode,0x0f,0x38,$opcodelet{$1}; + push @opcode,0x44|(($3&7)<<3),0x24; # ModR/M + push @opcode,($off=~/^0/?oct($off):$off)&0xff; + return ".byte\t".join(',',@opcode); + } return $line; } +sub movbe { + ".byte 0x0f,0x38,0xf1,0x44,0x24,".shift; +} + $code =~ s/\`([^\`]*)\`/eval($1)/gem; $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; +#$code =~ s/\bmovbe\s+%eax/bswap %eax; mov %eax/gm; # debugging artefact +$code =~ s/\bmovbe\s+%eax,\s*([0-9]+)\(%rsp\)/movbe($1)/gem; print $code; diff --git a/crypto/aes/asm/aesp8-ppc.pl b/crypto/aes/asm/aesp8-ppc.pl new file mode 100755 index 000000000000..a1891cc03caa --- /dev/null +++ b/crypto/aes/asm/aesp8-ppc.pl @@ -0,0 +1,1942 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements support for AES instructions as per PowerISA +# specification version 2.07, first implemented by POWER8 processor. +# The module is endian-agnostic in sense that it supports both big- +# and little-endian cases. Data alignment in parallelizable modes is +# handled with VSX loads and stores, which implies MSR.VSX flag being +# set. It should also be noted that ISA specification doesn't prohibit +# alignment exceptions for these instructions on page boundaries. +# Initially alignment was handled in pure AltiVec/VMX way [when data +# is aligned programmatically, which in turn guarantees exception- +# free execution], but it turned to hamper performance when vcipher +# instructions are interleaved. It's reckoned that eventual +# misalignment penalties at page boundaries are in average lower +# than additional overhead in pure AltiVec approach. + +$flavour = shift; + +if ($flavour =~ /64/) { + $SIZE_T =8; + $LRSAVE =2*$SIZE_T; + $STU ="stdu"; + $POP ="ld"; + $PUSH ="std"; + $UCMP ="cmpld"; + $SHL ="sldi"; +} elsif ($flavour =~ /32/) { + $SIZE_T =4; + $LRSAVE =$SIZE_T; + $STU ="stwu"; + $POP ="lwz"; + $PUSH ="stw"; + $UCMP ="cmplw"; + $SHL ="slwi"; +} else { die "nonsense $flavour"; } + +$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!"; + +$FRAME=8*$SIZE_T; +$prefix="aes_p8"; + +$sp="r1"; +$vrsave="r12"; + +######################################################################### +{{{ # Key setup procedures # +my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8)); +my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6)); +my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11)); + +$code.=<<___; +.machine "any" + +.text + +.align 7 +rcon: +.long 0x01000000, 0x01000000, 0x01000000, 0x01000000 ?rev +.long 0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000 ?rev +.long 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c ?rev +.long 0,0,0,0 ?asis +Lconsts: + mflr r0 + bcl 20,31,\$+4 + mflr $ptr #vvvvv "distance between . and rcon + addi $ptr,$ptr,-0x48 + mtlr r0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +.asciz "AES for PowerISA 2.07, CRYPTOGAMS by " + +.globl .${prefix}_set_encrypt_key +.align 5 +.${prefix}_set_encrypt_key: +Lset_encrypt_key: + mflr r11 + $PUSH r11,$LRSAVE($sp) + + li $ptr,-1 + ${UCMP}i $inp,0 + beq- Lenc_key_abort # if ($inp==0) return -1; + ${UCMP}i $out,0 + beq- Lenc_key_abort # if ($out==0) return -1; + li $ptr,-2 + cmpwi $bits,128 + blt- Lenc_key_abort + cmpwi $bits,256 + bgt- Lenc_key_abort + andi. r0,$bits,0x3f + bne- Lenc_key_abort + + lis r0,0xfff0 + mfspr $vrsave,256 + mtspr 256,r0 + + bl Lconsts + mtlr r11 + + neg r9,$inp + lvx $in0,0,$inp + addi $inp,$inp,15 # 15 is not typo + lvsr $key,0,r9 # borrow $key + li r8,0x20 + cmpwi $bits,192 + lvx $in1,0,$inp + le?vspltisb $mask,0x0f # borrow $mask + lvx $rcon,0,$ptr + le?vxor $key,$key,$mask # adjust for byte swap + lvx $mask,r8,$ptr + addi $ptr,$ptr,0x10 + vperm $in0,$in0,$in1,$key # align [and byte swap in LE] + li $cnt,8 + vxor $zero,$zero,$zero + mtctr $cnt + + ?lvsr $outperm,0,$out + vspltisb $outmask,-1 + lvx $outhead,0,$out + ?vperm $outmask,$zero,$outmask,$outperm + + blt Loop128 + addi $inp,$inp,8 + beq L192 + addi $inp,$inp,8 + b L256 + +.align 4 +Loop128: + vperm $key,$in0,$in0,$mask # rotate-n-splat + vsldoi $tmp,$zero,$in0,12 # >>32 + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + vcipherlast $key,$key,$rcon + stvx $stage,0,$out + addi $out,$out,16 + + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vadduwm $rcon,$rcon,$rcon + vxor $in0,$in0,$key + bdnz Loop128 + + lvx $rcon,0,$ptr # last two round keys + + vperm $key,$in0,$in0,$mask # rotate-n-splat + vsldoi $tmp,$zero,$in0,12 # >>32 + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + vcipherlast $key,$key,$rcon + stvx $stage,0,$out + addi $out,$out,16 + + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vadduwm $rcon,$rcon,$rcon + vxor $in0,$in0,$key + + vperm $key,$in0,$in0,$mask # rotate-n-splat + vsldoi $tmp,$zero,$in0,12 # >>32 + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + vcipherlast $key,$key,$rcon + stvx $stage,0,$out + addi $out,$out,16 + + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vxor $in0,$in0,$key + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + stvx $stage,0,$out + + addi $inp,$out,15 # 15 is not typo + addi $out,$out,0x50 + + li $rounds,10 + b Ldone + +.align 4 +L192: + lvx $tmp,0,$inp + li $cnt,4 + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + stvx $stage,0,$out + addi $out,$out,16 + vperm $in1,$in1,$tmp,$key # align [and byte swap in LE] + vspltisb $key,8 # borrow $key + mtctr $cnt + vsububm $mask,$mask,$key # adjust the mask + +Loop192: + vperm $key,$in1,$in1,$mask # roate-n-splat + vsldoi $tmp,$zero,$in0,12 # >>32 + vcipherlast $key,$key,$rcon + + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + + vsldoi $stage,$zero,$in1,8 + vspltw $tmp,$in0,3 + vxor $tmp,$tmp,$in1 + vsldoi $in1,$zero,$in1,12 # >>32 + vadduwm $rcon,$rcon,$rcon + vxor $in1,$in1,$tmp + vxor $in0,$in0,$key + vxor $in1,$in1,$key + vsldoi $stage,$stage,$in0,8 + + vperm $key,$in1,$in1,$mask # rotate-n-splat + vsldoi $tmp,$zero,$in0,12 # >>32 + vperm $outtail,$stage,$stage,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + vcipherlast $key,$key,$rcon + stvx $stage,0,$out + addi $out,$out,16 + + vsldoi $stage,$in0,$in1,8 + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vperm $outtail,$stage,$stage,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + stvx $stage,0,$out + addi $out,$out,16 + + vspltw $tmp,$in0,3 + vxor $tmp,$tmp,$in1 + vsldoi $in1,$zero,$in1,12 # >>32 + vadduwm $rcon,$rcon,$rcon + vxor $in1,$in1,$tmp + vxor $in0,$in0,$key + vxor $in1,$in1,$key + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + stvx $stage,0,$out + addi $inp,$out,15 # 15 is not typo + addi $out,$out,16 + bdnz Loop192 + + li $rounds,12 + addi $out,$out,0x20 + b Ldone + +.align 4 +L256: + lvx $tmp,0,$inp + li $cnt,7 + li $rounds,14 + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + stvx $stage,0,$out + addi $out,$out,16 + vperm $in1,$in1,$tmp,$key # align [and byte swap in LE] + mtctr $cnt + +Loop256: + vperm $key,$in1,$in1,$mask # rotate-n-splat + vsldoi $tmp,$zero,$in0,12 # >>32 + vperm $outtail,$in1,$in1,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + vcipherlast $key,$key,$rcon + stvx $stage,0,$out + addi $out,$out,16 + + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in0,$in0,$tmp + vadduwm $rcon,$rcon,$rcon + vxor $in0,$in0,$key + vperm $outtail,$in0,$in0,$outperm # rotate + vsel $stage,$outhead,$outtail,$outmask + vmr $outhead,$outtail + stvx $stage,0,$out + addi $inp,$out,15 # 15 is not typo + addi $out,$out,16 + bdz Ldone + + vspltw $key,$in0,3 # just splat + vsldoi $tmp,$zero,$in1,12 # >>32 + vsbox $key,$key + + vxor $in1,$in1,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in1,$in1,$tmp + vsldoi $tmp,$zero,$tmp,12 # >>32 + vxor $in1,$in1,$tmp + + vxor $in1,$in1,$key + b Loop256 + +.align 4 +Ldone: + lvx $in1,0,$inp # redundant in aligned case + vsel $in1,$outhead,$in1,$outmask + stvx $in1,0,$inp + li $ptr,0 + mtspr 256,$vrsave + stw $rounds,0($out) + +Lenc_key_abort: + mr r3,$ptr + blr + .long 0 + .byte 0,12,0x14,1,0,0,3,0 + .long 0 +.size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key + +.globl .${prefix}_set_decrypt_key +.align 5 +.${prefix}_set_decrypt_key: + $STU $sp,-$FRAME($sp) + mflr r10 + $PUSH r10,$FRAME+$LRSAVE($sp) + bl Lset_encrypt_key + mtlr r10 + + cmpwi r3,0 + bne- Ldec_key_abort + + slwi $cnt,$rounds,4 + subi $inp,$out,240 # first round key + srwi $rounds,$rounds,1 + add $out,$inp,$cnt # last round key + mtctr $rounds + +Ldeckey: + lwz r0, 0($inp) + lwz r6, 4($inp) + lwz r7, 8($inp) + lwz r8, 12($inp) + addi $inp,$inp,16 + lwz r9, 0($out) + lwz r10,4($out) + lwz r11,8($out) + lwz r12,12($out) + stw r0, 0($out) + stw r6, 4($out) + stw r7, 8($out) + stw r8, 12($out) + subi $out,$out,16 + stw r9, -16($inp) + stw r10,-12($inp) + stw r11,-8($inp) + stw r12,-4($inp) + bdnz Ldeckey + + xor r3,r3,r3 # return value +Ldec_key_abort: + addi $sp,$sp,$FRAME + blr + .long 0 + .byte 0,12,4,1,0x80,0,3,0 + .long 0 +.size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key +___ +}}} +######################################################################### +{{{ # Single block en- and decrypt procedures # +sub gen_block () { +my $dir = shift; +my $n = $dir eq "de" ? "n" : ""; +my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7)); + +$code.=<<___; +.globl .${prefix}_${dir}crypt +.align 5 +.${prefix}_${dir}crypt: + lwz $rounds,240($key) + lis r0,0xfc00 + mfspr $vrsave,256 + li $idx,15 # 15 is not typo + mtspr 256,r0 + + lvx v0,0,$inp + neg r11,$out + lvx v1,$idx,$inp + lvsl v2,0,$inp # inpperm + le?vspltisb v4,0x0f + ?lvsl v3,0,r11 # outperm + le?vxor v2,v2,v4 + li $idx,16 + vperm v0,v0,v1,v2 # align [and byte swap in LE] + lvx v1,0,$key + ?lvsl v5,0,$key # keyperm + srwi $rounds,$rounds,1 + lvx v2,$idx,$key + addi $idx,$idx,16 + subi $rounds,$rounds,1 + ?vperm v1,v1,v2,v5 # align round key + + vxor v0,v0,v1 + lvx v1,$idx,$key + addi $idx,$idx,16 + mtctr $rounds + +Loop_${dir}c: + ?vperm v2,v2,v1,v5 + v${n}cipher v0,v0,v2 + lvx v2,$idx,$key + addi $idx,$idx,16 + ?vperm v1,v1,v2,v5 + v${n}cipher v0,v0,v1 + lvx v1,$idx,$key + addi $idx,$idx,16 + bdnz Loop_${dir}c + + ?vperm v2,v2,v1,v5 + v${n}cipher v0,v0,v2 + lvx v2,$idx,$key + ?vperm v1,v1,v2,v5 + v${n}cipherlast v0,v0,v1 + + vspltisb v2,-1 + vxor v1,v1,v1 + li $idx,15 # 15 is not typo + ?vperm v2,v1,v2,v3 # outmask + le?vxor v3,v3,v4 + lvx v1,0,$out # outhead + vperm v0,v0,v0,v3 # rotate [and byte swap in LE] + vsel v1,v1,v0,v2 + lvx v4,$idx,$out + stvx v1,0,$out + vsel v0,v0,v4,v2 + stvx v0,$idx,$out + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,3,0 + .long 0 +.size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt +___ +} +&gen_block("en"); +&gen_block("de"); +}}} +######################################################################### +{{{ # CBC en- and decrypt procedures # +my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10)); +my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3)); +my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)= + map("v$_",(4..10)); +$code.=<<___; +.globl .${prefix}_cbc_encrypt +.align 5 +.${prefix}_cbc_encrypt: + ${UCMP}i $len,16 + bltlr- + + cmpwi $enc,0 # test direction + lis r0,0xffe0 + mfspr $vrsave,256 + mtspr 256,r0 + + li $idx,15 + vxor $rndkey0,$rndkey0,$rndkey0 + le?vspltisb $tmp,0x0f + + lvx $ivec,0,$ivp # load [unaligned] iv + lvsl $inpperm,0,$ivp + lvx $inptail,$idx,$ivp + le?vxor $inpperm,$inpperm,$tmp + vperm $ivec,$ivec,$inptail,$inpperm + + neg r11,$inp + ?lvsl $keyperm,0,$key # prepare for unaligned key + lwz $rounds,240($key) + + lvsr $inpperm,0,r11 # prepare for unaligned load + lvx $inptail,0,$inp + addi $inp,$inp,15 # 15 is not typo + le?vxor $inpperm,$inpperm,$tmp + + ?lvsr $outperm,0,$out # prepare for unaligned store + vspltisb $outmask,-1 + lvx $outhead,0,$out + ?vperm $outmask,$rndkey0,$outmask,$outperm + le?vxor $outperm,$outperm,$tmp + + srwi $rounds,$rounds,1 + li $idx,16 + subi $rounds,$rounds,1 + beq Lcbc_dec + +Lcbc_enc: + vmr $inout,$inptail + lvx $inptail,0,$inp + addi $inp,$inp,16 + mtctr $rounds + subi $len,$len,16 # len-=16 + + lvx $rndkey0,0,$key + vperm $inout,$inout,$inptail,$inpperm + lvx $rndkey1,$idx,$key + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key + addi $idx,$idx,16 + vxor $inout,$inout,$ivec + +Loop_cbc_enc: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key + addi $idx,$idx,16 + bdnz Loop_cbc_enc + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipherlast $ivec,$inout,$rndkey0 + ${UCMP}i $len,16 + + vperm $tmp,$ivec,$ivec,$outperm + vsel $inout,$outhead,$tmp,$outmask + vmr $outhead,$tmp + stvx $inout,0,$out + addi $out,$out,16 + bge Lcbc_enc + + b Lcbc_done + +.align 4 +Lcbc_dec: + ${UCMP}i $len,128 + bge _aesp8_cbc_decrypt8x + vmr $tmp,$inptail + lvx $inptail,0,$inp + addi $inp,$inp,16 + mtctr $rounds + subi $len,$len,16 # len-=16 + + lvx $rndkey0,0,$key + vperm $tmp,$tmp,$inptail,$inpperm + lvx $rndkey1,$idx,$key + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$tmp,$rndkey0 + lvx $rndkey0,$idx,$key + addi $idx,$idx,16 + +Loop_cbc_dec: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vncipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key + addi $idx,$idx,16 + bdnz Loop_cbc_dec + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vncipherlast $inout,$inout,$rndkey0 + ${UCMP}i $len,16 + + vxor $inout,$inout,$ivec + vmr $ivec,$tmp + vperm $tmp,$inout,$inout,$outperm + vsel $inout,$outhead,$tmp,$outmask + vmr $outhead,$tmp + stvx $inout,0,$out + addi $out,$out,16 + bge Lcbc_dec + +Lcbc_done: + addi $out,$out,-1 + lvx $inout,0,$out # redundant in aligned case + vsel $inout,$outhead,$inout,$outmask + stvx $inout,0,$out + + neg $enc,$ivp # write [unaligned] iv + li $idx,15 # 15 is not typo + vxor $rndkey0,$rndkey0,$rndkey0 + vspltisb $outmask,-1 + le?vspltisb $tmp,0x0f + ?lvsl $outperm,0,$enc + ?vperm $outmask,$rndkey0,$outmask,$outperm + le?vxor $outperm,$outperm,$tmp + lvx $outhead,0,$ivp + vperm $ivec,$ivec,$ivec,$outperm + vsel $inout,$outhead,$ivec,$outmask + lvx $inptail,$idx,$ivp + stvx $inout,0,$ivp + vsel $inout,$ivec,$inptail,$outmask + stvx $inout,$idx,$ivp + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,6,0 + .long 0 +___ +######################################################################### +{{ # Optimized CBC decrypt procedure # +my $key_="r11"; +my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31)); + $x00=0 if ($flavour =~ /osx/); +my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13)); +my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21)); +my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys + # v26-v31 last 6 round keys +my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment + +$code.=<<___; +.align 5 +_aesp8_cbc_decrypt8x: + $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) + li r10,`$FRAME+8*16+15` + li r11,`$FRAME+8*16+31` + stvx v20,r10,$sp # ABI says so + addi r10,r10,32 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + li r0,-1 + stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave + li $x10,0x10 + $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) + li $x20,0x20 + $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) + li $x30,0x30 + $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) + li $x40,0x40 + $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) + li $x50,0x50 + $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) + li $x60,0x60 + $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) + li $x70,0x70 + mtspr 256,r0 + + subi $rounds,$rounds,3 # -4 in total + subi $len,$len,128 # bias + + lvx $rndkey0,$x00,$key # load key schedule + lvx v30,$x10,$key + addi $key,$key,0x20 + lvx v31,$x00,$key + ?vperm $rndkey0,$rndkey0,v30,$keyperm + addi $key_,$sp,$FRAME+15 + mtctr $rounds + +Load_cbc_dec_key: + ?vperm v24,v30,v31,$keyperm + lvx v30,$x10,$key + addi $key,$key,0x20 + stvx v24,$x00,$key_ # off-load round[1] + ?vperm v25,v31,v30,$keyperm + lvx v31,$x00,$key + stvx v25,$x10,$key_ # off-load round[2] + addi $key_,$key_,0x20 + bdnz Load_cbc_dec_key + + lvx v26,$x10,$key + ?vperm v24,v30,v31,$keyperm + lvx v27,$x20,$key + stvx v24,$x00,$key_ # off-load round[3] + ?vperm v25,v31,v26,$keyperm + lvx v28,$x30,$key + stvx v25,$x10,$key_ # off-load round[4] + addi $key_,$sp,$FRAME+15 # rewind $key_ + ?vperm v26,v26,v27,$keyperm + lvx v29,$x40,$key + ?vperm v27,v27,v28,$keyperm + lvx v30,$x50,$key + ?vperm v28,v28,v29,$keyperm + lvx v31,$x60,$key + ?vperm v29,v29,v30,$keyperm + lvx $out0,$x70,$key # borrow $out0 + ?vperm v30,v30,v31,$keyperm + lvx v24,$x00,$key_ # pre-load round[1] + ?vperm v31,v31,$out0,$keyperm + lvx v25,$x10,$key_ # pre-load round[2] + + #lvx $inptail,0,$inp # "caller" already did this + #addi $inp,$inp,15 # 15 is not typo + subi $inp,$inp,15 # undo "caller" + + le?li $idx,8 + lvx_u $in0,$x00,$inp # load first 8 "words" + le?lvsl $inpperm,0,$idx + le?vspltisb $tmp,0x0f + lvx_u $in1,$x10,$inp + le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u + lvx_u $in2,$x20,$inp + le?vperm $in0,$in0,$in0,$inpperm + lvx_u $in3,$x30,$inp + le?vperm $in1,$in1,$in1,$inpperm + lvx_u $in4,$x40,$inp + le?vperm $in2,$in2,$in2,$inpperm + vxor $out0,$in0,$rndkey0 + lvx_u $in5,$x50,$inp + le?vperm $in3,$in3,$in3,$inpperm + vxor $out1,$in1,$rndkey0 + lvx_u $in6,$x60,$inp + le?vperm $in4,$in4,$in4,$inpperm + vxor $out2,$in2,$rndkey0 + lvx_u $in7,$x70,$inp + addi $inp,$inp,0x80 + le?vperm $in5,$in5,$in5,$inpperm + vxor $out3,$in3,$rndkey0 + le?vperm $in6,$in6,$in6,$inpperm + vxor $out4,$in4,$rndkey0 + le?vperm $in7,$in7,$in7,$inpperm + vxor $out5,$in5,$rndkey0 + vxor $out6,$in6,$rndkey0 + vxor $out7,$in7,$rndkey0 + + mtctr $rounds + b Loop_cbc_dec8x +.align 5 +Loop_cbc_dec8x: + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + vncipher $out6,$out6,v24 + vncipher $out7,$out7,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + vncipher $out6,$out6,v25 + vncipher $out7,$out7,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_cbc_dec8x + + subic $len,$len,128 # $len-=128 + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + vncipher $out6,$out6,v24 + vncipher $out7,$out7,v24 + + subfe. r0,r0,r0 # borrow?-1:0 + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + vncipher $out6,$out6,v25 + vncipher $out7,$out7,v25 + + and r0,r0,$len + vncipher $out0,$out0,v26 + vncipher $out1,$out1,v26 + vncipher $out2,$out2,v26 + vncipher $out3,$out3,v26 + vncipher $out4,$out4,v26 + vncipher $out5,$out5,v26 + vncipher $out6,$out6,v26 + vncipher $out7,$out7,v26 + + add $inp,$inp,r0 # $inp is adjusted in such + # way that at exit from the + # loop inX-in7 are loaded + # with last "words" + vncipher $out0,$out0,v27 + vncipher $out1,$out1,v27 + vncipher $out2,$out2,v27 + vncipher $out3,$out3,v27 + vncipher $out4,$out4,v27 + vncipher $out5,$out5,v27 + vncipher $out6,$out6,v27 + vncipher $out7,$out7,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vncipher $out0,$out0,v28 + vncipher $out1,$out1,v28 + vncipher $out2,$out2,v28 + vncipher $out3,$out3,v28 + vncipher $out4,$out4,v28 + vncipher $out5,$out5,v28 + vncipher $out6,$out6,v28 + vncipher $out7,$out7,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + + vncipher $out0,$out0,v29 + vncipher $out1,$out1,v29 + vncipher $out2,$out2,v29 + vncipher $out3,$out3,v29 + vncipher $out4,$out4,v29 + vncipher $out5,$out5,v29 + vncipher $out6,$out6,v29 + vncipher $out7,$out7,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + + vncipher $out0,$out0,v30 + vxor $ivec,$ivec,v31 # xor with last round key + vncipher $out1,$out1,v30 + vxor $in0,$in0,v31 + vncipher $out2,$out2,v30 + vxor $in1,$in1,v31 + vncipher $out3,$out3,v30 + vxor $in2,$in2,v31 + vncipher $out4,$out4,v30 + vxor $in3,$in3,v31 + vncipher $out5,$out5,v30 + vxor $in4,$in4,v31 + vncipher $out6,$out6,v30 + vxor $in5,$in5,v31 + vncipher $out7,$out7,v30 + vxor $in6,$in6,v31 + + vncipherlast $out0,$out0,$ivec + vncipherlast $out1,$out1,$in0 + lvx_u $in0,$x00,$inp # load next input block + vncipherlast $out2,$out2,$in1 + lvx_u $in1,$x10,$inp + vncipherlast $out3,$out3,$in2 + le?vperm $in0,$in0,$in0,$inpperm + lvx_u $in2,$x20,$inp + vncipherlast $out4,$out4,$in3 + le?vperm $in1,$in1,$in1,$inpperm + lvx_u $in3,$x30,$inp + vncipherlast $out5,$out5,$in4 + le?vperm $in2,$in2,$in2,$inpperm + lvx_u $in4,$x40,$inp + vncipherlast $out6,$out6,$in5 + le?vperm $in3,$in3,$in3,$inpperm + lvx_u $in5,$x50,$inp + vncipherlast $out7,$out7,$in6 + le?vperm $in4,$in4,$in4,$inpperm + lvx_u $in6,$x60,$inp + vmr $ivec,$in7 + le?vperm $in5,$in5,$in5,$inpperm + lvx_u $in7,$x70,$inp + addi $inp,$inp,0x80 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + le?vperm $in6,$in6,$in6,$inpperm + vxor $out0,$in0,$rndkey0 + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x10,$out + le?vperm $in7,$in7,$in7,$inpperm + vxor $out1,$in1,$rndkey0 + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x20,$out + vxor $out2,$in2,$rndkey0 + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x30,$out + vxor $out3,$in3,$rndkey0 + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x40,$out + vxor $out4,$in4,$rndkey0 + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x50,$out + vxor $out5,$in5,$rndkey0 + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x60,$out + vxor $out6,$in6,$rndkey0 + stvx_u $out7,$x70,$out + addi $out,$out,0x80 + vxor $out7,$in7,$rndkey0 + + mtctr $rounds + beq Loop_cbc_dec8x # did $len-=128 borrow? + + addic. $len,$len,128 + beq Lcbc_dec8x_done + nop + nop + +Loop_cbc_dec8x_tail: # up to 7 "words" tail... + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + vncipher $out6,$out6,v24 + vncipher $out7,$out7,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + vncipher $out6,$out6,v25 + vncipher $out7,$out7,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_cbc_dec8x_tail + + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + vncipher $out6,$out6,v24 + vncipher $out7,$out7,v24 + + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + vncipher $out6,$out6,v25 + vncipher $out7,$out7,v25 + + vncipher $out1,$out1,v26 + vncipher $out2,$out2,v26 + vncipher $out3,$out3,v26 + vncipher $out4,$out4,v26 + vncipher $out5,$out5,v26 + vncipher $out6,$out6,v26 + vncipher $out7,$out7,v26 + + vncipher $out1,$out1,v27 + vncipher $out2,$out2,v27 + vncipher $out3,$out3,v27 + vncipher $out4,$out4,v27 + vncipher $out5,$out5,v27 + vncipher $out6,$out6,v27 + vncipher $out7,$out7,v27 + + vncipher $out1,$out1,v28 + vncipher $out2,$out2,v28 + vncipher $out3,$out3,v28 + vncipher $out4,$out4,v28 + vncipher $out5,$out5,v28 + vncipher $out6,$out6,v28 + vncipher $out7,$out7,v28 + + vncipher $out1,$out1,v29 + vncipher $out2,$out2,v29 + vncipher $out3,$out3,v29 + vncipher $out4,$out4,v29 + vncipher $out5,$out5,v29 + vncipher $out6,$out6,v29 + vncipher $out7,$out7,v29 + + vncipher $out1,$out1,v30 + vxor $ivec,$ivec,v31 # last round key + vncipher $out2,$out2,v30 + vxor $in1,$in1,v31 + vncipher $out3,$out3,v30 + vxor $in2,$in2,v31 + vncipher $out4,$out4,v30 + vxor $in3,$in3,v31 + vncipher $out5,$out5,v30 + vxor $in4,$in4,v31 + vncipher $out6,$out6,v30 + vxor $in5,$in5,v31 + vncipher $out7,$out7,v30 + vxor $in6,$in6,v31 + + cmplwi $len,32 # switch($len) + blt Lcbc_dec8x_one + nop + beq Lcbc_dec8x_two + cmplwi $len,64 + blt Lcbc_dec8x_three + nop + beq Lcbc_dec8x_four + cmplwi $len,96 + blt Lcbc_dec8x_five + nop + beq Lcbc_dec8x_six + +Lcbc_dec8x_seven: + vncipherlast $out1,$out1,$ivec + vncipherlast $out2,$out2,$in1 + vncipherlast $out3,$out3,$in2 + vncipherlast $out4,$out4,$in3 + vncipherlast $out5,$out5,$in4 + vncipherlast $out6,$out6,$in5 + vncipherlast $out7,$out7,$in6 + vmr $ivec,$in7 + + le?vperm $out1,$out1,$out1,$inpperm + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x00,$out + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x10,$out + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x20,$out + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x30,$out + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x40,$out + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x50,$out + stvx_u $out7,$x60,$out + addi $out,$out,0x70 + b Lcbc_dec8x_done + +.align 5 +Lcbc_dec8x_six: + vncipherlast $out2,$out2,$ivec + vncipherlast $out3,$out3,$in2 + vncipherlast $out4,$out4,$in3 + vncipherlast $out5,$out5,$in4 + vncipherlast $out6,$out6,$in5 + vncipherlast $out7,$out7,$in6 + vmr $ivec,$in7 + + le?vperm $out2,$out2,$out2,$inpperm + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x00,$out + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x10,$out + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x20,$out + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x30,$out + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x40,$out + stvx_u $out7,$x50,$out + addi $out,$out,0x60 + b Lcbc_dec8x_done + +.align 5 +Lcbc_dec8x_five: + vncipherlast $out3,$out3,$ivec + vncipherlast $out4,$out4,$in3 + vncipherlast $out5,$out5,$in4 + vncipherlast $out6,$out6,$in5 + vncipherlast $out7,$out7,$in6 + vmr $ivec,$in7 + + le?vperm $out3,$out3,$out3,$inpperm + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x00,$out + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x10,$out + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x20,$out + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x30,$out + stvx_u $out7,$x40,$out + addi $out,$out,0x50 + b Lcbc_dec8x_done + +.align 5 +Lcbc_dec8x_four: + vncipherlast $out4,$out4,$ivec + vncipherlast $out5,$out5,$in4 + vncipherlast $out6,$out6,$in5 + vncipherlast $out7,$out7,$in6 + vmr $ivec,$in7 + + le?vperm $out4,$out4,$out4,$inpperm + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x00,$out + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x10,$out + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x20,$out + stvx_u $out7,$x30,$out + addi $out,$out,0x40 + b Lcbc_dec8x_done + +.align 5 +Lcbc_dec8x_three: + vncipherlast $out5,$out5,$ivec + vncipherlast $out6,$out6,$in5 + vncipherlast $out7,$out7,$in6 + vmr $ivec,$in7 + + le?vperm $out5,$out5,$out5,$inpperm + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x00,$out + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x10,$out + stvx_u $out7,$x20,$out + addi $out,$out,0x30 + b Lcbc_dec8x_done + +.align 5 +Lcbc_dec8x_two: + vncipherlast $out6,$out6,$ivec + vncipherlast $out7,$out7,$in6 + vmr $ivec,$in7 + + le?vperm $out6,$out6,$out6,$inpperm + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x00,$out + stvx_u $out7,$x10,$out + addi $out,$out,0x20 + b Lcbc_dec8x_done + +.align 5 +Lcbc_dec8x_one: + vncipherlast $out7,$out7,$ivec + vmr $ivec,$in7 + + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out7,0,$out + addi $out,$out,0x10 + +Lcbc_dec8x_done: + le?vperm $ivec,$ivec,$ivec,$inpperm + stvx_u $ivec,0,$ivp # write [unaligned] iv + + li r10,`$FRAME+15` + li r11,`$FRAME+31` + stvx $inpperm,r10,$sp # wipe copies of round keys + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + stvx $inpperm,r10,$sp + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + stvx $inpperm,r10,$sp + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + stvx $inpperm,r10,$sp + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + + mtspr 256,$vrsave + lvx v20,r10,$sp # ABI says so + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) + $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) + $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) + $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) + $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) + $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) + addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` + blr + .long 0 + .byte 0,12,0x04,0,0x80,6,6,0 + .long 0 +.size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt +___ +}} }}} + +######################################################################### +{{{ # CTR procedure[s] # +my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10)); +my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3)); +my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)= + map("v$_",(4..11)); +my $dat=$tmp; + +$code.=<<___; +.globl .${prefix}_ctr32_encrypt_blocks +.align 5 +.${prefix}_ctr32_encrypt_blocks: + ${UCMP}i $len,1 + bltlr- + + lis r0,0xfff0 + mfspr $vrsave,256 + mtspr 256,r0 + + li $idx,15 + vxor $rndkey0,$rndkey0,$rndkey0 + le?vspltisb $tmp,0x0f + + lvx $ivec,0,$ivp # load [unaligned] iv + lvsl $inpperm,0,$ivp + lvx $inptail,$idx,$ivp + vspltisb $one,1 + le?vxor $inpperm,$inpperm,$tmp + vperm $ivec,$ivec,$inptail,$inpperm + vsldoi $one,$rndkey0,$one,1 + + neg r11,$inp + ?lvsl $keyperm,0,$key # prepare for unaligned key + lwz $rounds,240($key) + + lvsr $inpperm,0,r11 # prepare for unaligned load + lvx $inptail,0,$inp + addi $inp,$inp,15 # 15 is not typo + le?vxor $inpperm,$inpperm,$tmp + + srwi $rounds,$rounds,1 + li $idx,16 + subi $rounds,$rounds,1 + + ${UCMP}i $len,8 + bge _aesp8_ctr32_encrypt8x + + ?lvsr $outperm,0,$out # prepare for unaligned store + vspltisb $outmask,-1 + lvx $outhead,0,$out + ?vperm $outmask,$rndkey0,$outmask,$outperm + le?vxor $outperm,$outperm,$tmp + + lvx $rndkey0,0,$key + mtctr $rounds + lvx $rndkey1,$idx,$key + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$ivec,$rndkey0 + lvx $rndkey0,$idx,$key + addi $idx,$idx,16 + b Loop_ctr32_enc + +.align 5 +Loop_ctr32_enc: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key + addi $idx,$idx,16 + bdnz Loop_ctr32_enc + + vadduwm $ivec,$ivec,$one + vmr $dat,$inptail + lvx $inptail,0,$inp + addi $inp,$inp,16 + subic. $len,$len,1 # blocks-- + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key + vperm $dat,$dat,$inptail,$inpperm + li $idx,16 + ?vperm $rndkey1,$rndkey0,$rndkey1,$keyperm + lvx $rndkey0,0,$key + vxor $dat,$dat,$rndkey1 # last round key + vcipherlast $inout,$inout,$dat + + lvx $rndkey1,$idx,$key + addi $idx,$idx,16 + vperm $inout,$inout,$inout,$outperm + vsel $dat,$outhead,$inout,$outmask + mtctr $rounds + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vmr $outhead,$inout + vxor $inout,$ivec,$rndkey0 + lvx $rndkey0,$idx,$key + addi $idx,$idx,16 + stvx $dat,0,$out + addi $out,$out,16 + bne Loop_ctr32_enc + + addi $out,$out,-1 + lvx $inout,0,$out # redundant in aligned case + vsel $inout,$outhead,$inout,$outmask + stvx $inout,0,$out + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,6,0 + .long 0 +___ +######################################################################### +{{ # Optimized CTR procedure # +my $key_="r11"; +my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31)); + $x00=0 if ($flavour =~ /osx/); +my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14)); +my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22)); +my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys + # v26-v31 last 6 round keys +my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment +my ($two,$three,$four)=($outhead,$outperm,$outmask); + +$code.=<<___; +.align 5 +_aesp8_ctr32_encrypt8x: + $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) + li r10,`$FRAME+8*16+15` + li r11,`$FRAME+8*16+31` + stvx v20,r10,$sp # ABI says so + addi r10,r10,32 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + li r0,-1 + stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave + li $x10,0x10 + $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) + li $x20,0x20 + $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) + li $x30,0x30 + $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) + li $x40,0x40 + $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) + li $x50,0x50 + $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) + li $x60,0x60 + $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) + li $x70,0x70 + mtspr 256,r0 + + subi $rounds,$rounds,3 # -4 in total + + lvx $rndkey0,$x00,$key # load key schedule + lvx v30,$x10,$key + addi $key,$key,0x20 + lvx v31,$x00,$key + ?vperm $rndkey0,$rndkey0,v30,$keyperm + addi $key_,$sp,$FRAME+15 + mtctr $rounds + +Load_ctr32_enc_key: + ?vperm v24,v30,v31,$keyperm + lvx v30,$x10,$key + addi $key,$key,0x20 + stvx v24,$x00,$key_ # off-load round[1] + ?vperm v25,v31,v30,$keyperm + lvx v31,$x00,$key + stvx v25,$x10,$key_ # off-load round[2] + addi $key_,$key_,0x20 + bdnz Load_ctr32_enc_key + + lvx v26,$x10,$key + ?vperm v24,v30,v31,$keyperm + lvx v27,$x20,$key + stvx v24,$x00,$key_ # off-load round[3] + ?vperm v25,v31,v26,$keyperm + lvx v28,$x30,$key + stvx v25,$x10,$key_ # off-load round[4] + addi $key_,$sp,$FRAME+15 # rewind $key_ + ?vperm v26,v26,v27,$keyperm + lvx v29,$x40,$key + ?vperm v27,v27,v28,$keyperm + lvx v30,$x50,$key + ?vperm v28,v28,v29,$keyperm + lvx v31,$x60,$key + ?vperm v29,v29,v30,$keyperm + lvx $out0,$x70,$key # borrow $out0 + ?vperm v30,v30,v31,$keyperm + lvx v24,$x00,$key_ # pre-load round[1] + ?vperm v31,v31,$out0,$keyperm + lvx v25,$x10,$key_ # pre-load round[2] + + vadduwm $two,$one,$one + subi $inp,$inp,15 # undo "caller" + $SHL $len,$len,4 + + vadduwm $out1,$ivec,$one # counter values ... + vadduwm $out2,$ivec,$two + vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] + le?li $idx,8 + vadduwm $out3,$out1,$two + vxor $out1,$out1,$rndkey0 + le?lvsl $inpperm,0,$idx + vadduwm $out4,$out2,$two + vxor $out2,$out2,$rndkey0 + le?vspltisb $tmp,0x0f + vadduwm $out5,$out3,$two + vxor $out3,$out3,$rndkey0 + le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u + vadduwm $out6,$out4,$two + vxor $out4,$out4,$rndkey0 + vadduwm $out7,$out5,$two + vxor $out5,$out5,$rndkey0 + vadduwm $ivec,$out6,$two # next counter value + vxor $out6,$out6,$rndkey0 + vxor $out7,$out7,$rndkey0 + + mtctr $rounds + b Loop_ctr32_enc8x +.align 5 +Loop_ctr32_enc8x: + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + vcipher $out5,$out5,v24 + vcipher $out6,$out6,v24 + vcipher $out7,$out7,v24 +Loop_ctr32_enc8x_middle: + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + vcipher $out5,$out5,v25 + vcipher $out6,$out6,v25 + vcipher $out7,$out7,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_ctr32_enc8x + + subic r11,$len,256 # $len-256, borrow $key_ + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + vcipher $out5,$out5,v24 + vcipher $out6,$out6,v24 + vcipher $out7,$out7,v24 + + subfe r0,r0,r0 # borrow?-1:0 + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + vcipher $out5,$out5,v25 + vcipher $out6,$out6,v25 + vcipher $out7,$out7,v25 + + and r0,r0,r11 + addi $key_,$sp,$FRAME+15 # rewind $key_ + vcipher $out0,$out0,v26 + vcipher $out1,$out1,v26 + vcipher $out2,$out2,v26 + vcipher $out3,$out3,v26 + vcipher $out4,$out4,v26 + vcipher $out5,$out5,v26 + vcipher $out6,$out6,v26 + vcipher $out7,$out7,v26 + lvx v24,$x00,$key_ # re-pre-load round[1] + + subic $len,$len,129 # $len-=129 + vcipher $out0,$out0,v27 + addi $len,$len,1 # $len-=128 really + vcipher $out1,$out1,v27 + vcipher $out2,$out2,v27 + vcipher $out3,$out3,v27 + vcipher $out4,$out4,v27 + vcipher $out5,$out5,v27 + vcipher $out6,$out6,v27 + vcipher $out7,$out7,v27 + lvx v25,$x10,$key_ # re-pre-load round[2] + + vcipher $out0,$out0,v28 + lvx_u $in0,$x00,$inp # load input + vcipher $out1,$out1,v28 + lvx_u $in1,$x10,$inp + vcipher $out2,$out2,v28 + lvx_u $in2,$x20,$inp + vcipher $out3,$out3,v28 + lvx_u $in3,$x30,$inp + vcipher $out4,$out4,v28 + lvx_u $in4,$x40,$inp + vcipher $out5,$out5,v28 + lvx_u $in5,$x50,$inp + vcipher $out6,$out6,v28 + lvx_u $in6,$x60,$inp + vcipher $out7,$out7,v28 + lvx_u $in7,$x70,$inp + addi $inp,$inp,0x80 + + vcipher $out0,$out0,v29 + le?vperm $in0,$in0,$in0,$inpperm + vcipher $out1,$out1,v29 + le?vperm $in1,$in1,$in1,$inpperm + vcipher $out2,$out2,v29 + le?vperm $in2,$in2,$in2,$inpperm + vcipher $out3,$out3,v29 + le?vperm $in3,$in3,$in3,$inpperm + vcipher $out4,$out4,v29 + le?vperm $in4,$in4,$in4,$inpperm + vcipher $out5,$out5,v29 + le?vperm $in5,$in5,$in5,$inpperm + vcipher $out6,$out6,v29 + le?vperm $in6,$in6,$in6,$inpperm + vcipher $out7,$out7,v29 + le?vperm $in7,$in7,$in7,$inpperm + + add $inp,$inp,r0 # $inp is adjusted in such + # way that at exit from the + # loop inX-in7 are loaded + # with last "words" + subfe. r0,r0,r0 # borrow?-1:0 + vcipher $out0,$out0,v30 + vxor $in0,$in0,v31 # xor with last round key + vcipher $out1,$out1,v30 + vxor $in1,$in1,v31 + vcipher $out2,$out2,v30 + vxor $in2,$in2,v31 + vcipher $out3,$out3,v30 + vxor $in3,$in3,v31 + vcipher $out4,$out4,v30 + vxor $in4,$in4,v31 + vcipher $out5,$out5,v30 + vxor $in5,$in5,v31 + vcipher $out6,$out6,v30 + vxor $in6,$in6,v31 + vcipher $out7,$out7,v30 + vxor $in7,$in7,v31 + + bne Lctr32_enc8x_break # did $len-129 borrow? + + vcipherlast $in0,$out0,$in0 + vcipherlast $in1,$out1,$in1 + vadduwm $out1,$ivec,$one # counter values ... + vcipherlast $in2,$out2,$in2 + vadduwm $out2,$ivec,$two + vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] + vcipherlast $in3,$out3,$in3 + vadduwm $out3,$out1,$two + vxor $out1,$out1,$rndkey0 + vcipherlast $in4,$out4,$in4 + vadduwm $out4,$out2,$two + vxor $out2,$out2,$rndkey0 + vcipherlast $in5,$out5,$in5 + vadduwm $out5,$out3,$two + vxor $out3,$out3,$rndkey0 + vcipherlast $in6,$out6,$in6 + vadduwm $out6,$out4,$two + vxor $out4,$out4,$rndkey0 + vcipherlast $in7,$out7,$in7 + vadduwm $out7,$out5,$two + vxor $out5,$out5,$rndkey0 + le?vperm $in0,$in0,$in0,$inpperm + vadduwm $ivec,$out6,$two # next counter value + vxor $out6,$out6,$rndkey0 + le?vperm $in1,$in1,$in1,$inpperm + vxor $out7,$out7,$rndkey0 + mtctr $rounds + + vcipher $out0,$out0,v24 + stvx_u $in0,$x00,$out + le?vperm $in2,$in2,$in2,$inpperm + vcipher $out1,$out1,v24 + stvx_u $in1,$x10,$out + le?vperm $in3,$in3,$in3,$inpperm + vcipher $out2,$out2,v24 + stvx_u $in2,$x20,$out + le?vperm $in4,$in4,$in4,$inpperm + vcipher $out3,$out3,v24 + stvx_u $in3,$x30,$out + le?vperm $in5,$in5,$in5,$inpperm + vcipher $out4,$out4,v24 + stvx_u $in4,$x40,$out + le?vperm $in6,$in6,$in6,$inpperm + vcipher $out5,$out5,v24 + stvx_u $in5,$x50,$out + le?vperm $in7,$in7,$in7,$inpperm + vcipher $out6,$out6,v24 + stvx_u $in6,$x60,$out + vcipher $out7,$out7,v24 + stvx_u $in7,$x70,$out + addi $out,$out,0x80 + + b Loop_ctr32_enc8x_middle + +.align 5 +Lctr32_enc8x_break: + cmpwi $len,-0x60 + blt Lctr32_enc8x_one + nop + beq Lctr32_enc8x_two + cmpwi $len,-0x40 + blt Lctr32_enc8x_three + nop + beq Lctr32_enc8x_four + cmpwi $len,-0x20 + blt Lctr32_enc8x_five + nop + beq Lctr32_enc8x_six + cmpwi $len,0x00 + blt Lctr32_enc8x_seven + +Lctr32_enc8x_eight: + vcipherlast $out0,$out0,$in0 + vcipherlast $out1,$out1,$in1 + vcipherlast $out2,$out2,$in2 + vcipherlast $out3,$out3,$in3 + vcipherlast $out4,$out4,$in4 + vcipherlast $out5,$out5,$in5 + vcipherlast $out6,$out6,$in6 + vcipherlast $out7,$out7,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x20,$out + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x30,$out + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x40,$out + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x50,$out + le?vperm $out7,$out7,$out7,$inpperm + stvx_u $out6,$x60,$out + stvx_u $out7,$x70,$out + addi $out,$out,0x80 + b Lctr32_enc8x_done + +.align 5 +Lctr32_enc8x_seven: + vcipherlast $out0,$out0,$in1 + vcipherlast $out1,$out1,$in2 + vcipherlast $out2,$out2,$in3 + vcipherlast $out3,$out3,$in4 + vcipherlast $out4,$out4,$in5 + vcipherlast $out5,$out5,$in6 + vcipherlast $out6,$out6,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x20,$out + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x30,$out + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x40,$out + le?vperm $out6,$out6,$out6,$inpperm + stvx_u $out5,$x50,$out + stvx_u $out6,$x60,$out + addi $out,$out,0x70 + b Lctr32_enc8x_done + +.align 5 +Lctr32_enc8x_six: + vcipherlast $out0,$out0,$in2 + vcipherlast $out1,$out1,$in3 + vcipherlast $out2,$out2,$in4 + vcipherlast $out3,$out3,$in5 + vcipherlast $out4,$out4,$in6 + vcipherlast $out5,$out5,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x20,$out + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x30,$out + le?vperm $out5,$out5,$out5,$inpperm + stvx_u $out4,$x40,$out + stvx_u $out5,$x50,$out + addi $out,$out,0x60 + b Lctr32_enc8x_done + +.align 5 +Lctr32_enc8x_five: + vcipherlast $out0,$out0,$in3 + vcipherlast $out1,$out1,$in4 + vcipherlast $out2,$out2,$in5 + vcipherlast $out3,$out3,$in6 + vcipherlast $out4,$out4,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x20,$out + le?vperm $out4,$out4,$out4,$inpperm + stvx_u $out3,$x30,$out + stvx_u $out4,$x40,$out + addi $out,$out,0x50 + b Lctr32_enc8x_done + +.align 5 +Lctr32_enc8x_four: + vcipherlast $out0,$out0,$in4 + vcipherlast $out1,$out1,$in5 + vcipherlast $out2,$out2,$in6 + vcipherlast $out3,$out3,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$inpperm + stvx_u $out2,$x20,$out + stvx_u $out3,$x30,$out + addi $out,$out,0x40 + b Lctr32_enc8x_done + +.align 5 +Lctr32_enc8x_three: + vcipherlast $out0,$out0,$in5 + vcipherlast $out1,$out1,$in6 + vcipherlast $out2,$out2,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + le?vperm $out2,$out2,$out2,$inpperm + stvx_u $out1,$x10,$out + stvx_u $out2,$x20,$out + addi $out,$out,0x30 + b Lcbc_dec8x_done + +.align 5 +Lctr32_enc8x_two: + vcipherlast $out0,$out0,$in6 + vcipherlast $out1,$out1,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + le?vperm $out1,$out1,$out1,$inpperm + stvx_u $out0,$x00,$out + stvx_u $out1,$x10,$out + addi $out,$out,0x20 + b Lcbc_dec8x_done + +.align 5 +Lctr32_enc8x_one: + vcipherlast $out0,$out0,$in7 + + le?vperm $out0,$out0,$out0,$inpperm + stvx_u $out0,0,$out + addi $out,$out,0x10 + +Lctr32_enc8x_done: + li r10,`$FRAME+15` + li r11,`$FRAME+31` + stvx $inpperm,r10,$sp # wipe copies of round keys + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + stvx $inpperm,r10,$sp + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + stvx $inpperm,r10,$sp + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + stvx $inpperm,r10,$sp + addi r10,r10,32 + stvx $inpperm,r11,$sp + addi r11,r11,32 + + mtspr 256,$vrsave + lvx v20,r10,$sp # ABI says so + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) + $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) + $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) + $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) + $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) + $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) + addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` + blr + .long 0 + .byte 0,12,0x04,0,0x80,6,6,0 + .long 0 +.size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks +___ +}} }}} + +my $consts=1; +foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/geo; + + # constants table endian-specific conversion + if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) { + my $conv=$3; + my @bytes=(); + + # convert to endian-agnostic format + if ($1 eq "long") { + foreach (split(/,\s*/,$2)) { + my $l = /^0/?oct:int; + push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff; + } + } else { + @bytes = map(/^0/?oct:int,split(/,\s*/,$2)); + } + + # little-endian conversion + if ($flavour =~ /le$/o) { + SWITCH: for($conv) { + /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; }; + /\?rev/ && do { @bytes=reverse(@bytes); last; }; + } + } + + #emit + print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n"; + next; + } + $consts=0 if (m/Lconsts:/o); # end of table + + # instructions prefixed with '?' are endian-specific and need + # to be adjusted accordingly... + if ($flavour =~ /le$/o) { # little-endian + s/le\?//o or + s/be\?/#be#/o or + s/\?lvsr/lvsl/o or + s/\?lvsl/lvsr/o or + s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or + s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or + s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o; + } else { # big-endian + s/le\?/#le#/o or + s/be\?//o or + s/\?([a-z]+)/$1/o; + } + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/aes/asm/aest4-sparcv9.pl b/crypto/aes/asm/aest4-sparcv9.pl new file mode 100755 index 000000000000..536f23b47c70 --- /dev/null +++ b/crypto/aes/asm/aest4-sparcv9.pl @@ -0,0 +1,919 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by David S. Miller and Andy Polyakov +# . The module is licensed under 2-clause BSD +# license. October 2012. All rights reserved. +# ==================================================================== + +###################################################################### +# AES for SPARC T4. +# +# AES round instructions complete in 3 cycles and can be issued every +# cycle. It means that round calculations should take 4*rounds cycles, +# because any given round instruction depends on result of *both* +# previous instructions: +# +# |0 |1 |2 |3 |4 +# |01|01|01| +# |23|23|23| +# |01|01|... +# |23|... +# +# Provided that fxor [with IV] takes 3 cycles to complete, critical +# path length for CBC encrypt would be 3+4*rounds, or in other words +# it should process one byte in at least (3+4*rounds)/16 cycles. This +# estimate doesn't account for "collateral" instructions, such as +# fetching input from memory, xor-ing it with zero-round key and +# storing the result. Yet, *measured* performance [for data aligned +# at 64-bit boundary!] deviates from this equation by less than 0.5%: +# +# 128-bit key 192- 256- +# CBC encrypt 2.70/2.90(*) 3.20/3.40 3.70/3.90 +# (*) numbers after slash are for +# misaligned data. +# +# Out-of-order execution logic managed to fully overlap "collateral" +# instructions with those on critical path. Amazing! +# +# As with Intel AES-NI, question is if it's possible to improve +# performance of parallelizeable modes by interleaving round +# instructions. Provided round instruction latency and throughput +# optimal interleave factor is 2. But can we expect 2x performance +# improvement? Well, as round instructions can be issued one per +# cycle, they don't saturate the 2-way issue pipeline and therefore +# there is room for "collateral" calculations... Yet, 2x speed-up +# over CBC encrypt remains unattaintable: +# +# 128-bit key 192- 256- +# CBC decrypt 1.64/2.11 1.89/2.37 2.23/2.61 +# CTR 1.64/2.08(*) 1.89/2.33 2.23/2.61 +# (*) numbers after slash are for +# misaligned data. +# +# Estimates based on amount of instructions under assumption that +# round instructions are not pairable with any other instruction +# suggest that latter is the actual case and pipeline runs +# underutilized. It should be noted that T4 out-of-order execution +# logic is so capable that performance gain from 2x interleave is +# not even impressive, ~7-13% over non-interleaved code, largest +# for 256-bit keys. + +# To anchor to something else, software implementation processes +# one byte in 29 cycles with 128-bit key on same processor. Intel +# Sandy Bridge encrypts byte in 5.07 cycles in CBC mode and decrypts +# in 0.93, naturally with AES-NI. + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +push(@INC,"${dir}","${dir}../../perlasm"); +require "sparcv9_modes.pl"; + +&asm_init(@ARGV); + +$::evp=1; # if $evp is set to 0, script generates module with +# AES_[en|de]crypt, AES_set_[en|de]crypt_key and AES_cbc_encrypt entry +# points. These however are not fully compatible with openssl/aes.h, +# because they expect AES_KEY to be aligned at 64-bit boundary. When +# used through EVP, alignment is arranged at EVP layer. Second thing +# that is arranged by EVP is at least 32-bit alignment of IV. + +###################################################################### +# single-round subroutines +# +{ +my ($inp,$out,$key,$rounds,$tmp,$mask)=map("%o$_",(0..5)); + +$code.=<<___ if ($::abibits==64); +.register %g2,#scratch +.register %g3,#scratch + +___ +$code.=<<___; +.text + +.globl aes_t4_encrypt +.align 32 +aes_t4_encrypt: + andcc $inp, 7, %g1 ! is input aligned? + andn $inp, 7, $inp + + ldx [$key + 0], %g4 + ldx [$key + 8], %g5 + + ldx [$inp + 0], %o4 + bz,pt %icc, 1f + ldx [$inp + 8], %o5 + ldx [$inp + 16], $inp + sll %g1, 3, %g1 + sub %g0, %g1, %o3 + sllx %o4, %g1, %o4 + sllx %o5, %g1, %g1 + srlx %o5, %o3, %o5 + srlx $inp, %o3, %o3 + or %o5, %o4, %o4 + or %o3, %g1, %o5 +1: + ld [$key + 240], $rounds + ldd [$key + 16], %f12 + ldd [$key + 24], %f14 + xor %g4, %o4, %o4 + xor %g5, %o5, %o5 + movxtod %o4, %f0 + movxtod %o5, %f2 + srl $rounds, 1, $rounds + ldd [$key + 32], %f16 + sub $rounds, 1, $rounds + ldd [$key + 40], %f18 + add $key, 48, $key + +.Lenc: + aes_eround01 %f12, %f0, %f2, %f4 + aes_eround23 %f14, %f0, %f2, %f2 + ldd [$key + 0], %f12 + ldd [$key + 8], %f14 + sub $rounds,1,$rounds + aes_eround01 %f16, %f4, %f2, %f0 + aes_eround23 %f18, %f4, %f2, %f2 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + brnz,pt $rounds, .Lenc + add $key, 32, $key + + andcc $out, 7, $tmp ! is output aligned? + aes_eround01 %f12, %f0, %f2, %f4 + aes_eround23 %f14, %f0, %f2, %f2 + aes_eround01_l %f16, %f4, %f2, %f0 + aes_eround23_l %f18, %f4, %f2, %f2 + + bnz,pn %icc, 2f + nop + + std %f0, [$out + 0] + retl + std %f2, [$out + 8] + +2: alignaddrl $out, %g0, $out + mov 0xff, $mask + srl $mask, $tmp, $mask + + faligndata %f0, %f0, %f4 + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + + stda %f4, [$out + $mask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $mask, $mask + retl + stda %f8, [$out + $mask]0xc0 ! partial store +.type aes_t4_encrypt,#function +.size aes_t4_encrypt,.-aes_t4_encrypt + +.globl aes_t4_decrypt +.align 32 +aes_t4_decrypt: + andcc $inp, 7, %g1 ! is input aligned? + andn $inp, 7, $inp + + ldx [$key + 0], %g4 + ldx [$key + 8], %g5 + + ldx [$inp + 0], %o4 + bz,pt %icc, 1f + ldx [$inp + 8], %o5 + ldx [$inp + 16], $inp + sll %g1, 3, %g1 + sub %g0, %g1, %o3 + sllx %o4, %g1, %o4 + sllx %o5, %g1, %g1 + srlx %o5, %o3, %o5 + srlx $inp, %o3, %o3 + or %o5, %o4, %o4 + or %o3, %g1, %o5 +1: + ld [$key + 240], $rounds + ldd [$key + 16], %f12 + ldd [$key + 24], %f14 + xor %g4, %o4, %o4 + xor %g5, %o5, %o5 + movxtod %o4, %f0 + movxtod %o5, %f2 + srl $rounds, 1, $rounds + ldd [$key + 32], %f16 + sub $rounds, 1, $rounds + ldd [$key + 40], %f18 + add $key, 48, $key + +.Ldec: + aes_dround01 %f12, %f0, %f2, %f4 + aes_dround23 %f14, %f0, %f2, %f2 + ldd [$key + 0], %f12 + ldd [$key + 8], %f14 + sub $rounds,1,$rounds + aes_dround01 %f16, %f4, %f2, %f0 + aes_dround23 %f18, %f4, %f2, %f2 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + brnz,pt $rounds, .Ldec + add $key, 32, $key + + andcc $out, 7, $tmp ! is output aligned? + aes_dround01 %f12, %f0, %f2, %f4 + aes_dround23 %f14, %f0, %f2, %f2 + aes_dround01_l %f16, %f4, %f2, %f0 + aes_dround23_l %f18, %f4, %f2, %f2 + + bnz,pn %icc, 2f + nop + + std %f0, [$out + 0] + retl + std %f2, [$out + 8] + +2: alignaddrl $out, %g0, $out + mov 0xff, $mask + srl $mask, $tmp, $mask + + faligndata %f0, %f0, %f4 + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + + stda %f4, [$out + $mask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $mask, $mask + retl + stda %f8, [$out + $mask]0xc0 ! partial store +.type aes_t4_decrypt,#function +.size aes_t4_decrypt,.-aes_t4_decrypt +___ +} + +###################################################################### +# key setup subroutines +# +{ +my ($inp,$bits,$out,$tmp)=map("%o$_",(0..5)); +$code.=<<___; +.globl aes_t4_set_encrypt_key +.align 32 +aes_t4_set_encrypt_key: +.Lset_encrypt_key: + and $inp, 7, $tmp + alignaddr $inp, %g0, $inp + cmp $bits, 192 + ldd [$inp + 0], %f0 + bl,pt %icc,.L128 + ldd [$inp + 8], %f2 + + be,pt %icc,.L192 + ldd [$inp + 16], %f4 + brz,pt $tmp, .L256aligned + ldd [$inp + 24], %f6 + + ldd [$inp + 32], %f8 + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 + faligndata %f4, %f6, %f4 + faligndata %f6, %f8, %f6 +.L256aligned: +___ +for ($i=0; $i<6; $i++) { + $code.=<<___; + std %f0, [$out + `32*$i+0`] + aes_kexpand1 %f0, %f6, $i, %f0 + std %f2, [$out + `32*$i+8`] + aes_kexpand2 %f2, %f0, %f2 + std %f4, [$out + `32*$i+16`] + aes_kexpand0 %f4, %f2, %f4 + std %f6, [$out + `32*$i+24`] + aes_kexpand2 %f6, %f4, %f6 +___ +} +$code.=<<___; + std %f0, [$out + `32*$i+0`] + aes_kexpand1 %f0, %f6, $i, %f0 + std %f2, [$out + `32*$i+8`] + aes_kexpand2 %f2, %f0, %f2 + std %f4, [$out + `32*$i+16`] + std %f6, [$out + `32*$i+24`] + std %f0, [$out + `32*$i+32`] + std %f2, [$out + `32*$i+40`] + + mov 14, $tmp + st $tmp, [$out + 240] + retl + xor %o0, %o0, %o0 + +.align 16 +.L192: + brz,pt $tmp, .L192aligned + nop + + ldd [$inp + 24], %f6 + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 + faligndata %f4, %f6, %f4 +.L192aligned: +___ +for ($i=0; $i<7; $i++) { + $code.=<<___; + std %f0, [$out + `24*$i+0`] + aes_kexpand1 %f0, %f4, $i, %f0 + std %f2, [$out + `24*$i+8`] + aes_kexpand2 %f2, %f0, %f2 + std %f4, [$out + `24*$i+16`] + aes_kexpand2 %f4, %f2, %f4 +___ +} +$code.=<<___; + std %f0, [$out + `24*$i+0`] + aes_kexpand1 %f0, %f4, $i, %f0 + std %f2, [$out + `24*$i+8`] + aes_kexpand2 %f2, %f0, %f2 + std %f4, [$out + `24*$i+16`] + std %f0, [$out + `24*$i+24`] + std %f2, [$out + `24*$i+32`] + + mov 12, $tmp + st $tmp, [$out + 240] + retl + xor %o0, %o0, %o0 + +.align 16 +.L128: + brz,pt $tmp, .L128aligned + nop + + ldd [$inp + 16], %f4 + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 +.L128aligned: +___ +for ($i=0; $i<10; $i++) { + $code.=<<___; + std %f0, [$out + `16*$i+0`] + aes_kexpand1 %f0, %f2, $i, %f0 + std %f2, [$out + `16*$i+8`] + aes_kexpand2 %f2, %f0, %f2 +___ +} +$code.=<<___; + std %f0, [$out + `16*$i+0`] + std %f2, [$out + `16*$i+8`] + + mov 10, $tmp + st $tmp, [$out + 240] + retl + xor %o0, %o0, %o0 +.type aes_t4_set_encrypt_key,#function +.size aes_t4_set_encrypt_key,.-aes_t4_set_encrypt_key + +.globl aes_t4_set_decrypt_key +.align 32 +aes_t4_set_decrypt_key: + mov %o7, %o5 + call .Lset_encrypt_key + nop + + mov %o5, %o7 + sll $tmp, 4, $inp ! $tmp is number of rounds + add $tmp, 2, $tmp + add $out, $inp, $inp ! $inp=$out+16*rounds + srl $tmp, 2, $tmp ! $tmp=(rounds+2)/4 + +.Lkey_flip: + ldd [$out + 0], %f0 + ldd [$out + 8], %f2 + ldd [$out + 16], %f4 + ldd [$out + 24], %f6 + ldd [$inp + 0], %f8 + ldd [$inp + 8], %f10 + ldd [$inp - 16], %f12 + ldd [$inp - 8], %f14 + sub $tmp, 1, $tmp + std %f0, [$inp + 0] + std %f2, [$inp + 8] + std %f4, [$inp - 16] + std %f6, [$inp - 8] + std %f8, [$out + 0] + std %f10, [$out + 8] + std %f12, [$out + 16] + std %f14, [$out + 24] + add $out, 32, $out + brnz $tmp, .Lkey_flip + sub $inp, 32, $inp + + retl + xor %o0, %o0, %o0 +.type aes_t4_set_decrypt_key,#function +.size aes_t4_set_decrypt_key,.-aes_t4_set_decrypt_key +___ +} + +{{{ +my ($inp,$out,$len,$key,$ivec,$enc)=map("%i$_",(0..5)); +my ($ileft,$iright,$ooff,$omask,$ivoff)=map("%l$_",(1..7)); + +$code.=<<___; +.align 32 +_aes128_encrypt_1x: +___ +for ($i=0; $i<4; $i++) { + $code.=<<___; + aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f4 + aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_eround01 %f`16+8*$i+4`, %f4, %f2, %f0 + aes_eround23 %f`16+8*$i+6`, %f4, %f2, %f2 +___ +} +$code.=<<___; + aes_eround01 %f48, %f0, %f2, %f4 + aes_eround23 %f50, %f0, %f2, %f2 + aes_eround01_l %f52, %f4, %f2, %f0 + retl + aes_eround23_l %f54, %f4, %f2, %f2 +.type _aes128_encrypt_1x,#function +.size _aes128_encrypt_1x,.-_aes128_encrypt_1x + +.align 32 +_aes128_encrypt_2x: +___ +for ($i=0; $i<4; $i++) { + $code.=<<___; + aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f8 + aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_eround01 %f`16+8*$i+0`, %f4, %f6, %f10 + aes_eround23 %f`16+8*$i+2`, %f4, %f6, %f6 + aes_eround01 %f`16+8*$i+4`, %f8, %f2, %f0 + aes_eround23 %f`16+8*$i+6`, %f8, %f2, %f2 + aes_eround01 %f`16+8*$i+4`, %f10, %f6, %f4 + aes_eround23 %f`16+8*$i+6`, %f10, %f6, %f6 +___ +} +$code.=<<___; + aes_eround01 %f48, %f0, %f2, %f8 + aes_eround23 %f50, %f0, %f2, %f2 + aes_eround01 %f48, %f4, %f6, %f10 + aes_eround23 %f50, %f4, %f6, %f6 + aes_eround01_l %f52, %f8, %f2, %f0 + aes_eround23_l %f54, %f8, %f2, %f2 + aes_eround01_l %f52, %f10, %f6, %f4 + retl + aes_eround23_l %f54, %f10, %f6, %f6 +.type _aes128_encrypt_2x,#function +.size _aes128_encrypt_2x,.-_aes128_encrypt_2x + +.align 32 +_aes128_loadkey: + ldx [$key + 0], %g4 + ldx [$key + 8], %g5 +___ +for ($i=2; $i<22;$i++) { # load key schedule + $code.=<<___; + ldd [$key + `8*$i`], %f`12+2*$i` +___ +} +$code.=<<___; + retl + nop +.type _aes128_loadkey,#function +.size _aes128_loadkey,.-_aes128_loadkey +_aes128_load_enckey=_aes128_loadkey +_aes128_load_deckey=_aes128_loadkey + +___ + +&alg_cbc_encrypt_implement("aes",128); +if ($::evp) { + &alg_ctr32_implement("aes",128); + &alg_xts_implement("aes",128,"en"); + &alg_xts_implement("aes",128,"de"); +} +&alg_cbc_decrypt_implement("aes",128); + +$code.=<<___; +.align 32 +_aes128_decrypt_1x: +___ +for ($i=0; $i<4; $i++) { + $code.=<<___; + aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f4 + aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_dround01 %f`16+8*$i+4`, %f4, %f2, %f0 + aes_dround23 %f`16+8*$i+6`, %f4, %f2, %f2 +___ +} +$code.=<<___; + aes_dround01 %f48, %f0, %f2, %f4 + aes_dround23 %f50, %f0, %f2, %f2 + aes_dround01_l %f52, %f4, %f2, %f0 + retl + aes_dround23_l %f54, %f4, %f2, %f2 +.type _aes128_decrypt_1x,#function +.size _aes128_decrypt_1x,.-_aes128_decrypt_1x + +.align 32 +_aes128_decrypt_2x: +___ +for ($i=0; $i<4; $i++) { + $code.=<<___; + aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f8 + aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_dround01 %f`16+8*$i+0`, %f4, %f6, %f10 + aes_dround23 %f`16+8*$i+2`, %f4, %f6, %f6 + aes_dround01 %f`16+8*$i+4`, %f8, %f2, %f0 + aes_dround23 %f`16+8*$i+6`, %f8, %f2, %f2 + aes_dround01 %f`16+8*$i+4`, %f10, %f6, %f4 + aes_dround23 %f`16+8*$i+6`, %f10, %f6, %f6 +___ +} +$code.=<<___; + aes_dround01 %f48, %f0, %f2, %f8 + aes_dround23 %f50, %f0, %f2, %f2 + aes_dround01 %f48, %f4, %f6, %f10 + aes_dround23 %f50, %f4, %f6, %f6 + aes_dround01_l %f52, %f8, %f2, %f0 + aes_dround23_l %f54, %f8, %f2, %f2 + aes_dround01_l %f52, %f10, %f6, %f4 + retl + aes_dround23_l %f54, %f10, %f6, %f6 +.type _aes128_decrypt_2x,#function +.size _aes128_decrypt_2x,.-_aes128_decrypt_2x +___ + +$code.=<<___; +.align 32 +_aes192_encrypt_1x: +___ +for ($i=0; $i<5; $i++) { + $code.=<<___; + aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f4 + aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_eround01 %f`16+8*$i+4`, %f4, %f2, %f0 + aes_eround23 %f`16+8*$i+6`, %f4, %f2, %f2 +___ +} +$code.=<<___; + aes_eround01 %f56, %f0, %f2, %f4 + aes_eround23 %f58, %f0, %f2, %f2 + aes_eround01_l %f60, %f4, %f2, %f0 + retl + aes_eround23_l %f62, %f4, %f2, %f2 +.type _aes192_encrypt_1x,#function +.size _aes192_encrypt_1x,.-_aes192_encrypt_1x + +.align 32 +_aes192_encrypt_2x: +___ +for ($i=0; $i<5; $i++) { + $code.=<<___; + aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f8 + aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_eround01 %f`16+8*$i+0`, %f4, %f6, %f10 + aes_eround23 %f`16+8*$i+2`, %f4, %f6, %f6 + aes_eround01 %f`16+8*$i+4`, %f8, %f2, %f0 + aes_eround23 %f`16+8*$i+6`, %f8, %f2, %f2 + aes_eround01 %f`16+8*$i+4`, %f10, %f6, %f4 + aes_eround23 %f`16+8*$i+6`, %f10, %f6, %f6 +___ +} +$code.=<<___; + aes_eround01 %f56, %f0, %f2, %f8 + aes_eround23 %f58, %f0, %f2, %f2 + aes_eround01 %f56, %f4, %f6, %f10 + aes_eround23 %f58, %f4, %f6, %f6 + aes_eround01_l %f60, %f8, %f2, %f0 + aes_eround23_l %f62, %f8, %f2, %f2 + aes_eround01_l %f60, %f10, %f6, %f4 + retl + aes_eround23_l %f62, %f10, %f6, %f6 +.type _aes192_encrypt_2x,#function +.size _aes192_encrypt_2x,.-_aes192_encrypt_2x + +.align 32 +_aes256_encrypt_1x: + aes_eround01 %f16, %f0, %f2, %f4 + aes_eround23 %f18, %f0, %f2, %f2 + ldd [$key + 208], %f16 + ldd [$key + 216], %f18 + aes_eround01 %f20, %f4, %f2, %f0 + aes_eround23 %f22, %f4, %f2, %f2 + ldd [$key + 224], %f20 + ldd [$key + 232], %f22 +___ +for ($i=1; $i<6; $i++) { + $code.=<<___; + aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f4 + aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_eround01 %f`16+8*$i+4`, %f4, %f2, %f0 + aes_eround23 %f`16+8*$i+6`, %f4, %f2, %f2 +___ +} +$code.=<<___; + aes_eround01 %f16, %f0, %f2, %f4 + aes_eround23 %f18, %f0, %f2, %f2 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + aes_eround01_l %f20, %f4, %f2, %f0 + aes_eround23_l %f22, %f4, %f2, %f2 + ldd [$key + 32], %f20 + retl + ldd [$key + 40], %f22 +.type _aes256_encrypt_1x,#function +.size _aes256_encrypt_1x,.-_aes256_encrypt_1x + +.align 32 +_aes256_encrypt_2x: + aes_eround01 %f16, %f0, %f2, %f8 + aes_eround23 %f18, %f0, %f2, %f2 + aes_eround01 %f16, %f4, %f6, %f10 + aes_eround23 %f18, %f4, %f6, %f6 + ldd [$key + 208], %f16 + ldd [$key + 216], %f18 + aes_eround01 %f20, %f8, %f2, %f0 + aes_eround23 %f22, %f8, %f2, %f2 + aes_eround01 %f20, %f10, %f6, %f4 + aes_eround23 %f22, %f10, %f6, %f6 + ldd [$key + 224], %f20 + ldd [$key + 232], %f22 +___ +for ($i=1; $i<6; $i++) { + $code.=<<___; + aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f8 + aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_eround01 %f`16+8*$i+0`, %f4, %f6, %f10 + aes_eround23 %f`16+8*$i+2`, %f4, %f6, %f6 + aes_eround01 %f`16+8*$i+4`, %f8, %f2, %f0 + aes_eround23 %f`16+8*$i+6`, %f8, %f2, %f2 + aes_eround01 %f`16+8*$i+4`, %f10, %f6, %f4 + aes_eround23 %f`16+8*$i+6`, %f10, %f6, %f6 +___ +} +$code.=<<___; + aes_eround01 %f16, %f0, %f2, %f8 + aes_eround23 %f18, %f0, %f2, %f2 + aes_eround01 %f16, %f4, %f6, %f10 + aes_eround23 %f18, %f4, %f6, %f6 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + aes_eround01_l %f20, %f8, %f2, %f0 + aes_eround23_l %f22, %f8, %f2, %f2 + aes_eround01_l %f20, %f10, %f6, %f4 + aes_eround23_l %f22, %f10, %f6, %f6 + ldd [$key + 32], %f20 + retl + ldd [$key + 40], %f22 +.type _aes256_encrypt_2x,#function +.size _aes256_encrypt_2x,.-_aes256_encrypt_2x + +.align 32 +_aes192_loadkey: + ldx [$key + 0], %g4 + ldx [$key + 8], %g5 +___ +for ($i=2; $i<26;$i++) { # load key schedule + $code.=<<___; + ldd [$key + `8*$i`], %f`12+2*$i` +___ +} +$code.=<<___; + retl + nop +.type _aes192_loadkey,#function +.size _aes192_loadkey,.-_aes192_loadkey +_aes256_loadkey=_aes192_loadkey +_aes192_load_enckey=_aes192_loadkey +_aes192_load_deckey=_aes192_loadkey +_aes256_load_enckey=_aes192_loadkey +_aes256_load_deckey=_aes192_loadkey +___ + +&alg_cbc_encrypt_implement("aes",256); +&alg_cbc_encrypt_implement("aes",192); +if ($::evp) { + &alg_ctr32_implement("aes",256); + &alg_xts_implement("aes",256,"en"); + &alg_xts_implement("aes",256,"de"); + &alg_ctr32_implement("aes",192); +} +&alg_cbc_decrypt_implement("aes",192); +&alg_cbc_decrypt_implement("aes",256); + +$code.=<<___; +.align 32 +_aes256_decrypt_1x: + aes_dround01 %f16, %f0, %f2, %f4 + aes_dround23 %f18, %f0, %f2, %f2 + ldd [$key + 208], %f16 + ldd [$key + 216], %f18 + aes_dround01 %f20, %f4, %f2, %f0 + aes_dround23 %f22, %f4, %f2, %f2 + ldd [$key + 224], %f20 + ldd [$key + 232], %f22 +___ +for ($i=1; $i<6; $i++) { + $code.=<<___; + aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f4 + aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_dround01 %f`16+8*$i+4`, %f4, %f2, %f0 + aes_dround23 %f`16+8*$i+6`, %f4, %f2, %f2 +___ +} +$code.=<<___; + aes_dround01 %f16, %f0, %f2, %f4 + aes_dround23 %f18, %f0, %f2, %f2 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + aes_dround01_l %f20, %f4, %f2, %f0 + aes_dround23_l %f22, %f4, %f2, %f2 + ldd [$key + 32], %f20 + retl + ldd [$key + 40], %f22 +.type _aes256_decrypt_1x,#function +.size _aes256_decrypt_1x,.-_aes256_decrypt_1x + +.align 32 +_aes256_decrypt_2x: + aes_dround01 %f16, %f0, %f2, %f8 + aes_dround23 %f18, %f0, %f2, %f2 + aes_dround01 %f16, %f4, %f6, %f10 + aes_dround23 %f18, %f4, %f6, %f6 + ldd [$key + 208], %f16 + ldd [$key + 216], %f18 + aes_dround01 %f20, %f8, %f2, %f0 + aes_dround23 %f22, %f8, %f2, %f2 + aes_dround01 %f20, %f10, %f6, %f4 + aes_dround23 %f22, %f10, %f6, %f6 + ldd [$key + 224], %f20 + ldd [$key + 232], %f22 +___ +for ($i=1; $i<6; $i++) { + $code.=<<___; + aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f8 + aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_dround01 %f`16+8*$i+0`, %f4, %f6, %f10 + aes_dround23 %f`16+8*$i+2`, %f4, %f6, %f6 + aes_dround01 %f`16+8*$i+4`, %f8, %f2, %f0 + aes_dround23 %f`16+8*$i+6`, %f8, %f2, %f2 + aes_dround01 %f`16+8*$i+4`, %f10, %f6, %f4 + aes_dround23 %f`16+8*$i+6`, %f10, %f6, %f6 +___ +} +$code.=<<___; + aes_dround01 %f16, %f0, %f2, %f8 + aes_dround23 %f18, %f0, %f2, %f2 + aes_dround01 %f16, %f4, %f6, %f10 + aes_dround23 %f18, %f4, %f6, %f6 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + aes_dround01_l %f20, %f8, %f2, %f0 + aes_dround23_l %f22, %f8, %f2, %f2 + aes_dround01_l %f20, %f10, %f6, %f4 + aes_dround23_l %f22, %f10, %f6, %f6 + ldd [$key + 32], %f20 + retl + ldd [$key + 40], %f22 +.type _aes256_decrypt_2x,#function +.size _aes256_decrypt_2x,.-_aes256_decrypt_2x + +.align 32 +_aes192_decrypt_1x: +___ +for ($i=0; $i<5; $i++) { + $code.=<<___; + aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f4 + aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_dround01 %f`16+8*$i+4`, %f4, %f2, %f0 + aes_dround23 %f`16+8*$i+6`, %f4, %f2, %f2 +___ +} +$code.=<<___; + aes_dround01 %f56, %f0, %f2, %f4 + aes_dround23 %f58, %f0, %f2, %f2 + aes_dround01_l %f60, %f4, %f2, %f0 + retl + aes_dround23_l %f62, %f4, %f2, %f2 +.type _aes192_decrypt_1x,#function +.size _aes192_decrypt_1x,.-_aes192_decrypt_1x + +.align 32 +_aes192_decrypt_2x: +___ +for ($i=0; $i<5; $i++) { + $code.=<<___; + aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f8 + aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 + aes_dround01 %f`16+8*$i+0`, %f4, %f6, %f10 + aes_dround23 %f`16+8*$i+2`, %f4, %f6, %f6 + aes_dround01 %f`16+8*$i+4`, %f8, %f2, %f0 + aes_dround23 %f`16+8*$i+6`, %f8, %f2, %f2 + aes_dround01 %f`16+8*$i+4`, %f10, %f6, %f4 + aes_dround23 %f`16+8*$i+6`, %f10, %f6, %f6 +___ +} +$code.=<<___; + aes_dround01 %f56, %f0, %f2, %f8 + aes_dround23 %f58, %f0, %f2, %f2 + aes_dround01 %f56, %f4, %f6, %f10 + aes_dround23 %f58, %f4, %f6, %f6 + aes_dround01_l %f60, %f8, %f2, %f0 + aes_dround23_l %f62, %f8, %f2, %f2 + aes_dround01_l %f60, %f10, %f6, %f4 + retl + aes_dround23_l %f62, %f10, %f6, %f6 +.type _aes192_decrypt_2x,#function +.size _aes192_decrypt_2x,.-_aes192_decrypt_2x +___ +}}} + +if (!$::evp) { +$code.=<<___; +.global AES_encrypt +AES_encrypt=aes_t4_encrypt +.global AES_decrypt +AES_decrypt=aes_t4_decrypt +.global AES_set_encrypt_key +.align 32 +AES_set_encrypt_key: + andcc %o2, 7, %g0 ! check alignment + bnz,a,pn %icc, 1f + mov -1, %o0 + brz,a,pn %o0, 1f + mov -1, %o0 + brz,a,pn %o2, 1f + mov -1, %o0 + andncc %o1, 0x1c0, %g0 + bnz,a,pn %icc, 1f + mov -2, %o0 + cmp %o1, 128 + bl,a,pn %icc, 1f + mov -2, %o0 + b aes_t4_set_encrypt_key + nop +1: retl + nop +.type AES_set_encrypt_key,#function +.size AES_set_encrypt_key,.-AES_set_encrypt_key + +.global AES_set_decrypt_key +.align 32 +AES_set_decrypt_key: + andcc %o2, 7, %g0 ! check alignment + bnz,a,pn %icc, 1f + mov -1, %o0 + brz,a,pn %o0, 1f + mov -1, %o0 + brz,a,pn %o2, 1f + mov -1, %o0 + andncc %o1, 0x1c0, %g0 + bnz,a,pn %icc, 1f + mov -2, %o0 + cmp %o1, 128 + bl,a,pn %icc, 1f + mov -2, %o0 + b aes_t4_set_decrypt_key + nop +1: retl + nop +.type AES_set_decrypt_key,#function +.size AES_set_decrypt_key,.-AES_set_decrypt_key +___ + +my ($inp,$out,$len,$key,$ivec,$enc)=map("%o$_",(0..5)); + +$code.=<<___; +.globl AES_cbc_encrypt +.align 32 +AES_cbc_encrypt: + ld [$key + 240], %g1 + nop + brz $enc, .Lcbc_decrypt + cmp %g1, 12 + + bl,pt %icc, aes128_t4_cbc_encrypt + nop + be,pn %icc, aes192_t4_cbc_encrypt + nop + ba aes256_t4_cbc_encrypt + nop + +.Lcbc_decrypt: + bl,pt %icc, aes128_t4_cbc_decrypt + nop + be,pn %icc, aes192_t4_cbc_decrypt + nop + ba aes256_t4_cbc_decrypt + nop +.type AES_cbc_encrypt,#function +.size AES_cbc_encrypt,.-AES_cbc_encrypt +___ +} +$code.=<<___; +.asciz "AES for SPARC T4, David S. Miller, Andy Polyakov" +.align 4 +___ + +&emit_assembler(); + +close STDOUT; diff --git a/crypto/aes/asm/aesv8-armx.pl b/crypto/aes/asm/aesv8-armx.pl new file mode 100755 index 000000000000..95ebae3beb9b --- /dev/null +++ b/crypto/aes/asm/aesv8-armx.pl @@ -0,0 +1,989 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements support for ARMv8 AES instructions. The +# module is endian-agnostic in sense that it supports both big- and +# little-endian cases. As does it support both 32- and 64-bit modes +# of operation. Latter is achieved by limiting amount of utilized +# registers to 16, which implies additional NEON load and integer +# instructions. This has no effect on mighty Apple A7, where results +# are literally equal to the theoretical estimates based on AES +# instruction latencies and issue rates. On Cortex-A53, an in-order +# execution core, this costs up to 10-15%, which is partially +# compensated by implementing dedicated code path for 128-bit +# CBC encrypt case. On Cortex-A57 parallelizable mode performance +# seems to be limited by sheer amount of NEON instructions... +# +# Performance in cycles per byte processed with 128-bit key: +# +# CBC enc CBC dec CTR +# Apple A7 2.39 1.20 1.20 +# Cortex-A53 1.32 1.29 1.46 +# Cortex-A57(*) 1.95 0.85 0.93 +# Denver 1.96 0.86 0.80 +# +# (*) original 3.64/1.34/1.32 results were for r0p0 revision +# and are still same even for updated module; + +$flavour = shift; +open STDOUT,">".shift; + +$prefix="aes_v8"; + +$code=<<___; +#include "arm_arch.h" + +#if __ARM_MAX_ARCH__>=7 +.text +___ +$code.=".arch armv8-a+crypto\n" if ($flavour =~ /64/); +$code.=".arch armv7-a\n.fpu neon\n.code 32\n" if ($flavour !~ /64/); + #^^^^^^ this is done to simplify adoption by not depending + # on latest binutils. + +# Assembler mnemonics are an eclectic mix of 32- and 64-bit syntax, +# NEON is mostly 32-bit mnemonics, integer - mostly 64. Goal is to +# maintain both 32- and 64-bit codes within single module and +# transliterate common code to either flavour with regex vodoo. +# +{{{ +my ($inp,$bits,$out,$ptr,$rounds)=("x0","w1","x2","x3","w12"); +my ($zero,$rcon,$mask,$in0,$in1,$tmp,$key)= + $flavour=~/64/? map("q$_",(0..6)) : map("q$_",(0..3,8..10)); + + +$code.=<<___; +.align 5 +rcon: +.long 0x01,0x01,0x01,0x01 +.long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d // rotate-n-splat +.long 0x1b,0x1b,0x1b,0x1b + +.globl ${prefix}_set_encrypt_key +.type ${prefix}_set_encrypt_key,%function +.align 5 +${prefix}_set_encrypt_key: +.Lenc_key: +___ +$code.=<<___ if ($flavour =~ /64/); + stp x29,x30,[sp,#-16]! + add x29,sp,#0 +___ +$code.=<<___; + mov $ptr,#-1 + cmp $inp,#0 + b.eq .Lenc_key_abort + cmp $out,#0 + b.eq .Lenc_key_abort + mov $ptr,#-2 + cmp $bits,#128 + b.lt .Lenc_key_abort + cmp $bits,#256 + b.gt .Lenc_key_abort + tst $bits,#0x3f + b.ne .Lenc_key_abort + + adr $ptr,rcon + cmp $bits,#192 + + veor $zero,$zero,$zero + vld1.8 {$in0},[$inp],#16 + mov $bits,#8 // reuse $bits + vld1.32 {$rcon,$mask},[$ptr],#32 + + b.lt .Loop128 + b.eq .L192 + b .L256 + +.align 4 +.Loop128: + vtbl.8 $key,{$in0},$mask + vext.8 $tmp,$zero,$in0,#12 + vst1.32 {$in0},[$out],#16 + aese $key,$zero + subs $bits,$bits,#1 + + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $key,$key,$rcon + veor $in0,$in0,$tmp + vshl.u8 $rcon,$rcon,#1 + veor $in0,$in0,$key + b.ne .Loop128 + + vld1.32 {$rcon},[$ptr] + + vtbl.8 $key,{$in0},$mask + vext.8 $tmp,$zero,$in0,#12 + vst1.32 {$in0},[$out],#16 + aese $key,$zero + + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $key,$key,$rcon + veor $in0,$in0,$tmp + vshl.u8 $rcon,$rcon,#1 + veor $in0,$in0,$key + + vtbl.8 $key,{$in0},$mask + vext.8 $tmp,$zero,$in0,#12 + vst1.32 {$in0},[$out],#16 + aese $key,$zero + + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $key,$key,$rcon + veor $in0,$in0,$tmp + veor $in0,$in0,$key + vst1.32 {$in0},[$out] + add $out,$out,#0x50 + + mov $rounds,#10 + b .Ldone + +.align 4 +.L192: + vld1.8 {$in1},[$inp],#8 + vmov.i8 $key,#8 // borrow $key + vst1.32 {$in0},[$out],#16 + vsub.i8 $mask,$mask,$key // adjust the mask + +.Loop192: + vtbl.8 $key,{$in1},$mask + vext.8 $tmp,$zero,$in0,#12 + vst1.32 {$in1},[$out],#8 + aese $key,$zero + subs $bits,$bits,#1 + + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in0,$in0,$tmp + + vdup.32 $tmp,${in0}[3] + veor $tmp,$tmp,$in1 + veor $key,$key,$rcon + vext.8 $in1,$zero,$in1,#12 + vshl.u8 $rcon,$rcon,#1 + veor $in1,$in1,$tmp + veor $in0,$in0,$key + veor $in1,$in1,$key + vst1.32 {$in0},[$out],#16 + b.ne .Loop192 + + mov $rounds,#12 + add $out,$out,#0x20 + b .Ldone + +.align 4 +.L256: + vld1.8 {$in1},[$inp] + mov $bits,#7 + mov $rounds,#14 + vst1.32 {$in0},[$out],#16 + +.Loop256: + vtbl.8 $key,{$in1},$mask + vext.8 $tmp,$zero,$in0,#12 + vst1.32 {$in1},[$out],#16 + aese $key,$zero + subs $bits,$bits,#1 + + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in0,$in0,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $key,$key,$rcon + veor $in0,$in0,$tmp + vshl.u8 $rcon,$rcon,#1 + veor $in0,$in0,$key + vst1.32 {$in0},[$out],#16 + b.eq .Ldone + + vdup.32 $key,${in0}[3] // just splat + vext.8 $tmp,$zero,$in1,#12 + aese $key,$zero + + veor $in1,$in1,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in1,$in1,$tmp + vext.8 $tmp,$zero,$tmp,#12 + veor $in1,$in1,$tmp + + veor $in1,$in1,$key + b .Loop256 + +.Ldone: + str $rounds,[$out] + mov $ptr,#0 + +.Lenc_key_abort: + mov x0,$ptr // return value + `"ldr x29,[sp],#16" if ($flavour =~ /64/)` + ret +.size ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key + +.globl ${prefix}_set_decrypt_key +.type ${prefix}_set_decrypt_key,%function +.align 5 +${prefix}_set_decrypt_key: +___ +$code.=<<___ if ($flavour =~ /64/); + stp x29,x30,[sp,#-16]! + add x29,sp,#0 +___ +$code.=<<___ if ($flavour !~ /64/); + stmdb sp!,{r4,lr} +___ +$code.=<<___; + bl .Lenc_key + + cmp x0,#0 + b.ne .Ldec_key_abort + + sub $out,$out,#240 // restore original $out + mov x4,#-16 + add $inp,$out,x12,lsl#4 // end of key schedule + + vld1.32 {v0.16b},[$out] + vld1.32 {v1.16b},[$inp] + vst1.32 {v0.16b},[$inp],x4 + vst1.32 {v1.16b},[$out],#16 + +.Loop_imc: + vld1.32 {v0.16b},[$out] + vld1.32 {v1.16b},[$inp] + aesimc v0.16b,v0.16b + aesimc v1.16b,v1.16b + vst1.32 {v0.16b},[$inp],x4 + vst1.32 {v1.16b},[$out],#16 + cmp $inp,$out + b.hi .Loop_imc + + vld1.32 {v0.16b},[$out] + aesimc v0.16b,v0.16b + vst1.32 {v0.16b},[$inp] + + eor x0,x0,x0 // return value +.Ldec_key_abort: +___ +$code.=<<___ if ($flavour !~ /64/); + ldmia sp!,{r4,pc} +___ +$code.=<<___ if ($flavour =~ /64/); + ldp x29,x30,[sp],#16 + ret +___ +$code.=<<___; +.size ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key +___ +}}} +{{{ +sub gen_block () { +my $dir = shift; +my ($e,$mc) = $dir eq "en" ? ("e","mc") : ("d","imc"); +my ($inp,$out,$key)=map("x$_",(0..2)); +my $rounds="w3"; +my ($rndkey0,$rndkey1,$inout)=map("q$_",(0..3)); + +$code.=<<___; +.globl ${prefix}_${dir}crypt +.type ${prefix}_${dir}crypt,%function +.align 5 +${prefix}_${dir}crypt: + ldr $rounds,[$key,#240] + vld1.32 {$rndkey0},[$key],#16 + vld1.8 {$inout},[$inp] + sub $rounds,$rounds,#2 + vld1.32 {$rndkey1},[$key],#16 + +.Loop_${dir}c: + aes$e $inout,$rndkey0 + aes$mc $inout,$inout + vld1.32 {$rndkey0},[$key],#16 + subs $rounds,$rounds,#2 + aes$e $inout,$rndkey1 + aes$mc $inout,$inout + vld1.32 {$rndkey1},[$key],#16 + b.gt .Loop_${dir}c + + aes$e $inout,$rndkey0 + aes$mc $inout,$inout + vld1.32 {$rndkey0},[$key] + aes$e $inout,$rndkey1 + veor $inout,$inout,$rndkey0 + + vst1.8 {$inout},[$out] + ret +.size ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt +___ +} +&gen_block("en"); +&gen_block("de"); +}}} +{{{ +my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); my $enc="w5"; +my ($rounds,$cnt,$key_,$step,$step1)=($enc,"w6","x7","x8","x12"); +my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7)); + +my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1); +my ($key4,$key5,$key6,$key7)=("x6","x12","x14",$key); + +### q8-q15 preloaded key schedule + +$code.=<<___; +.globl ${prefix}_cbc_encrypt +.type ${prefix}_cbc_encrypt,%function +.align 5 +${prefix}_cbc_encrypt: +___ +$code.=<<___ if ($flavour =~ /64/); + stp x29,x30,[sp,#-16]! + add x29,sp,#0 +___ +$code.=<<___ if ($flavour !~ /64/); + mov ip,sp + stmdb sp!,{r4-r8,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + ldmia ip,{r4-r5} @ load remaining args +___ +$code.=<<___; + subs $len,$len,#16 + mov $step,#16 + b.lo .Lcbc_abort + cclr $step,eq + + cmp $enc,#0 // en- or decrypting? + ldr $rounds,[$key,#240] + and $len,$len,#-16 + vld1.8 {$ivec},[$ivp] + vld1.8 {$dat},[$inp],$step + + vld1.32 {q8-q9},[$key] // load key schedule... + sub $rounds,$rounds,#6 + add $key_,$key,x5,lsl#4 // pointer to last 7 round keys + sub $rounds,$rounds,#2 + vld1.32 {q10-q11},[$key_],#32 + vld1.32 {q12-q13},[$key_],#32 + vld1.32 {q14-q15},[$key_],#32 + vld1.32 {$rndlast},[$key_] + + add $key_,$key,#32 + mov $cnt,$rounds + b.eq .Lcbc_dec + + cmp $rounds,#2 + veor $dat,$dat,$ivec + veor $rndzero_n_last,q8,$rndlast + b.eq .Lcbc_enc128 + + vld1.32 {$in0-$in1},[$key_] + add $key_,$key,#16 + add $key4,$key,#16*4 + add $key5,$key,#16*5 + aese $dat,q8 + aesmc $dat,$dat + add $key6,$key,#16*6 + add $key7,$key,#16*7 + b .Lenter_cbc_enc + +.align 4 +.Loop_cbc_enc: + aese $dat,q8 + aesmc $dat,$dat + vst1.8 {$ivec},[$out],#16 +.Lenter_cbc_enc: + aese $dat,q9 + aesmc $dat,$dat + aese $dat,$in0 + aesmc $dat,$dat + vld1.32 {q8},[$key4] + cmp $rounds,#4 + aese $dat,$in1 + aesmc $dat,$dat + vld1.32 {q9},[$key5] + b.eq .Lcbc_enc192 + + aese $dat,q8 + aesmc $dat,$dat + vld1.32 {q8},[$key6] + aese $dat,q9 + aesmc $dat,$dat + vld1.32 {q9},[$key7] + nop + +.Lcbc_enc192: + aese $dat,q8 + aesmc $dat,$dat + subs $len,$len,#16 + aese $dat,q9 + aesmc $dat,$dat + cclr $step,eq + aese $dat,q10 + aesmc $dat,$dat + aese $dat,q11 + aesmc $dat,$dat + vld1.8 {q8},[$inp],$step + aese $dat,q12 + aesmc $dat,$dat + veor q8,q8,$rndzero_n_last + aese $dat,q13 + aesmc $dat,$dat + vld1.32 {q9},[$key_] // re-pre-load rndkey[1] + aese $dat,q14 + aesmc $dat,$dat + aese $dat,q15 + veor $ivec,$dat,$rndlast + b.hs .Loop_cbc_enc + + vst1.8 {$ivec},[$out],#16 + b .Lcbc_done + +.align 5 +.Lcbc_enc128: + vld1.32 {$in0-$in1},[$key_] + aese $dat,q8 + aesmc $dat,$dat + b .Lenter_cbc_enc128 +.Loop_cbc_enc128: + aese $dat,q8 + aesmc $dat,$dat + vst1.8 {$ivec},[$out],#16 +.Lenter_cbc_enc128: + aese $dat,q9 + aesmc $dat,$dat + subs $len,$len,#16 + aese $dat,$in0 + aesmc $dat,$dat + cclr $step,eq + aese $dat,$in1 + aesmc $dat,$dat + aese $dat,q10 + aesmc $dat,$dat + aese $dat,q11 + aesmc $dat,$dat + vld1.8 {q8},[$inp],$step + aese $dat,q12 + aesmc $dat,$dat + aese $dat,q13 + aesmc $dat,$dat + aese $dat,q14 + aesmc $dat,$dat + veor q8,q8,$rndzero_n_last + aese $dat,q15 + veor $ivec,$dat,$rndlast + b.hs .Loop_cbc_enc128 + + vst1.8 {$ivec},[$out],#16 + b .Lcbc_done +___ +{ +my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9)); +$code.=<<___; +.align 5 +.Lcbc_dec: + vld1.8 {$dat2},[$inp],#16 + subs $len,$len,#32 // bias + add $cnt,$rounds,#2 + vorr $in1,$dat,$dat + vorr $dat1,$dat,$dat + vorr $in2,$dat2,$dat2 + b.lo .Lcbc_dec_tail + + vorr $dat1,$dat2,$dat2 + vld1.8 {$dat2},[$inp],#16 + vorr $in0,$dat,$dat + vorr $in1,$dat1,$dat1 + vorr $in2,$dat2,$dat2 + +.Loop3x_cbc_dec: + aesd $dat0,q8 + aesimc $dat0,$dat0 + aesd $dat1,q8 + aesimc $dat1,$dat1 + aesd $dat2,q8 + aesimc $dat2,$dat2 + vld1.32 {q8},[$key_],#16 + subs $cnt,$cnt,#2 + aesd $dat0,q9 + aesimc $dat0,$dat0 + aesd $dat1,q9 + aesimc $dat1,$dat1 + aesd $dat2,q9 + aesimc $dat2,$dat2 + vld1.32 {q9},[$key_],#16 + b.gt .Loop3x_cbc_dec + + aesd $dat0,q8 + aesimc $dat0,$dat0 + aesd $dat1,q8 + aesimc $dat1,$dat1 + aesd $dat2,q8 + aesimc $dat2,$dat2 + veor $tmp0,$ivec,$rndlast + subs $len,$len,#0x30 + veor $tmp1,$in0,$rndlast + mov.lo x6,$len // x6, $cnt, is zero at this point + aesd $dat0,q9 + aesimc $dat0,$dat0 + aesd $dat1,q9 + aesimc $dat1,$dat1 + aesd $dat2,q9 + aesimc $dat2,$dat2 + veor $tmp2,$in1,$rndlast + add $inp,$inp,x6 // $inp is adjusted in such way that + // at exit from the loop $dat1-$dat2 + // are loaded with last "words" + vorr $ivec,$in2,$in2 + mov $key_,$key + aesd $dat0,q12 + aesimc $dat0,$dat0 + aesd $dat1,q12 + aesimc $dat1,$dat1 + aesd $dat2,q12 + aesimc $dat2,$dat2 + vld1.8 {$in0},[$inp],#16 + aesd $dat0,q13 + aesimc $dat0,$dat0 + aesd $dat1,q13 + aesimc $dat1,$dat1 + aesd $dat2,q13 + aesimc $dat2,$dat2 + vld1.8 {$in1},[$inp],#16 + aesd $dat0,q14 + aesimc $dat0,$dat0 + aesd $dat1,q14 + aesimc $dat1,$dat1 + aesd $dat2,q14 + aesimc $dat2,$dat2 + vld1.8 {$in2},[$inp],#16 + aesd $dat0,q15 + aesd $dat1,q15 + aesd $dat2,q15 + vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0] + add $cnt,$rounds,#2 + veor $tmp0,$tmp0,$dat0 + veor $tmp1,$tmp1,$dat1 + veor $dat2,$dat2,$tmp2 + vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1] + vst1.8 {$tmp0},[$out],#16 + vorr $dat0,$in0,$in0 + vst1.8 {$tmp1},[$out],#16 + vorr $dat1,$in1,$in1 + vst1.8 {$dat2},[$out],#16 + vorr $dat2,$in2,$in2 + b.hs .Loop3x_cbc_dec + + cmn $len,#0x30 + b.eq .Lcbc_done + nop + +.Lcbc_dec_tail: + aesd $dat1,q8 + aesimc $dat1,$dat1 + aesd $dat2,q8 + aesimc $dat2,$dat2 + vld1.32 {q8},[$key_],#16 + subs $cnt,$cnt,#2 + aesd $dat1,q9 + aesimc $dat1,$dat1 + aesd $dat2,q9 + aesimc $dat2,$dat2 + vld1.32 {q9},[$key_],#16 + b.gt .Lcbc_dec_tail + + aesd $dat1,q8 + aesimc $dat1,$dat1 + aesd $dat2,q8 + aesimc $dat2,$dat2 + aesd $dat1,q9 + aesimc $dat1,$dat1 + aesd $dat2,q9 + aesimc $dat2,$dat2 + aesd $dat1,q12 + aesimc $dat1,$dat1 + aesd $dat2,q12 + aesimc $dat2,$dat2 + cmn $len,#0x20 + aesd $dat1,q13 + aesimc $dat1,$dat1 + aesd $dat2,q13 + aesimc $dat2,$dat2 + veor $tmp1,$ivec,$rndlast + aesd $dat1,q14 + aesimc $dat1,$dat1 + aesd $dat2,q14 + aesimc $dat2,$dat2 + veor $tmp2,$in1,$rndlast + aesd $dat1,q15 + aesd $dat2,q15 + b.eq .Lcbc_dec_one + veor $tmp1,$tmp1,$dat1 + veor $tmp2,$tmp2,$dat2 + vorr $ivec,$in2,$in2 + vst1.8 {$tmp1},[$out],#16 + vst1.8 {$tmp2},[$out],#16 + b .Lcbc_done + +.Lcbc_dec_one: + veor $tmp1,$tmp1,$dat2 + vorr $ivec,$in2,$in2 + vst1.8 {$tmp1},[$out],#16 + +.Lcbc_done: + vst1.8 {$ivec},[$ivp] +.Lcbc_abort: +___ +} +$code.=<<___ if ($flavour !~ /64/); + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r8,pc} +___ +$code.=<<___ if ($flavour =~ /64/); + ldr x29,[sp],#16 + ret +___ +$code.=<<___; +.size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt +___ +}}} +{{{ +my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); +my ($rounds,$cnt,$key_)=("w5","w6","x7"); +my ($ctr,$tctr0,$tctr1,$tctr2)=map("w$_",(8..10,12)); +my $step="x12"; # aliases with $tctr2 + +my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7)); +my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9)); + +my ($dat,$tmp)=($dat0,$tmp0); + +### q8-q15 preloaded key schedule + +$code.=<<___; +.globl ${prefix}_ctr32_encrypt_blocks +.type ${prefix}_ctr32_encrypt_blocks,%function +.align 5 +${prefix}_ctr32_encrypt_blocks: +___ +$code.=<<___ if ($flavour =~ /64/); + stp x29,x30,[sp,#-16]! + add x29,sp,#0 +___ +$code.=<<___ if ($flavour !~ /64/); + mov ip,sp + stmdb sp!,{r4-r10,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + ldr r4, [ip] @ load remaining arg +___ +$code.=<<___; + ldr $rounds,[$key,#240] + + ldr $ctr, [$ivp, #12] + vld1.32 {$dat0},[$ivp] + + vld1.32 {q8-q9},[$key] // load key schedule... + sub $rounds,$rounds,#4 + mov $step,#16 + cmp $len,#2 + add $key_,$key,x5,lsl#4 // pointer to last 5 round keys + sub $rounds,$rounds,#2 + vld1.32 {q12-q13},[$key_],#32 + vld1.32 {q14-q15},[$key_],#32 + vld1.32 {$rndlast},[$key_] + add $key_,$key,#32 + mov $cnt,$rounds + cclr $step,lo +#ifndef __ARMEB__ + rev $ctr, $ctr +#endif + vorr $dat1,$dat0,$dat0 + add $tctr1, $ctr, #1 + vorr $dat2,$dat0,$dat0 + add $ctr, $ctr, #2 + vorr $ivec,$dat0,$dat0 + rev $tctr1, $tctr1 + vmov.32 ${dat1}[3],$tctr1 + b.ls .Lctr32_tail + rev $tctr2, $ctr + sub $len,$len,#3 // bias + vmov.32 ${dat2}[3],$tctr2 + b .Loop3x_ctr32 + +.align 4 +.Loop3x_ctr32: + aese $dat0,q8 + aesmc $dat0,$dat0 + aese $dat1,q8 + aesmc $dat1,$dat1 + aese $dat2,q8 + aesmc $dat2,$dat2 + vld1.32 {q8},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,q9 + aesmc $dat0,$dat0 + aese $dat1,q9 + aesmc $dat1,$dat1 + aese $dat2,q9 + aesmc $dat2,$dat2 + vld1.32 {q9},[$key_],#16 + b.gt .Loop3x_ctr32 + + aese $dat0,q8 + aesmc $tmp0,$dat0 + aese $dat1,q8 + aesmc $tmp1,$dat1 + vld1.8 {$in0},[$inp],#16 + vorr $dat0,$ivec,$ivec + aese $dat2,q8 + aesmc $dat2,$dat2 + vld1.8 {$in1},[$inp],#16 + vorr $dat1,$ivec,$ivec + aese $tmp0,q9 + aesmc $tmp0,$tmp0 + aese $tmp1,q9 + aesmc $tmp1,$tmp1 + vld1.8 {$in2},[$inp],#16 + mov $key_,$key + aese $dat2,q9 + aesmc $tmp2,$dat2 + vorr $dat2,$ivec,$ivec + add $tctr0,$ctr,#1 + aese $tmp0,q12 + aesmc $tmp0,$tmp0 + aese $tmp1,q12 + aesmc $tmp1,$tmp1 + veor $in0,$in0,$rndlast + add $tctr1,$ctr,#2 + aese $tmp2,q12 + aesmc $tmp2,$tmp2 + veor $in1,$in1,$rndlast + add $ctr,$ctr,#3 + aese $tmp0,q13 + aesmc $tmp0,$tmp0 + aese $tmp1,q13 + aesmc $tmp1,$tmp1 + veor $in2,$in2,$rndlast + rev $tctr0,$tctr0 + aese $tmp2,q13 + aesmc $tmp2,$tmp2 + vmov.32 ${dat0}[3], $tctr0 + rev $tctr1,$tctr1 + aese $tmp0,q14 + aesmc $tmp0,$tmp0 + aese $tmp1,q14 + aesmc $tmp1,$tmp1 + vmov.32 ${dat1}[3], $tctr1 + rev $tctr2,$ctr + aese $tmp2,q14 + aesmc $tmp2,$tmp2 + vmov.32 ${dat2}[3], $tctr2 + subs $len,$len,#3 + aese $tmp0,q15 + aese $tmp1,q15 + aese $tmp2,q15 + + veor $in0,$in0,$tmp0 + vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0] + vst1.8 {$in0},[$out],#16 + veor $in1,$in1,$tmp1 + mov $cnt,$rounds + vst1.8 {$in1},[$out],#16 + veor $in2,$in2,$tmp2 + vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1] + vst1.8 {$in2},[$out],#16 + b.hs .Loop3x_ctr32 + + adds $len,$len,#3 + b.eq .Lctr32_done + cmp $len,#1 + mov $step,#16 + cclr $step,eq + +.Lctr32_tail: + aese $dat0,q8 + aesmc $dat0,$dat0 + aese $dat1,q8 + aesmc $dat1,$dat1 + vld1.32 {q8},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,q9 + aesmc $dat0,$dat0 + aese $dat1,q9 + aesmc $dat1,$dat1 + vld1.32 {q9},[$key_],#16 + b.gt .Lctr32_tail + + aese $dat0,q8 + aesmc $dat0,$dat0 + aese $dat1,q8 + aesmc $dat1,$dat1 + aese $dat0,q9 + aesmc $dat0,$dat0 + aese $dat1,q9 + aesmc $dat1,$dat1 + vld1.8 {$in0},[$inp],$step + aese $dat0,q12 + aesmc $dat0,$dat0 + aese $dat1,q12 + aesmc $dat1,$dat1 + vld1.8 {$in1},[$inp] + aese $dat0,q13 + aesmc $dat0,$dat0 + aese $dat1,q13 + aesmc $dat1,$dat1 + veor $in0,$in0,$rndlast + aese $dat0,q14 + aesmc $dat0,$dat0 + aese $dat1,q14 + aesmc $dat1,$dat1 + veor $in1,$in1,$rndlast + aese $dat0,q15 + aese $dat1,q15 + + cmp $len,#1 + veor $in0,$in0,$dat0 + veor $in1,$in1,$dat1 + vst1.8 {$in0},[$out],#16 + b.eq .Lctr32_done + vst1.8 {$in1},[$out] + +.Lctr32_done: +___ +$code.=<<___ if ($flavour !~ /64/); + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r10,pc} +___ +$code.=<<___ if ($flavour =~ /64/); + ldr x29,[sp],#16 + ret +___ +$code.=<<___; +.size ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks +___ +}}} +$code.=<<___; +#endif +___ +######################################## +if ($flavour =~ /64/) { ######## 64-bit code + my %opcode = ( + "aesd" => 0x4e285800, "aese" => 0x4e284800, + "aesimc"=> 0x4e287800, "aesmc" => 0x4e286800 ); + + local *unaes = sub { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5), + $mnemonic,$arg; + }; + + foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/geo; + + s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo; # old->new registers + s/@\s/\/\//o; # old->new style commentary + + #s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or + s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel $1$2,$1zr,$1$2,$3/o or + s/mov\.([a-z]+)\s+([wx][0-9]+),\s*([wx][0-9]+)/csel $2,$3,$2,$1/o or + s/vmov\.i8/movi/o or # fix up legacy mnemonics + s/vext\.8/ext/o or + s/vrev32\.8/rev32/o or + s/vtst\.8/cmtst/o or + s/vshr/ushr/o or + s/^(\s+)v/$1/o or # strip off v prefix + s/\bbx\s+lr\b/ret/o; + + # fix up remainig legacy suffixes + s/\.[ui]?8//o; + m/\],#8/o and s/\.16b/\.8b/go; + s/\.[ui]?32//o and s/\.16b/\.4s/go; + s/\.[ui]?64//o and s/\.16b/\.2d/go; + s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o; + + print $_,"\n"; + } +} else { ######## 32-bit code + my %opcode = ( + "aesd" => 0xf3b00340, "aese" => 0xf3b00300, + "aesimc"=> 0xf3b003c0, "aesmc" => 0xf3b00380 ); + + local *unaes = sub { + my ($mnemonic,$arg)=@_; + + if ($arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o) { + my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19) + |(($2&7)<<1) |(($2&8)<<2); + # since ARMv7 instructions are always encoded little-endian. + # correct solution is to use .inst directive, but older + # assemblers don't implement it:-( + sprintf ".byte\t0x%02x,0x%02x,0x%02x,0x%02x\t@ %s %s", + $word&0xff,($word>>8)&0xff, + ($word>>16)&0xff,($word>>24)&0xff, + $mnemonic,$arg; + } + }; + + sub unvtbl { + my $arg=shift; + + $arg =~ m/q([0-9]+),\s*\{q([0-9]+)\},\s*q([0-9]+)/o && + sprintf "vtbl.8 d%d,{q%d},d%d\n\t". + "vtbl.8 d%d,{q%d},d%d", 2*$1,$2,2*$3, 2*$1+1,$2,2*$3+1; + } + + sub unvdup32 { + my $arg=shift; + + $arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o && + sprintf "vdup.32 q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1; + } + + sub unvmov32 { + my $arg=shift; + + $arg =~ m/q([0-9]+)\[([0-3])\],(.*)/o && + sprintf "vmov.32 d%d[%d],%s",2*$1+($2>>1),$2&1,$3; + } + + foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/geo; + + s/\b[wx]([0-9]+)\b/r$1/go; # new->old registers + s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go; # new->old registers + s/\/\/\s?/@ /o; # new->old style commentary + + # fix up remainig new-style suffixes + s/\{q([0-9]+)\},\s*\[(.+)\],#8/sprintf "{d%d},[$2]!",2*$1/eo or + s/\],#[0-9]+/]!/o; + + s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or + s/cclr\s+([^,]+),\s*([a-z]+)/mov$2 $1,#0/o or + s/vtbl\.8\s+(.*)/unvtbl($1)/geo or + s/vdup\.32\s+(.*)/unvdup32($1)/geo or + s/vmov\.32\s+(.*)/unvmov32($1)/geo or + s/^(\s+)b\./$1b/o or + s/^(\s+)mov\./$1mov/o or + s/^(\s+)ret/$1bx\tlr/o; + + print $_,"\n"; + } +} + +close STDOUT; diff --git a/crypto/aes/asm/bsaes-armv7.pl b/crypto/aes/asm/bsaes-armv7.pl new file mode 100755 index 000000000000..fcc81d1a4933 --- /dev/null +++ b/crypto/aes/asm/bsaes-armv7.pl @@ -0,0 +1,2469 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# +# Specific modes and adaptation for Linux kernel by Ard Biesheuvel +# . Permission to use under GPL terms is +# granted. +# ==================================================================== + +# Bit-sliced AES for ARM NEON +# +# February 2012. +# +# This implementation is direct adaptation of bsaes-x86_64 module for +# ARM NEON. Except that this module is endian-neutral [in sense that +# it can be compiled for either endianness] by courtesy of vld1.8's +# neutrality. Initial version doesn't implement interface to OpenSSL, +# only low-level primitives and unsupported entry points, just enough +# to collect performance results, which for Cortex-A8 core are: +# +# encrypt 19.5 cycles per byte processed with 128-bit key +# decrypt 22.1 cycles per byte processed with 128-bit key +# key conv. 440 cycles per 128-bit key/0.18 of 8x block +# +# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, +# which is [much] worse than anticipated (for further details see +# http://www.openssl.org/~appro/Snapdragon-S4.html). +# +# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code +# manages in 20.0 cycles]. +# +# When comparing to x86_64 results keep in mind that NEON unit is +# [mostly] single-issue and thus can't [fully] benefit from +# instruction-level parallelism. And when comparing to aes-armv4 +# results keep in mind key schedule conversion overhead (see +# bsaes-x86_64.pl for further details)... +# +# + +# April-August 2013 +# +# Add CBC, CTR and XTS subroutines, adapt for kernel use. +# +# + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +my ($inp,$out,$len,$key)=("r0","r1","r2","r3"); +my @XMM=map("q$_",(0..15)); + +{ +my ($key,$rounds,$const)=("r4","r5","r6"); + +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } + +sub Sbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InBasisChange (@b); + &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s); + &OutBasisChange (@b[7,1,4,2,6,5,0,3]); +} + +sub InBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[2], @b[2], @b[1] + veor @b[5], @b[5], @b[6] + veor @b[3], @b[3], @b[0] + veor @b[6], @b[6], @b[2] + veor @b[5], @b[5], @b[0] + + veor @b[6], @b[6], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[4], @b[4], @b[5] + + veor @b[2], @b[2], @b[7] + veor @b[3], @b[3], @b[1] + veor @b[1], @b[1], @b[5] +___ +} + +sub OutBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] + veor @b[4], @b[4], @b[6] + veor @b[2], @b[2], @b[0] + veor @b[6], @b[6], @b[1] + + veor @b[1], @b[1], @b[5] + veor @b[5], @b[5], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[2], @b[2], @b[5] + + veor @b[4], @b[4], @b[7] +___ +} + +sub InvSbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InvInBasisChange (@b); + &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s); + &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]); +} + +sub InvInBasisChange { # OutBasisChange in reverse (with twist) +my @b=@_[5,1,2,6,3,7,0,4]; +$code.=<<___ + veor @b[1], @b[1], @b[7] + veor @b[4], @b[4], @b[7] + + veor @b[7], @b[7], @b[5] + veor @b[1], @b[1], @b[3] + veor @b[2], @b[2], @b[5] + veor @b[3], @b[3], @b[7] + + veor @b[6], @b[6], @b[1] + veor @b[2], @b[2], @b[0] + veor @b[5], @b[5], @b[3] + veor @b[4], @b[4], @b[6] + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] +___ +} + +sub InvOutBasisChange { # InBasisChange in reverse +my @b=@_[2,5,7,3,6,1,0,4]; +$code.=<<___; + veor @b[1], @b[1], @b[5] + veor @b[2], @b[2], @b[7] + + veor @b[3], @b[3], @b[1] + veor @b[4], @b[4], @b[5] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[5], @b[5], @b[0] + veor @b[3], @b[3], @b[7] + veor @b[6], @b[6], @b[2] + veor @b[2], @b[2], @b[1] + veor @b[6], @b[6], @b[3] + + veor @b[3], @b[3], @b[0] + veor @b[5], @b[5], @b[6] +___ +} + +sub Mul_GF4 { +#;************************************************************* +#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) * +#;************************************************************* +my ($x0,$x1,$y0,$y1,$t0,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $t1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $t1, $t0 + veor $x0, $x0, $t1 +___ +} + +sub Mul_GF4_N { # not used, see next subroutine +# multiply and scale by N +my ($x0,$x1,$y0,$y1,$t0)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $x1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $x1, $x0 + veor $x0, $x0, $t0 +___ +} + +sub Mul_GF4_N_GF4 { +# interleaved Mul_GF4_N and Mul_GF4 +my ($x0,$x1,$y0,$y1,$t0, + $x2,$x3,$y2,$y3,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + veor $t1, $y2, $y3 + vand $t0, $t0, $x0 + vand $t1, $t1, $x2 + veor $x0, $x0, $x1 + veor $x2, $x2, $x3 + vand $x1, $x1, $y0 + vand $x3, $x3, $y2 + vand $x0, $x0, $y1 + vand $x2, $x2, $y3 + veor $x1, $x1, $x0 + veor $x2, $x2, $x3 + veor $x0, $x0, $t0 + veor $x3, $x3, $t1 +___ +} +sub Mul_GF16_2 { +my @x=@_[0..7]; +my @y=@_[8..11]; +my @t=@_[12..15]; +$code.=<<___; + veor @t[0], @x[0], @x[2] + veor @t[1], @x[1], @x[3] +___ + &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[2], @x[3], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @x[0], @x[0], @t[0] + veor @x[2], @x[2], @t[0] + veor @x[1], @x[1], @t[1] + veor @x[3], @x[3], @t[1] + + veor @t[0], @x[4], @x[6] + veor @t[1], @x[5], @x[7] +___ + &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[6], @x[7], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @x[4], @x[4], @t[0] + veor @x[6], @x[6], @t[0] + veor @x[5], @x[5], @t[1] + veor @x[7], @x[7], @t[1] +___ +} +sub Inv_GF256 { +#;******************************************************************** +#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) * +#;******************************************************************** +my @x=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; +# direct optimizations from hardware +$code.=<<___; + veor @t[3], @x[4], @x[6] + veor @t[2], @x[5], @x[7] + veor @t[1], @x[1], @x[3] + veor @s[1], @x[7], @x[6] + vmov @t[0], @t[2] + veor @s[0], @x[0], @x[2] + + vorr @t[2], @t[2], @t[1] + veor @s[3], @t[3], @t[0] + vand @s[2], @t[3], @s[0] + vorr @t[3], @t[3], @s[0] + veor @s[0], @s[0], @t[1] + vand @t[0], @t[0], @t[1] + veor @t[1], @x[3], @x[2] + vand @s[3], @s[3], @s[0] + vand @s[1], @s[1], @t[1] + veor @t[1], @x[4], @x[5] + veor @s[0], @x[1], @x[0] + veor @t[3], @t[3], @s[1] + veor @t[2], @t[2], @s[1] + vand @s[1], @t[1], @s[0] + vorr @t[1], @t[1], @s[0] + veor @t[3], @t[3], @s[3] + veor @t[0], @t[0], @s[1] + veor @t[2], @t[2], @s[2] + veor @t[1], @t[1], @s[3] + veor @t[0], @t[0], @s[2] + vand @s[0], @x[7], @x[3] + veor @t[1], @t[1], @s[2] + vand @s[1], @x[6], @x[2] + vand @s[2], @x[5], @x[1] + vorr @s[3], @x[4], @x[0] + veor @t[3], @t[3], @s[0] + veor @t[1], @t[1], @s[2] + veor @t[0], @t[0], @s[3] + veor @t[2], @t[2], @s[1] + + @ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3 + + @ new smaller inversion + + vand @s[2], @t[3], @t[1] + vmov @s[0], @t[0] + + veor @s[1], @t[2], @s[2] + veor @s[3], @t[0], @s[2] + veor @s[2], @t[0], @s[2] @ @s[2]=@s[3] + + vbsl @s[1], @t[1], @t[0] + vbsl @s[3], @t[3], @t[2] + veor @t[3], @t[3], @t[2] + + vbsl @s[0], @s[1], @s[2] + vbsl @t[0], @s[2], @s[1] + + vand @s[2], @s[0], @s[3] + veor @t[1], @t[1], @t[0] + + veor @s[2], @s[2], @t[3] +___ +# output in s3, s2, s1, t1 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3 + &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]); + +### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb +} + +# AES linear components + +sub ShiftRows { +my @x=@_[0..7]; +my @t=@_[8..11]; +my $mask=pop; +$code.=<<___; + vldmia $key!, {@t[0]-@t[3]} + veor @t[0], @t[0], @x[0] + veor @t[1], @t[1], @x[1] + vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)` + vldmia $key!, {@t[0]} + veor @t[2], @t[2], @x[2] + vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)` + vldmia $key!, {@t[1]} + veor @t[3], @t[3], @x[3] + vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)` + vldmia $key!, {@t[2]} + vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)` + vldmia $key!, {@t[3]} + veor @t[0], @t[0], @x[4] + veor @t[1], @t[1], @x[5] + vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)` + veor @t[2], @t[2], @x[6] + vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)` + veor @t[3], @t[3], @x[7] + vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)` + vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)` +___ +} + +sub MixColumns { +# modified to emit output in order suitable for feeding back to aesenc[last] +my @x=@_[0..7]; +my @t=@_[8..15]; +my $inv=@_[16]; # optional +$code.=<<___; + vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32) + vext.8 @t[2], @x[2], @x[2], #12 + veor @x[1], @x[1], @t[1] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[2], @x[2], @t[2] + vext.8 @t[4], @x[4], @x[4], #12 + veor @x[3], @x[3], @t[3] + vext.8 @t[5], @x[5], @x[5], #12 + veor @x[4], @x[4], @t[4] + vext.8 @t[6], @x[6], @x[6], #12 + veor @x[5], @x[5], @t[5] + vext.8 @t[7], @x[7], @x[7], #12 + veor @x[6], @x[6], @t[6] + + veor @t[1], @t[1], @x[0] + veor @x[7], @x[7], @t[7] + vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor @t[2], @t[2], @x[1] + veor @t[0], @t[0], @x[7] + veor @t[1], @t[1], @x[7] + vext.8 @x[1], @x[1], @x[1], #8 + veor @t[5], @t[5], @x[4] + veor @x[0], @x[0], @t[0] + veor @t[6], @t[6], @x[5] + veor @x[1], @x[1], @t[1] + vext.8 @t[0], @x[4], @x[4], #8 + veor @t[4], @t[4], @x[3] + vext.8 @t[1], @x[5], @x[5], #8 + veor @t[7], @t[7], @x[6] + vext.8 @x[4], @x[3], @x[3], #8 + veor @t[3], @t[3], @x[2] + vext.8 @x[5], @x[7], @x[7], #8 + veor @t[4], @t[4], @x[7] + vext.8 @x[3], @x[6], @x[6], #8 + veor @t[3], @t[3], @x[7] + vext.8 @x[6], @x[2], @x[2], #8 + veor @x[7], @t[1], @t[5] +___ +$code.=<<___ if (!$inv); + veor @x[2], @t[0], @t[4] + veor @x[4], @x[4], @t[3] + veor @x[5], @x[5], @t[7] + veor @x[3], @x[3], @t[6] + @ vmov @x[2], @t[0] + veor @x[6], @x[6], @t[2] + @ vmov @x[7], @t[1] +___ +$code.=<<___ if ($inv); + veor @t[3], @t[3], @x[4] + veor @x[5], @x[5], @t[7] + veor @x[2], @x[3], @t[6] + veor @x[3], @t[0], @t[4] + veor @x[4], @x[6], @t[2] + vmov @x[6], @t[3] + @ vmov @x[7], @t[1] +___ +} + +sub InvMixColumns_orig { +my @x=@_[0..7]; +my @t=@_[8..15]; + +$code.=<<___; + @ multiplication by 0x0e + vext.8 @t[7], @x[7], @x[7], #12 + vmov @t[2], @x[2] + veor @x[2], @x[2], @x[5] @ 2 5 + veor @x[7], @x[7], @x[5] @ 7 5 + vext.8 @t[0], @x[0], @x[0], #12 + vmov @t[5], @x[5] + veor @x[5], @x[5], @x[0] @ 5 0 [1] + veor @x[0], @x[0], @x[1] @ 0 1 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[1], @x[1], @x[2] @ 1 25 + veor @x[0], @x[0], @x[6] @ 01 6 [2] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[1], @x[1], @x[3] @ 125 3 [4] + veor @x[2], @x[2], @x[0] @ 25 016 [3] + veor @x[3], @x[3], @x[7] @ 3 75 + veor @x[7], @x[7], @x[6] @ 75 6 [0] + vext.8 @t[6], @x[6], @x[6], #12 + vmov @t[4], @x[4] + veor @x[6], @x[6], @x[4] @ 6 4 + veor @x[4], @x[4], @x[3] @ 4 375 [6] + veor @x[3], @x[3], @x[7] @ 375 756=36 + veor @x[6], @x[6], @t[5] @ 64 5 [7] + veor @x[3], @x[3], @t[2] @ 36 2 + vext.8 @t[5], @t[5], @t[5], #12 + veor @x[3], @x[3], @t[4] @ 362 4 [5] +___ + my @y = @x[7,5,0,2,1,3,4,6]; +$code.=<<___; + @ multiplication by 0x0b + veor @y[1], @y[1], @y[0] + veor @y[0], @y[0], @t[0] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[5] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[1], @y[1], @t[6] + veor @y[0], @y[0], @t[7] + veor @t[7], @t[7], @t[6] @ clobber t[7] + + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @y[0] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[2], @y[2], @t[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + veor @y[2], @y[2], @t[2] + veor @y[3], @y[3], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[7] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[3], @y[3], @t[3] + veor @y[6], @y[6], @t[3] + veor @y[4], @y[4], @t[3] + veor @y[7], @y[7], @t[4] + vext.8 @t[3], @t[3], @t[3], #12 + veor @y[5], @y[5], @t[4] + veor @y[7], @y[7], @t[7] + veor @t[7], @t[7], @t[5] @ clobber t[7] even more + veor @y[3], @y[3], @t[5] + veor @y[4], @y[4], @t[4] + + veor @y[5], @y[5], @t[7] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[6], @y[6], @t[7] + veor @y[4], @y[4], @t[7] + + veor @t[7], @t[7], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + + @ multiplication by 0x0d + veor @y[4], @y[4], @y[7] + veor @t[7], @t[7], @t[6] @ restore t[7] + veor @y[7], @y[7], @t[4] + vext.8 @t[6], @t[6], @t[6], #12 + veor @y[2], @y[2], @t[0] + veor @y[7], @y[7], @t[5] + vext.8 @t[7], @t[7], @t[7], #12 + veor @y[2], @y[2], @t[2] + + veor @y[3], @y[3], @y[1] + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[0] + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @t[5] + veor @y[0], @y[0], @t[5] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[1], @y[1], @t[7] + veor @y[0], @y[0], @t[6] + veor @y[3], @y[3], @y[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + + veor @y[7], @y[7], @t[7] + veor @y[4], @y[4], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[6] + veor @t[6], @t[6], @t[3] @ clobber t[6] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[4], @y[4], @y[7] + veor @y[3], @y[3], @t[6] + + veor @y[6], @y[6], @t[6] + veor @y[5], @y[5], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + veor @y[6], @y[6], @t[4] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[5], @y[5], @t[6] + veor @y[6], @y[6], @t[7] + vext.8 @t[7], @t[7], @t[7], #12 + veor @t[6], @t[6], @t[3] @ restore t[6] + vext.8 @t[3], @t[3], @t[3], #12 + + @ multiplication by 0x09 + veor @y[4], @y[4], @y[1] + veor @t[1], @t[1], @y[1] @ t[1]=y[1] + veor @t[0], @t[0], @t[5] @ clobber t[0] + vext.8 @t[6], @t[6], @t[6], #12 + veor @t[1], @t[1], @t[5] + veor @y[3], @y[3], @t[0] + veor @t[0], @t[0], @y[0] @ t[0]=y[0] + veor @t[1], @t[1], @t[6] + veor @t[6], @t[6], @t[7] @ clobber t[6] + veor @y[4], @y[4], @t[1] + veor @y[7], @y[7], @t[4] + veor @y[6], @y[6], @t[3] + veor @y[5], @y[5], @t[2] + veor @t[4], @t[4], @y[4] @ t[4]=y[4] + veor @t[3], @t[3], @y[3] @ t[3]=y[3] + veor @t[5], @t[5], @y[5] @ t[5]=y[5] + veor @t[2], @t[2], @y[2] @ t[2]=y[2] + veor @t[3], @t[3], @t[7] + veor @XMM[5], @t[5], @t[6] + veor @XMM[6], @t[6], @y[6] @ t[6]=y[6] + veor @XMM[2], @t[2], @t[6] + veor @XMM[7], @t[7], @y[7] @ t[7]=y[7] + + vmov @XMM[0], @t[0] + vmov @XMM[1], @t[1] + @ vmov @XMM[2], @t[2] + vmov @XMM[3], @t[3] + vmov @XMM[4], @t[4] + @ vmov @XMM[5], @t[5] + @ vmov @XMM[6], @t[6] + @ vmov @XMM[7], @t[7] +___ +} + +sub InvMixColumns { +my @x=@_[0..7]; +my @t=@_[8..15]; + +# Thanks to Jussi Kivilinna for providing pointer to +# +# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 | +# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 | +# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 | +# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 | + +$code.=<<___; + @ multiplication by 0x05-0x00-0x04-0x00 + vext.8 @t[0], @x[0], @x[0], #8 + vext.8 @t[6], @x[6], @x[6], #8 + vext.8 @t[7], @x[7], @x[7], #8 + veor @t[0], @t[0], @x[0] + vext.8 @t[1], @x[1], @x[1], #8 + veor @t[6], @t[6], @x[6] + vext.8 @t[2], @x[2], @x[2], #8 + veor @t[7], @t[7], @x[7] + vext.8 @t[3], @x[3], @x[3], #8 + veor @t[1], @t[1], @x[1] + vext.8 @t[4], @x[4], @x[4], #8 + veor @t[2], @t[2], @x[2] + vext.8 @t[5], @x[5], @x[5], #8 + veor @t[3], @t[3], @x[3] + veor @t[4], @t[4], @x[4] + veor @t[5], @t[5], @x[5] + + veor @x[0], @x[0], @t[6] + veor @x[1], @x[1], @t[6] + veor @x[2], @x[2], @t[0] + veor @x[4], @x[4], @t[2] + veor @x[3], @x[3], @t[1] + veor @x[1], @x[1], @t[7] + veor @x[2], @x[2], @t[7] + veor @x[4], @x[4], @t[6] + veor @x[5], @x[5], @t[3] + veor @x[3], @x[3], @t[6] + veor @x[6], @x[6], @t[4] + veor @x[4], @x[4], @t[7] + veor @x[5], @x[5], @t[7] + veor @x[7], @x[7], @t[5] +___ + &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6 +} + +sub swapmove { +my ($a,$b,$n,$mask,$t)=@_; +$code.=<<___; + vshr.u64 $t, $b, #$n + veor $t, $t, $a + vand $t, $t, $mask + veor $a, $a, $t + vshl.u64 $t, $t, #$n + veor $b, $b, $t +___ +} +sub swapmove2x { +my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_; +$code.=<<___; + vshr.u64 $t0, $b0, #$n + vshr.u64 $t1, $b1, #$n + veor $t0, $t0, $a0 + veor $t1, $t1, $a1 + vand $t0, $t0, $mask + vand $t1, $t1, $mask + veor $a0, $a0, $t0 + vshl.u64 $t0, $t0, #$n + veor $a1, $a1, $t1 + vshl.u64 $t1, $t1, #$n + veor $b0, $b0, $t0 + veor $b1, $b1, $t1 +___ +} + +sub bitslice { +my @x=reverse(@_[0..7]); +my ($t0,$t1,$t2,$t3)=@_[8..11]; +$code.=<<___; + vmov.i8 $t0,#0x55 @ compose .LBS0 + vmov.i8 $t1,#0x33 @ compose .LBS1 +___ + &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3); + &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); +$code.=<<___; + vmov.i8 $t0,#0x0f @ compose .LBS2 +___ + &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3); + &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + + &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3); + &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3); +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" + +# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} +# define VFP_ABI_POP vldmia sp!,{d8-d15} +# define VFP_ABI_FRAME 0x40 +#else +# define VFP_ABI_PUSH +# define VFP_ABI_POP +# define VFP_ABI_FRAME 0 +# define BSAES_ASM_EXTENDED_KEY +# define XTS_CHAIN_TWEAK +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ +#endif + +#ifdef __thumb__ +# define adrl adr +#endif + +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.text +.syntax unified @ ARMv7-capable assembler is expected to handle this +#ifdef __thumb2__ +.thumb +#else +.code 32 +#endif + +.type _bsaes_decrypt8,%function +.align 4 +_bsaes_decrypt8: + adr $const,_bsaes_decrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + add $const,$const,#.LM0ISR-_bsaes_decrypt8 + + vldmia $const!, {@XMM[8]} @ .LM0ISR + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Ldec_sbox +.align 4 +.Ldec_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Ldec_sbox:\n"; + &InvSbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Ldec_done +___ + &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LISR + ite eq @ Thumb2 thing, sanity check in ARM + addeq $const,$const,#0x10 + bne .Ldec_loop + vldmia $const, {@XMM[12]} @ .LISRM0 + b .Ldec_loop +.align 4 +.Ldec_done: +___ + &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.type _bsaes_const,%object +.align 6 +_bsaes_const: +.LM0ISR: @ InvShiftRows constants + .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: + .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: + .quad 0x01040b0e0205080f, 0x0306090c00070a0d +.LM0SR: @ ShiftRows constants + .quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: + .quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: + .quad 0x0304090e00050a0f, 0x01060b0c0207080d +.LM0: + .quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LREVM0SR: + .quad 0x090d01050c000408, 0x03070b0f060a0e02 +.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by " +.align 6 +.size _bsaes_const,.-_bsaes_const + +.type _bsaes_encrypt8,%function +.align 4 +_bsaes_encrypt8: + adr $const,_bsaes_encrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + sub $const,$const,#_bsaes_encrypt8-.LM0SR + + vldmia $const!, {@XMM[8]} @ .LM0SR +_bsaes_encrypt8_alt: + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +_bsaes_encrypt8_bitslice: +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Lenc_sbox +.align 4 +.Lenc_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Lenc_sbox:\n"; + &Sbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Lenc_done +___ + &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LSR + ite eq @ Thumb2 thing, samity check in ARM + addeq $const,$const,#0x10 + bne .Lenc_loop + vldmia $const, {@XMM[12]} @ .LSRM0 + b .Lenc_loop +.align 4 +.Lenc_done: +___ + # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb + &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_encrypt8,.-_bsaes_encrypt8 +___ +} +{ +my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6"); + +sub bitslice_key { +my @x=reverse(@_[0..7]); +my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12]; + + &swapmove (@x[0,1],1,$bs0,$t2,$t3); +$code.=<<___; + @ &swapmove(@x[2,3],1,$t0,$t2,$t3); + vmov @x[2], @x[0] + vmov @x[3], @x[1] +___ + #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); + + &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3); +$code.=<<___; + @ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + vmov @x[4], @x[0] + vmov @x[6], @x[2] + vmov @x[5], @x[1] + vmov @x[7], @x[3] +___ + &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3); + &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3); +} + +$code.=<<___; +.type _bsaes_key_convert,%function +.align 4 +_bsaes_key_convert: + adr $const,_bsaes_key_convert + vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key + sub $const,$const,#_bsaes_key_convert-.LM0 + vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key + + vmov.i8 @XMM[8], #0x01 @ bit masks + vmov.i8 @XMM[9], #0x02 + vmov.i8 @XMM[10], #0x04 + vmov.i8 @XMM[11], #0x08 + vmov.i8 @XMM[12], #0x10 + vmov.i8 @XMM[13], #0x20 + vldmia $const, {@XMM[14]} @ .LM0 + +#ifdef __ARMEL__ + vrev32.8 @XMM[7], @XMM[7] + vrev32.8 @XMM[15], @XMM[15] +#endif + sub $rounds,$rounds,#1 + vstmia $out!, {@XMM[7]} @ save round 0 key + b .Lkey_loop + +.align 4 +.Lkey_loop: + vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])` + vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])` + vmov.i8 @XMM[6], #0x40 + vmov.i8 @XMM[15], #0x80 + + vtst.8 @XMM[0], @XMM[7], @XMM[8] + vtst.8 @XMM[1], @XMM[7], @XMM[9] + vtst.8 @XMM[2], @XMM[7], @XMM[10] + vtst.8 @XMM[3], @XMM[7], @XMM[11] + vtst.8 @XMM[4], @XMM[7], @XMM[12] + vtst.8 @XMM[5], @XMM[7], @XMM[13] + vtst.8 @XMM[6], @XMM[7], @XMM[6] + vtst.8 @XMM[7], @XMM[7], @XMM[15] + vld1.8 {@XMM[15]}, [$inp]! @ load next round key + vmvn @XMM[0], @XMM[0] @ "pnot" + vmvn @XMM[1], @XMM[1] + vmvn @XMM[5], @XMM[5] + vmvn @XMM[6], @XMM[6] +#ifdef __ARMEL__ + vrev32.8 @XMM[15], @XMM[15] +#endif + subs $rounds,$rounds,#1 + vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key + bne .Lkey_loop + + vmov.i8 @XMM[7],#0x63 @ compose .L63 + @ don't save last round key + bx lr +.size _bsaes_key_convert,.-_bsaes_key_convert +___ +} + +if (0) { # following four functions are unsupported interface + # used for benchmarking... +$code.=<<___; +.globl bsaes_enc_key_convert +.type bsaes_enc_key_convert,%function +.align 4 +bsaes_enc_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_enc_key_convert,.-bsaes_enc_key_convert + +.globl bsaes_encrypt_128 +.type bsaes_encrypt_128,%function +.align 4 +bsaes_encrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Lenc128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_encrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Lenc128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_encrypt_128,.-bsaes_encrypt_128 + +.globl bsaes_dec_key_convert +.type bsaes_dec_key_convert,%function +.align 4 +bsaes_dec_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + vldmia $out, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $out, {@XMM[7]} + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_dec_key_convert,.-bsaes_dec_key_convert + +.globl bsaes_decrypt_128 +.type bsaes_decrypt_128,%function +.align 4 +bsaes_decrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Ldec128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_decrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Ldec128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_decrypt_128,.-bsaes_decrypt_128 +___ +} +{ +my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10)); +my ($keysched)=("sp"); + +$code.=<<___; +.extern AES_cbc_encrypt +.extern AES_decrypt + +.global bsaes_cbc_encrypt +.type bsaes_cbc_encrypt,%function +.align 5 +bsaes_cbc_encrypt: +#ifndef __KERNEL__ + cmp $len, #128 +#ifndef __thumb__ + blo AES_cbc_encrypt +#else + bhs 1f + b AES_cbc_encrypt +1: +#endif +#endif + + @ it is up to the caller to make sure we are called with enc == 0 + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ivp, [ip] @ IV is 1st arg on the stack + mov $len, $len, lsr#4 @ len in 16 byte blocks + sub sp, #0x10 @ scratch space to carry over the IV + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ sifze of bit-slices key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + vldmia $keysched, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $keysched, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: +#endif + + vld1.8 {@XMM[15]}, [$ivp] @ load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs $len, $len, #0x8 + bmi .Lcbc_dec_loop_finish + + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5, $rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp] + sub $inp, $inp, #0x60 + vstmia $fp, {@XMM[15]} @ put aside IV + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + veor @XMM[5], @XMM[5], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[5]}, [$out]! + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds $len, $len, #8 + beq .Lcbc_dec_done + + vld1.8 {@XMM[0]}, [$inp]! @ load input + cmp $len, #2 + blo .Lcbc_dec_one + vld1.8 {@XMM[1]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + mov r5, $rounds + vstmia $fp, {@XMM[15]} @ put aside IV + beq .Lcbc_dec_two + vld1.8 {@XMM[2]}, [$inp]! + cmp $len, #4 + blo .Lcbc_dec_three + vld1.8 {@XMM[3]}, [$inp]! + beq .Lcbc_dec_four + vld1.8 {@XMM[4]}, [$inp]! + cmp $len, #6 + blo .Lcbc_dec_five + vld1.8 {@XMM[5]}, [$inp]! + beq .Lcbc_dec_six + vld1.8 {@XMM[6]}, [$inp]! + sub $inp, $inp, #0x70 + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub $inp, $inp, #0x60 + bl _bsaes_decrypt8 + vldmia $fp,{@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub $inp, $inp, #0x50 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[2], @XMM[2], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub $inp, $inp, #0x40 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub $inp, $inp, #0x30 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub $inp, $inp, #0x20 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! @ reload input + veor @XMM[1], @XMM[1], @XMM[8] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub $inp, $inp, #0x10 + mov $rounds, $out @ save original out pointer + mov $out, $fp @ use the iv scratch space as out buffer + mov r2, $key + vmov @XMM[4],@XMM[15] @ just in case ensure that IV + vmov @XMM[5],@XMM[0] @ and input are preserved + bl AES_decrypt + vld1.8 {@XMM[0]}, [$fp,:64] @ load result + veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV + vmov @XMM[15], @XMM[5] @ @XMM[5] holds input + vst1.8 {@XMM[0]}, [$rounds] @ write output + +.Lcbc_dec_done: +#ifndef BSAES_ASM_EXTENDED_KEY + vmov.i32 q0, #0 + vmov.i32 q1, #0 +.Lcbc_dec_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lcbc_dec_bzero +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + vst1.8 {@XMM[15]}, [$ivp] @ return IV + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} +.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt +___ +} +{ +my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10))); +my $const = "r6"; # shared with _bsaes_encrypt8_alt +my $keysched = "sp"; + +$code.=<<___; +.extern AES_encrypt +.global bsaes_ctr32_encrypt_blocks +.type bsaes_ctr32_encrypt_blocks,%function +.align 5 +bsaes_ctr32_encrypt_blocks: + cmp $len, #8 @ use plain AES for + blo .Lctr_enc_short @ small sizes + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ctr, [ip] @ ctr is 1st arg on the stack + sub sp, sp, #0x10 @ scratch space to carry over the ctr + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ size of bit-sliced key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vld1.8 {@XMM[0]}, [$ctr] @ load counter + add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr + vldmia $keysched, {@XMM[4]} @ load round0 key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + +.align 2 +0: add r12, $key, #248 + vld1.8 {@XMM[0]}, [$ctr] @ load counter + adrl $ctr, .LREVM0SR @ borrow $ctr + vldmia r12, {@XMM[4]} @ load round0 key + sub sp, #0x10 @ place for adjusted round0 key +#endif + + vmov.i32 @XMM[8],#1 @ compose 1<<96 + veor @XMM[9],@XMM[9],@XMM[9] + vrev32.8 @XMM[0],@XMM[0] + vext.8 @XMM[8],@XMM[9],@XMM[8],#4 + vrev32.8 @XMM[4],@XMM[4] + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vstmia $keysched, {@XMM[4]} @ save adjusted round0 key + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96 + vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1 + vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2 + vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3 + vadd.u32 @XMM[4], @XMM[1], @XMM[10] + vadd.u32 @XMM[5], @XMM[2], @XMM[10] + vadd.u32 @XMM[6], @XMM[3], @XMM[10] + vadd.u32 @XMM[7], @XMM[4], @XMM[10] + vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter + + @ Borrow prologue from _bsaes_encrypt8 to use the opportunity + @ to flip byte order in 32-bit counter + + vldmia $keysched, {@XMM[9]} @ load round0 key +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, $keysched, #0x10 @ pass next round key +#else + add r4, $key, #`248+16` +#endif + vldmia $ctr, {@XMM[8]} @ .LREVM0SR + mov r5, $rounds @ pass rounds + vstmia $fp, {@XMM[10]} @ save next counter + sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants + + bl _bsaes_encrypt8_alt + + subs $len, $len, #8 + blo .Lctr_enc_loop_done + + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[0], @XMM[8] + veor @XMM[1], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[10] + veor @XMM[6], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[7], @XMM[13] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + veor @XMM[5], @XMM[15] + vst1.8 {@XMM[6]}, [$out]! + vmov.i32 @XMM[8], #1 @ compose 1<<96 + vst1.8 {@XMM[3]}, [$out]! + veor @XMM[9], @XMM[9], @XMM[9] + vst1.8 {@XMM[7]}, [$out]! + vext.8 @XMM[8], @XMM[9], @XMM[8], #4 + vst1.8 {@XMM[2]}, [$out]! + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vst1.8 {@XMM[5]}, [$out]! + vldmia $fp, {@XMM[0]} @ load counter + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add $len, $len, #8 + vld1.8 {@XMM[8]}, [$inp]! @ load input + veor @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! @ write output + cmp $len, #2 + blo .Lctr_enc_done + vld1.8 {@XMM[9]}, [$inp]! + veor @XMM[1], @XMM[9] + vst1.8 {@XMM[1]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[4], @XMM[10] + vst1.8 {@XMM[4]}, [$out]! + cmp $len, #4 + blo .Lctr_enc_done + vld1.8 {@XMM[11]}, [$inp]! + veor @XMM[6], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[3]}, [$out]! + cmp $len, #6 + blo .Lctr_enc_done + vld1.8 {@XMM[13]}, [$inp]! + veor @XMM[7], @XMM[13] + vst1.8 {@XMM[7]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[14]}, [$inp] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[2]}, [$out]! + +.Lctr_enc_done: + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifndef BSAES_ASM_EXTENDED_KEY +.Lctr_enc_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lctr_enc_bzero +#else + vstmia $keysched, {q0-q1} +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.align 4 +.Lctr_enc_short: + ldr ip, [sp] @ ctr pointer is passed on stack + stmdb sp!, {r4-r8, lr} + + mov r4, $inp @ copy arguments + mov r5, $out + mov r6, $len + mov r7, $key + ldr r8, [ip, #12] @ load counter LSW + vld1.8 {@XMM[1]}, [ip] @ load whole counter value +#ifdef __ARMEL__ + rev r8, r8 +#endif + sub sp, sp, #0x10 + vst1.8 {@XMM[1]}, [sp,:64] @ copy counter value + sub sp, sp, #0x10 + +.Lctr_enc_short_loop: + add r0, sp, #0x10 @ input counter value + mov r1, sp @ output on the stack + mov r2, r7 @ key + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [r4]! @ load input + vld1.8 {@XMM[1]}, [sp,:64] @ load encrypted counter + add r8, r8, #1 +#ifdef __ARMEL__ + rev r0, r8 + str r0, [sp, #0x1c] @ next counter value +#else + str r8, [sp, #0x1c] @ next counter value +#endif + veor @XMM[0],@XMM[0],@XMM[1] + vst1.8 {@XMM[0]}, [r5]! @ store output + subs r6, r6, #1 + bne .Lctr_enc_short_loop + + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vstmia sp!, {q0-q1} + + ldmia sp!, {r4-r8, pc} +.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks +___ +} +{ +###################################################################### +# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len, +# const AES_KEY *key1, const AES_KEY *key2, +# const unsigned char iv[16]); +# +my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3))); +my $const="r6"; # returned by _bsaes_key_convert +my $twmask=@XMM[5]; +my @T=@XMM[6..7]; + +$code.=<<___; +.globl bsaes_xts_encrypt +.type bsaes_xts_encrypt,%function +.align 4 +bsaes_xts_encrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0,sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + subs $len, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds $len, #0x70 + bmi .Lxts_enc_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_enc_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done + +@ put this in range for both ARM and Thumb mode adr instructions +.align 5 +.Lxts_magic: + .quad 1, 0x87 + +.align 5 +.Lxts_enc_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_enc_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_enc_ret + sub r6, $out, #0x10 + +.Lxts_enc_steal: + ldrb r0, [$inp], #1 + ldrb r1, [$out, #-0x10] + strb r0, [$out, #-0x10] + strb r1, [$out], #1 + + subs $len, #1 + bhi .Lxts_enc_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_enc_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_enc_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_encrypt,.-bsaes_xts_encrypt + +.globl bsaes_xts_decrypt +.type bsaes_xts_decrypt,%function +.align 4 +bsaes_xts_decrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0, sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + add r4, sp, #0x90 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + tst $len, #0xf @ if not multiple of 16 + it ne @ Thumb2 thing, sanity check in ARM + subne $len, #0x10 @ subtract another 16 bytes + subs $len, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds $len, #0x70 + bmi .Lxts_dec_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_dec_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + mov r5, $magic @ preserve magic + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + mov $magic, r5 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_dec_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_dec_ret + + @ calculate one round of extra tweak for the stolen ciphertext + vldmia $magic, {$twmask} + vshr.s64 @XMM[6], @XMM[8], #63 + vand @XMM[6], @XMM[6], $twmask + vadd.u64 @XMM[9], @XMM[8], @XMM[8] + vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")` + veor @XMM[9], @XMM[9], @XMM[6] + + @ perform the final decryption with the last tweak value + vld1.8 {@XMM[0]}, [$inp]! + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[9] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[9] + vst1.8 {@XMM[0]}, [$out] + + mov r6, $out +.Lxts_dec_steal: + ldrb r1, [$out] + ldrb r0, [$inp], #1 + strb r1, [$out, #0x10] + strb r0, [$out], #1 + + subs $len, #1 + bhi .Lxts_dec_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_dec_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_dec_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_decrypt,.-bsaes_xts_decrypt +___ +} +$code.=<<___; +#endif +___ + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +print $code; + +close STDOUT; diff --git a/crypto/aes/asm/bsaes-x86_64.pl b/crypto/aes/asm/bsaes-x86_64.pl index 41b90f08443f..3f7d33c45bce 100755 --- a/crypto/aes/asm/bsaes-x86_64.pl +++ b/crypto/aes/asm/bsaes-x86_64.pl @@ -38,8 +38,9 @@ # Emilia's this(*) difference # # Core 2 9.30 8.69 +7% -# Nehalem(**) 7.63 6.98 +9% -# Atom 17.1 17.4 -2%(***) +# Nehalem(**) 7.63 6.88 +11% +# Atom 17.1 16.4 +4% +# Silvermont - 12.9 # # (*) Comparison is not completely fair, because "this" is ECB, # i.e. no extra processing such as counter values calculation @@ -50,14 +51,6 @@ # (**) Results were collected on Westmere, which is considered to # be equivalent to Nehalem for this code. # -# (***) Slowdown on Atom is rather strange per se, because original -# implementation has a number of 9+-bytes instructions, which -# are bad for Atom front-end, and which I eliminated completely. -# In attempt to address deterioration sbox() was tested in FP -# SIMD "domain" (movaps instead of movdqa, xorps instead of -# pxor, etc.). While it resulted in nominal 4% improvement on -# Atom, it hurted Westmere by more than 2x factor. -# # As for key schedule conversion subroutine. Interface to OpenSSL # relies on per-invocation on-the-fly conversion. This naturally # has impact on performance, especially for short inputs. Conversion @@ -67,7 +60,7 @@ # conversion conversion/8x block # Core 2 240 0.22 # Nehalem 180 0.20 -# Atom 430 0.19 +# Atom 430 0.20 # # The ratio values mean that 128-byte blocks will be processed # 16-18% slower, 256-byte blocks - 9-10%, 384-byte blocks - 6-7%, @@ -83,9 +76,10 @@ # Add decryption procedure. Performance in CPU cycles spent to decrypt # one byte out of 4096-byte buffer with 128-bit key is: # -# Core 2 9.83 -# Nehalem 7.74 -# Atom 19.0 +# Core 2 9.98 +# Nehalem 7.80 +# Atom 17.9 +# Silvermont 14.0 # # November 2011. # @@ -434,21 +428,21 @@ my $mask=pop; $code.=<<___; pxor 0x00($key),@x[0] pxor 0x10($key),@x[1] - pshufb $mask,@x[0] pxor 0x20($key),@x[2] - pshufb $mask,@x[1] pxor 0x30($key),@x[3] - pshufb $mask,@x[2] + pshufb $mask,@x[0] + pshufb $mask,@x[1] pxor 0x40($key),@x[4] - pshufb $mask,@x[3] pxor 0x50($key),@x[5] - pshufb $mask,@x[4] + pshufb $mask,@x[2] + pshufb $mask,@x[3] pxor 0x60($key),@x[6] - pshufb $mask,@x[5] pxor 0x70($key),@x[7] + pshufb $mask,@x[4] + pshufb $mask,@x[5] pshufb $mask,@x[6] - lea 0x80($key),$key pshufb $mask,@x[7] + lea 0x80($key),$key ___ } @@ -820,18 +814,18 @@ _bsaes_encrypt8: movdqa 0x50($const), @XMM[8] # .LM0SR pxor @XMM[9], @XMM[0] # xor with round0 key pxor @XMM[9], @XMM[1] - pshufb @XMM[8], @XMM[0] pxor @XMM[9], @XMM[2] - pshufb @XMM[8], @XMM[1] pxor @XMM[9], @XMM[3] - pshufb @XMM[8], @XMM[2] + pshufb @XMM[8], @XMM[0] + pshufb @XMM[8], @XMM[1] pxor @XMM[9], @XMM[4] - pshufb @XMM[8], @XMM[3] pxor @XMM[9], @XMM[5] - pshufb @XMM[8], @XMM[4] + pshufb @XMM[8], @XMM[2] + pshufb @XMM[8], @XMM[3] pxor @XMM[9], @XMM[6] - pshufb @XMM[8], @XMM[5] pxor @XMM[9], @XMM[7] + pshufb @XMM[8], @XMM[4] + pshufb @XMM[8], @XMM[5] pshufb @XMM[8], @XMM[6] pshufb @XMM[8], @XMM[7] _bsaes_encrypt8_bitslice: @@ -884,18 +878,18 @@ _bsaes_decrypt8: movdqa -0x30($const), @XMM[8] # .LM0ISR pxor @XMM[9], @XMM[0] # xor with round0 key pxor @XMM[9], @XMM[1] - pshufb @XMM[8], @XMM[0] pxor @XMM[9], @XMM[2] - pshufb @XMM[8], @XMM[1] pxor @XMM[9], @XMM[3] - pshufb @XMM[8], @XMM[2] + pshufb @XMM[8], @XMM[0] + pshufb @XMM[8], @XMM[1] pxor @XMM[9], @XMM[4] - pshufb @XMM[8], @XMM[3] pxor @XMM[9], @XMM[5] - pshufb @XMM[8], @XMM[4] + pshufb @XMM[8], @XMM[2] + pshufb @XMM[8], @XMM[3] pxor @XMM[9], @XMM[6] - pshufb @XMM[8], @XMM[5] pxor @XMM[9], @XMM[7] + pshufb @XMM[8], @XMM[4] + pshufb @XMM[8], @XMM[5] pshufb @XMM[8], @XMM[6] pshufb @XMM[8], @XMM[7] ___ @@ -1937,21 +1931,21 @@ $code.=<<___; movdqa -0x10(%r11), @XMM[8] # .LSWPUPM0SR pxor @XMM[9], @XMM[0] # xor with round0 key pxor @XMM[9], @XMM[1] - pshufb @XMM[8], @XMM[0] pxor @XMM[9], @XMM[2] - pshufb @XMM[8], @XMM[1] pxor @XMM[9], @XMM[3] - pshufb @XMM[8], @XMM[2] + pshufb @XMM[8], @XMM[0] + pshufb @XMM[8], @XMM[1] pxor @XMM[9], @XMM[4] - pshufb @XMM[8], @XMM[3] pxor @XMM[9], @XMM[5] - pshufb @XMM[8], @XMM[4] + pshufb @XMM[8], @XMM[2] + pshufb @XMM[8], @XMM[3] pxor @XMM[9], @XMM[6] - pshufb @XMM[8], @XMM[5] pxor @XMM[9], @XMM[7] + pshufb @XMM[8], @XMM[4] + pshufb @XMM[8], @XMM[5] pshufb @XMM[8], @XMM[6] - lea .LBS0(%rip), %r11 # constants table pshufb @XMM[8], @XMM[7] + lea .LBS0(%rip), %r11 # constants table mov %ebx,%r10d # pass rounds call _bsaes_encrypt8_bitslice diff --git a/crypto/aes/asm/vpaes-ppc.pl b/crypto/aes/asm/vpaes-ppc.pl new file mode 100755 index 000000000000..7fda60ed9e4d --- /dev/null +++ b/crypto/aes/asm/vpaes-ppc.pl @@ -0,0 +1,1512 @@ +#!/usr/bin/env perl + +###################################################################### +## Constant-time SSSE3 AES core implementation. +## version 0.1 +## +## By Mike Hamburg (Stanford University), 2009 +## Public domain. +## +## For details see http://shiftleft.org/papers/vector_aes/ and +## http://crypto.stanford.edu/vpaes/. + +# CBC encrypt/decrypt performance in cycles per byte processed with +# 128-bit key. +# +# aes-ppc.pl this +# G4e 35.5/52.1/(23.8) 11.9(*)/15.4 +# POWER6 42.7/54.3/(28.2) 63.0/92.8(**) +# POWER7 32.3/42.9/(18.4) 18.5/23.3 +# +# (*) This is ~10% worse than reported in paper. The reason is +# twofold. This module doesn't make any assumption about +# key schedule (or data for that matter) alignment and handles +# it in-line. Secondly it, being transliterated from +# vpaes-x86_64.pl, relies on "nested inversion" better suited +# for Intel CPUs. +# (**) Inadequate POWER6 performance is due to astronomic AltiVec +# latency, 9 cycles per simple logical operation. + +$flavour = shift; + +if ($flavour =~ /64/) { + $SIZE_T =8; + $LRSAVE =2*$SIZE_T; + $STU ="stdu"; + $POP ="ld"; + $PUSH ="std"; + $UCMP ="cmpld"; +} elsif ($flavour =~ /32/) { + $SIZE_T =4; + $LRSAVE =$SIZE_T; + $STU ="stwu"; + $POP ="lwz"; + $PUSH ="stw"; + $UCMP ="cmplw"; +} else { die "nonsense $flavour"; } + +$sp="r1"; +$FRAME=6*$SIZE_T+13*16; # 13*16 is for v20-v31 offload + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!"; + +$code.=<<___; +.machine "any" + +.text + +.align 7 # totally strategic alignment +_vpaes_consts: +Lk_mc_forward: # mc_forward + .long 0x01020300, 0x05060704, 0x090a0b08, 0x0d0e0f0c ?inv + .long 0x05060704, 0x090a0b08, 0x0d0e0f0c, 0x01020300 ?inv + .long 0x090a0b08, 0x0d0e0f0c, 0x01020300, 0x05060704 ?inv + .long 0x0d0e0f0c, 0x01020300, 0x05060704, 0x090a0b08 ?inv +Lk_mc_backward: # mc_backward + .long 0x03000102, 0x07040506, 0x0b08090a, 0x0f0c0d0e ?inv + .long 0x0f0c0d0e, 0x03000102, 0x07040506, 0x0b08090a ?inv + .long 0x0b08090a, 0x0f0c0d0e, 0x03000102, 0x07040506 ?inv + .long 0x07040506, 0x0b08090a, 0x0f0c0d0e, 0x03000102 ?inv +Lk_sr: # sr + .long 0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f ?inv + .long 0x00050a0f, 0x04090e03, 0x080d0207, 0x0c01060b ?inv + .long 0x0009020b, 0x040d060f, 0x08010a03, 0x0c050e07 ?inv + .long 0x000d0a07, 0x04010e0b, 0x0805020f, 0x0c090603 ?inv + +## +## "Hot" constants +## +Lk_inv: # inv, inva + .long 0xf001080d, 0x0f06050e, 0x020c0b0a, 0x09030704 ?rev + .long 0xf0070b0f, 0x060a0401, 0x09080502, 0x0c0e0d03 ?rev +Lk_ipt: # input transform (lo, hi) + .long 0x00702a5a, 0x98e8b2c2, 0x08782252, 0x90e0baca ?rev + .long 0x004d7c31, 0x7d30014c, 0x81ccfdb0, 0xfcb180cd ?rev +Lk_sbo: # sbou, sbot + .long 0x00c7bd6f, 0x176dd2d0, 0x78a802c5, 0x7abfaa15 ?rev + .long 0x006abb5f, 0xa574e4cf, 0xfa352b41, 0xd1901e8e ?rev +Lk_sb1: # sb1u, sb1t + .long 0x0023e2fa, 0x15d41836, 0xefd92e0d, 0xc1ccf73b ?rev + .long 0x003e50cb, 0x8fe19bb1, 0x44f52a14, 0x6e7adfa5 ?rev +Lk_sb2: # sb2u, sb2t + .long 0x0029e10a, 0x4088eb69, 0x4a2382ab, 0xc863a1c2 ?rev + .long 0x0024710b, 0xc6937ae2, 0xcd2f98bc, 0x55e9b75e ?rev + +## +## Decryption stuff +## +Lk_dipt: # decryption input transform + .long 0x005f540b, 0x045b500f, 0x1a454e11, 0x1e414a15 ?rev + .long 0x00650560, 0xe683e386, 0x94f191f4, 0x72177712 ?rev +Lk_dsbo: # decryption sbox final output + .long 0x0040f97e, 0x53ea8713, 0x2d3e94d4, 0xb96daac7 ?rev + .long 0x001d4493, 0x0f56d712, 0x9c8ec5d8, 0x59814bca ?rev +Lk_dsb9: # decryption sbox output *9*u, *9*t + .long 0x00d6869a, 0x53031c85, 0xc94c994f, 0x501fd5ca ?rev + .long 0x0049d7ec, 0x89173bc0, 0x65a5fbb2, 0x9e2c5e72 ?rev +Lk_dsbd: # decryption sbox output *D*u, *D*t + .long 0x00a2b1e6, 0xdfcc577d, 0x39442a88, 0x139b6ef5 ?rev + .long 0x00cbc624, 0xf7fae23c, 0xd3efde15, 0x0d183129 ?rev +Lk_dsbb: # decryption sbox output *B*u, *B*t + .long 0x0042b496, 0x926422d0, 0x04d4f2b0, 0xf6462660 ?rev + .long 0x006759cd, 0xa69894c1, 0x6baa5532, 0x3e0cfff3 ?rev +Lk_dsbe: # decryption sbox output *E*u, *E*t + .long 0x00d0d426, 0x9692f246, 0xb0f6b464, 0x04604222 ?rev + .long 0x00c1aaff, 0xcda6550c, 0x323e5998, 0x6bf36794 ?rev + +## +## Key schedule constants +## +Lk_dksd: # decryption key schedule: invskew x*D + .long 0x0047e4a3, 0x5d1ab9fe, 0xf9be1d5a, 0xa4e34007 ?rev + .long 0x008336b5, 0xf477c241, 0x1e9d28ab, 0xea69dc5f ?rev +Lk_dksb: # decryption key schedule: invskew x*B + .long 0x00d55085, 0x1fca4f9a, 0x994cc91c, 0x8653d603 ?rev + .long 0x004afcb6, 0xa7ed5b11, 0xc882347e, 0x6f2593d9 ?rev +Lk_dkse: # decryption key schedule: invskew x*E + 0x63 + .long 0x00d6c91f, 0xca1c03d5, 0x86504f99, 0x4c9a8553 ?rev + .long 0xe87bdc4f, 0x059631a2, 0x8714b320, 0x6af95ecd ?rev +Lk_dks9: # decryption key schedule: invskew x*9 + .long 0x00a7d97e, 0xc86f11b6, 0xfc5b2582, 0x3493ed4a ?rev + .long 0x00331427, 0x62517645, 0xcefddae9, 0xac9fb88b ?rev + +Lk_rcon: # rcon + .long 0xb6ee9daf, 0xb991831f, 0x817d7c4d, 0x08982a70 ?asis +Lk_s63: + .long 0x5b5b5b5b, 0x5b5b5b5b, 0x5b5b5b5b, 0x5b5b5b5b ?asis + +Lk_opt: # output transform + .long 0x0060b6d6, 0x29499fff, 0x0868bede, 0x214197f7 ?rev + .long 0x00ecbc50, 0x51bded01, 0xe00c5cb0, 0xb15d0de1 ?rev +Lk_deskew: # deskew tables: inverts the sbox's "skew" + .long 0x00e3a447, 0x40a3e407, 0x1af9be5d, 0x5ab9fe1d ?rev + .long 0x0069ea83, 0xdcb5365f, 0x771e9df4, 0xabc24128 ?rev +.align 5 +Lconsts: + mflr r0 + bcl 20,31,\$+4 + mflr r12 #vvvvv "distance between . and _vpaes_consts + addi r12,r12,-0x308 + mtlr r0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +.asciz "Vector Permutation AES for AltiVec, Mike Hamburg (Stanford University)" +.align 6 +___ + +my ($inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm) = map("v$_",(26..31)); +{ +my ($inp,$out,$key) = map("r$_",(3..5)); + +my ($invlo,$invhi,$iptlo,$ipthi,$sbou,$sbot) = map("v$_",(10..15)); +my ($sb1u,$sb1t,$sb2u,$sb2t) = map("v$_",(16..19)); +my ($sb9u,$sb9t,$sbdu,$sbdt,$sbbu,$sbbt,$sbeu,$sbet)=map("v$_",(16..23)); + +$code.=<<___; +## +## _aes_preheat +## +## Fills register %r10 -> .aes_consts (so you can -fPIC) +## and %xmm9-%xmm15 as specified below. +## +.align 4 +_vpaes_encrypt_preheat: + mflr r8 + bl Lconsts + mtlr r8 + li r11, 0xc0 # Lk_inv + li r10, 0xd0 + li r9, 0xe0 # Lk_ipt + li r8, 0xf0 + vxor v7, v7, v7 # 0x00..00 + vspltisb v8,4 # 0x04..04 + vspltisb v9,0x0f # 0x0f..0f + lvx $invlo, r12, r11 + li r11, 0x100 + lvx $invhi, r12, r10 + li r10, 0x110 + lvx $iptlo, r12, r9 + li r9, 0x120 + lvx $ipthi, r12, r8 + li r8, 0x130 + lvx $sbou, r12, r11 + li r11, 0x140 + lvx $sbot, r12, r10 + li r10, 0x150 + lvx $sb1u, r12, r9 + lvx $sb1t, r12, r8 + lvx $sb2u, r12, r11 + lvx $sb2t, r12, r10 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +## +## _aes_encrypt_core +## +## AES-encrypt %xmm0. +## +## Inputs: +## %xmm0 = input +## %xmm9-%xmm15 as in _vpaes_preheat +## (%rdx) = scheduled keys +## +## Output in %xmm0 +## Clobbers %xmm1-%xmm6, %r9, %r10, %r11, %rax +## +## +.align 5 +_vpaes_encrypt_core: + lwz r8, 240($key) # pull rounds + li r9, 16 + lvx v5, 0, $key # vmovdqu (%r9), %xmm5 # round0 key + li r11, 0x10 + lvx v6, r9, $key + addi r9, r9, 16 + ?vperm v5, v5, v6, $keyperm # align round key + addi r10, r11, 0x40 + vsrb v1, v0, v8 # vpsrlb \$4, %xmm0, %xmm0 + vperm v0, $iptlo, $iptlo, v0 # vpshufb %xmm1, %xmm2, %xmm1 + vperm v1, $ipthi, $ipthi, v1 # vpshufb %xmm0, %xmm3, %xmm2 + vxor v0, v0, v5 # vpxor %xmm5, %xmm1, %xmm0 + vxor v0, v0, v1 # vpxor %xmm2, %xmm0, %xmm0 + mtctr r8 + b Lenc_entry + +.align 4 +Lenc_loop: + # middle of middle round + vperm v4, $sb1t, v7, v2 # vpshufb %xmm2, %xmm13, %xmm4 # 4 = sb1u + lvx v1, r12, r11 # vmovdqa -0x40(%r11,%r10), %xmm1 # .Lk_mc_forward[] + addi r11, r11, 16 + vperm v0, $sb1u, v7, v3 # vpshufb %xmm3, %xmm12, %xmm0 # 0 = sb1t + vxor v4, v4, v5 # vpxor %xmm5, %xmm4, %xmm4 # 4 = sb1u + k + andi. r11, r11, 0x30 # and \$0x30, %r11 # ... mod 4 + vperm v5, $sb2t, v7, v2 # vpshufb %xmm2, %xmm15, %xmm5 # 4 = sb2u + vxor v0, v0, v4 # vpxor %xmm4, %xmm0, %xmm0 # 0 = A + vperm v2, $sb2u, v7, v3 # vpshufb %xmm3, %xmm14, %xmm2 # 2 = sb2t + lvx v4, r12, r10 # vmovdqa (%r11,%r10), %xmm4 # .Lk_mc_backward[] + addi r10, r11, 0x40 + vperm v3, v0, v7, v1 # vpshufb %xmm1, %xmm0, %xmm3 # 0 = B + vxor v2, v2, v5 # vpxor %xmm5, %xmm2, %xmm2 # 2 = 2A + vperm v0, v0, v7, v4 # vpshufb %xmm4, %xmm0, %xmm0 # 3 = D + vxor v3, v3, v2 # vpxor %xmm2, %xmm3, %xmm3 # 0 = 2A+B + vperm v4, v3, v7, v1 # vpshufb %xmm1, %xmm3, %xmm4 # 0 = 2B+C + vxor v0, v0, v3 # vpxor %xmm3, %xmm0, %xmm0 # 3 = 2A+B+D + vxor v0, v0, v4 # vpxor %xmm4, %xmm0, %xmm0 # 0 = 2A+3B+C+D + +Lenc_entry: + # top of round + vsrb v1, v0, v8 # vpsrlb \$4, %xmm0, %xmm0 # 1 = i + vperm v5, $invhi, $invhi, v0 # vpshufb %xmm1, %xmm11, %xmm5 # 2 = a/k + vxor v0, v0, v1 # vpxor %xmm0, %xmm1, %xmm1 # 0 = j + vperm v3, $invlo, $invlo, v1 # vpshufb %xmm0, %xmm10, %xmm3 # 3 = 1/i + vperm v4, $invlo, $invlo, v0 # vpshufb %xmm1, %xmm10, %xmm4 # 4 = 1/j + vand v0, v0, v9 + vxor v3, v3, v5 # vpxor %xmm5, %xmm3, %xmm3 # 3 = iak = 1/i + a/k + vxor v4, v4, v5 # vpxor %xmm5, %xmm4, %xmm4 # 4 = jak = 1/j + a/k + vperm v2, $invlo, v7, v3 # vpshufb %xmm3, %xmm10, %xmm2 # 2 = 1/iak + vmr v5, v6 + lvx v6, r9, $key # vmovdqu (%r9), %xmm5 + vperm v3, $invlo, v7, v4 # vpshufb %xmm4, %xmm10, %xmm3 # 3 = 1/jak + addi r9, r9, 16 + vxor v2, v2, v0 # vpxor %xmm1, %xmm2, %xmm2 # 2 = io + ?vperm v5, v5, v6, $keyperm # align round key + vxor v3, v3, v1 # vpxor %xmm0, %xmm3, %xmm3 # 3 = jo + bdnz Lenc_loop + + # middle of last round + addi r10, r11, 0x80 + # vmovdqa -0x60(%r10), %xmm4 # 3 : sbou .Lk_sbo + # vmovdqa -0x50(%r10), %xmm0 # 0 : sbot .Lk_sbo+16 + vperm v4, $sbou, v7, v2 # vpshufb %xmm2, %xmm4, %xmm4 # 4 = sbou + lvx v1, r12, r10 # vmovdqa 0x40(%r11,%r10), %xmm1 # .Lk_sr[] + vperm v0, $sbot, v7, v3 # vpshufb %xmm3, %xmm0, %xmm0 # 0 = sb1t + vxor v4, v4, v5 # vpxor %xmm5, %xmm4, %xmm4 # 4 = sb1u + k + vxor v0, v0, v4 # vpxor %xmm4, %xmm0, %xmm0 # 0 = A + vperm v0, v0, v7, v1 # vpshufb %xmm1, %xmm0, %xmm0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +.globl .vpaes_encrypt +.align 5 +.vpaes_encrypt: + $STU $sp,-$FRAME($sp) + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mflr r6 + mfspr r7, 256 # save vrsave + stvx v20,r10,$sp + addi r10,r10,32 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + stw r7,`$FRAME-4`($sp) # save vrsave + li r0, -1 + $PUSH r6,`$FRAME+$LRSAVE`($sp) + mtspr 256, r0 # preserve all AltiVec registers + + bl _vpaes_encrypt_preheat + + ?lvsl $inpperm, 0, $inp # prepare for unaligned access + lvx v0, 0, $inp + addi $inp, $inp, 15 # 15 is not a typo + ?lvsr $outperm, 0, $out + ?lvsl $keyperm, 0, $key # prepare for unaligned access + vnor $outmask, v7, v7 # 0xff..ff + lvx $inptail, 0, $inp # redundant in aligned case + ?vperm $outmask, v7, $outmask, $outperm + lvx $outhead, 0, $out + ?vperm v0, v0, $inptail, $inpperm + + bl _vpaes_encrypt_core + + vperm v0, v0, v0, $outperm # rotate right/left + vsel v1, $outhead, v0, $outmask + vmr $outhead, v0 + stvx v1, 0, $out + addi $out, $out, 15 # 15 is not a typo + ######## + + lvx v1, 0, $out # redundant in aligned case + vsel v1, $outhead, v1, $outmask + stvx v1, 0, $out + + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mtlr r6 + mtspr 256, r7 # restore vrsave + lvx v20,r10,$sp + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + addi $sp,$sp,$FRAME + blr + .long 0 + .byte 0,12,0x04,1,0x80,0,3,0 + .long 0 +.size .vpaes_encrypt,.-.vpaes_encrypt + +.align 4 +_vpaes_decrypt_preheat: + mflr r8 + bl Lconsts + mtlr r8 + li r11, 0xc0 # Lk_inv + li r10, 0xd0 + li r9, 0x160 # Ldipt + li r8, 0x170 + vxor v7, v7, v7 # 0x00..00 + vspltisb v8,4 # 0x04..04 + vspltisb v9,0x0f # 0x0f..0f + lvx $invlo, r12, r11 + li r11, 0x180 + lvx $invhi, r12, r10 + li r10, 0x190 + lvx $iptlo, r12, r9 + li r9, 0x1a0 + lvx $ipthi, r12, r8 + li r8, 0x1b0 + lvx $sbou, r12, r11 + li r11, 0x1c0 + lvx $sbot, r12, r10 + li r10, 0x1d0 + lvx $sb9u, r12, r9 + li r9, 0x1e0 + lvx $sb9t, r12, r8 + li r8, 0x1f0 + lvx $sbdu, r12, r11 + li r11, 0x200 + lvx $sbdt, r12, r10 + li r10, 0x210 + lvx $sbbu, r12, r9 + lvx $sbbt, r12, r8 + lvx $sbeu, r12, r11 + lvx $sbet, r12, r10 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +## +## Decryption core +## +## Same API as encryption core. +## +.align 4 +_vpaes_decrypt_core: + lwz r8, 240($key) # pull rounds + li r9, 16 + lvx v5, 0, $key # vmovdqu (%r9), %xmm4 # round0 key + li r11, 0x30 + lvx v6, r9, $key + addi r9, r9, 16 + ?vperm v5, v5, v6, $keyperm # align round key + vsrb v1, v0, v8 # vpsrlb \$4, %xmm0, %xmm0 + vperm v0, $iptlo, $iptlo, v0 # vpshufb %xmm1, %xmm2, %xmm2 + vperm v1, $ipthi, $ipthi, v1 # vpshufb %xmm0, %xmm1, %xmm0 + vxor v0, v0, v5 # vpxor %xmm4, %xmm2, %xmm2 + vxor v0, v0, v1 # vpxor %xmm2, %xmm0, %xmm0 + mtctr r8 + b Ldec_entry + +.align 4 +Ldec_loop: +# +# Inverse mix columns +# + lvx v0, r12, r11 # v5 and v0 are flipped + # vmovdqa -0x20(%r10),%xmm4 # 4 : sb9u + # vmovdqa -0x10(%r10),%xmm1 # 0 : sb9t + vperm v4, $sb9u, v7, v2 # vpshufb %xmm2, %xmm4, %xmm4 # 4 = sb9u + subi r11, r11, 16 + vperm v1, $sb9t, v7, v3 # vpshufb %xmm3, %xmm1, %xmm1 # 0 = sb9t + andi. r11, r11, 0x30 + vxor v5, v5, v4 # vpxor %xmm4, %xmm0, %xmm0 + # vmovdqa 0x00(%r10),%xmm4 # 4 : sbdu + vxor v5, v5, v1 # vpxor %xmm1, %xmm0, %xmm0 # 0 = ch + # vmovdqa 0x10(%r10),%xmm1 # 0 : sbdt + + vperm v4, $sbdu, v7, v2 # vpshufb %xmm2, %xmm4, %xmm4 # 4 = sbdu + vperm v5, v5, v7, v0 # vpshufb %xmm5, %xmm0, %xmm0 # MC ch + vperm v1, $sbdt, v7, v3 # vpshufb %xmm3, %xmm1, %xmm1 # 0 = sbdt + vxor v5, v5, v4 # vpxor %xmm4, %xmm0, %xmm0 # 4 = ch + # vmovdqa 0x20(%r10), %xmm4 # 4 : sbbu + vxor v5, v5, v1 # vpxor %xmm1, %xmm0, %xmm0 # 0 = ch + # vmovdqa 0x30(%r10), %xmm1 # 0 : sbbt + + vperm v4, $sbbu, v7, v2 # vpshufb %xmm2, %xmm4, %xmm4 # 4 = sbbu + vperm v5, v5, v7, v0 # vpshufb %xmm5, %xmm0, %xmm0 # MC ch + vperm v1, $sbbt, v7, v3 # vpshufb %xmm3, %xmm1, %xmm1 # 0 = sbbt + vxor v5, v5, v4 # vpxor %xmm4, %xmm0, %xmm0 # 4 = ch + # vmovdqa 0x40(%r10), %xmm4 # 4 : sbeu + vxor v5, v5, v1 # vpxor %xmm1, %xmm0, %xmm0 # 0 = ch + # vmovdqa 0x50(%r10), %xmm1 # 0 : sbet + + vperm v4, $sbeu, v7, v2 # vpshufb %xmm2, %xmm4, %xmm4 # 4 = sbeu + vperm v5, v5, v7, v0 # vpshufb %xmm5, %xmm0, %xmm0 # MC ch + vperm v1, $sbet, v7, v3 # vpshufb %xmm3, %xmm1, %xmm1 # 0 = sbet + vxor v0, v5, v4 # vpxor %xmm4, %xmm0, %xmm0 # 4 = ch + vxor v0, v0, v1 # vpxor %xmm1, %xmm0, %xmm0 # 0 = ch + +Ldec_entry: + # top of round + vsrb v1, v0, v8 # vpsrlb \$4, %xmm0, %xmm0 # 1 = i + vperm v2, $invhi, $invhi, v0 # vpshufb %xmm1, %xmm11, %xmm2 # 2 = a/k + vxor v0, v0, v1 # vpxor %xmm0, %xmm1, %xmm1 # 0 = j + vperm v3, $invlo, $invlo, v1 # vpshufb %xmm0, %xmm10, %xmm3 # 3 = 1/i + vperm v4, $invlo, $invlo, v0 # vpshufb %xmm1, %xmm10, %xmm4 # 4 = 1/j + vand v0, v0, v9 + vxor v3, v3, v2 # vpxor %xmm2, %xmm3, %xmm3 # 3 = iak = 1/i + a/k + vxor v4, v4, v2 # vpxor %xmm2, %xmm4, %xmm4 # 4 = jak = 1/j + a/k + vperm v2, $invlo, v7, v3 # vpshufb %xmm3, %xmm10, %xmm2 # 2 = 1/iak + vmr v5, v6 + lvx v6, r9, $key # vmovdqu (%r9), %xmm0 + vperm v3, $invlo, v7, v4 # vpshufb %xmm4, %xmm10, %xmm3 # 3 = 1/jak + addi r9, r9, 16 + vxor v2, v2, v0 # vpxor %xmm1, %xmm2, %xmm2 # 2 = io + ?vperm v5, v5, v6, $keyperm # align round key + vxor v3, v3, v1 # vpxor %xmm0, %xmm3, %xmm3 # 3 = jo + bdnz Ldec_loop + + # middle of last round + addi r10, r11, 0x80 + # vmovdqa 0x60(%r10), %xmm4 # 3 : sbou + vperm v4, $sbou, v7, v2 # vpshufb %xmm2, %xmm4, %xmm4 # 4 = sbou + # vmovdqa 0x70(%r10), %xmm1 # 0 : sbot + lvx v2, r12, r10 # vmovdqa -0x160(%r11), %xmm2 # .Lk_sr-.Lk_dsbd=-0x160 + vperm v1, $sbot, v7, v3 # vpshufb %xmm3, %xmm1, %xmm1 # 0 = sb1t + vxor v4, v4, v5 # vpxor %xmm0, %xmm4, %xmm4 # 4 = sb1u + k + vxor v0, v1, v4 # vpxor %xmm4, %xmm1, %xmm0 # 0 = A + vperm v0, v0, v7, v2 # vpshufb %xmm2, %xmm0, %xmm0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +.globl .vpaes_decrypt +.align 5 +.vpaes_decrypt: + $STU $sp,-$FRAME($sp) + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mflr r6 + mfspr r7, 256 # save vrsave + stvx v20,r10,$sp + addi r10,r10,32 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + stw r7,`$FRAME-4`($sp) # save vrsave + li r0, -1 + $PUSH r6,`$FRAME+$LRSAVE`($sp) + mtspr 256, r0 # preserve all AltiVec registers + + bl _vpaes_decrypt_preheat + + ?lvsl $inpperm, 0, $inp # prepare for unaligned access + lvx v0, 0, $inp + addi $inp, $inp, 15 # 15 is not a typo + ?lvsr $outperm, 0, $out + ?lvsl $keyperm, 0, $key + vnor $outmask, v7, v7 # 0xff..ff + lvx $inptail, 0, $inp # redundant in aligned case + ?vperm $outmask, v7, $outmask, $outperm + lvx $outhead, 0, $out + ?vperm v0, v0, $inptail, $inpperm + + bl _vpaes_decrypt_core + + vperm v0, v0, v0, $outperm # rotate right/left + vsel v1, $outhead, v0, $outmask + vmr $outhead, v0 + stvx v1, 0, $out + addi $out, $out, 15 # 15 is not a typo + ######## + + lvx v1, 0, $out # redundant in aligned case + vsel v1, $outhead, v1, $outmask + stvx v1, 0, $out + + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mtlr r6 + mtspr 256, r7 # restore vrsave + lvx v20,r10,$sp + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + addi $sp,$sp,$FRAME + blr + .long 0 + .byte 0,12,0x04,1,0x80,0,3,0 + .long 0 +.size .vpaes_decrypt,.-.vpaes_decrypt + +.globl .vpaes_cbc_encrypt +.align 5 +.vpaes_cbc_encrypt: + ${UCMP}i r5,16 + bltlr- + + $STU $sp,-`($FRAME+2*$SIZE_T)`($sp) + mflr r0 + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mfspr r12, 256 + stvx v20,r10,$sp + addi r10,r10,32 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + stw r12,`$FRAME-4`($sp) # save vrsave + $PUSH r30,`$FRAME+$SIZE_T*0`($sp) + $PUSH r31,`$FRAME+$SIZE_T*1`($sp) + li r9, -16 + $PUSH r0, `$FRAME+$SIZE_T*2+$LRSAVE`($sp) + + and r30, r5, r9 # copy length&-16 + mr r5, r6 # copy pointer to key + mr r31, r7 # copy pointer to iv + blt Lcbc_abort + cmpwi r8, 0 # test direction + li r6, -1 + mr r7, r12 # copy vrsave + mtspr 256, r6 # preserve all AltiVec registers + + lvx v24, 0, r31 # load [potentially unaligned] iv + li r9, 15 + ?lvsl $inpperm, 0, r31 + lvx v25, r9, r31 + ?vperm v24, v24, v25, $inpperm + + neg r8, $inp # prepare for unaligned access + vxor v7, v7, v7 + ?lvsl $keyperm, 0, $key + ?lvsr $outperm, 0, $out + ?lvsr $inpperm, 0, r8 # -$inp + vnor $outmask, v7, v7 # 0xff..ff + lvx $inptail, 0, $inp + ?vperm $outmask, v7, $outmask, $outperm + addi $inp, $inp, 15 # 15 is not a typo + lvx $outhead, 0, $out + + beq Lcbc_decrypt + + bl _vpaes_encrypt_preheat + li r0, 16 + +Lcbc_enc_loop: + vmr v0, $inptail + lvx $inptail, 0, $inp + addi $inp, $inp, 16 + ?vperm v0, v0, $inptail, $inpperm + vxor v0, v0, v24 # ^= iv + + bl _vpaes_encrypt_core + + vmr v24, v0 # put aside iv + sub. r30, r30, r0 # len -= 16 + vperm v0, v0, v0, $outperm # rotate right/left + vsel v1, $outhead, v0, $outmask + vmr $outhead, v0 + stvx v1, 0, $out + addi $out, $out, 16 + bne Lcbc_enc_loop + + b Lcbc_done + +.align 5 +Lcbc_decrypt: + bl _vpaes_decrypt_preheat + li r0, 16 + +Lcbc_dec_loop: + vmr v0, $inptail + lvx $inptail, 0, $inp + addi $inp, $inp, 16 + ?vperm v0, v0, $inptail, $inpperm + vmr v25, v0 # put aside input + + bl _vpaes_decrypt_core + + vxor v0, v0, v24 # ^= iv + vmr v24, v25 + sub. r30, r30, r0 # len -= 16 + vperm v0, v0, v0, $outperm # rotate right/left + vsel v1, $outhead, v0, $outmask + vmr $outhead, v0 + stvx v1, 0, $out + addi $out, $out, 16 + bne Lcbc_dec_loop + +Lcbc_done: + addi $out, $out, -1 + lvx v1, 0, $out # redundant in aligned case + vsel v1, $outhead, v1, $outmask + stvx v1, 0, $out + + neg r8, r31 # write [potentially unaligned] iv + ?lvsl $outperm, 0, r8 + li r6, 15 + vnor $outmask, v7, v7 # 0xff..ff + ?vperm $outmask, v7, $outmask, $outperm + lvx $outhead, 0, r31 + vperm v24, v24, v24, $outperm # rotate right/left + vsel v0, $outhead, v24, $outmask + lvx v1, r6, r31 + stvx v0, 0, r31 + vsel v1, v24, v1, $outmask + stvx v1, r6, r31 + + mtspr 256, r7 # restore vrsave + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + lvx v20,r10,$sp + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp +Lcbc_abort: + $POP r0, `$FRAME+$SIZE_T*2+$LRSAVE`($sp) + $POP r30,`$FRAME+$SIZE_T*0`($sp) + $POP r31,`$FRAME+$SIZE_T*1`($sp) + mtlr r0 + addi $sp,$sp,`$FRAME+$SIZE_T*2` + blr + .long 0 + .byte 0,12,0x04,1,0x80,2,6,0 + .long 0 +.size .vpaes_cbc_encrypt,.-.vpaes_cbc_encrypt +___ +} +{ +my ($inp,$bits,$out)=map("r$_",(3..5)); +my $dir="cr1"; +my ($invlo,$invhi,$iptlo,$ipthi,$rcon) = map("v$_",(10..13,24)); + +$code.=<<___; +######################################################## +## ## +## AES key schedule ## +## ## +######################################################## +.align 4 +_vpaes_key_preheat: + mflr r8 + bl Lconsts + mtlr r8 + li r11, 0xc0 # Lk_inv + li r10, 0xd0 + li r9, 0xe0 # L_ipt + li r8, 0xf0 + + vspltisb v8,4 # 0x04..04 + vxor v9,v9,v9 # 0x00..00 + lvx $invlo, r12, r11 # Lk_inv + li r11, 0x120 + lvx $invhi, r12, r10 + li r10, 0x130 + lvx $iptlo, r12, r9 # Lk_ipt + li r9, 0x220 + lvx $ipthi, r12, r8 + li r8, 0x230 + + lvx v14, r12, r11 # Lk_sb1 + li r11, 0x240 + lvx v15, r12, r10 + li r10, 0x250 + + lvx v16, r12, r9 # Lk_dksd + li r9, 0x260 + lvx v17, r12, r8 + li r8, 0x270 + lvx v18, r12, r11 # Lk_dksb + li r11, 0x280 + lvx v19, r12, r10 + li r10, 0x290 + lvx v20, r12, r9 # Lk_dkse + li r9, 0x2a0 + lvx v21, r12, r8 + li r8, 0x2b0 + lvx v22, r12, r11 # Lk_dks9 + lvx v23, r12, r10 + + lvx v24, r12, r9 # Lk_rcon + lvx v25, 0, r12 # Lk_mc_forward[0] + lvx v26, r12, r8 # Lks63 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +.align 4 +_vpaes_schedule_core: + mflr r7 + + bl _vpaes_key_preheat # load the tables + + #lvx v0, 0, $inp # vmovdqu (%rdi), %xmm0 # load key (unaligned) + neg r8, $inp # prepare for unaligned access + lvx v0, 0, $inp + addi $inp, $inp, 15 # 15 is not typo + ?lvsr $inpperm, 0, r8 # -$inp + lvx v6, 0, $inp # v6 serves as inptail + addi $inp, $inp, 8 + ?vperm v0, v0, v6, $inpperm + + # input transform + vmr v3, v0 # vmovdqa %xmm0, %xmm3 + bl _vpaes_schedule_transform + vmr v7, v0 # vmovdqa %xmm0, %xmm7 + + bne $dir, Lschedule_am_decrypting + + # encrypting, output zeroth round key after transform + li r8, 0x30 # mov \$0x30,%r8d + addi r10, r12, 0x80 # lea .Lk_sr(%rip),%r10 + + ?lvsr $outperm, 0, $out # prepare for unaligned access + vnor $outmask, v9, v9 # 0xff..ff + lvx $outhead, 0, $out + ?vperm $outmask, v9, $outmask, $outperm + + #stvx v0, 0, $out # vmovdqu %xmm0, (%rdx) + vperm v1, v0, v0, $outperm # rotate right/left + vsel v2, $outhead, v1, $outmask + vmr $outhead, v1 + stvx v2, 0, $out + b Lschedule_go + +Lschedule_am_decrypting: + srwi r8, $bits, 1 # shr \$1,%r8d + andi. r8, r8, 32 # and \$32,%r8d + xori r8, r8, 32 # xor \$32,%r8d # nbits==192?0:32 + addi r10, r12, 0x80 # lea .Lk_sr(%rip),%r10 + # decrypting, output zeroth round key after shiftrows + lvx v1, r8, r10 # vmovdqa (%r8,%r10), %xmm1 + vperm v4, v3, v3, v1 # vpshufb %xmm1, %xmm3, %xmm3 + + neg r0, $out # prepare for unaligned access + ?lvsl $outperm, 0, r0 + addi $out, $out, 15 # 15 is not typo + vnor $outmask, v9, v9 # 0xff..ff + lvx $outhead, 0, $out + ?vperm $outmask, $outmask, v9, $outperm + + #stvx v4, 0, $out # vmovdqu %xmm3, (%rdx) + vperm v4, v4, v4, $outperm # rotate right/left + vsel v2, $outhead, v4, $outmask + vmr $outhead, v4 + stvx v2, 0, $out + xori r8, r8, 0x30 # xor \$0x30, %r8 + +Lschedule_go: + cmplwi $bits, 192 # cmp \$192, %esi + bgt Lschedule_256 + beq Lschedule_192 + # 128: fall though + +## +## .schedule_128 +## +## 128-bit specific part of key schedule. +## +## This schedule is really simple, because all its parts +## are accomplished by the subroutines. +## +Lschedule_128: + li r0, 10 # mov \$10, %esi + mtctr r0 + +Loop_schedule_128: + bl _vpaes_schedule_round + bdz Lschedule_mangle_last # dec %esi + bl _vpaes_schedule_mangle # write output + b Loop_schedule_128 + +## +## .aes_schedule_192 +## +## 192-bit specific part of key schedule. +## +## The main body of this schedule is the same as the 128-bit +## schedule, but with more smearing. The long, high side is +## stored in %xmm7 as before, and the short, low side is in +## the high bits of %xmm6. +## +## This schedule is somewhat nastier, however, because each +## round produces 192 bits of key material, or 1.5 round keys. +## Therefore, on each cycle we do 2 rounds and produce 3 round +## keys. +## +.align 4 +Lschedule_192: + li r0, 4 # mov \$4, %esi + lvx v0, 0, $inp + ?vperm v0, v6, v0, $inpperm + ?vsldoi v0, v3, v0, 8 # vmovdqu 8(%rdi),%xmm0 # load key part 2 (very unaligned) + bl _vpaes_schedule_transform # input transform + ?vsldoi v6, v0, v9, 8 + ?vsldoi v6, v9, v6, 8 # clobber "low" side with zeros + mtctr r0 + +Loop_schedule_192: + bl _vpaes_schedule_round + ?vsldoi v0, v6, v0, 8 # vpalignr \$8,%xmm6,%xmm0,%xmm0 + bl _vpaes_schedule_mangle # save key n + bl _vpaes_schedule_192_smear + bl _vpaes_schedule_mangle # save key n+1 + bl _vpaes_schedule_round + bdz Lschedule_mangle_last # dec %esi + bl _vpaes_schedule_mangle # save key n+2 + bl _vpaes_schedule_192_smear + b Loop_schedule_192 + +## +## .aes_schedule_256 +## +## 256-bit specific part of key schedule. +## +## The structure here is very similar to the 128-bit +## schedule, but with an additional "low side" in +## %xmm6. The low side's rounds are the same as the +## high side's, except no rcon and no rotation. +## +.align 4 +Lschedule_256: + li r0, 7 # mov \$7, %esi + addi $inp, $inp, 8 + lvx v0, 0, $inp # vmovdqu 16(%rdi),%xmm0 # load key part 2 (unaligned) + ?vperm v0, v6, v0, $inpperm + bl _vpaes_schedule_transform # input transform + mtctr r0 + +Loop_schedule_256: + bl _vpaes_schedule_mangle # output low result + vmr v6, v0 # vmovdqa %xmm0, %xmm6 # save cur_lo in xmm6 + + # high round + bl _vpaes_schedule_round + bdz Lschedule_mangle_last # dec %esi + bl _vpaes_schedule_mangle + + # low round. swap xmm7 and xmm6 + ?vspltw v0, v0, 3 # vpshufd \$0xFF, %xmm0, %xmm0 + vmr v5, v7 # vmovdqa %xmm7, %xmm5 + vmr v7, v6 # vmovdqa %xmm6, %xmm7 + bl _vpaes_schedule_low_round + vmr v7, v5 # vmovdqa %xmm5, %xmm7 + + b Loop_schedule_256 +## +## .aes_schedule_mangle_last +## +## Mangler for last round of key schedule +## Mangles %xmm0 +## when encrypting, outputs out(%xmm0) ^ 63 +## when decrypting, outputs unskew(%xmm0) +## +## Always called right before return... jumps to cleanup and exits +## +.align 4 +Lschedule_mangle_last: + # schedule last round key from xmm0 + li r11, 0x2e0 # lea .Lk_deskew(%rip),%r11 + li r9, 0x2f0 + bne $dir, Lschedule_mangle_last_dec + + # encrypting + lvx v1, r8, r10 # vmovdqa (%r8,%r10),%xmm1 + li r11, 0x2c0 # lea .Lk_opt(%rip), %r11 # prepare to output transform + li r9, 0x2d0 # prepare to output transform + vperm v0, v0, v0, v1 # vpshufb %xmm1, %xmm0, %xmm0 # output permute + + lvx $iptlo, r11, r12 # reload $ipt + lvx $ipthi, r9, r12 + addi $out, $out, 16 # add \$16, %rdx + vxor v0, v0, v26 # vpxor .Lk_s63(%rip), %xmm0, %xmm0 + bl _vpaes_schedule_transform # output transform + + #stvx v0, r0, $out # vmovdqu %xmm0, (%rdx) # save last key + vperm v0, v0, v0, $outperm # rotate right/left + vsel v2, $outhead, v0, $outmask + vmr $outhead, v0 + stvx v2, 0, $out + + addi $out, $out, 15 # 15 is not typo + lvx v1, 0, $out # redundant in aligned case + vsel v1, $outhead, v1, $outmask + stvx v1, 0, $out + b Lschedule_mangle_done + +.align 4 +Lschedule_mangle_last_dec: + lvx $iptlo, r11, r12 # reload $ipt + lvx $ipthi, r9, r12 + addi $out, $out, -16 # add \$-16, %rdx + vxor v0, v0, v26 # vpxor .Lk_s63(%rip), %xmm0, %xmm0 + bl _vpaes_schedule_transform # output transform + + #stvx v0, r0, $out # vmovdqu %xmm0, (%rdx) # save last key + vperm v0, v0, v0, $outperm # rotate right/left + vsel v2, $outhead, v0, $outmask + vmr $outhead, v0 + stvx v2, 0, $out + + addi $out, $out, -15 # -15 is not typo + lvx v1, 0, $out # redundant in aligned case + vsel v1, $outhead, v1, $outmask + stvx v1, 0, $out + +Lschedule_mangle_done: + mtlr r7 + # cleanup + vxor v0, v0, v0 # vpxor %xmm0, %xmm0, %xmm0 + vxor v1, v1, v1 # vpxor %xmm1, %xmm1, %xmm1 + vxor v2, v2, v2 # vpxor %xmm2, %xmm2, %xmm2 + vxor v3, v3, v3 # vpxor %xmm3, %xmm3, %xmm3 + vxor v4, v4, v4 # vpxor %xmm4, %xmm4, %xmm4 + vxor v5, v5, v5 # vpxor %xmm5, %xmm5, %xmm5 + vxor v6, v6, v6 # vpxor %xmm6, %xmm6, %xmm6 + vxor v7, v7, v7 # vpxor %xmm7, %xmm7, %xmm7 + + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +## +## .aes_schedule_192_smear +## +## Smear the short, low side in the 192-bit key schedule. +## +## Inputs: +## %xmm7: high side, b a x y +## %xmm6: low side, d c 0 0 +## %xmm13: 0 +## +## Outputs: +## %xmm6: b+c+d b+c 0 0 +## %xmm0: b+c+d b+c b a +## +.align 4 +_vpaes_schedule_192_smear: + ?vspltw v0, v7, 3 + ?vsldoi v1, v9, v6, 12 # vpshufd \$0x80, %xmm6, %xmm1 # d c 0 0 -> c 0 0 0 + ?vsldoi v0, v7, v0, 8 # vpshufd \$0xFE, %xmm7, %xmm0 # b a _ _ -> b b b a + vxor v6, v6, v1 # vpxor %xmm1, %xmm6, %xmm6 # -> c+d c 0 0 + vxor v6, v6, v0 # vpxor %xmm0, %xmm6, %xmm6 # -> b+c+d b+c b a + vmr v0, v6 + ?vsldoi v6, v6, v9, 8 + ?vsldoi v6, v9, v6, 8 # clobber low side with zeros + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +## +## .aes_schedule_round +## +## Runs one main round of the key schedule on %xmm0, %xmm7 +## +## Specifically, runs subbytes on the high dword of %xmm0 +## then rotates it by one byte and xors into the low dword of +## %xmm7. +## +## Adds rcon from low byte of %xmm8, then rotates %xmm8 for +## next rcon. +## +## Smears the dwords of %xmm7 by xoring the low into the +## second low, result into third, result into highest. +## +## Returns results in %xmm7 = %xmm0. +## Clobbers %xmm1-%xmm4, %r11. +## +.align 4 +_vpaes_schedule_round: + # extract rcon from xmm8 + #vxor v4, v4, v4 # vpxor %xmm4, %xmm4, %xmm4 + ?vsldoi v1, $rcon, v9, 15 # vpalignr \$15, %xmm8, %xmm4, %xmm1 + ?vsldoi $rcon, $rcon, $rcon, 15 # vpalignr \$15, %xmm8, %xmm8, %xmm8 + vxor v7, v7, v1 # vpxor %xmm1, %xmm7, %xmm7 + + # rotate + ?vspltw v0, v0, 3 # vpshufd \$0xFF, %xmm0, %xmm0 + ?vsldoi v0, v0, v0, 1 # vpalignr \$1, %xmm0, %xmm0, %xmm0 + + # fall through... + + # low round: same as high round, but no rotation and no rcon. +_vpaes_schedule_low_round: + # smear xmm7 + ?vsldoi v1, v9, v7, 12 # vpslldq \$4, %xmm7, %xmm1 + vxor v7, v7, v1 # vpxor %xmm1, %xmm7, %xmm7 + vspltisb v1, 0x0f # 0x0f..0f + ?vsldoi v4, v9, v7, 8 # vpslldq \$8, %xmm7, %xmm4 + + # subbytes + vand v1, v1, v0 # vpand %xmm9, %xmm0, %xmm1 # 0 = k + vsrb v0, v0, v8 # vpsrlb \$4, %xmm0, %xmm0 # 1 = i + vxor v7, v7, v4 # vpxor %xmm4, %xmm7, %xmm7 + vperm v2, $invhi, v9, v1 # vpshufb %xmm1, %xmm11, %xmm2 # 2 = a/k + vxor v1, v1, v0 # vpxor %xmm0, %xmm1, %xmm1 # 0 = j + vperm v3, $invlo, v9, v0 # vpshufb %xmm0, %xmm10, %xmm3 # 3 = 1/i + vxor v3, v3, v2 # vpxor %xmm2, %xmm3, %xmm3 # 3 = iak = 1/i + a/k + vperm v4, $invlo, v9, v1 # vpshufb %xmm1, %xmm10, %xmm4 # 4 = 1/j + vxor v7, v7, v26 # vpxor .Lk_s63(%rip), %xmm7, %xmm7 + vperm v3, $invlo, v9, v3 # vpshufb %xmm3, %xmm10, %xmm3 # 2 = 1/iak + vxor v4, v4, v2 # vpxor %xmm2, %xmm4, %xmm4 # 4 = jak = 1/j + a/k + vperm v2, $invlo, v9, v4 # vpshufb %xmm4, %xmm10, %xmm2 # 3 = 1/jak + vxor v3, v3, v1 # vpxor %xmm1, %xmm3, %xmm3 # 2 = io + vxor v2, v2, v0 # vpxor %xmm0, %xmm2, %xmm2 # 3 = jo + vperm v4, v15, v9, v3 # vpshufb %xmm3, %xmm13, %xmm4 # 4 = sbou + vperm v1, v14, v9, v2 # vpshufb %xmm2, %xmm12, %xmm1 # 0 = sb1t + vxor v1, v1, v4 # vpxor %xmm4, %xmm1, %xmm1 # 0 = sbox output + + # add in smeared stuff + vxor v0, v1, v7 # vpxor %xmm7, %xmm1, %xmm0 + vxor v7, v1, v7 # vmovdqa %xmm0, %xmm7 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +## +## .aes_schedule_transform +## +## Linear-transform %xmm0 according to tables at (%r11) +## +## Requires that %xmm9 = 0x0F0F... as in preheat +## Output in %xmm0 +## Clobbers %xmm2 +## +.align 4 +_vpaes_schedule_transform: + #vand v1, v0, v9 # vpand %xmm9, %xmm0, %xmm1 + vsrb v2, v0, v8 # vpsrlb \$4, %xmm0, %xmm0 + # vmovdqa (%r11), %xmm2 # lo + vperm v0, $iptlo, $iptlo, v0 # vpshufb %xmm1, %xmm2, %xmm2 + # vmovdqa 16(%r11), %xmm1 # hi + vperm v2, $ipthi, $ipthi, v2 # vpshufb %xmm0, %xmm1, %xmm0 + vxor v0, v0, v2 # vpxor %xmm2, %xmm0, %xmm0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +## +## .aes_schedule_mangle +## +## Mangle xmm0 from (basis-transformed) standard version +## to our version. +## +## On encrypt, +## xor with 0x63 +## multiply by circulant 0,1,1,1 +## apply shiftrows transform +## +## On decrypt, +## xor with 0x63 +## multiply by "inverse mixcolumns" circulant E,B,D,9 +## deskew +## apply shiftrows transform +## +## +## Writes out to (%rdx), and increments or decrements it +## Keeps track of round number mod 4 in %r8 +## Preserves xmm0 +## Clobbers xmm1-xmm5 +## +.align 4 +_vpaes_schedule_mangle: + #vmr v4, v0 # vmovdqa %xmm0, %xmm4 # save xmm0 for later + # vmovdqa .Lk_mc_forward(%rip),%xmm5 + bne $dir, Lschedule_mangle_dec + + # encrypting + vxor v4, v0, v26 # vpxor .Lk_s63(%rip), %xmm0, %xmm4 + addi $out, $out, 16 # add \$16, %rdx + vperm v4, v4, v4, v25 # vpshufb %xmm5, %xmm4, %xmm4 + vperm v1, v4, v4, v25 # vpshufb %xmm5, %xmm4, %xmm1 + vperm v3, v1, v1, v25 # vpshufb %xmm5, %xmm1, %xmm3 + vxor v4, v4, v1 # vpxor %xmm1, %xmm4, %xmm4 + lvx v1, r8, r10 # vmovdqa (%r8,%r10), %xmm1 + vxor v3, v3, v4 # vpxor %xmm4, %xmm3, %xmm3 + + vperm v3, v3, v3, v1 # vpshufb %xmm1, %xmm3, %xmm3 + addi r8, r8, -16 # add \$-16, %r8 + andi. r8, r8, 0x30 # and \$0x30, %r8 + + #stvx v3, 0, $out # vmovdqu %xmm3, (%rdx) + vperm v1, v3, v3, $outperm # rotate right/left + vsel v2, $outhead, v1, $outmask + vmr $outhead, v1 + stvx v2, 0, $out + blr + +.align 4 +Lschedule_mangle_dec: + # inverse mix columns + # lea .Lk_dksd(%rip),%r11 + vsrb v1, v0, v8 # vpsrlb \$4, %xmm4, %xmm1 # 1 = hi + #and v4, v0, v9 # vpand %xmm9, %xmm4, %xmm4 # 4 = lo + + # vmovdqa 0x00(%r11), %xmm2 + vperm v2, v16, v16, v0 # vpshufb %xmm4, %xmm2, %xmm2 + # vmovdqa 0x10(%r11), %xmm3 + vperm v3, v17, v17, v1 # vpshufb %xmm1, %xmm3, %xmm3 + vxor v3, v3, v2 # vpxor %xmm2, %xmm3, %xmm3 + vperm v3, v3, v9, v25 # vpshufb %xmm5, %xmm3, %xmm3 + + # vmovdqa 0x20(%r11), %xmm2 + vperm v2, v18, v18, v0 # vpshufb %xmm4, %xmm2, %xmm2 + vxor v2, v2, v3 # vpxor %xmm3, %xmm2, %xmm2 + # vmovdqa 0x30(%r11), %xmm3 + vperm v3, v19, v19, v1 # vpshufb %xmm1, %xmm3, %xmm3 + vxor v3, v3, v2 # vpxor %xmm2, %xmm3, %xmm3 + vperm v3, v3, v9, v25 # vpshufb %xmm5, %xmm3, %xmm3 + + # vmovdqa 0x40(%r11), %xmm2 + vperm v2, v20, v20, v0 # vpshufb %xmm4, %xmm2, %xmm2 + vxor v2, v2, v3 # vpxor %xmm3, %xmm2, %xmm2 + # vmovdqa 0x50(%r11), %xmm3 + vperm v3, v21, v21, v1 # vpshufb %xmm1, %xmm3, %xmm3 + vxor v3, v3, v2 # vpxor %xmm2, %xmm3, %xmm3 + + # vmovdqa 0x60(%r11), %xmm2 + vperm v2, v22, v22, v0 # vpshufb %xmm4, %xmm2, %xmm2 + vperm v3, v3, v9, v25 # vpshufb %xmm5, %xmm3, %xmm3 + # vmovdqa 0x70(%r11), %xmm4 + vperm v4, v23, v23, v1 # vpshufb %xmm1, %xmm4, %xmm4 + lvx v1, r8, r10 # vmovdqa (%r8,%r10), %xmm1 + vxor v2, v2, v3 # vpxor %xmm3, %xmm2, %xmm2 + vxor v3, v4, v2 # vpxor %xmm2, %xmm4, %xmm3 + + addi $out, $out, -16 # add \$-16, %rdx + + vperm v3, v3, v3, v1 # vpshufb %xmm1, %xmm3, %xmm3 + addi r8, r8, -16 # add \$-16, %r8 + andi. r8, r8, 0x30 # and \$0x30, %r8 + + #stvx v3, 0, $out # vmovdqu %xmm3, (%rdx) + vperm v1, v3, v3, $outperm # rotate right/left + vsel v2, $outhead, v1, $outmask + vmr $outhead, v1 + stvx v2, 0, $out + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +.globl .vpaes_set_encrypt_key +.align 5 +.vpaes_set_encrypt_key: + $STU $sp,-$FRAME($sp) + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mflr r0 + mfspr r6, 256 # save vrsave + stvx v20,r10,$sp + addi r10,r10,32 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + stw r6,`$FRAME-4`($sp) # save vrsave + li r7, -1 + $PUSH r0, `$FRAME+$LRSAVE`($sp) + mtspr 256, r7 # preserve all AltiVec registers + + srwi r9, $bits, 5 # shr \$5,%eax + addi r9, r9, 6 # add \$5,%eax + stw r9, 240($out) # mov %eax,240(%rdx) # AES_KEY->rounds = nbits/32+5; + + cmplw $dir, $bits, $bits # set encrypt direction + li r8, 0x30 # mov \$0x30,%r8d + bl _vpaes_schedule_core + + $POP r0, `$FRAME+$LRSAVE`($sp) + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mtspr 256, r6 # restore vrsave + mtlr r0 + xor r3, r3, r3 + lvx v20,r10,$sp + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + addi $sp,$sp,$FRAME + blr + .long 0 + .byte 0,12,0x04,1,0x80,0,3,0 + .long 0 +.size .vpaes_set_encrypt_key,.-.vpaes_set_encrypt_key + +.globl .vpaes_set_decrypt_key +.align 4 +.vpaes_set_decrypt_key: + $STU $sp,-$FRAME($sp) + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mflr r0 + mfspr r6, 256 # save vrsave + stvx v20,r10,$sp + addi r10,r10,32 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + stw r6,`$FRAME-4`($sp) # save vrsave + li r7, -1 + $PUSH r0, `$FRAME+$LRSAVE`($sp) + mtspr 256, r7 # preserve all AltiVec registers + + srwi r9, $bits, 5 # shr \$5,%eax + addi r9, r9, 6 # add \$5,%eax + stw r9, 240($out) # mov %eax,240(%rdx) # AES_KEY->rounds = nbits/32+5; + + slwi r9, r9, 4 # shl \$4,%eax + add $out, $out, r9 # lea (%rdx,%rax),%rdx + + cmplwi $dir, $bits, 0 # set decrypt direction + srwi r8, $bits, 1 # shr \$1,%r8d + andi. r8, r8, 32 # and \$32,%r8d + xori r8, r8, 32 # xor \$32,%r8d # nbits==192?0:32 + bl _vpaes_schedule_core + + $POP r0, `$FRAME+$LRSAVE`($sp) + li r10,`15+6*$SIZE_T` + li r11,`31+6*$SIZE_T` + mtspr 256, r6 # restore vrsave + mtlr r0 + xor r3, r3, r3 + lvx v20,r10,$sp + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + addi $sp,$sp,$FRAME + blr + .long 0 + .byte 0,12,0x04,1,0x80,0,3,0 + .long 0 +.size .vpaes_set_decrypt_key,.-.vpaes_set_decrypt_key +___ +} + +my $consts=1; +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + # constants table endian-specific conversion + if ($consts && m/\.long\s+(.+)\s+(\?[a-z]*)$/o) { + my $conv=$2; + my @bytes=(); + + # convert to endian-agnostic format + foreach (split(/,\s+/,$1)) { + my $l = /^0/?oct:int; + push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff; + } + + # little-endian conversion + if ($flavour =~ /le$/o) { + SWITCH: for($conv) { + /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; }; + /\?rev/ && do { @bytes=reverse(@bytes); last; }; + } + } + + #emit + print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n"; + next; + } + $consts=0 if (m/Lconsts:/o); # end of table + + # instructions prefixed with '?' are endian-specific and need + # to be adjusted accordingly... + if ($flavour =~ /le$/o) { # little-endian + s/\?lvsr/lvsl/o or + s/\?lvsl/lvsr/o or + s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or + s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or + s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o; + } else { # big-endian + s/\?([a-z]+)/$1/o; + } + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/aes/asm/vpaes-x86.pl b/crypto/aes/asm/vpaes-x86.pl index 1533e2c3042c..2ba149c3f9d5 100755 --- a/crypto/aes/asm/vpaes-x86.pl +++ b/crypto/aes/asm/vpaes-x86.pl @@ -27,9 +27,10 @@ # # aes-586.pl vpaes-x86.pl # -# Core 2(**) 29.1/42.3/18.3 22.0/25.6(***) -# Nehalem 27.9/40.4/18.1 10.3/12.0 -# Atom 102./119./60.1 64.5/85.3(***) +# Core 2(**) 28.1/41.4/18.3 21.9/25.2(***) +# Nehalem 27.9/40.4/18.1 10.2/11.9 +# Atom 70.7/92.1/60.1 61.1/75.4(***) +# Silvermont 45.4/62.9/24.1 49.2/61.1(***) # # (*) "Hyper-threading" in the context refers rather to cache shared # among multiple cores, than to specifically Intel HTT. As vast @@ -40,8 +41,8 @@ # (**) "Core 2" refers to initial 65nm design, a.k.a. Conroe. # # (***) Less impressive improvement on Core 2 and Atom is due to slow -# pshufb, yet it's respectable +32%/65% improvement on Core 2 -# and +58%/40% on Atom (as implied, over "hyper-threading-safe" +# pshufb, yet it's respectable +28%/64% improvement on Core 2 +# and +15% on Atom (as implied, over "hyper-threading-safe" # code path). # # @@ -183,35 +184,35 @@ $k_dsbo=0x2c0; # decryption sbox final output &movdqa ("xmm1","xmm6") &movdqa ("xmm2",&QWP($k_ipt,$const)); &pandn ("xmm1","xmm0"); - &movdqu ("xmm5",&QWP(0,$key)); - &psrld ("xmm1",4); &pand ("xmm0","xmm6"); + &movdqu ("xmm5",&QWP(0,$key)); &pshufb ("xmm2","xmm0"); &movdqa ("xmm0",&QWP($k_ipt+16,$const)); - &pshufb ("xmm0","xmm1"); &pxor ("xmm2","xmm5"); - &pxor ("xmm0","xmm2"); + &psrld ("xmm1",4); &add ($key,16); + &pshufb ("xmm0","xmm1"); &lea ($base,&DWP($k_mc_backward,$const)); + &pxor ("xmm0","xmm2"); &jmp (&label("enc_entry")); &set_label("enc_loop",16); # middle of middle round &movdqa ("xmm4",&QWP($k_sb1,$const)); # 4 : sb1u - &pshufb ("xmm4","xmm2"); # 4 = sb1u - &pxor ("xmm4","xmm5"); # 4 = sb1u + k &movdqa ("xmm0",&QWP($k_sb1+16,$const));# 0 : sb1t + &pshufb ("xmm4","xmm2"); # 4 = sb1u &pshufb ("xmm0","xmm3"); # 0 = sb1t - &pxor ("xmm0","xmm4"); # 0 = A + &pxor ("xmm4","xmm5"); # 4 = sb1u + k &movdqa ("xmm5",&QWP($k_sb2,$const)); # 4 : sb2u - &pshufb ("xmm5","xmm2"); # 4 = sb2u + &pxor ("xmm0","xmm4"); # 0 = A &movdqa ("xmm1",&QWP(-0x40,$base,$magic));# .Lk_mc_forward[] + &pshufb ("xmm5","xmm2"); # 4 = sb2u &movdqa ("xmm2",&QWP($k_sb2+16,$const));# 2 : sb2t - &pshufb ("xmm2","xmm3"); # 2 = sb2t - &pxor ("xmm2","xmm5"); # 2 = 2A &movdqa ("xmm4",&QWP(0,$base,$magic)); # .Lk_mc_backward[] + &pshufb ("xmm2","xmm3"); # 2 = sb2t &movdqa ("xmm3","xmm0"); # 3 = A + &pxor ("xmm2","xmm5"); # 2 = 2A &pshufb ("xmm0","xmm1"); # 0 = B &add ($key,16); # next key &pxor ("xmm0","xmm2"); # 0 = 2A+B @@ -220,30 +221,30 @@ $k_dsbo=0x2c0; # decryption sbox final output &pxor ("xmm3","xmm0"); # 3 = 2A+B+D &pshufb ("xmm0","xmm1"); # 0 = 2B+C &and ($magic,0x30); # ... mod 4 - &pxor ("xmm0","xmm3"); # 0 = 2A+3B+C+D &sub ($round,1); # nr-- + &pxor ("xmm0","xmm3"); # 0 = 2A+3B+C+D &set_label("enc_entry"); # top of round &movdqa ("xmm1","xmm6"); # 1 : i + &movdqa ("xmm5",&QWP($k_inv+16,$const));# 2 : a/k &pandn ("xmm1","xmm0"); # 1 = i<<4 &psrld ("xmm1",4); # 1 = i &pand ("xmm0","xmm6"); # 0 = k - &movdqa ("xmm5",&QWP($k_inv+16,$const));# 2 : a/k &pshufb ("xmm5","xmm0"); # 2 = a/k - &pxor ("xmm0","xmm1"); # 0 = j &movdqa ("xmm3","xmm7"); # 3 : 1/i + &pxor ("xmm0","xmm1"); # 0 = j &pshufb ("xmm3","xmm1"); # 3 = 1/i - &pxor ("xmm3","xmm5"); # 3 = iak = 1/i + a/k &movdqa ("xmm4","xmm7"); # 4 : 1/j + &pxor ("xmm3","xmm5"); # 3 = iak = 1/i + a/k &pshufb ("xmm4","xmm0"); # 4 = 1/j - &pxor ("xmm4","xmm5"); # 4 = jak = 1/j + a/k &movdqa ("xmm2","xmm7"); # 2 : 1/iak + &pxor ("xmm4","xmm5"); # 4 = jak = 1/j + a/k &pshufb ("xmm2","xmm3"); # 2 = 1/iak - &pxor ("xmm2","xmm0"); # 2 = io &movdqa ("xmm3","xmm7"); # 3 : 1/jak - &movdqu ("xmm5",&QWP(0,$key)); + &pxor ("xmm2","xmm0"); # 2 = io &pshufb ("xmm3","xmm4"); # 3 = 1/jak + &movdqu ("xmm5",&QWP(0,$key)); &pxor ("xmm3","xmm1"); # 3 = jo &jnz (&label("enc_loop")); @@ -265,8 +266,8 @@ $k_dsbo=0x2c0; # decryption sbox final output ## Same API as encryption core. ## &function_begin_B("_vpaes_decrypt_core"); - &mov ($round,&DWP(240,$key)); &lea ($base,&DWP($k_dsbd,$const)); + &mov ($round,&DWP(240,$key)); &movdqa ("xmm1","xmm6"); &movdqa ("xmm2",&QWP($k_dipt-$k_dsbd,$base)); &pandn ("xmm1","xmm0"); @@ -292,62 +293,61 @@ $k_dsbo=0x2c0; # decryption sbox final output ## Inverse mix columns ## &movdqa ("xmm4",&QWP(-0x20,$base)); # 4 : sb9u + &movdqa ("xmm1",&QWP(-0x10,$base)); # 0 : sb9t &pshufb ("xmm4","xmm2"); # 4 = sb9u - &pxor ("xmm4","xmm0"); - &movdqa ("xmm0",&QWP(-0x10,$base)); # 0 : sb9t - &pshufb ("xmm0","xmm3"); # 0 = sb9t - &pxor ("xmm0","xmm4"); # 0 = ch - &add ($key,16); # next round key - - &pshufb ("xmm0","xmm5"); # MC ch + &pshufb ("xmm1","xmm3"); # 0 = sb9t + &pxor ("xmm0","xmm4"); &movdqa ("xmm4",&QWP(0,$base)); # 4 : sbdu - &pshufb ("xmm4","xmm2"); # 4 = sbdu - &pxor ("xmm4","xmm0"); # 4 = ch - &movdqa ("xmm0",&QWP(0x10,$base)); # 0 : sbdt - &pshufb ("xmm0","xmm3"); # 0 = sbdt - &pxor ("xmm0","xmm4"); # 0 = ch - &sub ($round,1); # nr-- + &pxor ("xmm0","xmm1"); # 0 = ch + &movdqa ("xmm1",&QWP(0x10,$base)); # 0 : sbdt + &pshufb ("xmm4","xmm2"); # 4 = sbdu &pshufb ("xmm0","xmm5"); # MC ch + &pshufb ("xmm1","xmm3"); # 0 = sbdt + &pxor ("xmm0","xmm4"); # 4 = ch &movdqa ("xmm4",&QWP(0x20,$base)); # 4 : sbbu - &pshufb ("xmm4","xmm2"); # 4 = sbbu - &pxor ("xmm4","xmm0"); # 4 = ch - &movdqa ("xmm0",&QWP(0x30,$base)); # 0 : sbbt - &pshufb ("xmm0","xmm3"); # 0 = sbbt - &pxor ("xmm0","xmm4"); # 0 = ch + &pxor ("xmm0","xmm1"); # 0 = ch + &movdqa ("xmm1",&QWP(0x30,$base)); # 0 : sbbt + &pshufb ("xmm4","xmm2"); # 4 = sbbu &pshufb ("xmm0","xmm5"); # MC ch + &pshufb ("xmm1","xmm3"); # 0 = sbbt + &pxor ("xmm0","xmm4"); # 4 = ch &movdqa ("xmm4",&QWP(0x40,$base)); # 4 : sbeu - &pshufb ("xmm4","xmm2"); # 4 = sbeu - &pxor ("xmm4","xmm0"); # 4 = ch - &movdqa ("xmm0",&QWP(0x50,$base)); # 0 : sbet - &pshufb ("xmm0","xmm3"); # 0 = sbet - &pxor ("xmm0","xmm4"); # 0 = ch + &pxor ("xmm0","xmm1"); # 0 = ch + &movdqa ("xmm1",&QWP(0x50,$base)); # 0 : sbet + &pshufb ("xmm4","xmm2"); # 4 = sbeu + &pshufb ("xmm0","xmm5"); # MC ch + &pshufb ("xmm1","xmm3"); # 0 = sbet + &pxor ("xmm0","xmm4"); # 4 = ch + &add ($key,16); # next round key &palignr("xmm5","xmm5",12); + &pxor ("xmm0","xmm1"); # 0 = ch + &sub ($round,1); # nr-- &set_label("dec_entry"); # top of round &movdqa ("xmm1","xmm6"); # 1 : i + &movdqa ("xmm2",&QWP($k_inv+16,$const));# 2 : a/k &pandn ("xmm1","xmm0"); # 1 = i<<4 - &psrld ("xmm1",4); # 1 = i &pand ("xmm0","xmm6"); # 0 = k - &movdqa ("xmm2",&QWP($k_inv+16,$const));# 2 : a/k + &psrld ("xmm1",4); # 1 = i &pshufb ("xmm2","xmm0"); # 2 = a/k - &pxor ("xmm0","xmm1"); # 0 = j &movdqa ("xmm3","xmm7"); # 3 : 1/i + &pxor ("xmm0","xmm1"); # 0 = j &pshufb ("xmm3","xmm1"); # 3 = 1/i - &pxor ("xmm3","xmm2"); # 3 = iak = 1/i + a/k &movdqa ("xmm4","xmm7"); # 4 : 1/j + &pxor ("xmm3","xmm2"); # 3 = iak = 1/i + a/k &pshufb ("xmm4","xmm0"); # 4 = 1/j &pxor ("xmm4","xmm2"); # 4 = jak = 1/j + a/k &movdqa ("xmm2","xmm7"); # 2 : 1/iak &pshufb ("xmm2","xmm3"); # 2 = 1/iak - &pxor ("xmm2","xmm0"); # 2 = io &movdqa ("xmm3","xmm7"); # 3 : 1/jak + &pxor ("xmm2","xmm0"); # 2 = io &pshufb ("xmm3","xmm4"); # 3 = 1/jak - &pxor ("xmm3","xmm1"); # 3 = jo &movdqu ("xmm0",&QWP(0,$key)); + &pxor ("xmm3","xmm1"); # 3 = jo &jnz (&label("dec_loop")); # middle of last round @@ -542,12 +542,12 @@ $k_dsbo=0x2c0; # decryption sbox final output ## %xmm0: b+c+d b+c b a ## &function_begin_B("_vpaes_schedule_192_smear"); - &pshufd ("xmm0","xmm6",0x80); # d c 0 0 -> c 0 0 0 - &pxor ("xmm6","xmm0"); # -> c+d c 0 0 + &pshufd ("xmm1","xmm6",0x80); # d c 0 0 -> c 0 0 0 &pshufd ("xmm0","xmm7",0xFE); # b a _ _ -> b b b a + &pxor ("xmm6","xmm1"); # -> c+d c 0 0 + &pxor ("xmm1","xmm1"); &pxor ("xmm6","xmm0"); # -> b+c+d b+c b a &movdqa ("xmm0","xmm6"); - &pxor ("xmm1","xmm1"); &movhlps("xmm6","xmm1"); # clobber low side with zeros &ret (); &function_end_B("_vpaes_schedule_192_smear"); diff --git a/crypto/aes/asm/vpaes-x86_64.pl b/crypto/aes/asm/vpaes-x86_64.pl index bd7f45b85091..f2ef318fae4e 100755 --- a/crypto/aes/asm/vpaes-x86_64.pl +++ b/crypto/aes/asm/vpaes-x86_64.pl @@ -27,9 +27,10 @@ # # aes-x86_64.pl vpaes-x86_64.pl # -# Core 2(**) 30.5/43.7/14.3 21.8/25.7(***) -# Nehalem 30.5/42.2/14.6 9.8/11.8 -# Atom 63.9/79.0/32.1 64.0/84.8(***) +# Core 2(**) 29.6/41.1/14.3 21.9/25.2(***) +# Nehalem 29.6/40.3/14.6 10.0/11.8 +# Atom 57.3/74.2/32.1 60.9/77.2(***) +# Silvermont 52.7/64.0/19.5 48.8/60.8(***) # # (*) "Hyper-threading" in the context refers rather to cache shared # among multiple cores, than to specifically Intel HTT. As vast @@ -40,7 +41,7 @@ # (**) "Core 2" refers to initial 65nm design, a.k.a. Conroe. # # (***) Less impressive improvement on Core 2 and Atom is due to slow -# pshufb, yet it's respectable +40%/78% improvement on Core 2 +# pshufb, yet it's respectable +36%/62% improvement on Core 2 # (as implied, over "hyper-threading-safe" code path). # # @@ -95,8 +96,8 @@ _vpaes_encrypt_core: movdqa .Lk_ipt+16(%rip), %xmm0 # ipthi pshufb %xmm1, %xmm0 pxor %xmm5, %xmm2 - pxor %xmm2, %xmm0 add \$16, %r9 + pxor %xmm2, %xmm0 lea .Lk_mc_backward(%rip),%r10 jmp .Lenc_entry @@ -104,19 +105,19 @@ _vpaes_encrypt_core: .Lenc_loop: # middle of middle round movdqa %xmm13, %xmm4 # 4 : sb1u - pshufb %xmm2, %xmm4 # 4 = sb1u - pxor %xmm5, %xmm4 # 4 = sb1u + k movdqa %xmm12, %xmm0 # 0 : sb1t + pshufb %xmm2, %xmm4 # 4 = sb1u pshufb %xmm3, %xmm0 # 0 = sb1t - pxor %xmm4, %xmm0 # 0 = A + pxor %xmm5, %xmm4 # 4 = sb1u + k movdqa %xmm15, %xmm5 # 4 : sb2u - pshufb %xmm2, %xmm5 # 4 = sb2u + pxor %xmm4, %xmm0 # 0 = A movdqa -0x40(%r11,%r10), %xmm1 # .Lk_mc_forward[] + pshufb %xmm2, %xmm5 # 4 = sb2u + movdqa (%r11,%r10), %xmm4 # .Lk_mc_backward[] movdqa %xmm14, %xmm2 # 2 : sb2t pshufb %xmm3, %xmm2 # 2 = sb2t - pxor %xmm5, %xmm2 # 2 = 2A - movdqa (%r11,%r10), %xmm4 # .Lk_mc_backward[] movdqa %xmm0, %xmm3 # 3 = A + pxor %xmm5, %xmm2 # 2 = 2A pshufb %xmm1, %xmm0 # 0 = B add \$16, %r9 # next key pxor %xmm2, %xmm0 # 0 = 2A+B @@ -125,30 +126,30 @@ _vpaes_encrypt_core: pxor %xmm0, %xmm3 # 3 = 2A+B+D pshufb %xmm1, %xmm0 # 0 = 2B+C and \$0x30, %r11 # ... mod 4 - pxor %xmm3, %xmm0 # 0 = 2A+3B+C+D sub \$1,%rax # nr-- + pxor %xmm3, %xmm0 # 0 = 2A+3B+C+D .Lenc_entry: # top of round movdqa %xmm9, %xmm1 # 1 : i + movdqa %xmm11, %xmm5 # 2 : a/k pandn %xmm0, %xmm1 # 1 = i<<4 psrld \$4, %xmm1 # 1 = i pand %xmm9, %xmm0 # 0 = k - movdqa %xmm11, %xmm5 # 2 : a/k pshufb %xmm0, %xmm5 # 2 = a/k - pxor %xmm1, %xmm0 # 0 = j movdqa %xmm10, %xmm3 # 3 : 1/i + pxor %xmm1, %xmm0 # 0 = j pshufb %xmm1, %xmm3 # 3 = 1/i - pxor %xmm5, %xmm3 # 3 = iak = 1/i + a/k movdqa %xmm10, %xmm4 # 4 : 1/j + pxor %xmm5, %xmm3 # 3 = iak = 1/i + a/k pshufb %xmm0, %xmm4 # 4 = 1/j - pxor %xmm5, %xmm4 # 4 = jak = 1/j + a/k movdqa %xmm10, %xmm2 # 2 : 1/iak + pxor %xmm5, %xmm4 # 4 = jak = 1/j + a/k pshufb %xmm3, %xmm2 # 2 = 1/iak - pxor %xmm0, %xmm2 # 2 = io movdqa %xmm10, %xmm3 # 3 : 1/jak - movdqu (%r9), %xmm5 + pxor %xmm0, %xmm2 # 2 = io pshufb %xmm4, %xmm3 # 3 = 1/jak + movdqu (%r9), %xmm5 pxor %xmm1, %xmm3 # 3 = jo jnz .Lenc_loop @@ -201,62 +202,61 @@ _vpaes_decrypt_core: ## Inverse mix columns ## movdqa -0x20(%r10),%xmm4 # 4 : sb9u + movdqa -0x10(%r10),%xmm1 # 0 : sb9t pshufb %xmm2, %xmm4 # 4 = sb9u - pxor %xmm0, %xmm4 - movdqa -0x10(%r10),%xmm0 # 0 : sb9t - pshufb %xmm3, %xmm0 # 0 = sb9t - pxor %xmm4, %xmm0 # 0 = ch - add \$16, %r9 # next round key - - pshufb %xmm5, %xmm0 # MC ch + pshufb %xmm3, %xmm1 # 0 = sb9t + pxor %xmm4, %xmm0 movdqa 0x00(%r10),%xmm4 # 4 : sbdu + pxor %xmm1, %xmm0 # 0 = ch + movdqa 0x10(%r10),%xmm1 # 0 : sbdt + pshufb %xmm2, %xmm4 # 4 = sbdu - pxor %xmm0, %xmm4 # 4 = ch - movdqa 0x10(%r10),%xmm0 # 0 : sbdt - pshufb %xmm3, %xmm0 # 0 = sbdt - pxor %xmm4, %xmm0 # 0 = ch - sub \$1,%rax # nr-- - pshufb %xmm5, %xmm0 # MC ch + pshufb %xmm3, %xmm1 # 0 = sbdt + pxor %xmm4, %xmm0 # 4 = ch movdqa 0x20(%r10),%xmm4 # 4 : sbbu + pxor %xmm1, %xmm0 # 0 = ch + movdqa 0x30(%r10),%xmm1 # 0 : sbbt + pshufb %xmm2, %xmm4 # 4 = sbbu - pxor %xmm0, %xmm4 # 4 = ch - movdqa 0x30(%r10),%xmm0 # 0 : sbbt - pshufb %xmm3, %xmm0 # 0 = sbbt - pxor %xmm4, %xmm0 # 0 = ch - pshufb %xmm5, %xmm0 # MC ch + pshufb %xmm3, %xmm1 # 0 = sbbt + pxor %xmm4, %xmm0 # 4 = ch movdqa 0x40(%r10),%xmm4 # 4 : sbeu - pshufb %xmm2, %xmm4 # 4 = sbeu - pxor %xmm0, %xmm4 # 4 = ch - movdqa 0x50(%r10),%xmm0 # 0 : sbet - pshufb %xmm3, %xmm0 # 0 = sbet - pxor %xmm4, %xmm0 # 0 = ch + pxor %xmm1, %xmm0 # 0 = ch + movdqa 0x50(%r10),%xmm1 # 0 : sbet + pshufb %xmm2, %xmm4 # 4 = sbeu + pshufb %xmm5, %xmm0 # MC ch + pshufb %xmm3, %xmm1 # 0 = sbet + pxor %xmm4, %xmm0 # 4 = ch + add \$16, %r9 # next round key palignr \$12, %xmm5, %xmm5 - + pxor %xmm1, %xmm0 # 0 = ch + sub \$1,%rax # nr-- + .Ldec_entry: # top of round movdqa %xmm9, %xmm1 # 1 : i pandn %xmm0, %xmm1 # 1 = i<<4 + movdqa %xmm11, %xmm2 # 2 : a/k psrld \$4, %xmm1 # 1 = i pand %xmm9, %xmm0 # 0 = k - movdqa %xmm11, %xmm2 # 2 : a/k pshufb %xmm0, %xmm2 # 2 = a/k - pxor %xmm1, %xmm0 # 0 = j movdqa %xmm10, %xmm3 # 3 : 1/i + pxor %xmm1, %xmm0 # 0 = j pshufb %xmm1, %xmm3 # 3 = 1/i - pxor %xmm2, %xmm3 # 3 = iak = 1/i + a/k movdqa %xmm10, %xmm4 # 4 : 1/j + pxor %xmm2, %xmm3 # 3 = iak = 1/i + a/k pshufb %xmm0, %xmm4 # 4 = 1/j pxor %xmm2, %xmm4 # 4 = jak = 1/j + a/k movdqa %xmm10, %xmm2 # 2 : 1/iak pshufb %xmm3, %xmm2 # 2 = 1/iak - pxor %xmm0, %xmm2 # 2 = io movdqa %xmm10, %xmm3 # 3 : 1/jak + pxor %xmm0, %xmm2 # 2 = io pshufb %xmm4, %xmm3 # 3 = 1/jak - pxor %xmm1, %xmm3 # 3 = jo movdqu (%r9), %xmm0 + pxor %xmm1, %xmm3 # 3 = jo jnz .Ldec_loop # middle of last round @@ -464,12 +464,12 @@ _vpaes_schedule_core: .type _vpaes_schedule_192_smear,\@abi-omnipotent .align 16 _vpaes_schedule_192_smear: - pshufd \$0x80, %xmm6, %xmm0 # d c 0 0 -> c 0 0 0 - pxor %xmm0, %xmm6 # -> c+d c 0 0 + pshufd \$0x80, %xmm6, %xmm1 # d c 0 0 -> c 0 0 0 pshufd \$0xFE, %xmm7, %xmm0 # b a _ _ -> b b b a + pxor %xmm1, %xmm6 # -> c+d c 0 0 + pxor %xmm1, %xmm1 pxor %xmm0, %xmm6 # -> b+c+d b+c b a movdqa %xmm6, %xmm0 - pxor %xmm1, %xmm1 movhlps %xmm1, %xmm6 # clobber low side with zeros ret .size _vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear diff --git a/crypto/arm64cpuid.S b/crypto/arm64cpuid.S new file mode 100644 index 000000000000..4778ac1deacc --- /dev/null +++ b/crypto/arm64cpuid.S @@ -0,0 +1,46 @@ +#include "arm_arch.h" + +.text +.arch armv8-a+crypto + +.align 5 +.global _armv7_neon_probe +.type _armv7_neon_probe,%function +_armv7_neon_probe: + orr v15.16b, v15.16b, v15.16b + ret +.size _armv7_neon_probe,.-_armv7_neon_probe + +.global _armv7_tick +.type _armv7_tick,%function +_armv7_tick: + mrs x0, CNTVCT_EL0 + ret +.size _armv7_tick,.-_armv7_tick + +.global _armv8_aes_probe +.type _armv8_aes_probe,%function +_armv8_aes_probe: + aese v0.16b, v0.16b + ret +.size _armv8_aes_probe,.-_armv8_aes_probe + +.global _armv8_sha1_probe +.type _armv8_sha1_probe,%function +_armv8_sha1_probe: + sha1h s0, s0 + ret +.size _armv8_sha1_probe,.-_armv8_sha1_probe + +.global _armv8_sha256_probe +.type _armv8_sha256_probe,%function +_armv8_sha256_probe: + sha256su0 v0.4s, v0.4s + ret +.size _armv8_sha256_probe,.-_armv8_sha256_probe +.global _armv8_pmull_probe +.type _armv8_pmull_probe,%function +_armv8_pmull_probe: + pmull v0.1q, v0.1d, v0.1d + ret +.size _armv8_pmull_probe,.-_armv8_pmull_probe diff --git a/crypto/arm_arch.h b/crypto/arm_arch.h index b65437162596..9d6e58880d0e 100644 --- a/crypto/arm_arch.h +++ b/crypto/arm_arch.h @@ -10,13 +10,24 @@ # define __ARMEL__ # endif # elif defined(__GNUC__) +# if defined(__aarch64__) +# define __ARM_ARCH__ 8 +# if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define __ARMEB__ +# else +# define __ARMEL__ +# endif /* * Why doesn't gcc define __ARM_ARCH__? Instead it defines * bunch of below macros. See all_architectires[] table in * gcc/config/arm/arm.c. On a side note it defines * __ARMEL__/__ARMEB__ for little-/big-endian. */ -# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ +# elif defined(__ARM_ARCH) +# define __ARM_ARCH__ __ARM_ARCH +# elif defined(__ARM_ARCH_8A__) +# define __ARM_ARCH__ 8 +# elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__) || \ defined(__ARM_ARCH_7EM__) # define __ARM_ARCH__ 7 @@ -41,11 +52,27 @@ # include # endif +# if !defined(__ARM_MAX_ARCH__) +# define __ARM_MAX_ARCH__ __ARM_ARCH__ +# endif + +# if __ARM_MAX_ARCH__<__ARM_ARCH__ +# error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__" +# elif __ARM_MAX_ARCH__!=__ARM_ARCH__ +# if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__) +# error "can't build universal big-endian binary" +# endif +# endif + # if !__ASSEMBLER__ extern unsigned int OPENSSL_armcap_P; - -# define ARMV7_NEON (1<<0) -# define ARMV7_TICK (1<<1) # endif +# define ARMV7_NEON (1<<0) +# define ARMV7_TICK (1<<1) +# define ARMV8_AES (1<<2) +# define ARMV8_SHA1 (1<<3) +# define ARMV8_SHA256 (1<<4) +# define ARMV8_PMULL (1<<5) + #endif diff --git a/crypto/armcap.c b/crypto/armcap.c index 28522ea86784..356fa152871f 100644 --- a/crypto/armcap.c +++ b/crypto/armcap.c @@ -7,8 +7,18 @@ #include "arm_arch.h" -unsigned int OPENSSL_armcap_P; +unsigned int OPENSSL_armcap_P = 0; +#if __ARM_MAX_ARCH__<7 +void OPENSSL_cpuid_setup(void) +{ +} + +unsigned long OPENSSL_rdtsc(void) +{ + return 0; +} +#else static sigset_t all_masked; static sigjmp_buf ill_jmp; @@ -22,9 +32,13 @@ static void ill_handler(int sig) * ARM compilers support inline assembler... */ void _armv7_neon_probe(void); -unsigned int _armv7_tick(void); +void _armv8_aes_probe(void); +void _armv8_sha1_probe(void); +void _armv8_sha256_probe(void); +void _armv8_pmull_probe(void); +unsigned long _armv7_tick(void); -unsigned int OPENSSL_rdtsc(void) +unsigned long OPENSSL_rdtsc(void) { if (OPENSSL_armcap_P & ARMV7_TICK) return _armv7_tick(); @@ -32,9 +46,44 @@ unsigned int OPENSSL_rdtsc(void) return 0; } -#if defined(__GNUC__) && __GNUC__>=2 +/* + * Use a weak reference to getauxval() so we can use it if it is available but + * don't break the build if it is not. + */ +# if defined(__GNUC__) && __GNUC__>=2 void OPENSSL_cpuid_setup(void) __attribute__ ((constructor)); -#endif +extern unsigned long getauxval(unsigned long type) __attribute__ ((weak)); +# else +static unsigned long (*getauxval) (unsigned long) = NULL; +# endif + +/* + * ARM puts the the feature bits for Crypto Extensions in AT_HWCAP2, whereas + * AArch64 used AT_HWCAP. + */ +# if defined(__arm__) || defined (__arm) +# define HWCAP 16 + /* AT_HWCAP */ +# define HWCAP_NEON (1 << 12) + +# define HWCAP_CE 26 + /* AT_HWCAP2 */ +# define HWCAP_CE_AES (1 << 0) +# define HWCAP_CE_PMULL (1 << 1) +# define HWCAP_CE_SHA1 (1 << 2) +# define HWCAP_CE_SHA256 (1 << 3) +# elif defined(__aarch64__) +# define HWCAP 16 + /* AT_HWCAP */ +# define HWCAP_NEON (1 << 1) + +# define HWCAP_CE HWCAP +# define HWCAP_CE_AES (1 << 3) +# define HWCAP_CE_PMULL (1 << 4) +# define HWCAP_CE_SHA1 (1 << 5) +# define HWCAP_CE_SHA256 (1 << 6) +# endif + void OPENSSL_cpuid_setup(void) { char *e; @@ -47,7 +96,7 @@ void OPENSSL_cpuid_setup(void) trigger = 1; if ((e = getenv("OPENSSL_armcap"))) { - OPENSSL_armcap_P = strtoul(e, NULL, 0); + OPENSSL_armcap_P = (unsigned int)strtoul(e, NULL, 0); return; } @@ -67,9 +116,42 @@ void OPENSSL_cpuid_setup(void) sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset); sigaction(SIGILL, &ill_act, &ill_oact); - if (sigsetjmp(ill_jmp, 1) == 0) { + if (getauxval != NULL) { + if (getauxval(HWCAP) & HWCAP_NEON) { + unsigned long hwcap = getauxval(HWCAP_CE); + + OPENSSL_armcap_P |= ARMV7_NEON; + + if (hwcap & HWCAP_CE_AES) + OPENSSL_armcap_P |= ARMV8_AES; + + if (hwcap & HWCAP_CE_PMULL) + OPENSSL_armcap_P |= ARMV8_PMULL; + + if (hwcap & HWCAP_CE_SHA1) + OPENSSL_armcap_P |= ARMV8_SHA1; + + if (hwcap & HWCAP_CE_SHA256) + OPENSSL_armcap_P |= ARMV8_SHA256; + } + } else if (sigsetjmp(ill_jmp, 1) == 0) { _armv7_neon_probe(); OPENSSL_armcap_P |= ARMV7_NEON; + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_pmull_probe(); + OPENSSL_armcap_P |= ARMV8_PMULL | ARMV8_AES; + } else if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_aes_probe(); + OPENSSL_armcap_P |= ARMV8_AES; + } + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_sha1_probe(); + OPENSSL_armcap_P |= ARMV8_SHA1; + } + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_sha256_probe(); + OPENSSL_armcap_P |= ARMV8_SHA256; + } } if (sigsetjmp(ill_jmp, 1) == 0) { _armv7_tick(); @@ -79,3 +161,4 @@ void OPENSSL_cpuid_setup(void) sigaction(SIGILL, &ill_oact, NULL); sigprocmask(SIG_SETMASK, &oset, NULL); } +#endif diff --git a/crypto/armv4cpuid.S b/crypto/armv4cpuid.S index 2d618deaa43e..65010ae4fe06 100644 --- a/crypto/armv4cpuid.S +++ b/crypto/armv4cpuid.S @@ -4,20 +4,6 @@ .code 32 .align 5 -.global _armv7_neon_probe -.type _armv7_neon_probe,%function -_armv7_neon_probe: - .word 0xf26ee1fe @ vorr q15,q15,q15 - .word 0xe12fff1e @ bx lr -.size _armv7_neon_probe,.-_armv7_neon_probe - -.global _armv7_tick -.type _armv7_tick,%function -_armv7_tick: - mrc p15,0,r0,c9,c13,0 - .word 0xe12fff1e @ bx lr -.size _armv7_tick,.-_armv7_tick - .global OPENSSL_atomic_add .type OPENSSL_atomic_add,%function OPENSSL_atomic_add: @@ -28,7 +14,7 @@ OPENSSL_atomic_add: cmp r2,#0 bne .Ladd mov r0,r3 - .word 0xe12fff1e @ bx lr + bx lr #else stmdb sp!,{r4-r6,lr} ldr r2,.Lspinlock @@ -81,62 +67,131 @@ OPENSSL_cleanse: adds r1,r1,#4 bne .Little .Lcleanse_done: +#if __ARM_ARCH__>=5 + bx lr +#else tst lr,#1 moveq pc,lr .word 0xe12fff1e @ bx lr +#endif .size OPENSSL_cleanse,.-OPENSSL_cleanse +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.align 5 +.global _armv7_neon_probe +.type _armv7_neon_probe,%function +_armv7_neon_probe: + vorr q0,q0,q0 + bx lr +.size _armv7_neon_probe,.-_armv7_neon_probe + +.global _armv7_tick +.type _armv7_tick,%function +_armv7_tick: + mrrc p15,1,r0,r1,c14 @ CNTVCT + bx lr +.size _armv7_tick,.-_armv7_tick + +.global _armv8_aes_probe +.type _armv8_aes_probe,%function +_armv8_aes_probe: + .byte 0x00,0x03,0xb0,0xf3 @ aese.8 q0,q0 + bx lr +.size _armv8_aes_probe,.-_armv8_aes_probe + +.global _armv8_sha1_probe +.type _armv8_sha1_probe,%function +_armv8_sha1_probe: + .byte 0x40,0x0c,0x00,0xf2 @ sha1c.32 q0,q0,q0 + bx lr +.size _armv8_sha1_probe,.-_armv8_sha1_probe + +.global _armv8_sha256_probe +.type _armv8_sha256_probe,%function +_armv8_sha256_probe: + .byte 0x40,0x0c,0x00,0xf3 @ sha256h.32 q0,q0,q0 + bx lr +.size _armv8_sha256_probe,.-_armv8_sha256_probe +.global _armv8_pmull_probe +.type _armv8_pmull_probe,%function +_armv8_pmull_probe: + .byte 0x00,0x0e,0xa0,0xf2 @ vmull.p64 q0,d0,d0 + bx lr +.size _armv8_pmull_probe,.-_armv8_pmull_probe +#endif + .global OPENSSL_wipe_cpu .type OPENSSL_wipe_cpu,%function OPENSSL_wipe_cpu: +#if __ARM_MAX_ARCH__>=7 ldr r0,.LOPENSSL_armcap adr r1,.LOPENSSL_armcap ldr r0,[r1,r0] +#endif eor r2,r2,r2 eor r3,r3,r3 eor ip,ip,ip +#if __ARM_MAX_ARCH__>=7 tst r0,#1 beq .Lwipe_done - .word 0xf3000150 @ veor q0, q0, q0 - .word 0xf3022152 @ veor q1, q1, q1 - .word 0xf3044154 @ veor q2, q2, q2 - .word 0xf3066156 @ veor q3, q3, q3 - .word 0xf34001f0 @ veor q8, q8, q8 - .word 0xf34221f2 @ veor q9, q9, q9 - .word 0xf34441f4 @ veor q10, q10, q10 - .word 0xf34661f6 @ veor q11, q11, q11 - .word 0xf34881f8 @ veor q12, q12, q12 - .word 0xf34aa1fa @ veor q13, q13, q13 - .word 0xf34cc1fc @ veor q14, q14, q14 - .word 0xf34ee1fe @ veor q15, q15, q15 + veor q0, q0, q0 + veor q1, q1, q1 + veor q2, q2, q2 + veor q3, q3, q3 + veor q8, q8, q8 + veor q9, q9, q9 + veor q10, q10, q10 + veor q11, q11, q11 + veor q12, q12, q12 + veor q13, q13, q13 + veor q14, q14, q14 + veor q15, q15, q15 .Lwipe_done: +#endif mov r0,sp +#if __ARM_ARCH__>=5 + bx lr +#else tst lr,#1 moveq pc,lr .word 0xe12fff1e @ bx lr +#endif .size OPENSSL_wipe_cpu,.-OPENSSL_wipe_cpu .global OPENSSL_instrument_bus .type OPENSSL_instrument_bus,%function OPENSSL_instrument_bus: eor r0,r0,r0 +#if __ARM_ARCH__>=5 + bx lr +#else tst lr,#1 moveq pc,lr .word 0xe12fff1e @ bx lr +#endif .size OPENSSL_instrument_bus,.-OPENSSL_instrument_bus .global OPENSSL_instrument_bus2 .type OPENSSL_instrument_bus2,%function OPENSSL_instrument_bus2: eor r0,r0,r0 +#if __ARM_ARCH__>=5 + bx lr +#else tst lr,#1 moveq pc,lr .word 0xe12fff1e @ bx lr +#endif .size OPENSSL_instrument_bus2,.-OPENSSL_instrument_bus2 .align 5 +#if __ARM_MAX_ARCH__>=7 .LOPENSSL_armcap: .word OPENSSL_armcap_P-.LOPENSSL_armcap +#endif #if __ARM_ARCH__>=6 .align 5 #else diff --git a/crypto/asn1/Makefile b/crypto/asn1/Makefile index d774f7821238..330fe81b740c 100644 --- a/crypto/asn1/Makefile +++ b/crypto/asn1/Makefile @@ -176,7 +176,7 @@ a_gentm.o: ../../include/openssl/err.h ../../include/openssl/lhash.h a_gentm.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h a_gentm.o: ../../include/openssl/ossl_typ.h ../../include/openssl/safestack.h a_gentm.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h -a_gentm.o: ../cryptlib.h ../o_time.h a_gentm.c +a_gentm.o: ../cryptlib.h ../o_time.h a_gentm.c asn1_locl.h a_i2d_fp.o: ../../e_os.h ../../include/openssl/asn1.h a_i2d_fp.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h a_i2d_fp.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h @@ -277,6 +277,7 @@ a_time.o: ../../include/openssl/lhash.h ../../include/openssl/opensslconf.h a_time.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h a_time.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h a_time.o: ../../include/openssl/symhacks.h ../cryptlib.h ../o_time.h a_time.c +a_time.o: asn1_locl.h a_type.o: ../../e_os.h ../../include/openssl/asn1.h a_type.o: ../../include/openssl/asn1t.h ../../include/openssl/bio.h a_type.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h @@ -293,7 +294,7 @@ a_utctm.o: ../../include/openssl/err.h ../../include/openssl/lhash.h a_utctm.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h a_utctm.o: ../../include/openssl/ossl_typ.h ../../include/openssl/safestack.h a_utctm.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h -a_utctm.o: ../cryptlib.h ../o_time.h a_utctm.c +a_utctm.o: ../cryptlib.h ../o_time.h a_utctm.c asn1_locl.h a_utf8.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h a_utf8.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h a_utf8.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h diff --git a/crypto/asn1/a_gentm.c b/crypto/asn1/a_gentm.c index 8b3ef716e334..fa76dcac91f3 100644 --- a/crypto/asn1/a_gentm.c +++ b/crypto/asn1/a_gentm.c @@ -65,6 +65,7 @@ #include "cryptlib.h" #include "o_time.h" #include +#include "asn1_locl.h" #if 0 @@ -117,7 +118,7 @@ ASN1_GENERALIZEDTIME *d2i_ASN1_GENERALIZEDTIME(ASN1_GENERALIZEDTIME **a, #endif -int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *d) +int asn1_generalizedtime_to_tm(struct tm *tm, const ASN1_GENERALIZEDTIME *d) { static const int min[9] = { 0, 0, 1, 1, 0, 0, 0, 0, 0 }; static const int max[9] = { 99, 99, 12, 31, 23, 59, 59, 12, 59 }; @@ -139,6 +140,8 @@ int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *d) for (i = 0; i < 7; i++) { if ((i == 6) && ((a[o] == 'Z') || (a[o] == '+') || (a[o] == '-'))) { i++; + if (tm) + tm->tm_sec = 0; break; } if ((a[o] < '0') || (a[o] > '9')) @@ -155,6 +158,31 @@ int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *d) if ((n < min[i]) || (n > max[i])) goto err; + if (tm) { + switch (i) { + case 0: + tm->tm_year = n * 100 - 1900; + break; + case 1: + tm->tm_year += n; + break; + case 2: + tm->tm_mon = n - 1; + break; + case 3: + tm->tm_mday = n; + break; + case 4: + tm->tm_hour = n; + break; + case 5: + tm->tm_min = n; + break; + case 6: + tm->tm_sec = n; + break; + } + } } /* * Optional fractional seconds: decimal point followed by one or more @@ -174,6 +202,7 @@ int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *d) if (a[o] == 'Z') o++; else if ((a[o] == '+') || (a[o] == '-')) { + int offsign = a[o] == '-' ? -1 : 1, offset = 0; o++; if (o + 4 > l) goto err; @@ -187,9 +216,17 @@ int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *d) n = (n * 10) + a[o] - '0'; if ((n < min[i]) || (n > max[i])) goto err; + if (tm) { + if (i == 7) + offset = n * 3600; + else if (i == 8) + offset += n * 60; + } o++; } - } else { + if (offset && !OPENSSL_gmtime_adj(tm, 0, offset * offsign)) + return 0; + } else if (a[o]) { /* Missing time zone information. */ goto err; } @@ -198,6 +235,11 @@ int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *d) return (0); } +int ASN1_GENERALIZEDTIME_check(const ASN1_GENERALIZEDTIME *d) +{ + return asn1_generalizedtime_to_tm(NULL, d); +} + int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *s, const char *str) { ASN1_GENERALIZEDTIME t; diff --git a/crypto/asn1/a_time.c b/crypto/asn1/a_time.c index c81f0de5a9e8..fcb2d565cdd0 100644 --- a/crypto/asn1/a_time.c +++ b/crypto/asn1/a_time.c @@ -66,6 +66,7 @@ #include "cryptlib.h" #include "o_time.h" #include +#include "asn1_locl.h" IMPLEMENT_ASN1_MSTRING(ASN1_TIME, B_ASN1_TIME) @@ -196,3 +197,32 @@ int ASN1_TIME_set_string(ASN1_TIME *s, const char *str) return 1; } + +static int asn1_time_to_tm(struct tm *tm, const ASN1_TIME *t) +{ + if (t == NULL) { + time_t now_t; + time(&now_t); + if (OPENSSL_gmtime(&now_t, tm)) + return 1; + return 0; + } + + if (t->type == V_ASN1_UTCTIME) + return asn1_utctime_to_tm(tm, t); + else if (t->type == V_ASN1_GENERALIZEDTIME) + return asn1_generalizedtime_to_tm(tm, t); + + return 0; +} + +int ASN1_TIME_diff(int *pday, int *psec, + const ASN1_TIME *from, const ASN1_TIME *to) +{ + struct tm tm_from, tm_to; + if (!asn1_time_to_tm(&tm_from, from)) + return 0; + if (!asn1_time_to_tm(&tm_to, to)) + return 0; + return OPENSSL_gmtime_diff(pday, psec, &tm_from, &tm_to); +} diff --git a/crypto/asn1/a_utctm.c b/crypto/asn1/a_utctm.c index 179de6d39590..724a10be4ed6 100644 --- a/crypto/asn1/a_utctm.c +++ b/crypto/asn1/a_utctm.c @@ -61,6 +61,7 @@ #include "cryptlib.h" #include "o_time.h" #include +#include "asn1_locl.h" #if 0 int i2d_ASN1_UTCTIME(ASN1_UTCTIME *a, unsigned char **pp) @@ -109,7 +110,7 @@ ASN1_UTCTIME *d2i_ASN1_UTCTIME(ASN1_UTCTIME **a, unsigned char **pp, #endif -int ASN1_UTCTIME_check(ASN1_UTCTIME *d) +int asn1_utctime_to_tm(struct tm *tm, const ASN1_UTCTIME *d) { static const int min[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; static const int max[8] = { 99, 12, 31, 23, 59, 59, 12, 59 }; @@ -127,6 +128,8 @@ int ASN1_UTCTIME_check(ASN1_UTCTIME *d) for (i = 0; i < 6; i++) { if ((i == 5) && ((a[o] == 'Z') || (a[o] == '+') || (a[o] == '-'))) { i++; + if (tm) + tm->tm_sec = 0; break; } if ((a[o] < '0') || (a[o] > '9')) @@ -143,10 +146,33 @@ int ASN1_UTCTIME_check(ASN1_UTCTIME *d) if ((n < min[i]) || (n > max[i])) goto err; + if (tm) { + switch (i) { + case 0: + tm->tm_year = n < 50 ? n + 100 : n; + break; + case 1: + tm->tm_mon = n - 1; + break; + case 2: + tm->tm_mday = n; + break; + case 3: + tm->tm_hour = n; + break; + case 4: + tm->tm_min = n; + break; + case 5: + tm->tm_sec = n; + break; + } + } } if (a[o] == 'Z') o++; else if ((a[o] == '+') || (a[o] == '-')) { + int offsign = a[o] == '-' ? -1 : 1, offset = 0; o++; if (o + 4 > l) goto err; @@ -160,12 +186,25 @@ int ASN1_UTCTIME_check(ASN1_UTCTIME *d) n = (n * 10) + a[o] - '0'; if ((n < min[i]) || (n > max[i])) goto err; + if (tm) { + if (i == 6) + offset = n * 3600; + else if (i == 7) + offset += n * 60; + } o++; } + if (offset && !OPENSSL_gmtime_adj(tm, 0, offset * offsign)) + return 0; } - return (o == l); + return o == l; err: - return (0); + return 0; +} + +int ASN1_UTCTIME_check(const ASN1_UTCTIME *d) +{ + return asn1_utctime_to_tm(NULL, d); } int ASN1_UTCTIME_set_string(ASN1_UTCTIME *s, const char *str) @@ -249,43 +288,26 @@ ASN1_UTCTIME *ASN1_UTCTIME_adj(ASN1_UTCTIME *s, time_t t, int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *s, time_t t) { - struct tm *tm; - struct tm data; - int offset; - int year; - -#define g2(p) (((p)[0]-'0')*10+(p)[1]-'0') - - if (s->data[12] == 'Z') - offset = 0; - else { - offset = g2(s->data + 13) * 60 + g2(s->data + 15); - if (s->data[12] == '-') - offset = -offset; - } + struct tm stm, ttm; + int day, sec; - t -= offset * 60; /* FIXME: may overflow in extreme cases */ + if (!asn1_utctime_to_tm(&stm, s)) + return -2; - tm = OPENSSL_gmtime(&t, &data); - /* - * NB: -1, 0, 1 already valid return values so use -2 to indicate error. - */ - if (tm == NULL) + if (!OPENSSL_gmtime(&t, &ttm)) return -2; -#define return_cmp(a,b) if ((a)<(b)) return -1; else if ((a)>(b)) return 1 - year = g2(s->data); - if (year < 50) - year += 100; - return_cmp(year, tm->tm_year); - return_cmp(g2(s->data + 2) - 1, tm->tm_mon); - return_cmp(g2(s->data + 4), tm->tm_mday); - return_cmp(g2(s->data + 6), tm->tm_hour); - return_cmp(g2(s->data + 8), tm->tm_min); - return_cmp(g2(s->data + 10), tm->tm_sec); -#undef g2 -#undef return_cmp + if (!OPENSSL_gmtime_diff(&day, &sec, &ttm, &stm)) + return -2; + if (day > 0) + return 1; + if (day < 0) + return -1; + if (sec > 0) + return 1; + if (sec < 0) + return -1; return 0; } diff --git a/crypto/asn1/ameth_lib.c b/crypto/asn1/ameth_lib.c index 45f3f4056049..5389c0434740 100644 --- a/crypto/asn1/ameth_lib.c +++ b/crypto/asn1/ameth_lib.c @@ -68,6 +68,7 @@ extern const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[]; extern const EVP_PKEY_ASN1_METHOD dsa_asn1_meths[]; extern const EVP_PKEY_ASN1_METHOD dh_asn1_meth; +extern const EVP_PKEY_ASN1_METHOD dhx_asn1_meth; extern const EVP_PKEY_ASN1_METHOD eckey_asn1_meth; extern const EVP_PKEY_ASN1_METHOD hmac_asn1_meth; extern const EVP_PKEY_ASN1_METHOD cmac_asn1_meth; @@ -92,7 +93,10 @@ static const EVP_PKEY_ASN1_METHOD *standard_methods[] = { &eckey_asn1_meth, #endif &hmac_asn1_meth, - &cmac_asn1_meth + &cmac_asn1_meth, +#ifndef OPENSSL_NO_DH + &dhx_asn1_meth +#endif }; typedef int sk_cmp_fn_type(const char *const *a, const char *const *b); @@ -460,3 +464,21 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth, { ameth->pkey_ctrl = pkey_ctrl; } + +void EVP_PKEY_asn1_set_item(EVP_PKEY_ASN1_METHOD *ameth, + int (*item_verify) (EVP_MD_CTX *ctx, + const ASN1_ITEM *it, + void *asn, + X509_ALGOR *a, + ASN1_BIT_STRING *sig, + EVP_PKEY *pkey), + int (*item_sign) (EVP_MD_CTX *ctx, + const ASN1_ITEM *it, + void *asn, + X509_ALGOR *alg1, + X509_ALGOR *alg2, + ASN1_BIT_STRING *sig)) +{ + ameth->item_sign = item_sign; + ameth->item_verify = item_verify; +} diff --git a/crypto/asn1/asn1.h b/crypto/asn1/asn1.h index 39b7833f582f..68e791fcdbe8 100644 --- a/crypto/asn1/asn1.h +++ b/crypto/asn1/asn1.h @@ -207,13 +207,13 @@ typedef struct asn1_const_ctx_st { # define ASN1_OBJECT_FLAG_CRITICAL 0x02/* critical x509v3 object id */ # define ASN1_OBJECT_FLAG_DYNAMIC_STRINGS 0x04/* internal use */ # define ASN1_OBJECT_FLAG_DYNAMIC_DATA 0x08/* internal use */ -typedef struct asn1_object_st { +struct asn1_object_st { const char *sn, *ln; int nid; int length; const unsigned char *data; /* data remains const after init */ int flags; /* Should we free this one */ -} ASN1_OBJECT; +}; # define ASN1_STRING_FLAG_BITS_LEFT 0x08/* Set if 0x07 has bits left value */ /* @@ -843,7 +843,7 @@ int ASN1_INTEGER_cmp(const ASN1_INTEGER *x, const ASN1_INTEGER *y); DECLARE_ASN1_FUNCTIONS(ASN1_ENUMERATED) -int ASN1_UTCTIME_check(ASN1_UTCTIME *a); +int ASN1_UTCTIME_check(const ASN1_UTCTIME *a); ASN1_UTCTIME *ASN1_UTCTIME_set(ASN1_UTCTIME *s, time_t t); ASN1_UTCTIME *ASN1_UTCTIME_adj(ASN1_UTCTIME *s, time_t t, int offset_day, long offset_sec); @@ -853,13 +853,15 @@ int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *s, time_t t); time_t ASN1_UTCTIME_get(const ASN1_UTCTIME *s); # endif -int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *a); +int ASN1_GENERALIZEDTIME_check(const ASN1_GENERALIZEDTIME *a); ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_set(ASN1_GENERALIZEDTIME *s, time_t t); ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_adj(ASN1_GENERALIZEDTIME *s, time_t t, int offset_day, long offset_sec); int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *s, const char *str); +int ASN1_TIME_diff(int *pday, int *psec, + const ASN1_TIME *from, const ASN1_TIME *to); DECLARE_ASN1_FUNCTIONS(ASN1_OCTET_STRING) ASN1_OCTET_STRING *ASN1_OCTET_STRING_dup(const ASN1_OCTET_STRING *a); diff --git a/crypto/asn1/asn1_locl.h b/crypto/asn1/asn1_locl.h index 9f5ed8472718..4c004fab9a8d 100644 --- a/crypto/asn1/asn1_locl.h +++ b/crypto/asn1/asn1_locl.h @@ -59,6 +59,9 @@ /* Internal ASN1 structures and functions: not for application use */ +int asn1_utctime_to_tm(struct tm *tm, const ASN1_UTCTIME *d); +int asn1_generalizedtime_to_tm(struct tm *tm, const ASN1_GENERALIZEDTIME *d); + /* ASN1 print context structure */ struct asn1_pctx_st { diff --git a/crypto/asn1/t_x509.c b/crypto/asn1/t_x509.c index 4e7c45dd5ddc..8aab55130c95 100644 --- a/crypto/asn1/t_x509.c +++ b/crypto/asn1/t_x509.c @@ -228,6 +228,21 @@ int X509_print_ex(BIO *bp, X509 *x, unsigned long nmflags, } } + if (!(cflag & X509_FLAG_NO_IDS)) { + if (ci->issuerUID) { + if (BIO_printf(bp, "%8sIssuer Unique ID: ", "") <= 0) + goto err; + if (!X509_signature_dump(bp, ci->issuerUID, 12)) + goto err; + } + if (ci->subjectUID) { + if (BIO_printf(bp, "%8sSubject Unique ID: ", "") <= 0) + goto err; + if (!X509_signature_dump(bp, ci->subjectUID, 12)) + goto err; + } + } + if (!(cflag & X509_FLAG_NO_EXTENSIONS)) X509V3_extensions_print(bp, "X509v3 extensions", ci->extensions, cflag, 8); diff --git a/crypto/asn1/x_crl.c b/crypto/asn1/x_crl.c index e258c714b249..027950330d8b 100644 --- a/crypto/asn1/x_crl.c +++ b/crypto/asn1/x_crl.c @@ -58,8 +58,8 @@ #include #include "cryptlib.h" -#include "asn1_locl.h" #include +#include "asn1_locl.h" #include #include @@ -341,6 +341,8 @@ ASN1_SEQUENCE_ref(X509_CRL, crl_cb, CRYPTO_LOCK_X509_CRL) = { IMPLEMENT_ASN1_FUNCTIONS(X509_REVOKED) +IMPLEMENT_ASN1_DUP_FUNCTION(X509_REVOKED) + IMPLEMENT_ASN1_FUNCTIONS(X509_CRL_INFO) IMPLEMENT_ASN1_FUNCTIONS(X509_CRL) diff --git a/crypto/asn1/x_x509.c b/crypto/asn1/x_x509.c index f56e837b3a08..5f266a26b4c2 100644 --- a/crypto/asn1/x_x509.c +++ b/crypto/asn1/x_x509.c @@ -208,3 +208,23 @@ int i2d_X509_AUX(X509 *a, unsigned char **pp) length += i2d_X509_CERT_AUX(a->aux, pp); return length; } + +int i2d_re_X509_tbs(X509 *x, unsigned char **pp) +{ + x->cert_info->enc.modified = 1; + return i2d_X509_CINF(x->cert_info, pp); +} + +void X509_get0_signature(ASN1_BIT_STRING **psig, X509_ALGOR **palg, + const X509 *x) +{ + if (psig) + *psig = x->signature; + if (palg) + *palg = x->sig_alg; +} + +int X509_get_signature_nid(const X509 *x) +{ + return OBJ_obj2nid(x->sig_alg->algorithm); +} diff --git a/crypto/bio/b_dump.c b/crypto/bio/b_dump.c index 3293c724c94f..ed8e521449a4 100644 --- a/crypto/bio/b_dump.c +++ b/crypto/bio/b_dump.c @@ -182,3 +182,28 @@ int BIO_dump_indent(BIO *bp, const char *s, int len, int indent) { return BIO_dump_indent_cb(write_bio, bp, s, len, indent); } + +int BIO_hex_string(BIO *out, int indent, int width, unsigned char *data, + int datalen) +{ + int i, j = 0; + + if (datalen < 1) + return 1; + + for (i = 0; i < datalen - 1; i++) { + if (i && !j) + BIO_printf(out, "%*s", indent, ""); + + BIO_printf(out, "%02X:", data[i]); + + j = (j + 1) % width; + if (!j) + BIO_printf(out, "\n"); + } + + if (i && !j) + BIO_printf(out, "%*s", indent, ""); + BIO_printf(out, "%02X", data[datalen - 1]); + return 1; +} diff --git a/crypto/bio/b_sock.c b/crypto/bio/b_sock.c index bda882c40b08..5bad0a2bada2 100644 --- a/crypto/bio/b_sock.c +++ b/crypto/bio/b_sock.c @@ -225,13 +225,17 @@ int BIO_get_port(const char *str, unsigned short *port_ptr) int BIO_sock_error(int sock) { int j, i; - int size; + union { + size_t s; + int i; + } size; # if defined(OPENSSL_SYS_BEOS_R5) return 0; # endif - size = sizeof(int); + /* heuristic way to adapt for platforms that expect 64-bit optlen */ + size.s = 0, size.i = sizeof(j); /* * Note: under Windows the third parameter is of type (char *) whereas * under other systems it is (void *) if you don't have a cast it will diff --git a/crypto/bio/bio.h b/crypto/bio/bio.h index d583cc108508..f78796b069f5 100644 --- a/crypto/bio/bio.h +++ b/crypto/bio/bio.h @@ -174,6 +174,7 @@ extern "C" { # define BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT 45/* Next DTLS handshake timeout * to adjust socket timeouts */ +# define BIO_CTRL_DGRAM_SET_DONT_FRAG 48 # define BIO_CTRL_DGRAM_GET_MTU_OVERHEAD 49 @@ -725,6 +726,9 @@ int BIO_dump_indent(BIO *b, const char *bytes, int len, int indent); int BIO_dump_fp(FILE *fp, const char *s, int len); int BIO_dump_indent_fp(FILE *fp, const char *s, int len, int indent); # endif +int BIO_hex_string(BIO *out, int indent, int width, unsigned char *data, + int datalen); + struct hostent *BIO_gethostbyname(const char *name); /*- * We might want a thread-safe interface too: @@ -761,8 +765,8 @@ int BIO_dgram_sctp_wait_for_dry(BIO *b); int BIO_dgram_sctp_msg_waiting(BIO *b); # endif BIO *BIO_new_fd(int fd, int close_flag); -BIO *BIO_new_connect(char *host_port); -BIO *BIO_new_accept(char *host_port); +BIO *BIO_new_connect(const char *host_port); +BIO *BIO_new_accept(const char *host_port); int BIO_new_bio_pair(BIO **bio1, size_t writebuf1, BIO **bio2, size_t writebuf2); diff --git a/crypto/bio/bio_err.c b/crypto/bio/bio_err.c index 6dd6162fc93a..d9007aa3d32d 100644 --- a/crypto/bio/bio_err.c +++ b/crypto/bio/bio_err.c @@ -1,6 +1,6 @@ /* crypto/bio/bio_err.c */ /* ==================================================================== - * Copyright (c) 1999-2011 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2015 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/crypto/bio/bss_acpt.c b/crypto/bio/bss_acpt.c index d08292c3e9ac..4a5e39bd387b 100644 --- a/crypto/bio/bss_acpt.c +++ b/crypto/bio/bss_acpt.c @@ -445,7 +445,7 @@ static int acpt_puts(BIO *bp, const char *str) return (ret); } -BIO *BIO_new_accept(char *str) +BIO *BIO_new_accept(const char *str) { BIO *ret; diff --git a/crypto/bio/bss_conn.c b/crypto/bio/bss_conn.c index 6a5e8de8812a..42d0afffbc6c 100644 --- a/crypto/bio/bss_conn.c +++ b/crypto/bio/bss_conn.c @@ -585,7 +585,7 @@ static int conn_puts(BIO *bp, const char *str) return (ret); } -BIO *BIO_new_connect(char *str) +BIO *BIO_new_connect(const char *str) { BIO *ret; diff --git a/crypto/bio/bss_dgram.c b/crypto/bio/bss_dgram.c index e3e3dd0503a8..7fcd831da06b 100644 --- a/crypto/bio/bss_dgram.c +++ b/crypto/bio/bss_dgram.c @@ -65,7 +65,7 @@ #include #ifndef OPENSSL_NO_DGRAM -# if defined(OPENSSL_SYS_WIN32) || defined(OPENSSL_SYS_VMS) +# if defined(OPENSSL_SYS_VMS) # include # endif @@ -80,6 +80,10 @@ # define IP_MTU 14 /* linux is lame */ # endif +# if OPENSSL_USE_IPV6 && !defined(IPPROTO_IPV6) +# define IPPROTO_IPV6 41 /* windows is lame */ +# endif + # if defined(__FreeBSD__) && defined(IN6_IS_ADDR_V4MAPPED) /* Standard definition causes type-punning problems. */ # undef IN6_IS_ADDR_V4MAPPED @@ -496,8 +500,8 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) int *ip; struct sockaddr *to = NULL; bio_dgram_data *data = NULL; -# if defined(OPENSSL_SYS_LINUX) && (defined(IP_MTU_DISCOVER) || defined(IP_MTU)) int sockopt_val = 0; +# if defined(OPENSSL_SYS_LINUX) && (defined(IP_MTU_DISCOVER) || defined(IP_MTU)) socklen_t sockopt_len; /* assume that system supporting IP_MTU is * modern enough to define socklen_t */ socklen_t addr_len; @@ -882,6 +886,61 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) ret = 0; break; # endif + case BIO_CTRL_DGRAM_SET_DONT_FRAG: + sockopt_val = num ? 1 : 0; + + switch (data->peer.sa.sa_family) { + case AF_INET: +# if defined(IP_DONTFRAG) + if ((ret = setsockopt(b->num, IPPROTO_IP, IP_DONTFRAG, + &sockopt_val, sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } +# elif defined(OPENSSL_SYS_LINUX) && defined(IP_MTU_DISCOVER) && defined (IP_PMTUDISC_PROBE) + if ((sockopt_val = num ? IP_PMTUDISC_PROBE : IP_PMTUDISC_DONT), + (ret = setsockopt(b->num, IPPROTO_IP, IP_MTU_DISCOVER, + &sockopt_val, sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } +# elif defined(OPENSSL_SYS_WINDOWS) && defined(IP_DONTFRAGMENT) + if ((ret = setsockopt(b->num, IPPROTO_IP, IP_DONTFRAGMENT, + (const char *)&sockopt_val, + sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } +# else + ret = -1; +# endif + break; +# if OPENSSL_USE_IPV6 + case AF_INET6: +# if defined(IPV6_DONTFRAG) + if ((ret = setsockopt(b->num, IPPROTO_IPV6, IPV6_DONTFRAG, + (const void *)&sockopt_val, + sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } +# elif defined(OPENSSL_SYS_LINUX) && defined(IPV6_MTUDISCOVER) + if ((sockopt_val = num ? IP_PMTUDISC_PROBE : IP_PMTUDISC_DONT), + (ret = setsockopt(b->num, IPPROTO_IPV6, IPV6_MTU_DISCOVER, + &sockopt_val, sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } +# else + ret = -1; +# endif + break; +# endif + default: + ret = -1; + break; + } + break; case BIO_CTRL_DGRAM_GET_MTU_OVERHEAD: ret = dgram_get_mtu_overhead(data); break; @@ -1995,11 +2054,22 @@ int BIO_dgram_non_fatal_error(int err) static void get_current_time(struct timeval *t) { -# ifdef OPENSSL_SYS_WIN32 - struct _timeb tb; - _ftime(&tb); - t->tv_sec = (long)tb.time; - t->tv_usec = (long)tb.millitm * 1000; +# if defined(_WIN32) + SYSTEMTIME st; + union { + unsigned __int64 ul; + FILETIME ft; + } now; + + GetSystemTime(&st); + SystemTimeToFileTime(&st, &now.ft); +# ifdef __MINGW32__ + now.ul -= 116444736000000000ULL; +# else + now.ul -= 116444736000000000UI64; /* re-bias to 1/1/1970 */ +# endif + t->tv_sec = (long)(now.ul / 10000000); + t->tv_usec = ((int)(now.ul % 10000000)) / 10; # elif defined(OPENSSL_SYS_VMS) struct timeb tb; ftime(&tb); diff --git a/crypto/bio/bss_fd.c b/crypto/bio/bss_fd.c index ccef578154d7..5f4e34481b0e 100644 --- a/crypto/bio/bss_fd.c +++ b/crypto/bio/bss_fd.c @@ -63,9 +63,27 @@ #if defined(OPENSSL_NO_POSIX_IO) /* - * One can argue that one should implement dummy placeholder for - * BIO_s_fd here... + * Dummy placeholder for BIO_s_fd... */ +BIO *BIO_new_fd(int fd, int close_flag) +{ + return NULL; +} + +int BIO_fd_non_fatal_error(int err) +{ + return 0; +} + +int BIO_fd_should_retry(int i) +{ + return 0; +} + +BIO_METHOD *BIO_s_fd(void) +{ + return NULL; +} #else /* * As for unconditional usage of "UPLINK" interface in this module. diff --git a/crypto/bn/Makefile b/crypto/bn/Makefile index 3d0158c15aea..215855ecae91 100644 --- a/crypto/bn/Makefile +++ b/crypto/bn/Makefile @@ -77,6 +77,12 @@ sparcv9a-mont.s: asm/sparcv9a-mont.pl $(PERL) asm/sparcv9a-mont.pl $(CFLAGS) > $@ sparcv9-mont.s: asm/sparcv9-mont.pl $(PERL) asm/sparcv9-mont.pl $(CFLAGS) > $@ +vis3-mont.s: asm/vis3-mont.pl + $(PERL) asm/vis3-mont.pl $(CFLAGS) > $@ +sparct4-mont.S: asm/sparct4-mont.pl + $(PERL) asm/sparct4-mont.pl $(CFLAGS) > $@ +sparcv9-gf2m.S: asm/sparcv9-gf2m.pl + $(PERL) asm/sparcv9-gf2m.pl $(CFLAGS) > $@ bn-mips3.o: asm/mips3.s @if [ "$(CC)" = "gcc" ]; then \ @@ -102,8 +108,10 @@ x86_64-mont5.s: asm/x86_64-mont5.pl $(PERL) asm/x86_64-mont5.pl $(PERLASM_SCHEME) > $@ x86_64-gf2m.s: asm/x86_64-gf2m.pl $(PERL) asm/x86_64-gf2m.pl $(PERLASM_SCHEME) > $@ -modexp512-x86_64.s: asm/modexp512-x86_64.pl - $(PERL) asm/modexp512-x86_64.pl $(PERLASM_SCHEME) > $@ +rsaz-x86_64.s: asm/rsaz-x86_64.pl + $(PERL) asm/rsaz-x86_64.pl $(PERLASM_SCHEME) > $@ +rsaz-avx2.s: asm/rsaz-avx2.pl + $(PERL) asm/rsaz-avx2.pl $(PERLASM_SCHEME) > $@ bn-ia64.s: asm/ia64.S $(CC) $(CFLAGS) -E asm/ia64.S > $@ @@ -125,14 +133,15 @@ ppc-mont.s: asm/ppc-mont.pl;$(PERL) asm/ppc-mont.pl $(PERLASM_SCHEME) $@ ppc64-mont.s: asm/ppc64-mont.pl;$(PERL) asm/ppc64-mont.pl $(PERLASM_SCHEME) $@ alpha-mont.s: asm/alpha-mont.pl - (preproc=/tmp/$$$$.$@; trap "rm $$preproc" INT; \ + (preproc=$$$$.$@.S; trap "rm $$preproc" INT; \ $(PERL) asm/alpha-mont.pl > $$preproc && \ - $(CC) -E $$preproc > $@ && rm $$preproc) + $(CC) -E -P $$preproc > $@ && rm $$preproc) # GNU make "catch all" -%-mont.s: asm/%-mont.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +%-mont.S: asm/%-mont.pl; $(PERL) $< $(PERLASM_SCHEME) $@ %-gf2m.S: asm/%-gf2m.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +armv4-mont.o: armv4-mont.S armv4-gf2m.o: armv4-gf2m.S files: @@ -244,6 +253,7 @@ bn_exp.o: ../../include/openssl/lhash.h ../../include/openssl/opensslconf.h bn_exp.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h bn_exp.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h bn_exp.o: ../../include/openssl/symhacks.h ../cryptlib.h bn_exp.c bn_lcl.h +bn_exp.o: rsaz_exp.h bn_exp2.o: ../../e_os.h ../../include/openssl/bio.h ../../include/openssl/bn.h bn_exp2.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h bn_exp2.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h diff --git a/crypto/bn/asm/armv4-gf2m.pl b/crypto/bn/asm/armv4-gf2m.pl index c52e0b75b5b6..8f529c95cf05 100755 --- a/crypto/bn/asm/armv4-gf2m.pl +++ b/crypto/bn/asm/armv4-gf2m.pl @@ -20,48 +20,26 @@ # length, more for longer keys. Even though NEON 1x1 multiplication # runs in even less cycles, ~30, improvement is measurable only on # longer keys. One has to optimize code elsewhere to get NEON glow... +# +# April 2014 +# +# Double bn_GF2m_mul_2x2 performance by using algorithm from paper +# referred below, which improves ECDH and ECDSA verify benchmarks +# by 18-40%. +# +# Câmara, D.; Gouvêa, C. P. L.; López, J. & Dahab, R.: Fast Software +# Polynomial Multiplication on ARM Processors using the NEON Engine. +# +# http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; -sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } -sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } -sub Q() { shift=~m|d([1-3]?[02468])|?"q".($1/2):""; } - $code=<<___; #include "arm_arch.h" .text .code 32 - -#if __ARM_ARCH__>=7 -.fpu neon - -.type mul_1x1_neon,%function -.align 5 -mul_1x1_neon: - vshl.u64 `&Dlo("q1")`,d16,#8 @ q1-q3 are slided $a - vmull.p8 `&Q("d0")`,d16,d17 @ a·bb - vshl.u64 `&Dlo("q2")`,d16,#16 - vmull.p8 q1,`&Dlo("q1")`,d17 @ a<<8·bb - vshl.u64 `&Dlo("q3")`,d16,#24 - vmull.p8 q2,`&Dlo("q2")`,d17 @ a<<16·bb - vshr.u64 `&Dlo("q1")`,#8 - vmull.p8 q3,`&Dlo("q3")`,d17 @ a<<24·bb - vshl.u64 `&Dhi("q1")`,#24 - veor d0,`&Dlo("q1")` - vshr.u64 `&Dlo("q2")`,#16 - veor d0,`&Dhi("q1")` - vshl.u64 `&Dhi("q2")`,#16 - veor d0,`&Dlo("q2")` - vshr.u64 `&Dlo("q3")`,#24 - veor d0,`&Dhi("q2")` - vshl.u64 `&Dhi("q3")`,#8 - veor d0,`&Dlo("q3")` - veor d0,`&Dhi("q3")` - bx lr -.size mul_1x1_neon,.-mul_1x1_neon -#endif ___ ################ # private interface to mul_1x1_ialu @@ -159,56 +137,17 @@ ___ # void bn_GF2m_mul_2x2(BN_ULONG *r, # BN_ULONG a1,BN_ULONG a0, # BN_ULONG b1,BN_ULONG b0); # r[3..0]=a1a0·b1b0 - -($A1,$B1,$A0,$B0,$A1B1,$A0B0)=map("d$_",(18..23)); - +{ $code.=<<___; .global bn_GF2m_mul_2x2 .type bn_GF2m_mul_2x2,%function .align 5 bn_GF2m_mul_2x2: -#if __ARM_ARCH__>=7 +#if __ARM_MAX_ARCH__>=7 ldr r12,.LOPENSSL_armcap .Lpic: ldr r12,[pc,r12] tst r12,#1 - beq .Lialu - - veor $A1,$A1 - vmov.32 $B1,r3,r3 @ two copies of b1 - vmov.32 ${A1}[0],r1 @ a1 - - veor $A0,$A0 - vld1.32 ${B0}[],[sp,:32] @ two copies of b0 - vmov.32 ${A0}[0],r2 @ a0 - mov r12,lr - - vmov d16,$A1 - vmov d17,$B1 - bl mul_1x1_neon @ a1·b1 - vmov $A1B1,d0 - - vmov d16,$A0 - vmov d17,$B0 - bl mul_1x1_neon @ a0·b0 - vmov $A0B0,d0 - - veor d16,$A0,$A1 - veor d17,$B0,$B1 - veor $A0,$A0B0,$A1B1 - bl mul_1x1_neon @ (a0+a1)·(b0+b1) - - veor d0,$A0 @ (a0+a1)·(b0+b1)-a0·b0-a1·b1 - vshl.u64 d1,d0,#32 - vshr.u64 d0,d0,#32 - veor $A0B0,d1 - veor $A1B1,d0 - vst1.32 {${A0B0}[0]},[r0,:32]! - vst1.32 {${A0B0}[1]},[r0,:32]! - vst1.32 {${A1B1}[0]},[r0,:32]! - vst1.32 {${A1B1}[1]},[r0,:32] - bx r12 -.align 4 -.Lialu: + bne .LNEON #endif ___ $ret="r10"; # reassigned 1st argument @@ -260,8 +199,72 @@ $code.=<<___; moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif +___ +} +{ +my ($r,$t0,$t1,$t2,$t3)=map("q$_",(0..3,8..12)); +my ($a,$b,$k48,$k32,$k16)=map("d$_",(26..31)); + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.align 5 +.LNEON: + ldr r12, [sp] @ 5th argument + vmov.32 $a, r2, r1 + vmov.32 $b, r12, r3 + vmov.i64 $k48, #0x0000ffffffffffff + vmov.i64 $k32, #0x00000000ffffffff + vmov.i64 $k16, #0x000000000000ffff + + vext.8 $t0#lo, $a, $a, #1 @ A1 + vmull.p8 $t0, $t0#lo, $b @ F = A1*B + vext.8 $r#lo, $b, $b, #1 @ B1 + vmull.p8 $r, $a, $r#lo @ E = A*B1 + vext.8 $t1#lo, $a, $a, #2 @ A2 + vmull.p8 $t1, $t1#lo, $b @ H = A2*B + vext.8 $t3#lo, $b, $b, #2 @ B2 + vmull.p8 $t3, $a, $t3#lo @ G = A*B2 + vext.8 $t2#lo, $a, $a, #3 @ A3 + veor $t0, $t0, $r @ L = E + F + vmull.p8 $t2, $t2#lo, $b @ J = A3*B + vext.8 $r#lo, $b, $b, #3 @ B3 + veor $t1, $t1, $t3 @ M = G + H + vmull.p8 $r, $a, $r#lo @ I = A*B3 + veor $t0#lo, $t0#lo, $t0#hi @ t0 = (L) (P0 + P1) << 8 + vand $t0#hi, $t0#hi, $k48 + vext.8 $t3#lo, $b, $b, #4 @ B4 + veor $t1#lo, $t1#lo, $t1#hi @ t1 = (M) (P2 + P3) << 16 + vand $t1#hi, $t1#hi, $k32 + vmull.p8 $t3, $a, $t3#lo @ K = A*B4 + veor $t2, $t2, $r @ N = I + J + veor $t0#lo, $t0#lo, $t0#hi + veor $t1#lo, $t1#lo, $t1#hi + veor $t2#lo, $t2#lo, $t2#hi @ t2 = (N) (P4 + P5) << 24 + vand $t2#hi, $t2#hi, $k16 + vext.8 $t0, $t0, $t0, #15 + veor $t3#lo, $t3#lo, $t3#hi @ t3 = (K) (P6 + P7) << 32 + vmov.i64 $t3#hi, #0 + vext.8 $t1, $t1, $t1, #14 + veor $t2#lo, $t2#lo, $t2#hi + vmull.p8 $r, $a, $b @ D = A*B + vext.8 $t3, $t3, $t3, #12 + vext.8 $t2, $t2, $t2, #13 + veor $t0, $t0, $t1 + veor $t2, $t2, $t3 + veor $r, $r, $t0 + veor $r, $r, $t2 + + vst1.32 {$r}, [r0] + ret @ bx lr +#endif +___ +} +$code.=<<___; .size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2 -#if __ARM_ARCH__>=7 +#if __ARM_MAX_ARCH__>=7 .align 5 .LOPENSSL_armcap: .word OPENSSL_armcap_P-(.Lpic+8) @@ -269,10 +272,18 @@ $code.=<<___; .asciz "GF(2^m) Multiplication for ARMv4/NEON, CRYPTOGAMS by " .align 5 +#if __ARM_MAX_ARCH__>=7 .comm OPENSSL_armcap_P,4,4 +#endif ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 -print $code; +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or + s/\bret\b/bx lr/go or + s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4 + + print $_,"\n"; +} close STDOUT; # enforce flush diff --git a/crypto/bn/asm/armv4-mont.pl b/crypto/bn/asm/armv4-mont.pl index f78a8b5f0f55..1d330e9f8aa3 100755 --- a/crypto/bn/asm/armv4-mont.pl +++ b/crypto/bn/asm/armv4-mont.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -23,6 +23,21 @@ # than 1/2KB. Windows CE port would be trivial, as it's exclusively # about decorations, ABI and instruction syntax are identical. +# November 2013 +# +# Add NEON code path, which handles lengths divisible by 8. RSA/DSA +# performance improvement on Cortex-A8 is ~45-100% depending on key +# length, more for longer keys. On Cortex-A15 the span is ~10-105%. +# On Snapdragon S4 improvement was measured to vary from ~70% to +# incredible ~380%, yes, 4.8x faster, for RSA4096 sign. But this is +# rather because original integer-only code seems to perform +# suboptimally on S4. Situation on Cortex-A9 is unfortunately +# different. It's being looked into, but the trouble is that +# performance for vectors longer than 256 bits is actually couple +# of percent worse than for integer-only code. The code is chosen +# for execution on all NEON-capable processors, because gain on +# others outweighs the marginal loss on Cortex-A9. + while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; @@ -52,16 +67,40 @@ $_n0="$num,#14*4"; $_num="$num,#15*4"; $_bpend=$_num; $code=<<___; +#include "arm_arch.h" + .text +.code 32 + +#if __ARM_MAX_ARCH__>=7 +.align 5 +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-bn_mul_mont +#endif .global bn_mul_mont .type bn_mul_mont,%function -.align 2 +.align 5 bn_mul_mont: + ldr ip,[sp,#4] @ load num stmdb sp!,{r0,r2} @ sp points at argument block - ldr $num,[sp,#3*4] @ load num - cmp $num,#2 +#if __ARM_MAX_ARCH__>=7 + tst ip,#7 + bne .Lialu + adr r0,bn_mul_mont + ldr r2,.LOPENSSL_armcap + ldr r0,[r0,r2] + tst r0,#1 @ NEON available? + ldmia sp, {r0,r2} + beq .Lialu + add sp,sp,#8 + b bn_mul8x_mont_neon +.align 4 +.Lialu: +#endif + cmp ip,#2 + mov $num,ip @ load num movlt r0,#0 addlt sp,sp,#2*4 blt .Labrt @@ -191,14 +230,447 @@ bn_mul_mont: ldmia sp!,{r4-r12,lr} @ restore registers add sp,sp,#2*4 @ skip over {r0,r2} mov r0,#1 -.Labrt: tst lr,#1 +.Labrt: +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) +#endif .size bn_mul_mont,.-bn_mul_mont -.asciz "Montgomery multiplication for ARMv4, CRYPTOGAMS by " +___ +{ +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } + +my ($A0,$A1,$A2,$A3)=map("d$_",(0..3)); +my ($N0,$N1,$N2,$N3)=map("d$_",(4..7)); +my ($Z,$Temp)=("q4","q5"); +my ($A0xB,$A1xB,$A2xB,$A3xB,$A4xB,$A5xB,$A6xB,$A7xB)=map("q$_",(6..13)); +my ($Bi,$Ni,$M0)=map("d$_",(28..31)); +my $zero=&Dlo($Z); +my $temp=&Dlo($Temp); + +my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("r$_",(0..5)); +my ($tinptr,$toutptr,$inner,$outer)=map("r$_",(6..9)); + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.type bn_mul8x_mont_neon,%function +.align 5 +bn_mul8x_mont_neon: + mov ip,sp + stmdb sp!,{r4-r11} + vstmdb sp!,{d8-d15} @ ABI specification says so + ldmia ip,{r4-r5} @ load rest of parameter block + + sub $toutptr,sp,#16 + vld1.32 {${Bi}[0]}, [$bptr,:32]! + sub $toutptr,$toutptr,$num,lsl#4 + vld1.32 {$A0-$A3}, [$aptr]! @ can't specify :32 :-( + and $toutptr,$toutptr,#-64 + vld1.32 {${M0}[0]}, [$n0,:32] + mov sp,$toutptr @ alloca + veor $zero,$zero,$zero + subs $inner,$num,#8 + vzip.16 $Bi,$zero + + vmull.u32 $A0xB,$Bi,${A0}[0] + vmull.u32 $A1xB,$Bi,${A0}[1] + vmull.u32 $A2xB,$Bi,${A1}[0] + vshl.i64 $temp,`&Dhi("$A0xB")`,#16 + vmull.u32 $A3xB,$Bi,${A1}[1] + + vadd.u64 $temp,$temp,`&Dlo("$A0xB")` + veor $zero,$zero,$zero + vmul.u32 $Ni,$temp,$M0 + + vmull.u32 $A4xB,$Bi,${A2}[0] + vld1.32 {$N0-$N3}, [$nptr]! + vmull.u32 $A5xB,$Bi,${A2}[1] + vmull.u32 $A6xB,$Bi,${A3}[0] + vzip.16 $Ni,$zero + vmull.u32 $A7xB,$Bi,${A3}[1] + + bne .LNEON_1st + + @ special case for num=8, everything is in register bank... + + vmlal.u32 $A0xB,$Ni,${N0}[0] + sub $outer,$num,#1 + vmlal.u32 $A1xB,$Ni,${N0}[1] + vmlal.u32 $A2xB,$Ni,${N1}[0] + vmlal.u32 $A3xB,$Ni,${N1}[1] + + vmlal.u32 $A4xB,$Ni,${N2}[0] + vmov $Temp,$A0xB + vmlal.u32 $A5xB,$Ni,${N2}[1] + vmov $A0xB,$A1xB + vmlal.u32 $A6xB,$Ni,${N3}[0] + vmov $A1xB,$A2xB + vmlal.u32 $A7xB,$Ni,${N3}[1] + vmov $A2xB,$A3xB + vmov $A3xB,$A4xB + vshr.u64 $temp,$temp,#16 + vmov $A4xB,$A5xB + vmov $A5xB,$A6xB + vadd.u64 $temp,$temp,`&Dhi("$Temp")` + vmov $A6xB,$A7xB + veor $A7xB,$A7xB + vshr.u64 $temp,$temp,#16 + + b .LNEON_outer8 + +.align 4 +.LNEON_outer8: + vld1.32 {${Bi}[0]}, [$bptr,:32]! + veor $zero,$zero,$zero + vzip.16 $Bi,$zero + vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp + + vmlal.u32 $A0xB,$Bi,${A0}[0] + vmlal.u32 $A1xB,$Bi,${A0}[1] + vmlal.u32 $A2xB,$Bi,${A1}[0] + vshl.i64 $temp,`&Dhi("$A0xB")`,#16 + vmlal.u32 $A3xB,$Bi,${A1}[1] + + vadd.u64 $temp,$temp,`&Dlo("$A0xB")` + veor $zero,$zero,$zero + subs $outer,$outer,#1 + vmul.u32 $Ni,$temp,$M0 + + vmlal.u32 $A4xB,$Bi,${A2}[0] + vmlal.u32 $A5xB,$Bi,${A2}[1] + vmlal.u32 $A6xB,$Bi,${A3}[0] + vzip.16 $Ni,$zero + vmlal.u32 $A7xB,$Bi,${A3}[1] + + vmlal.u32 $A0xB,$Ni,${N0}[0] + vmlal.u32 $A1xB,$Ni,${N0}[1] + vmlal.u32 $A2xB,$Ni,${N1}[0] + vmlal.u32 $A3xB,$Ni,${N1}[1] + + vmlal.u32 $A4xB,$Ni,${N2}[0] + vmov $Temp,$A0xB + vmlal.u32 $A5xB,$Ni,${N2}[1] + vmov $A0xB,$A1xB + vmlal.u32 $A6xB,$Ni,${N3}[0] + vmov $A1xB,$A2xB + vmlal.u32 $A7xB,$Ni,${N3}[1] + vmov $A2xB,$A3xB + vmov $A3xB,$A4xB + vshr.u64 $temp,$temp,#16 + vmov $A4xB,$A5xB + vmov $A5xB,$A6xB + vadd.u64 $temp,$temp,`&Dhi("$Temp")` + vmov $A6xB,$A7xB + veor $A7xB,$A7xB + vshr.u64 $temp,$temp,#16 + + bne .LNEON_outer8 + + vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp + mov $toutptr,sp + vshr.u64 $temp,`&Dlo("$A0xB")`,#16 + mov $inner,$num + vadd.u64 `&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp + add $tinptr,sp,#16 + vshr.u64 $temp,`&Dhi("$A0xB")`,#16 + vzip.16 `&Dlo("$A0xB")`,`&Dhi("$A0xB")` + + b .LNEON_tail2 + +.align 4 +.LNEON_1st: + vmlal.u32 $A0xB,$Ni,${N0}[0] + vld1.32 {$A0-$A3}, [$aptr]! + vmlal.u32 $A1xB,$Ni,${N0}[1] + subs $inner,$inner,#8 + vmlal.u32 $A2xB,$Ni,${N1}[0] + vmlal.u32 $A3xB,$Ni,${N1}[1] + + vmlal.u32 $A4xB,$Ni,${N2}[0] + vld1.32 {$N0-$N1}, [$nptr]! + vmlal.u32 $A5xB,$Ni,${N2}[1] + vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! + vmlal.u32 $A6xB,$Ni,${N3}[0] + vmlal.u32 $A7xB,$Ni,${N3}[1] + vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! + + vmull.u32 $A0xB,$Bi,${A0}[0] + vld1.32 {$N2-$N3}, [$nptr]! + vmull.u32 $A1xB,$Bi,${A0}[1] + vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! + vmull.u32 $A2xB,$Bi,${A1}[0] + vmull.u32 $A3xB,$Bi,${A1}[1] + vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! + + vmull.u32 $A4xB,$Bi,${A2}[0] + vmull.u32 $A5xB,$Bi,${A2}[1] + vmull.u32 $A6xB,$Bi,${A3}[0] + vmull.u32 $A7xB,$Bi,${A3}[1] + + bne .LNEON_1st + + vmlal.u32 $A0xB,$Ni,${N0}[0] + add $tinptr,sp,#16 + vmlal.u32 $A1xB,$Ni,${N0}[1] + sub $aptr,$aptr,$num,lsl#2 @ rewind $aptr + vmlal.u32 $A2xB,$Ni,${N1}[0] + vld1.64 {$Temp}, [sp,:128] + vmlal.u32 $A3xB,$Ni,${N1}[1] + sub $outer,$num,#1 + + vmlal.u32 $A4xB,$Ni,${N2}[0] + vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! + vmlal.u32 $A5xB,$Ni,${N2}[1] + vshr.u64 $temp,$temp,#16 + vld1.64 {$A0xB}, [$tinptr, :128]! + vmlal.u32 $A6xB,$Ni,${N3}[0] + vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! + vmlal.u32 $A7xB,$Ni,${N3}[1] + + vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! + vadd.u64 $temp,$temp,`&Dhi("$Temp")` + veor $Z,$Z,$Z + vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! + vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! + vst1.64 {$Z}, [$toutptr,:128] + vshr.u64 $temp,$temp,#16 + + b .LNEON_outer + +.align 4 +.LNEON_outer: + vld1.32 {${Bi}[0]}, [$bptr,:32]! + sub $nptr,$nptr,$num,lsl#2 @ rewind $nptr + vld1.32 {$A0-$A3}, [$aptr]! + veor $zero,$zero,$zero + mov $toutptr,sp + vzip.16 $Bi,$zero + sub $inner,$num,#8 + vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp + + vmlal.u32 $A0xB,$Bi,${A0}[0] + vld1.64 {$A3xB-$A4xB},[$tinptr,:256]! + vmlal.u32 $A1xB,$Bi,${A0}[1] + vmlal.u32 $A2xB,$Bi,${A1}[0] + vld1.64 {$A5xB-$A6xB},[$tinptr,:256]! + vmlal.u32 $A3xB,$Bi,${A1}[1] + + vshl.i64 $temp,`&Dhi("$A0xB")`,#16 + veor $zero,$zero,$zero + vadd.u64 $temp,$temp,`&Dlo("$A0xB")` + vld1.64 {$A7xB},[$tinptr,:128]! + vmul.u32 $Ni,$temp,$M0 + + vmlal.u32 $A4xB,$Bi,${A2}[0] + vld1.32 {$N0-$N3}, [$nptr]! + vmlal.u32 $A5xB,$Bi,${A2}[1] + vmlal.u32 $A6xB,$Bi,${A3}[0] + vzip.16 $Ni,$zero + vmlal.u32 $A7xB,$Bi,${A3}[1] + +.LNEON_inner: + vmlal.u32 $A0xB,$Ni,${N0}[0] + vld1.32 {$A0-$A3}, [$aptr]! + vmlal.u32 $A1xB,$Ni,${N0}[1] + subs $inner,$inner,#8 + vmlal.u32 $A2xB,$Ni,${N1}[0] + vmlal.u32 $A3xB,$Ni,${N1}[1] + vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! + + vmlal.u32 $A4xB,$Ni,${N2}[0] + vld1.64 {$A0xB}, [$tinptr, :128]! + vmlal.u32 $A5xB,$Ni,${N2}[1] + vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! + vmlal.u32 $A6xB,$Ni,${N3}[0] + vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! + vmlal.u32 $A7xB,$Ni,${N3}[1] + vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! + + vmlal.u32 $A0xB,$Bi,${A0}[0] + vld1.64 {$A3xB-$A4xB}, [$tinptr, :256]! + vmlal.u32 $A1xB,$Bi,${A0}[1] + vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! + vmlal.u32 $A2xB,$Bi,${A1}[0] + vld1.64 {$A5xB-$A6xB}, [$tinptr, :256]! + vmlal.u32 $A3xB,$Bi,${A1}[1] + vld1.32 {$N0-$N3}, [$nptr]! + + vmlal.u32 $A4xB,$Bi,${A2}[0] + vld1.64 {$A7xB}, [$tinptr, :128]! + vmlal.u32 $A5xB,$Bi,${A2}[1] + vmlal.u32 $A6xB,$Bi,${A3}[0] + vmlal.u32 $A7xB,$Bi,${A3}[1] + + bne .LNEON_inner + + vmlal.u32 $A0xB,$Ni,${N0}[0] + add $tinptr,sp,#16 + vmlal.u32 $A1xB,$Ni,${N0}[1] + sub $aptr,$aptr,$num,lsl#2 @ rewind $aptr + vmlal.u32 $A2xB,$Ni,${N1}[0] + vld1.64 {$Temp}, [sp,:128] + vmlal.u32 $A3xB,$Ni,${N1}[1] + subs $outer,$outer,#1 + + vmlal.u32 $A4xB,$Ni,${N2}[0] + vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! + vmlal.u32 $A5xB,$Ni,${N2}[1] + vld1.64 {$A0xB}, [$tinptr, :128]! + vshr.u64 $temp,$temp,#16 + vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! + vmlal.u32 $A6xB,$Ni,${N3}[0] + vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! + vmlal.u32 $A7xB,$Ni,${N3}[1] + + vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! + vadd.u64 $temp,$temp,`&Dhi("$Temp")` + vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! + vshr.u64 $temp,$temp,#16 + + bne .LNEON_outer + + mov $toutptr,sp + mov $inner,$num + +.LNEON_tail: + vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp + vld1.64 {$A3xB-$A4xB}, [$tinptr, :256]! + vshr.u64 $temp,`&Dlo("$A0xB")`,#16 + vadd.u64 `&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp + vld1.64 {$A5xB-$A6xB}, [$tinptr, :256]! + vshr.u64 $temp,`&Dhi("$A0xB")`,#16 + vld1.64 {$A7xB}, [$tinptr, :128]! + vzip.16 `&Dlo("$A0xB")`,`&Dhi("$A0xB")` + +.LNEON_tail2: + vadd.u64 `&Dlo("$A1xB")`,`&Dlo("$A1xB")`,$temp + vst1.32 {`&Dlo("$A0xB")`[0]}, [$toutptr, :32]! + vshr.u64 $temp,`&Dlo("$A1xB")`,#16 + vadd.u64 `&Dhi("$A1xB")`,`&Dhi("$A1xB")`,$temp + vshr.u64 $temp,`&Dhi("$A1xB")`,#16 + vzip.16 `&Dlo("$A1xB")`,`&Dhi("$A1xB")` + + vadd.u64 `&Dlo("$A2xB")`,`&Dlo("$A2xB")`,$temp + vst1.32 {`&Dlo("$A1xB")`[0]}, [$toutptr, :32]! + vshr.u64 $temp,`&Dlo("$A2xB")`,#16 + vadd.u64 `&Dhi("$A2xB")`,`&Dhi("$A2xB")`,$temp + vshr.u64 $temp,`&Dhi("$A2xB")`,#16 + vzip.16 `&Dlo("$A2xB")`,`&Dhi("$A2xB")` + + vadd.u64 `&Dlo("$A3xB")`,`&Dlo("$A3xB")`,$temp + vst1.32 {`&Dlo("$A2xB")`[0]}, [$toutptr, :32]! + vshr.u64 $temp,`&Dlo("$A3xB")`,#16 + vadd.u64 `&Dhi("$A3xB")`,`&Dhi("$A3xB")`,$temp + vshr.u64 $temp,`&Dhi("$A3xB")`,#16 + vzip.16 `&Dlo("$A3xB")`,`&Dhi("$A3xB")` + + vadd.u64 `&Dlo("$A4xB")`,`&Dlo("$A4xB")`,$temp + vst1.32 {`&Dlo("$A3xB")`[0]}, [$toutptr, :32]! + vshr.u64 $temp,`&Dlo("$A4xB")`,#16 + vadd.u64 `&Dhi("$A4xB")`,`&Dhi("$A4xB")`,$temp + vshr.u64 $temp,`&Dhi("$A4xB")`,#16 + vzip.16 `&Dlo("$A4xB")`,`&Dhi("$A4xB")` + + vadd.u64 `&Dlo("$A5xB")`,`&Dlo("$A5xB")`,$temp + vst1.32 {`&Dlo("$A4xB")`[0]}, [$toutptr, :32]! + vshr.u64 $temp,`&Dlo("$A5xB")`,#16 + vadd.u64 `&Dhi("$A5xB")`,`&Dhi("$A5xB")`,$temp + vshr.u64 $temp,`&Dhi("$A5xB")`,#16 + vzip.16 `&Dlo("$A5xB")`,`&Dhi("$A5xB")` + + vadd.u64 `&Dlo("$A6xB")`,`&Dlo("$A6xB")`,$temp + vst1.32 {`&Dlo("$A5xB")`[0]}, [$toutptr, :32]! + vshr.u64 $temp,`&Dlo("$A6xB")`,#16 + vadd.u64 `&Dhi("$A6xB")`,`&Dhi("$A6xB")`,$temp + vld1.64 {$A0xB}, [$tinptr, :128]! + vshr.u64 $temp,`&Dhi("$A6xB")`,#16 + vzip.16 `&Dlo("$A6xB")`,`&Dhi("$A6xB")` + + vadd.u64 `&Dlo("$A7xB")`,`&Dlo("$A7xB")`,$temp + vst1.32 {`&Dlo("$A6xB")`[0]}, [$toutptr, :32]! + vshr.u64 $temp,`&Dlo("$A7xB")`,#16 + vadd.u64 `&Dhi("$A7xB")`,`&Dhi("$A7xB")`,$temp + vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! + vshr.u64 $temp,`&Dhi("$A7xB")`,#16 + vzip.16 `&Dlo("$A7xB")`,`&Dhi("$A7xB")` + subs $inner,$inner,#8 + vst1.32 {`&Dlo("$A7xB")`[0]}, [$toutptr, :32]! + + bne .LNEON_tail + + vst1.32 {${temp}[0]}, [$toutptr, :32] @ top-most bit + sub $nptr,$nptr,$num,lsl#2 @ rewind $nptr + subs $aptr,sp,#0 @ clear carry flag + add $bptr,sp,$num,lsl#2 + +.LNEON_sub: + ldmia $aptr!, {r4-r7} + ldmia $nptr!, {r8-r11} + sbcs r8, r4,r8 + sbcs r9, r5,r9 + sbcs r10,r6,r10 + sbcs r11,r7,r11 + teq $aptr,$bptr @ preserves carry + stmia $rptr!, {r8-r11} + bne .LNEON_sub + + ldr r10, [$aptr] @ load top-most bit + veor q0,q0,q0 + sub r11,$bptr,sp @ this is num*4 + veor q1,q1,q1 + mov $aptr,sp + sub $rptr,$rptr,r11 @ rewind $rptr + mov $nptr,$bptr @ second 3/4th of frame + sbcs r10,r10,#0 @ result is carry flag + +.LNEON_copy_n_zap: + ldmia $aptr!, {r4-r7} + ldmia $rptr, {r8-r11} + movcc r8, r4 + vst1.64 {q0-q1}, [$nptr,:256]! @ wipe + movcc r9, r5 + movcc r10,r6 + vst1.64 {q0-q1}, [$nptr,:256]! @ wipe + movcc r11,r7 + ldmia $aptr, {r4-r7} + stmia $rptr!, {r8-r11} + sub $aptr,$aptr,#16 + ldmia $rptr, {r8-r11} + movcc r8, r4 + vst1.64 {q0-q1}, [$aptr,:256]! @ wipe + movcc r9, r5 + movcc r10,r6 + vst1.64 {q0-q1}, [$nptr,:256]! @ wipe + movcc r11,r7 + teq $aptr,$bptr @ preserves carry + stmia $rptr!, {r8-r11} + bne .LNEON_copy_n_zap + + sub sp,ip,#96 + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r11} + ret @ bx lr +.size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon +#endif +___ +} +$code.=<<___; +.asciz "Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by " .align 2 +#if __ARM_MAX_ARCH__>=7 +.comm OPENSSL_armcap_P,4,4 +#endif ___ +$code =~ s/\`([^\`]*)\`/eval $1/gem; $code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 +$code =~ s/\bret\b/bx lr/gm; print $code; close STDOUT; diff --git a/crypto/bn/asm/mips-mont.pl b/crypto/bn/asm/mips-mont.pl index caae04ed3a86..a33cdf411121 100755 --- a/crypto/bn/asm/mips-mont.pl +++ b/crypto/bn/asm/mips-mont.pl @@ -46,7 +46,7 @@ # ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); # ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); # -$flavour = shift; # supported flavours are o32,n32,64,nubi32,nubi64 +$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64 if ($flavour =~ /64|n32/i) { $PTR_ADD="dadd"; # incidentally works even on n32 diff --git a/crypto/bn/asm/mips.pl b/crypto/bn/asm/mips.pl index 215c9a748325..acafde5e5685 100755 --- a/crypto/bn/asm/mips.pl +++ b/crypto/bn/asm/mips.pl @@ -48,7 +48,7 @@ # has to content with 40-85% improvement depending on benchmark and # key length, more for longer keys. -$flavour = shift; +$flavour = shift || "o32"; while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; diff --git a/crypto/bn/asm/mips3.s b/crypto/bn/asm/mips3.s new file mode 100644 index 000000000000..dca4105c7db1 --- /dev/null +++ b/crypto/bn/asm/mips3.s @@ -0,0 +1,2201 @@ +.rdata +.asciiz "mips3.s, Version 1.1" +.asciiz "MIPS III/IV ISA artwork by Andy Polyakov " + +/* + * ==================================================================== + * Written by Andy Polyakov for the OpenSSL + * project. + * + * Rights for redistribution and usage in source and binary forms are + * granted according to the OpenSSL license. Warranty of any kind is + * disclaimed. + * ==================================================================== + */ + +/* + * This is my modest contributon to the OpenSSL project (see + * http://www.openssl.org/ for more information about it) and is + * a drop-in MIPS III/IV ISA replacement for crypto/bn/bn_asm.c + * module. For updates see http://fy.chalmers.se/~appro/hpe/. + * + * The module is designed to work with either of the "new" MIPS ABI(5), + * namely N32 or N64, offered by IRIX 6.x. It's not ment to work under + * IRIX 5.x not only because it doesn't support new ABIs but also + * because 5.x kernels put R4x00 CPU into 32-bit mode and all those + * 64-bit instructions (daddu, dmultu, etc.) found below gonna only + * cause illegal instruction exception:-( + * + * In addition the code depends on preprocessor flags set up by MIPSpro + * compiler driver (either as or cc) and therefore (probably?) can't be + * compiled by the GNU assembler. GNU C driver manages fine though... + * I mean as long as -mmips-as is specified or is the default option, + * because then it simply invokes /usr/bin/as which in turn takes + * perfect care of the preprocessor definitions. Another neat feature + * offered by the MIPSpro assembler is an optimization pass. This gave + * me the opportunity to have the code looking more regular as all those + * architecture dependent instruction rescheduling details were left to + * the assembler. Cool, huh? + * + * Performance improvement is astonishing! 'apps/openssl speed rsa dsa' + * goes way over 3 times faster! + * + * + */ +#include +#include + +#if _MIPS_ISA>=4 +#define MOVNZ(cond,dst,src) \ + movn dst,src,cond +#else +#define MOVNZ(cond,dst,src) \ + .set noreorder; \ + bnezl cond,.+8; \ + move dst,src; \ + .set reorder +#endif + +.text + +.set noat +.set reorder + +#define MINUS4 v1 + +.align 5 +LEAF(bn_mul_add_words) + .set noreorder + bgtzl a2,.L_bn_mul_add_words_proceed + ld t0,0(a1) + jr ra + move v0,zero + .set reorder + +.L_bn_mul_add_words_proceed: + li MINUS4,-4 + and ta0,a2,MINUS4 + move v0,zero + beqz ta0,.L_bn_mul_add_words_tail + +.L_bn_mul_add_words_loop: + dmultu t0,a3 + ld t1,0(a0) + ld t2,8(a1) + ld t3,8(a0) + ld ta0,16(a1) + ld ta1,16(a0) + daddu t1,v0 + sltu v0,t1,v0 /* All manuals say it "compares 32-bit + * values", but it seems to work fine + * even on 64-bit registers. */ + mflo AT + mfhi t0 + daddu t1,AT + daddu v0,t0 + sltu AT,t1,AT + sd t1,0(a0) + daddu v0,AT + + dmultu t2,a3 + ld ta2,24(a1) + ld ta3,24(a0) + daddu t3,v0 + sltu v0,t3,v0 + mflo AT + mfhi t2 + daddu t3,AT + daddu v0,t2 + sltu AT,t3,AT + sd t3,8(a0) + daddu v0,AT + + dmultu ta0,a3 + subu a2,4 + PTR_ADD a0,32 + PTR_ADD a1,32 + daddu ta1,v0 + sltu v0,ta1,v0 + mflo AT + mfhi ta0 + daddu ta1,AT + daddu v0,ta0 + sltu AT,ta1,AT + sd ta1,-16(a0) + daddu v0,AT + + + dmultu ta2,a3 + and ta0,a2,MINUS4 + daddu ta3,v0 + sltu v0,ta3,v0 + mflo AT + mfhi ta2 + daddu ta3,AT + daddu v0,ta2 + sltu AT,ta3,AT + sd ta3,-8(a0) + daddu v0,AT + .set noreorder + bgtzl ta0,.L_bn_mul_add_words_loop + ld t0,0(a1) + + bnezl a2,.L_bn_mul_add_words_tail + ld t0,0(a1) + .set reorder + +.L_bn_mul_add_words_return: + jr ra + +.L_bn_mul_add_words_tail: + dmultu t0,a3 + ld t1,0(a0) + subu a2,1 + daddu t1,v0 + sltu v0,t1,v0 + mflo AT + mfhi t0 + daddu t1,AT + daddu v0,t0 + sltu AT,t1,AT + sd t1,0(a0) + daddu v0,AT + beqz a2,.L_bn_mul_add_words_return + + ld t0,8(a1) + dmultu t0,a3 + ld t1,8(a0) + subu a2,1 + daddu t1,v0 + sltu v0,t1,v0 + mflo AT + mfhi t0 + daddu t1,AT + daddu v0,t0 + sltu AT,t1,AT + sd t1,8(a0) + daddu v0,AT + beqz a2,.L_bn_mul_add_words_return + + ld t0,16(a1) + dmultu t0,a3 + ld t1,16(a0) + daddu t1,v0 + sltu v0,t1,v0 + mflo AT + mfhi t0 + daddu t1,AT + daddu v0,t0 + sltu AT,t1,AT + sd t1,16(a0) + daddu v0,AT + jr ra +END(bn_mul_add_words) + +.align 5 +LEAF(bn_mul_words) + .set noreorder + bgtzl a2,.L_bn_mul_words_proceed + ld t0,0(a1) + jr ra + move v0,zero + .set reorder + +.L_bn_mul_words_proceed: + li MINUS4,-4 + and ta0,a2,MINUS4 + move v0,zero + beqz ta0,.L_bn_mul_words_tail + +.L_bn_mul_words_loop: + dmultu t0,a3 + ld t2,8(a1) + ld ta0,16(a1) + ld ta2,24(a1) + mflo AT + mfhi t0 + daddu v0,AT + sltu t1,v0,AT + sd v0,0(a0) + daddu v0,t1,t0 + + dmultu t2,a3 + subu a2,4 + PTR_ADD a0,32 + PTR_ADD a1,32 + mflo AT + mfhi t2 + daddu v0,AT + sltu t3,v0,AT + sd v0,-24(a0) + daddu v0,t3,t2 + + dmultu ta0,a3 + mflo AT + mfhi ta0 + daddu v0,AT + sltu ta1,v0,AT + sd v0,-16(a0) + daddu v0,ta1,ta0 + + + dmultu ta2,a3 + and ta0,a2,MINUS4 + mflo AT + mfhi ta2 + daddu v0,AT + sltu ta3,v0,AT + sd v0,-8(a0) + daddu v0,ta3,ta2 + .set noreorder + bgtzl ta0,.L_bn_mul_words_loop + ld t0,0(a1) + + bnezl a2,.L_bn_mul_words_tail + ld t0,0(a1) + .set reorder + +.L_bn_mul_words_return: + jr ra + +.L_bn_mul_words_tail: + dmultu t0,a3 + subu a2,1 + mflo AT + mfhi t0 + daddu v0,AT + sltu t1,v0,AT + sd v0,0(a0) + daddu v0,t1,t0 + beqz a2,.L_bn_mul_words_return + + ld t0,8(a1) + dmultu t0,a3 + subu a2,1 + mflo AT + mfhi t0 + daddu v0,AT + sltu t1,v0,AT + sd v0,8(a0) + daddu v0,t1,t0 + beqz a2,.L_bn_mul_words_return + + ld t0,16(a1) + dmultu t0,a3 + mflo AT + mfhi t0 + daddu v0,AT + sltu t1,v0,AT + sd v0,16(a0) + daddu v0,t1,t0 + jr ra +END(bn_mul_words) + +.align 5 +LEAF(bn_sqr_words) + .set noreorder + bgtzl a2,.L_bn_sqr_words_proceed + ld t0,0(a1) + jr ra + move v0,zero + .set reorder + +.L_bn_sqr_words_proceed: + li MINUS4,-4 + and ta0,a2,MINUS4 + move v0,zero + beqz ta0,.L_bn_sqr_words_tail + +.L_bn_sqr_words_loop: + dmultu t0,t0 + ld t2,8(a1) + ld ta0,16(a1) + ld ta2,24(a1) + mflo t1 + mfhi t0 + sd t1,0(a0) + sd t0,8(a0) + + dmultu t2,t2 + subu a2,4 + PTR_ADD a0,64 + PTR_ADD a1,32 + mflo t3 + mfhi t2 + sd t3,-48(a0) + sd t2,-40(a0) + + dmultu ta0,ta0 + mflo ta1 + mfhi ta0 + sd ta1,-32(a0) + sd ta0,-24(a0) + + + dmultu ta2,ta2 + and ta0,a2,MINUS4 + mflo ta3 + mfhi ta2 + sd ta3,-16(a0) + sd ta2,-8(a0) + + .set noreorder + bgtzl ta0,.L_bn_sqr_words_loop + ld t0,0(a1) + + bnezl a2,.L_bn_sqr_words_tail + ld t0,0(a1) + .set reorder + +.L_bn_sqr_words_return: + move v0,zero + jr ra + +.L_bn_sqr_words_tail: + dmultu t0,t0 + subu a2,1 + mflo t1 + mfhi t0 + sd t1,0(a0) + sd t0,8(a0) + beqz a2,.L_bn_sqr_words_return + + ld t0,8(a1) + dmultu t0,t0 + subu a2,1 + mflo t1 + mfhi t0 + sd t1,16(a0) + sd t0,24(a0) + beqz a2,.L_bn_sqr_words_return + + ld t0,16(a1) + dmultu t0,t0 + mflo t1 + mfhi t0 + sd t1,32(a0) + sd t0,40(a0) + jr ra +END(bn_sqr_words) + +.align 5 +LEAF(bn_add_words) + .set noreorder + bgtzl a3,.L_bn_add_words_proceed + ld t0,0(a1) + jr ra + move v0,zero + .set reorder + +.L_bn_add_words_proceed: + li MINUS4,-4 + and AT,a3,MINUS4 + move v0,zero + beqz AT,.L_bn_add_words_tail + +.L_bn_add_words_loop: + ld ta0,0(a2) + subu a3,4 + ld t1,8(a1) + and AT,a3,MINUS4 + ld t2,16(a1) + PTR_ADD a2,32 + ld t3,24(a1) + PTR_ADD a0,32 + ld ta1,-24(a2) + PTR_ADD a1,32 + ld ta2,-16(a2) + ld ta3,-8(a2) + daddu ta0,t0 + sltu t8,ta0,t0 + daddu t0,ta0,v0 + sltu v0,t0,ta0 + sd t0,-32(a0) + daddu v0,t8 + + daddu ta1,t1 + sltu t9,ta1,t1 + daddu t1,ta1,v0 + sltu v0,t1,ta1 + sd t1,-24(a0) + daddu v0,t9 + + daddu ta2,t2 + sltu t8,ta2,t2 + daddu t2,ta2,v0 + sltu v0,t2,ta2 + sd t2,-16(a0) + daddu v0,t8 + + daddu ta3,t3 + sltu t9,ta3,t3 + daddu t3,ta3,v0 + sltu v0,t3,ta3 + sd t3,-8(a0) + daddu v0,t9 + + .set noreorder + bgtzl AT,.L_bn_add_words_loop + ld t0,0(a1) + + bnezl a3,.L_bn_add_words_tail + ld t0,0(a1) + .set reorder + +.L_bn_add_words_return: + jr ra + +.L_bn_add_words_tail: + ld ta0,0(a2) + daddu ta0,t0 + subu a3,1 + sltu t8,ta0,t0 + daddu t0,ta0,v0 + sltu v0,t0,ta0 + sd t0,0(a0) + daddu v0,t8 + beqz a3,.L_bn_add_words_return + + ld t1,8(a1) + ld ta1,8(a2) + daddu ta1,t1 + subu a3,1 + sltu t9,ta1,t1 + daddu t1,ta1,v0 + sltu v0,t1,ta1 + sd t1,8(a0) + daddu v0,t9 + beqz a3,.L_bn_add_words_return + + ld t2,16(a1) + ld ta2,16(a2) + daddu ta2,t2 + sltu t8,ta2,t2 + daddu t2,ta2,v0 + sltu v0,t2,ta2 + sd t2,16(a0) + daddu v0,t8 + jr ra +END(bn_add_words) + +.align 5 +LEAF(bn_sub_words) + .set noreorder + bgtzl a3,.L_bn_sub_words_proceed + ld t0,0(a1) + jr ra + move v0,zero + .set reorder + +.L_bn_sub_words_proceed: + li MINUS4,-4 + and AT,a3,MINUS4 + move v0,zero + beqz AT,.L_bn_sub_words_tail + +.L_bn_sub_words_loop: + ld ta0,0(a2) + subu a3,4 + ld t1,8(a1) + and AT,a3,MINUS4 + ld t2,16(a1) + PTR_ADD a2,32 + ld t3,24(a1) + PTR_ADD a0,32 + ld ta1,-24(a2) + PTR_ADD a1,32 + ld ta2,-16(a2) + ld ta3,-8(a2) + sltu t8,t0,ta0 + dsubu t0,ta0 + dsubu ta0,t0,v0 + sd ta0,-32(a0) + MOVNZ (t0,v0,t8) + + sltu t9,t1,ta1 + dsubu t1,ta1 + dsubu ta1,t1,v0 + sd ta1,-24(a0) + MOVNZ (t1,v0,t9) + + + sltu t8,t2,ta2 + dsubu t2,ta2 + dsubu ta2,t2,v0 + sd ta2,-16(a0) + MOVNZ (t2,v0,t8) + + sltu t9,t3,ta3 + dsubu t3,ta3 + dsubu ta3,t3,v0 + sd ta3,-8(a0) + MOVNZ (t3,v0,t9) + + .set noreorder + bgtzl AT,.L_bn_sub_words_loop + ld t0,0(a1) + + bnezl a3,.L_bn_sub_words_tail + ld t0,0(a1) + .set reorder + +.L_bn_sub_words_return: + jr ra + +.L_bn_sub_words_tail: + ld ta0,0(a2) + subu a3,1 + sltu t8,t0,ta0 + dsubu t0,ta0 + dsubu ta0,t0,v0 + MOVNZ (t0,v0,t8) + sd ta0,0(a0) + beqz a3,.L_bn_sub_words_return + + ld t1,8(a1) + subu a3,1 + ld ta1,8(a2) + sltu t9,t1,ta1 + dsubu t1,ta1 + dsubu ta1,t1,v0 + MOVNZ (t1,v0,t9) + sd ta1,8(a0) + beqz a3,.L_bn_sub_words_return + + ld t2,16(a1) + ld ta2,16(a2) + sltu t8,t2,ta2 + dsubu t2,ta2 + dsubu ta2,t2,v0 + MOVNZ (t2,v0,t8) + sd ta2,16(a0) + jr ra +END(bn_sub_words) + +#undef MINUS4 + +.align 5 +LEAF(bn_div_3_words) + .set reorder + move a3,a0 /* we know that bn_div_words doesn't + * touch a3, ta2, ta3 and preserves a2 + * so that we can save two arguments + * and return address in registers + * instead of stack:-) + */ + ld a0,(a3) + move ta2,a1 + ld a1,-8(a3) + bne a0,a2,.L_bn_div_3_words_proceed + li v0,-1 + jr ra +.L_bn_div_3_words_proceed: + move ta3,ra + bal bn_div_words + move ra,ta3 + dmultu ta2,v0 + ld t2,-16(a3) + move ta0,zero + mfhi t1 + mflo t0 + sltu t8,t1,v1 +.L_bn_div_3_words_inner_loop: + bnez t8,.L_bn_div_3_words_inner_loop_done + sgeu AT,t2,t0 + seq t9,t1,v1 + and AT,t9 + sltu t3,t0,ta2 + daddu v1,a2 + dsubu t1,t3 + dsubu t0,ta2 + sltu t8,t1,v1 + sltu ta0,v1,a2 + or t8,ta0 + .set noreorder + beqzl AT,.L_bn_div_3_words_inner_loop + dsubu v0,1 + .set reorder +.L_bn_div_3_words_inner_loop_done: + jr ra +END(bn_div_3_words) + +.align 5 +LEAF(bn_div_words) + .set noreorder + bnezl a2,.L_bn_div_words_proceed + move v1,zero + jr ra + li v0,-1 /* I'd rather signal div-by-zero + * which can be done with 'break 7' */ + +.L_bn_div_words_proceed: + bltz a2,.L_bn_div_words_body + move t9,v1 + dsll a2,1 + bgtz a2,.-4 + addu t9,1 + + .set reorder + negu t1,t9 + li t2,-1 + dsll t2,t1 + and t2,a0 + dsrl AT,a1,t1 + .set noreorder + bnezl t2,.+8 + break 6 /* signal overflow */ + .set reorder + dsll a0,t9 + dsll a1,t9 + or a0,AT + +#define QT ta0 +#define HH ta1 +#define DH v1 +.L_bn_div_words_body: + dsrl DH,a2,32 + sgeu AT,a0,a2 + .set noreorder + bnezl AT,.+8 + dsubu a0,a2 + .set reorder + + li QT,-1 + dsrl HH,a0,32 + dsrl QT,32 /* q=0xffffffff */ + beq DH,HH,.L_bn_div_words_skip_div1 + ddivu zero,a0,DH + mflo QT +.L_bn_div_words_skip_div1: + dmultu a2,QT + dsll t3,a0,32 + dsrl AT,a1,32 + or t3,AT + mflo t0 + mfhi t1 +.L_bn_div_words_inner_loop1: + sltu t2,t3,t0 + seq t8,HH,t1 + sltu AT,HH,t1 + and t2,t8 + sltu v0,t0,a2 + or AT,t2 + .set noreorder + beqz AT,.L_bn_div_words_inner_loop1_done + dsubu t1,v0 + dsubu t0,a2 + b .L_bn_div_words_inner_loop1 + dsubu QT,1 + .set reorder +.L_bn_div_words_inner_loop1_done: + + dsll a1,32 + dsubu a0,t3,t0 + dsll v0,QT,32 + + li QT,-1 + dsrl HH,a0,32 + dsrl QT,32 /* q=0xffffffff */ + beq DH,HH,.L_bn_div_words_skip_div2 + ddivu zero,a0,DH + mflo QT +.L_bn_div_words_skip_div2: +#undef DH + dmultu a2,QT + dsll t3,a0,32 + dsrl AT,a1,32 + or t3,AT + mflo t0 + mfhi t1 +.L_bn_div_words_inner_loop2: + sltu t2,t3,t0 + seq t8,HH,t1 + sltu AT,HH,t1 + and t2,t8 + sltu v1,t0,a2 + or AT,t2 + .set noreorder + beqz AT,.L_bn_div_words_inner_loop2_done + dsubu t1,v1 + dsubu t0,a2 + b .L_bn_div_words_inner_loop2 + dsubu QT,1 + .set reorder +.L_bn_div_words_inner_loop2_done: +#undef HH + + dsubu a0,t3,t0 + or v0,QT + dsrl v1,a0,t9 /* v1 contains remainder if anybody wants it */ + dsrl a2,t9 /* restore a2 */ + jr ra +#undef QT +END(bn_div_words) + +#define a_0 t0 +#define a_1 t1 +#define a_2 t2 +#define a_3 t3 +#define b_0 ta0 +#define b_1 ta1 +#define b_2 ta2 +#define b_3 ta3 + +#define a_4 s0 +#define a_5 s2 +#define a_6 s4 +#define a_7 a1 /* once we load a[7] we don't need a anymore */ +#define b_4 s1 +#define b_5 s3 +#define b_6 s5 +#define b_7 a2 /* once we load b[7] we don't need b anymore */ + +#define t_1 t8 +#define t_2 t9 + +#define c_1 v0 +#define c_2 v1 +#define c_3 a3 + +#define FRAME_SIZE 48 + +.align 5 +LEAF(bn_mul_comba8) + .set noreorder + PTR_SUB sp,FRAME_SIZE + .frame sp,64,ra + .set reorder + ld a_0,0(a1) /* If compiled with -mips3 option on + * R5000 box assembler barks on this + * line with "shouldn't have mult/div + * as last instruction in bb (R10K + * bug)" warning. If anybody out there + * has a clue about how to circumvent + * this do send me a note. + * + */ + ld b_0,0(a2) + ld a_1,8(a1) + ld a_2,16(a1) + ld a_3,24(a1) + ld b_1,8(a2) + ld b_2,16(a2) + ld b_3,24(a2) + dmultu a_0,b_0 /* mul_add_c(a[0],b[0],c1,c2,c3); */ + sd s0,0(sp) + sd s1,8(sp) + sd s2,16(sp) + sd s3,24(sp) + sd s4,32(sp) + sd s5,40(sp) + mflo c_1 + mfhi c_2 + + dmultu a_0,b_1 /* mul_add_c(a[0],b[1],c2,c3,c1); */ + ld a_4,32(a1) + ld a_5,40(a1) + ld a_6,48(a1) + ld a_7,56(a1) + ld b_4,32(a2) + ld b_5,40(a2) + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu c_3,t_2,AT + dmultu a_1,b_0 /* mul_add_c(a[1],b[0],c2,c3,c1); */ + ld b_6,48(a2) + ld b_7,56(a2) + sd c_1,0(a0) /* r[0]=c1; */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu c_1,c_3,t_2 + sd c_2,8(a0) /* r[1]=c2; */ + + dmultu a_2,b_0 /* mul_add_c(a[2],b[0],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + dmultu a_1,b_1 /* mul_add_c(a[1],b[1],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu c_2,c_1,t_2 + dmultu a_0,b_2 /* mul_add_c(a[0],b[2],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,16(a0) /* r[2]=c3; */ + + dmultu a_0,b_3 /* mul_add_c(a[0],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu c_3,c_2,t_2 + dmultu a_1,b_2 /* mul_add_c(a[1],b[2],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_2,b_1 /* mul_add_c(a[2],b[1],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_3,b_0 /* mul_add_c(a[3],b[0],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,24(a0) /* r[3]=c1; */ + + dmultu a_4,b_0 /* mul_add_c(a[4],b[0],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu c_1,c_3,t_2 + dmultu a_3,b_1 /* mul_add_c(a[3],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_2,b_2 /* mul_add_c(a[2],b[2],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_1,b_3 /* mul_add_c(a[1],b[3],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_0,b_4 /* mul_add_c(a[0],b[4],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,32(a0) /* r[4]=c2; */ + + dmultu a_0,b_5 /* mul_add_c(a[0],b[5],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu c_2,c_1,t_2 + dmultu a_1,b_4 /* mul_add_c(a[1],b[4],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_2,b_3 /* mul_add_c(a[2],b[3],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_3,b_2 /* mul_add_c(a[3],b[2],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_4,b_1 /* mul_add_c(a[4],b[1],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_5,b_0 /* mul_add_c(a[5],b[0],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,40(a0) /* r[5]=c3; */ + + dmultu a_6,b_0 /* mul_add_c(a[6],b[0],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu c_3,c_2,t_2 + dmultu a_5,b_1 /* mul_add_c(a[5],b[1],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_4,b_2 /* mul_add_c(a[4],b[2],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_3,b_3 /* mul_add_c(a[3],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_2,b_4 /* mul_add_c(a[2],b[4],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_1,b_5 /* mul_add_c(a[1],b[5],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_0,b_6 /* mul_add_c(a[0],b[6],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,48(a0) /* r[6]=c1; */ + + dmultu a_0,b_7 /* mul_add_c(a[0],b[7],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu c_1,c_3,t_2 + dmultu a_1,b_6 /* mul_add_c(a[1],b[6],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_2,b_5 /* mul_add_c(a[2],b[5],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_3,b_4 /* mul_add_c(a[3],b[4],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_4,b_3 /* mul_add_c(a[4],b[3],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_5,b_2 /* mul_add_c(a[5],b[2],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_6,b_1 /* mul_add_c(a[6],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_7,b_0 /* mul_add_c(a[7],b[0],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,56(a0) /* r[7]=c2; */ + + dmultu a_7,b_1 /* mul_add_c(a[7],b[1],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu c_2,c_1,t_2 + dmultu a_6,b_2 /* mul_add_c(a[6],b[2],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_5,b_3 /* mul_add_c(a[5],b[3],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_4,b_4 /* mul_add_c(a[4],b[4],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_3,b_5 /* mul_add_c(a[3],b[5],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_2,b_6 /* mul_add_c(a[2],b[6],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_1,b_7 /* mul_add_c(a[1],b[7],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,64(a0) /* r[8]=c3; */ + + dmultu a_2,b_7 /* mul_add_c(a[2],b[7],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu c_3,c_2,t_2 + dmultu a_3,b_6 /* mul_add_c(a[3],b[6],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_4,b_5 /* mul_add_c(a[4],b[5],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_5,b_4 /* mul_add_c(a[5],b[4],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_6,b_3 /* mul_add_c(a[6],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_7,b_2 /* mul_add_c(a[7],b[2],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,72(a0) /* r[9]=c1; */ + + dmultu a_7,b_3 /* mul_add_c(a[7],b[3],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu c_1,c_3,t_2 + dmultu a_6,b_4 /* mul_add_c(a[6],b[4],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_5,b_5 /* mul_add_c(a[5],b[5],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_4,b_6 /* mul_add_c(a[4],b[6],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_3,b_7 /* mul_add_c(a[3],b[7],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,80(a0) /* r[10]=c2; */ + + dmultu a_4,b_7 /* mul_add_c(a[4],b[7],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu c_2,c_1,t_2 + dmultu a_5,b_6 /* mul_add_c(a[5],b[6],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_6,b_5 /* mul_add_c(a[6],b[5],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_7,b_4 /* mul_add_c(a[7],b[4],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,88(a0) /* r[11]=c3; */ + + dmultu a_7,b_5 /* mul_add_c(a[7],b[5],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu c_3,c_2,t_2 + dmultu a_6,b_6 /* mul_add_c(a[6],b[6],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_5,b_7 /* mul_add_c(a[5],b[7],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,96(a0) /* r[12]=c1; */ + + dmultu a_6,b_7 /* mul_add_c(a[6],b[7],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu c_1,c_3,t_2 + dmultu a_7,b_6 /* mul_add_c(a[7],b[6],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,104(a0) /* r[13]=c2; */ + + dmultu a_7,b_7 /* mul_add_c(a[7],b[7],c3,c1,c2); */ + ld s0,0(sp) + ld s1,8(sp) + ld s2,16(sp) + ld s3,24(sp) + ld s4,32(sp) + ld s5,40(sp) + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sd c_3,112(a0) /* r[14]=c3; */ + sd c_1,120(a0) /* r[15]=c1; */ + + PTR_ADD sp,FRAME_SIZE + + jr ra +END(bn_mul_comba8) + +.align 5 +LEAF(bn_mul_comba4) + .set reorder + ld a_0,0(a1) + ld b_0,0(a2) + ld a_1,8(a1) + ld a_2,16(a1) + dmultu a_0,b_0 /* mul_add_c(a[0],b[0],c1,c2,c3); */ + ld a_3,24(a1) + ld b_1,8(a2) + ld b_2,16(a2) + ld b_3,24(a2) + mflo c_1 + mfhi c_2 + sd c_1,0(a0) + + dmultu a_0,b_1 /* mul_add_c(a[0],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu c_3,t_2,AT + dmultu a_1,b_0 /* mul_add_c(a[1],b[0],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu c_1,c_3,t_2 + sd c_2,8(a0) + + dmultu a_2,b_0 /* mul_add_c(a[2],b[0],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + dmultu a_1,b_1 /* mul_add_c(a[1],b[1],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu c_2,c_1,t_2 + dmultu a_0,b_2 /* mul_add_c(a[0],b[2],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,16(a0) + + dmultu a_0,b_3 /* mul_add_c(a[0],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu c_3,c_2,t_2 + dmultu a_1,b_2 /* mul_add_c(a[1],b[2],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_2,b_1 /* mul_add_c(a[2],b[1],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_3,b_0 /* mul_add_c(a[3],b[0],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,24(a0) + + dmultu a_3,b_1 /* mul_add_c(a[3],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu c_1,c_3,t_2 + dmultu a_2,b_2 /* mul_add_c(a[2],b[2],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_1,b_3 /* mul_add_c(a[1],b[3],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,32(a0) + + dmultu a_2,b_3 /* mul_add_c(a[2],b[3],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu c_2,c_1,t_2 + dmultu a_3,b_2 /* mul_add_c(a[3],b[2],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,40(a0) + + dmultu a_3,b_3 /* mul_add_c(a[3],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sd c_1,48(a0) + sd c_2,56(a0) + + jr ra +END(bn_mul_comba4) + +#undef a_4 +#undef a_5 +#undef a_6 +#undef a_7 +#define a_4 b_0 +#define a_5 b_1 +#define a_6 b_2 +#define a_7 b_3 + +.align 5 +LEAF(bn_sqr_comba8) + .set reorder + ld a_0,0(a1) + ld a_1,8(a1) + ld a_2,16(a1) + ld a_3,24(a1) + + dmultu a_0,a_0 /* mul_add_c(a[0],b[0],c1,c2,c3); */ + ld a_4,32(a1) + ld a_5,40(a1) + ld a_6,48(a1) + ld a_7,56(a1) + mflo c_1 + mfhi c_2 + sd c_1,0(a0) + + dmultu a_0,a_1 /* mul_add_c2(a[0],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt c_1,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu c_3,t_2,AT + sd c_2,8(a0) + + dmultu a_2,a_0 /* mul_add_c2(a[2],b[0],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt c_2,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_1,a_1 /* mul_add_c(a[1],b[1],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,16(a0) + + dmultu a_0,a_3 /* mul_add_c2(a[0],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt c_3,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_1,a_2 /* mul_add_c2(a[1],b[2],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_3,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,24(a0) + + dmultu a_4,a_0 /* mul_add_c2(a[4],b[0],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt c_1,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_3,a_1 /* mul_add_c2(a[3],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_1,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_2,a_2 /* mul_add_c(a[2],b[2],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,32(a0) + + dmultu a_0,a_5 /* mul_add_c2(a[0],b[5],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt c_2,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_1,a_4 /* mul_add_c2(a[1],b[4],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_2,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_2,a_3 /* mul_add_c2(a[2],b[3],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_2,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,40(a0) + + dmultu a_6,a_0 /* mul_add_c2(a[6],b[0],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt c_3,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_5,a_1 /* mul_add_c2(a[5],b[1],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_3,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_4,a_2 /* mul_add_c2(a[4],b[2],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_3,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_3,a_3 /* mul_add_c(a[3],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,48(a0) + + dmultu a_0,a_7 /* mul_add_c2(a[0],b[7],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt c_1,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_1,a_6 /* mul_add_c2(a[1],b[6],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_1,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_2,a_5 /* mul_add_c2(a[2],b[5],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_1,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_3,a_4 /* mul_add_c2(a[3],b[4],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_1,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,56(a0) + + dmultu a_7,a_1 /* mul_add_c2(a[7],b[1],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt c_2,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_6,a_2 /* mul_add_c2(a[6],b[2],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_2,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_5,a_3 /* mul_add_c2(a[5],b[3],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_2,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_4,a_4 /* mul_add_c(a[4],b[4],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,64(a0) + + dmultu a_2,a_7 /* mul_add_c2(a[2],b[7],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt c_3,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_3,a_6 /* mul_add_c2(a[3],b[6],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_3,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_4,a_5 /* mul_add_c2(a[4],b[5],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_3,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,72(a0) + + dmultu a_7,a_3 /* mul_add_c2(a[7],b[3],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt c_1,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_6,a_4 /* mul_add_c2(a[6],b[4],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_1,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_5,a_5 /* mul_add_c(a[5],b[5],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,80(a0) + + dmultu a_4,a_7 /* mul_add_c2(a[4],b[7],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt c_2,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_5,a_6 /* mul_add_c2(a[5],b[6],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_2,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,88(a0) + + dmultu a_7,a_5 /* mul_add_c2(a[7],b[5],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt c_3,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_6,a_6 /* mul_add_c(a[6],b[6],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,96(a0) + + dmultu a_6,a_7 /* mul_add_c2(a[6],b[7],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt c_1,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,104(a0) + + dmultu a_7,a_7 /* mul_add_c(a[7],b[7],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sd c_3,112(a0) + sd c_1,120(a0) + + jr ra +END(bn_sqr_comba8) + +.align 5 +LEAF(bn_sqr_comba4) + .set reorder + ld a_0,0(a1) + ld a_1,8(a1) + ld a_2,16(a1) + ld a_3,24(a1) + dmultu a_0,a_0 /* mul_add_c(a[0],b[0],c1,c2,c3); */ + mflo c_1 + mfhi c_2 + sd c_1,0(a0) + + dmultu a_0,a_1 /* mul_add_c2(a[0],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt c_1,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu c_3,t_2,AT + sd c_2,8(a0) + + dmultu a_2,a_0 /* mul_add_c2(a[2],b[0],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt c_2,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + dmultu a_1,a_1 /* mul_add_c(a[1],b[1],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,16(a0) + + dmultu a_0,a_3 /* mul_add_c2(a[0],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt c_3,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + dmultu a_1,a_2 /* mul_add_c(a2[1],b[2],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + slt AT,t_2,zero + daddu c_3,AT + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sltu AT,c_2,t_2 + daddu c_3,AT + sd c_1,24(a0) + + dmultu a_3,a_1 /* mul_add_c2(a[3],b[1],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + slt c_1,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + dmultu a_2,a_2 /* mul_add_c(a[2],b[2],c2,c3,c1); */ + mflo t_1 + mfhi t_2 + daddu c_2,t_1 + sltu AT,c_2,t_1 + daddu t_2,AT + daddu c_3,t_2 + sltu AT,c_3,t_2 + daddu c_1,AT + sd c_2,32(a0) + + dmultu a_2,a_3 /* mul_add_c2(a[2],b[3],c3,c1,c2); */ + mflo t_1 + mfhi t_2 + slt c_2,t_2,zero + dsll t_2,1 + slt a2,t_1,zero + daddu t_2,a2 + dsll t_1,1 + daddu c_3,t_1 + sltu AT,c_3,t_1 + daddu t_2,AT + daddu c_1,t_2 + sltu AT,c_1,t_2 + daddu c_2,AT + sd c_3,40(a0) + + dmultu a_3,a_3 /* mul_add_c(a[3],b[3],c1,c2,c3); */ + mflo t_1 + mfhi t_2 + daddu c_1,t_1 + sltu AT,c_1,t_1 + daddu t_2,AT + daddu c_2,t_2 + sd c_1,48(a0) + sd c_2,56(a0) + + jr ra +END(bn_sqr_comba4) diff --git a/crypto/bn/asm/modexp512-x86_64.pl b/crypto/bn/asm/modexp512-x86_64.pl deleted file mode 100755 index bfd6e975416d..000000000000 --- a/crypto/bn/asm/modexp512-x86_64.pl +++ /dev/null @@ -1,1497 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (c) 2010-2011 Intel Corp. -# Author: Vinodh.Gopal@intel.com -# Jim Guilford -# Erdinc.Ozturk@intel.com -# Maxim.Perminov@intel.com -# -# More information about algorithm used can be found at: -# http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf -# -# ==================================================================== -# Copyright (c) 2011 The OpenSSL Project. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# -# 3. All advertising materials mentioning features or use of this -# software must display the following acknowledgment: -# "This product includes software developed by the OpenSSL Project -# for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" -# -# 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to -# endorse or promote products derived from this software without -# prior written permission. For written permission, please contact -# licensing@OpenSSL.org. -# -# 5. Products derived from this software may not be called "OpenSSL" -# nor may "OpenSSL" appear in their names without prior written -# permission of the OpenSSL Project. -# -# 6. Redistributions of any form whatsoever must retain the following -# acknowledgment: -# "This product includes software developed by the OpenSSL Project -# for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" -# -# THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR -# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -# OF THE POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - -$flavour = shift; -$output = shift; -if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } - -my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); - -$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -die "can't locate x86_64-xlate.pl"; - -open OUT,"| \"$^X\" $xlate $flavour $output"; -*STDOUT=*OUT; - -use strict; -my $code=".text\n\n"; -my $m=0; - -# -# Define x512 macros -# - -#MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2 -# -# uses rax, rdx, and args -sub MULSTEP_512_ADD -{ - my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_; - my @X=@$x; # make a copy -$code.=<<___; - mov (+8*0)($SRC2), %rax - mul $OP # rdx:rax = %OP * [0] - mov ($ASRC), $X[0] - add %rax, $X[0] - adc \$0, %rdx - mov $X[0], $DST -___ -for(my $i=1;$i<8;$i++) { -$code.=<<___; - mov %rdx, $TMP - - mov (+8*$i)($SRC2), %rax - mul $OP # rdx:rax = %OP * [$i] - mov (+8*$i)($ASRC), $X[$i] - add %rax, $X[$i] - adc \$0, %rdx - add $TMP, $X[$i] - adc \$0, %rdx -___ -} -$code.=<<___; - mov %rdx, $X[0] -___ -} - -#MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp -# -# uses rax, rdx, and args -sub MULSTEP_512 -{ - my ($x, $DST, $SRC2, $OP, $TMP)=@_; - my @X=@$x; # make a copy -$code.=<<___; - mov (+8*0)($SRC2), %rax - mul $OP # rdx:rax = %OP * [0] - add %rax, $X[0] - adc \$0, %rdx - mov $X[0], $DST -___ -for(my $i=1;$i<8;$i++) { -$code.=<<___; - mov %rdx, $TMP - - mov (+8*$i)($SRC2), %rax - mul $OP # rdx:rax = %OP * [$i] - add %rax, $X[$i] - adc \$0, %rdx - add $TMP, $X[$i] - adc \$0, %rdx -___ -} -$code.=<<___; - mov %rdx, $X[0] -___ -} - -# -# Swizzle Macros -# - -# macro to copy data from flat space to swizzled table -#MACRO swizzle pDst, pSrc, tmp1, tmp2 -# pDst and pSrc are modified -sub swizzle -{ - my ($pDst, $pSrc, $cnt, $d0)=@_; -$code.=<<___; - mov \$8, $cnt -loop_$m: - mov ($pSrc), $d0 - mov $d0#w, ($pDst) - shr \$16, $d0 - mov $d0#w, (+64*1)($pDst) - shr \$16, $d0 - mov $d0#w, (+64*2)($pDst) - shr \$16, $d0 - mov $d0#w, (+64*3)($pDst) - lea 8($pSrc), $pSrc - lea 64*4($pDst), $pDst - dec $cnt - jnz loop_$m -___ - - $m++; -} - -# macro to copy data from swizzled table to flat space -#MACRO unswizzle pDst, pSrc, tmp*3 -sub unswizzle -{ - my ($pDst, $pSrc, $cnt, $d0, $d1)=@_; -$code.=<<___; - mov \$4, $cnt -loop_$m: - movzxw (+64*3+256*0)($pSrc), $d0 - movzxw (+64*3+256*1)($pSrc), $d1 - shl \$16, $d0 - shl \$16, $d1 - mov (+64*2+256*0)($pSrc), $d0#w - mov (+64*2+256*1)($pSrc), $d1#w - shl \$16, $d0 - shl \$16, $d1 - mov (+64*1+256*0)($pSrc), $d0#w - mov (+64*1+256*1)($pSrc), $d1#w - shl \$16, $d0 - shl \$16, $d1 - mov (+64*0+256*0)($pSrc), $d0#w - mov (+64*0+256*1)($pSrc), $d1#w - mov $d0, (+8*0)($pDst) - mov $d1, (+8*1)($pDst) - lea 256*2($pSrc), $pSrc - lea 8*2($pDst), $pDst - sub \$1, $cnt - jnz loop_$m -___ - - $m++; -} - -# -# Data Structures -# - -# Reduce Data -# -# -# Offset Value -# 0C0 Carries -# 0B8 X2[10] -# 0B0 X2[9] -# 0A8 X2[8] -# 0A0 X2[7] -# 098 X2[6] -# 090 X2[5] -# 088 X2[4] -# 080 X2[3] -# 078 X2[2] -# 070 X2[1] -# 068 X2[0] -# 060 X1[12] P[10] -# 058 X1[11] P[9] Z[8] -# 050 X1[10] P[8] Z[7] -# 048 X1[9] P[7] Z[6] -# 040 X1[8] P[6] Z[5] -# 038 X1[7] P[5] Z[4] -# 030 X1[6] P[4] Z[3] -# 028 X1[5] P[3] Z[2] -# 020 X1[4] P[2] Z[1] -# 018 X1[3] P[1] Z[0] -# 010 X1[2] P[0] Y[2] -# 008 X1[1] Q[1] Y[1] -# 000 X1[0] Q[0] Y[0] - -my $X1_offset = 0; # 13 qwords -my $X2_offset = $X1_offset + 13*8; # 11 qwords -my $Carries_offset = $X2_offset + 11*8; # 1 qword -my $Q_offset = 0; # 2 qwords -my $P_offset = $Q_offset + 2*8; # 11 qwords -my $Y_offset = 0; # 3 qwords -my $Z_offset = $Y_offset + 3*8; # 9 qwords - -my $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords) - -# -# Stack Frame -# -# -# offset value -# ... -# ... -# 280 Garray - -# 278 tmp16[15] -# ... ... -# 200 tmp16[0] - -# 1F8 tmp[7] -# ... ... -# 1C0 tmp[0] - -# 1B8 GT[7] -# ... ... -# 180 GT[0] - -# 178 Reduce Data -# ... ... -# 0B8 Reduce Data -# 0B0 reserved -# 0A8 reserved -# 0A0 reserved -# 098 reserved -# 090 reserved -# 088 reduce result addr -# 080 exp[8] - -# ... -# 048 exp[1] -# 040 exp[0] - -# 038 reserved -# 030 loop_idx -# 028 pg -# 020 i -# 018 pData ; arg 4 -# 010 pG ; arg 2 -# 008 pResult ; arg 1 -# 000 rsp ; stack pointer before subtract - -my $rsp_offset = 0; -my $pResult_offset = 8*1 + $rsp_offset; -my $pG_offset = 8*1 + $pResult_offset; -my $pData_offset = 8*1 + $pG_offset; -my $i_offset = 8*1 + $pData_offset; -my $pg_offset = 8*1 + $i_offset; -my $loop_idx_offset = 8*1 + $pg_offset; -my $reserved1_offset = 8*1 + $loop_idx_offset; -my $exp_offset = 8*1 + $reserved1_offset; -my $red_result_addr_offset= 8*9 + $exp_offset; -my $reserved2_offset = 8*1 + $red_result_addr_offset; -my $Reduce_Data_offset = 8*5 + $reserved2_offset; -my $GT_offset = $Red_Data_Size + $Reduce_Data_offset; -my $tmp_offset = 8*8 + $GT_offset; -my $tmp16_offset = 8*8 + $tmp_offset; -my $garray_offset = 8*16 + $tmp16_offset; -my $mem_size = 8*8*32 + $garray_offset; - -# -# Offsets within Reduce Data -# -# -# struct MODF_2FOLD_MONT_512_C1_DATA { -# UINT64 t[8][8]; -# UINT64 m[8]; -# UINT64 m1[8]; /* 2^768 % m */ -# UINT64 m2[8]; /* 2^640 % m */ -# UINT64 k1[2]; /* (- 1/m) % 2^128 */ -# }; - -my $T = 0; -my $M = 512; # = 8 * 8 * 8 -my $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */ -my $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */ -my $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */ - -# -# FUNCTIONS -# - -{{{ -# -# MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords) -# and add 512-bits (8 qwords) -# to get 640 bits (10 qwords) -# Input: 128-bit mul source: [rdi+8*1], rbp -# 512-bit mul source: [rsi+8*n] -# 512-bit add source: r15, r14, ..., r9, r8 -# Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0] -# Clobbers all regs except: rcx, rsi, rdi -$code.=<<___; -.type MULADD_128x512,\@abi-omnipotent -.align 16 -MULADD_128x512: -___ - &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx"); -$code.=<<___; - mov (+8*1)(%rdi), %rbp -___ - &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx"); -$code.=<<___; - ret -.size MULADD_128x512,.-MULADD_128x512 -___ -}}} - -{{{ -#MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0 -# -# Inputs: pDst: Destination (768 bits, 12 qwords) -# pA: Multiplicand (1024 bits, 16 qwords) -# pB: Multiplicand (512 bits, 8 qwords) -# Dst = Ah * B + Al -# where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits) -# Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0] -# Uses registers: arguments, RAX, RDX -sub MULADD_256x512 -{ - my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_; -$code.=<<___; - mov (+8*12)($pA), $OP -___ - &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP); - push(@$X,shift(@$X)); - -$code.=<<___; - mov (+8*13)($pA), $OP -___ - &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP); - push(@$X,shift(@$X)); - -$code.=<<___; - mov (+8*14)($pA), $OP -___ - &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP); - push(@$X,shift(@$X)); - -$code.=<<___; - mov (+8*15)($pA), $OP -___ - &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP); - push(@$X,shift(@$X)); -} - -# -# mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */ -# UINT64 *m, /* 512 bits, 8 qwords */ -# MODF_2FOLD_MONT_512_C1_DATA *data, -# UINT64 *r) /* 512 bits, 8 qwords */ -# Input: x (number to be reduced): tmp16 (Implicit) -# m (modulus): [pM] (Implicit) -# data (reduce data): [pData] (Implicit) -# Output: r (result): Address in [red_res_addr] -# result also in: r9, r8, r15, r14, r13, r12, r11, r10 - -my @X=map("%r$_",(8..15)); - -$code.=<<___; -.type mont_reduce,\@abi-omnipotent -.align 16 -mont_reduce: -___ - -my $STACK_DEPTH = 8; - # - # X1 = Xh * M1 + Xl -$code.=<<___; - lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords - mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords - add \$$M1, %rsi - lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords - -___ - - &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times - # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0] - -$code.=<<___; - xor %rax, %rax - # X1 += xl - add (+8*8)(%rcx), $X[4] - adc (+8*9)(%rcx), $X[5] - adc (+8*10)(%rcx), $X[6] - adc (+8*11)(%rcx), $X[7] - adc \$0, %rax - # X1 is now rax, r11-r8, r15-r12, tmp16[3:0] - - # - # check for carry ;; carry stored in rax - mov $X[4], (+8*8)(%rdi) # rdi points to X1 - mov $X[5], (+8*9)(%rdi) - mov $X[6], %rbp - mov $X[7], (+8*11)(%rdi) - - mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) - - mov (+8*0)(%rdi), $X[4] - mov (+8*1)(%rdi), $X[5] - mov (+8*2)(%rdi), $X[6] - mov (+8*3)(%rdi), $X[7] - - # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8 - # rdi -> X1 - # rsi -> M1 - - # - # X2 = Xh * M2 + Xl - # do first part (X2 = Xh * M2) - add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords - # Xh is actually { [rdi+8*1], rbp } - add \$`$M2-$M1`, %rsi # rsi -> M2 - lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords -___ - unshift(@X,pop(@X)); unshift(@X,pop(@X)); -$code.=<<___; - - call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 - # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] - mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax - - # X2 += Xl - add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl - adc (+8*9-8*10)(%rdi), $X[7] - mov $X[6], (+8*8)(%rcx) - mov $X[7], (+8*9)(%rcx) - - adc %rax, %rax - mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) - - lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords - add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords - - # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half) - # B1:B0 = rsi[1:0] = K1[1:0] - # A1:A0 = rcx[1:0] = X2[1:0] - # Result = rdi[1],rbp = Q[1],rbp - mov (%rsi), %r8 # B0 - mov (+8*1)(%rsi), %rbx # B1 - - mov (%rcx), %rax # A0 - mul %r8 # B0 - mov %rax, %rbp - mov %rdx, %r9 - - mov (+8*1)(%rcx), %rax # A1 - mul %r8 # B0 - add %rax, %r9 - - mov (%rcx), %rax # A0 - mul %rbx # B1 - add %rax, %r9 - - mov %r9, (+8*1)(%rdi) - # end MUL_128x128t128 - - sub \$`$K1-$M`, %rsi - - mov (%rcx), $X[6] - mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0] - - call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 - # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] - - # load first half of m to rdx, rdi, rbx, rax - # moved this here for efficiency - mov (+8*0)(%rsi), %rax - mov (+8*1)(%rsi), %rbx - mov (+8*2)(%rsi), %rdi - mov (+8*3)(%rsi), %rdx - - # continue with reduction - mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp - - add (+8*8)(%rcx), $X[6] - adc (+8*9)(%rcx), $X[7] - - #accumulate the final carry to rbp - adc %rbp, %rbp - - # Add in overflow corrections: R = (X2>>128) += T[overflow] - # R = {r9, r8, r15, r14, ..., r10} - shl \$3, %rbp - mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T) - add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out - - # rsi will be used to generate a mask after the addition - xor %rsi, %rsi - - add (+8*8*0)(%rbp), $X[0] - adc (+8*8*1)(%rbp), $X[1] - adc (+8*8*2)(%rbp), $X[2] - adc (+8*8*3)(%rbp), $X[3] - adc (+8*8*4)(%rbp), $X[4] - adc (+8*8*5)(%rbp), $X[5] - adc (+8*8*6)(%rbp), $X[6] - adc (+8*8*7)(%rbp), $X[7] - - # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF - # if carry is clear: rsi = 0x0000000000000000 - sbb \$0, %rsi - - # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m - and %rsi, %rax - and %rsi, %rbx - and %rsi, %rdi - and %rsi, %rdx - - mov \$1, %rbp - sub %rax, $X[0] - sbb %rbx, $X[1] - sbb %rdi, $X[2] - sbb %rdx, $X[3] - - # if there is a borrow: rbp = 0 - # if there is no borrow: rbp = 1 - # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m - sbb \$0, %rbp - - #load second half of m to rdx, rdi, rbx, rax - - add \$$M, %rcx - mov (+8*4)(%rcx), %rax - mov (+8*5)(%rcx), %rbx - mov (+8*6)(%rcx), %rdi - mov (+8*7)(%rcx), %rdx - - # use the rsi mask as before - # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m - and %rsi, %rax - and %rsi, %rbx - and %rsi, %rdi - and %rsi, %rdx - - # if rbp = 0, there was a borrow before, it is moved to the carry flag - # if rbp = 1, there was not a borrow before, carry flag is cleared - sub \$1, %rbp - - sbb %rax, $X[4] - sbb %rbx, $X[5] - sbb %rdi, $X[6] - sbb %rdx, $X[7] - - # write R back to memory - - mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi - mov $X[0], (+8*0)(%rsi) - mov $X[1], (+8*1)(%rsi) - mov $X[2], (+8*2)(%rsi) - mov $X[3], (+8*3)(%rsi) - mov $X[4], (+8*4)(%rsi) - mov $X[5], (+8*5)(%rsi) - mov $X[6], (+8*6)(%rsi) - mov $X[7], (+8*7)(%rsi) - - ret -.size mont_reduce,.-mont_reduce -___ -}}} - -{{{ -#MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2 -# -# Inputs: pDst: Destination (1024 bits, 16 qwords) -# pA: Multiplicand (512 bits, 8 qwords) -# pB: Multiplicand (512 bits, 8 qwords) -# Uses registers rax, rdx, args -# B operand in [pB] and also in x7...x0 -sub MUL_512x512 -{ - my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_; - my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); - my @X=@$x; # make a copy - -$code.=<<___; - mov (+8*0)($pA), $OP - - mov $X[0], %rax - mul $OP # rdx:rax = %OP * [0] - mov %rax, (+$pDst_o+8*0)($pDst) - mov %rdx, $X[0] -___ -for(my $i=1;$i<8;$i++) { -$code.=<<___; - mov $X[$i], %rax - mul $OP # rdx:rax = %OP * [$i] - add %rax, $X[$i-1] - adc \$0, %rdx - mov %rdx, $X[$i] -___ -} - -for(my $i=1;$i<8;$i++) { -$code.=<<___; - mov (+8*$i)($pA), $OP -___ - - &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP); - push(@X,shift(@X)); -} - -$code.=<<___; - mov $X[0], (+$pDst_o+8*8)($pDst) - mov $X[1], (+$pDst_o+8*9)($pDst) - mov $X[2], (+$pDst_o+8*10)($pDst) - mov $X[3], (+$pDst_o+8*11)($pDst) - mov $X[4], (+$pDst_o+8*12)($pDst) - mov $X[5], (+$pDst_o+8*13)($pDst) - mov $X[6], (+$pDst_o+8*14)($pDst) - mov $X[7], (+$pDst_o+8*15)($pDst) -___ -} - -# -# mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits) -# Input: src1: Address of source 1: rdi -# src2: Address of source 2: rsi -# Output: dst: Address of destination: [red_res_addr] -# src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10 -# Temp: Clobbers [tmp16], all registers -$code.=<<___; -.type mont_mul_a3b,\@abi-omnipotent -.align 16 -mont_mul_a3b: - # - # multiply tmp = src1 * src2 - # For multiply: dst = rcx, src1 = rdi, src2 = rsi - # stack depth is extra 8 from call -___ - &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx"); -$code.=<<___; - # - # Dst = tmp % m - # Call reduce(tmp, m, data, dst) - - # tail recursion optimization: jmp to mont_reduce and return from there - jmp mont_reduce - # call mont_reduce - # ret -.size mont_mul_a3b,.-mont_mul_a3b -___ -}}} - -{{{ -#SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4 -# -# Input in memory [pA] and also in x7...x0 -# Uses all argument registers plus rax and rdx -# -# This version computes all of the off-diagonal terms into memory, -# and then it adds in the diagonal terms - -sub SQR_512 -{ - my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_; - my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); - my @X=@$x; # make a copy -$code.=<<___; - # ------------------ - # first pass 01...07 - # ------------------ - mov $X[0], $A - - mov $X[1],%rax - mul $A - mov %rax, (+$pDst_o+8*1)($pDst) -___ -for(my $i=2;$i<8;$i++) { -$code.=<<___; - mov %rdx, $X[$i-2] - mov $X[$i],%rax - mul $A - add %rax, $X[$i-2] - adc \$0, %rdx -___ -} -$code.=<<___; - mov %rdx, $x7 - - mov $X[0], (+$pDst_o+8*2)($pDst) - - # ------------------ - # second pass 12...17 - # ------------------ - - mov (+8*1)($pA), $A - - mov (+8*2)($pA),%rax - mul $A - add %rax, $X[1] - adc \$0, %rdx - mov $X[1], (+$pDst_o+8*3)($pDst) - - mov %rdx, $X[0] - mov (+8*3)($pA),%rax - mul $A - add %rax, $X[2] - adc \$0, %rdx - add $X[0], $X[2] - adc \$0, %rdx - mov $X[2], (+$pDst_o+8*4)($pDst) - - mov %rdx, $X[0] - mov (+8*4)($pA),%rax - mul $A - add %rax, $X[3] - adc \$0, %rdx - add $X[0], $X[3] - adc \$0, %rdx - - mov %rdx, $X[0] - mov (+8*5)($pA),%rax - mul $A - add %rax, $X[4] - adc \$0, %rdx - add $X[0], $X[4] - adc \$0, %rdx - - mov %rdx, $X[0] - mov $X[6],%rax - mul $A - add %rax, $X[5] - adc \$0, %rdx - add $X[0], $X[5] - adc \$0, %rdx - - mov %rdx, $X[0] - mov $X[7],%rax - mul $A - add %rax, $x7 - adc \$0, %rdx - add $X[0], $x7 - adc \$0, %rdx - - mov %rdx, $X[1] - - # ------------------ - # third pass 23...27 - # ------------------ - mov (+8*2)($pA), $A - - mov (+8*3)($pA),%rax - mul $A - add %rax, $X[3] - adc \$0, %rdx - mov $X[3], (+$pDst_o+8*5)($pDst) - - mov %rdx, $X[0] - mov (+8*4)($pA),%rax - mul $A - add %rax, $X[4] - adc \$0, %rdx - add $X[0], $X[4] - adc \$0, %rdx - mov $X[4], (+$pDst_o+8*6)($pDst) - - mov %rdx, $X[0] - mov (+8*5)($pA),%rax - mul $A - add %rax, $X[5] - adc \$0, %rdx - add $X[0], $X[5] - adc \$0, %rdx - - mov %rdx, $X[0] - mov $X[6],%rax - mul $A - add %rax, $x7 - adc \$0, %rdx - add $X[0], $x7 - adc \$0, %rdx - - mov %rdx, $X[0] - mov $X[7],%rax - mul $A - add %rax, $X[1] - adc \$0, %rdx - add $X[0], $X[1] - adc \$0, %rdx - - mov %rdx, $X[2] - - # ------------------ - # fourth pass 34...37 - # ------------------ - - mov (+8*3)($pA), $A - - mov (+8*4)($pA),%rax - mul $A - add %rax, $X[5] - adc \$0, %rdx - mov $X[5], (+$pDst_o+8*7)($pDst) - - mov %rdx, $X[0] - mov (+8*5)($pA),%rax - mul $A - add %rax, $x7 - adc \$0, %rdx - add $X[0], $x7 - adc \$0, %rdx - mov $x7, (+$pDst_o+8*8)($pDst) - - mov %rdx, $X[0] - mov $X[6],%rax - mul $A - add %rax, $X[1] - adc \$0, %rdx - add $X[0], $X[1] - adc \$0, %rdx - - mov %rdx, $X[0] - mov $X[7],%rax - mul $A - add %rax, $X[2] - adc \$0, %rdx - add $X[0], $X[2] - adc \$0, %rdx - - mov %rdx, $X[5] - - # ------------------ - # fifth pass 45...47 - # ------------------ - mov (+8*4)($pA), $A - - mov (+8*5)($pA),%rax - mul $A - add %rax, $X[1] - adc \$0, %rdx - mov $X[1], (+$pDst_o+8*9)($pDst) - - mov %rdx, $X[0] - mov $X[6],%rax - mul $A - add %rax, $X[2] - adc \$0, %rdx - add $X[0], $X[2] - adc \$0, %rdx - mov $X[2], (+$pDst_o+8*10)($pDst) - - mov %rdx, $X[0] - mov $X[7],%rax - mul $A - add %rax, $X[5] - adc \$0, %rdx - add $X[0], $X[5] - adc \$0, %rdx - - mov %rdx, $X[1] - - # ------------------ - # sixth pass 56...57 - # ------------------ - mov (+8*5)($pA), $A - - mov $X[6],%rax - mul $A - add %rax, $X[5] - adc \$0, %rdx - mov $X[5], (+$pDst_o+8*11)($pDst) - - mov %rdx, $X[0] - mov $X[7],%rax - mul $A - add %rax, $X[1] - adc \$0, %rdx - add $X[0], $X[1] - adc \$0, %rdx - mov $X[1], (+$pDst_o+8*12)($pDst) - - mov %rdx, $X[2] - - # ------------------ - # seventh pass 67 - # ------------------ - mov $X[6], $A - - mov $X[7],%rax - mul $A - add %rax, $X[2] - adc \$0, %rdx - mov $X[2], (+$pDst_o+8*13)($pDst) - - mov %rdx, (+$pDst_o+8*14)($pDst) - - # start finalize (add in squares, and double off-terms) - mov (+$pDst_o+8*1)($pDst), $X[0] - mov (+$pDst_o+8*2)($pDst), $X[1] - mov (+$pDst_o+8*3)($pDst), $X[2] - mov (+$pDst_o+8*4)($pDst), $X[3] - mov (+$pDst_o+8*5)($pDst), $X[4] - mov (+$pDst_o+8*6)($pDst), $X[5] - - mov (+8*3)($pA), %rax - mul %rax - mov %rax, $x6 - mov %rdx, $X[6] - - add $X[0], $X[0] - adc $X[1], $X[1] - adc $X[2], $X[2] - adc $X[3], $X[3] - adc $X[4], $X[4] - adc $X[5], $X[5] - adc \$0, $X[6] - - mov (+8*0)($pA), %rax - mul %rax - mov %rax, (+$pDst_o+8*0)($pDst) - mov %rdx, $A - - mov (+8*1)($pA), %rax - mul %rax - - add $A, $X[0] - adc %rax, $X[1] - adc \$0, %rdx - - mov %rdx, $A - mov $X[0], (+$pDst_o+8*1)($pDst) - mov $X[1], (+$pDst_o+8*2)($pDst) - - mov (+8*2)($pA), %rax - mul %rax - - add $A, $X[2] - adc %rax, $X[3] - adc \$0, %rdx - - mov %rdx, $A - - mov $X[2], (+$pDst_o+8*3)($pDst) - mov $X[3], (+$pDst_o+8*4)($pDst) - - xor $tmp, $tmp - add $A, $X[4] - adc $x6, $X[5] - adc \$0, $tmp - - mov $X[4], (+$pDst_o+8*5)($pDst) - mov $X[5], (+$pDst_o+8*6)($pDst) - - # %%tmp has 0/1 in column 7 - # %%A6 has a full value in column 7 - - mov (+$pDst_o+8*7)($pDst), $X[0] - mov (+$pDst_o+8*8)($pDst), $X[1] - mov (+$pDst_o+8*9)($pDst), $X[2] - mov (+$pDst_o+8*10)($pDst), $X[3] - mov (+$pDst_o+8*11)($pDst), $X[4] - mov (+$pDst_o+8*12)($pDst), $X[5] - mov (+$pDst_o+8*13)($pDst), $x6 - mov (+$pDst_o+8*14)($pDst), $x7 - - mov $X[7], %rax - mul %rax - mov %rax, $X[7] - mov %rdx, $A - - add $X[0], $X[0] - adc $X[1], $X[1] - adc $X[2], $X[2] - adc $X[3], $X[3] - adc $X[4], $X[4] - adc $X[5], $X[5] - adc $x6, $x6 - adc $x7, $x7 - adc \$0, $A - - add $tmp, $X[0] - - mov (+8*4)($pA), %rax - mul %rax - - add $X[6], $X[0] - adc %rax, $X[1] - adc \$0, %rdx - - mov %rdx, $tmp - - mov $X[0], (+$pDst_o+8*7)($pDst) - mov $X[1], (+$pDst_o+8*8)($pDst) - - mov (+8*5)($pA), %rax - mul %rax - - add $tmp, $X[2] - adc %rax, $X[3] - adc \$0, %rdx - - mov %rdx, $tmp - - mov $X[2], (+$pDst_o+8*9)($pDst) - mov $X[3], (+$pDst_o+8*10)($pDst) - - mov (+8*6)($pA), %rax - mul %rax - - add $tmp, $X[4] - adc %rax, $X[5] - adc \$0, %rdx - - mov $X[4], (+$pDst_o+8*11)($pDst) - mov $X[5], (+$pDst_o+8*12)($pDst) - - add %rdx, $x6 - adc $X[7], $x7 - adc \$0, $A - - mov $x6, (+$pDst_o+8*13)($pDst) - mov $x7, (+$pDst_o+8*14)($pDst) - mov $A, (+$pDst_o+8*15)($pDst) -___ -} - -# -# sqr_reduce: subroutine to compute Result = reduce(Result * Result) -# -# input and result also in: r9, r8, r15, r14, r13, r12, r11, r10 -# -$code.=<<___; -.type sqr_reduce,\@abi-omnipotent -.align 16 -sqr_reduce: - mov (+$pResult_offset+8)(%rsp), %rcx -___ - &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi"); -$code.=<<___; - # tail recursion optimization: jmp to mont_reduce and return from there - jmp mont_reduce - # call mont_reduce - # ret -.size sqr_reduce,.-sqr_reduce -___ -}}} - -# -# MAIN FUNCTION -# - -#mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */ -# UINT64 *g, /* 512 bits, 8 qwords */ -# UINT64 *exp, /* 512 bits, 8 qwords */ -# struct mod_ctx_512 *data) - -# window size = 5 -# table size = 2^5 = 32 -#table_entries equ 32 -#table_size equ table_entries * 8 -$code.=<<___; -.globl mod_exp_512 -.type mod_exp_512,\@function,4 -mod_exp_512: - push %rbp - push %rbx - push %r12 - push %r13 - push %r14 - push %r15 - - # adjust stack down and then align it with cache boundary - mov %rsp, %r8 - sub \$$mem_size, %rsp - and \$-64, %rsp - - # store previous stack pointer and arguments - mov %r8, (+$rsp_offset)(%rsp) - mov %rdi, (+$pResult_offset)(%rsp) - mov %rsi, (+$pG_offset)(%rsp) - mov %rcx, (+$pData_offset)(%rsp) -.Lbody: - # transform g into montgomery space - # GT = reduce(g * C2) = reduce(g * (2^256)) - # reduce expects to have the input in [tmp16] - pxor %xmm4, %xmm4 - movdqu (+16*0)(%rsi), %xmm0 - movdqu (+16*1)(%rsi), %xmm1 - movdqu (+16*2)(%rsi), %xmm2 - movdqu (+16*3)(%rsi), %xmm3 - movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp) - movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp) - movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) - movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) - movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp) - movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp) - movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp) - movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp) - - # load pExp before rdx gets blown away - movdqu (+16*0)(%rdx), %xmm0 - movdqu (+16*1)(%rdx), %xmm1 - movdqu (+16*2)(%rdx), %xmm2 - movdqu (+16*3)(%rdx), %xmm3 - - lea (+$GT_offset)(%rsp), %rbx - mov %rbx, (+$red_result_addr_offset)(%rsp) - call mont_reduce - - # Initialize tmp = C - lea (+$tmp_offset)(%rsp), %rcx - xor %rax, %rax - mov %rax, (+8*0)(%rcx) - mov %rax, (+8*1)(%rcx) - mov %rax, (+8*3)(%rcx) - mov %rax, (+8*4)(%rcx) - mov %rax, (+8*5)(%rcx) - mov %rax, (+8*6)(%rcx) - mov %rax, (+8*7)(%rcx) - mov %rax, (+$exp_offset+8*8)(%rsp) - movq \$1, (+8*2)(%rcx) - - lea (+$garray_offset)(%rsp), %rbp - mov %rcx, %rsi # pTmp - mov %rbp, %rdi # Garray[][0] -___ - - &swizzle("%rdi", "%rcx", "%rax", "%rbx"); - - # for (rax = 31; rax != 0; rax--) { - # tmp = reduce(tmp * G) - # swizzle(pg, tmp); - # pg += 2; } -$code.=<<___; - mov \$31, %rax - mov %rax, (+$i_offset)(%rsp) - mov %rbp, (+$pg_offset)(%rsp) - # rsi -> pTmp - mov %rsi, (+$red_result_addr_offset)(%rsp) - mov (+8*0)(%rsi), %r10 - mov (+8*1)(%rsi), %r11 - mov (+8*2)(%rsi), %r12 - mov (+8*3)(%rsi), %r13 - mov (+8*4)(%rsi), %r14 - mov (+8*5)(%rsi), %r15 - mov (+8*6)(%rsi), %r8 - mov (+8*7)(%rsi), %r9 -init_loop: - lea (+$GT_offset)(%rsp), %rdi - call mont_mul_a3b - lea (+$tmp_offset)(%rsp), %rsi - mov (+$pg_offset)(%rsp), %rbp - add \$2, %rbp - mov %rbp, (+$pg_offset)(%rsp) - mov %rsi, %rcx # rcx = rsi = addr of tmp -___ - - &swizzle("%rbp", "%rcx", "%rax", "%rbx"); -$code.=<<___; - mov (+$i_offset)(%rsp), %rax - sub \$1, %rax - mov %rax, (+$i_offset)(%rsp) - jne init_loop - - # - # Copy exponent onto stack - movdqa %xmm0, (+$exp_offset+16*0)(%rsp) - movdqa %xmm1, (+$exp_offset+16*1)(%rsp) - movdqa %xmm2, (+$exp_offset+16*2)(%rsp) - movdqa %xmm3, (+$exp_offset+16*3)(%rsp) - - - # - # Do exponentiation - # Initialize result to G[exp{511:507}] - mov (+$exp_offset+62)(%rsp), %eax - mov %rax, %rdx - shr \$11, %rax - and \$0x07FF, %edx - mov %edx, (+$exp_offset+62)(%rsp) - lea (+$garray_offset)(%rsp,%rax,2), %rsi - mov (+$pResult_offset)(%rsp), %rdx -___ - - &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); - - # - # Loop variables - # rcx = [loop_idx] = index: 510-5 to 0 by 5 -$code.=<<___; - movq \$505, (+$loop_idx_offset)(%rsp) - - mov (+$pResult_offset)(%rsp), %rcx - mov %rcx, (+$red_result_addr_offset)(%rsp) - mov (+8*0)(%rcx), %r10 - mov (+8*1)(%rcx), %r11 - mov (+8*2)(%rcx), %r12 - mov (+8*3)(%rcx), %r13 - mov (+8*4)(%rcx), %r14 - mov (+8*5)(%rcx), %r15 - mov (+8*6)(%rcx), %r8 - mov (+8*7)(%rcx), %r9 - jmp sqr_2 - -main_loop_a3b: - call sqr_reduce - call sqr_reduce - call sqr_reduce -sqr_2: - call sqr_reduce - call sqr_reduce - - # - # Do multiply, first look up proper value in Garray - mov (+$loop_idx_offset)(%rsp), %rcx # bit index - mov %rcx, %rax - shr \$4, %rax # rax is word pointer - mov (+$exp_offset)(%rsp,%rax,2), %edx - and \$15, %rcx - shrq %cl, %rdx - and \$0x1F, %rdx - - lea (+$garray_offset)(%rsp,%rdx,2), %rsi - lea (+$tmp_offset)(%rsp), %rdx - mov %rdx, %rdi -___ - - &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); - # rdi = tmp = pG - - # - # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData) - # result result pG M Data -$code.=<<___; - mov (+$pResult_offset)(%rsp), %rsi - call mont_mul_a3b - - # - # finish loop - mov (+$loop_idx_offset)(%rsp), %rcx - sub \$5, %rcx - mov %rcx, (+$loop_idx_offset)(%rsp) - jge main_loop_a3b - - # - -end_main_loop_a3b: - # transform result out of Montgomery space - # result = reduce(result) - mov (+$pResult_offset)(%rsp), %rdx - pxor %xmm4, %xmm4 - movdqu (+16*0)(%rdx), %xmm0 - movdqu (+16*1)(%rdx), %xmm1 - movdqu (+16*2)(%rdx), %xmm2 - movdqu (+16*3)(%rdx), %xmm3 - movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp) - movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp) - movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) - movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) - movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp) - movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp) - movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp) - movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp) - call mont_reduce - - # If result > m, subract m - # load result into r15:r8 - mov (+$pResult_offset)(%rsp), %rax - mov (+8*0)(%rax), %r8 - mov (+8*1)(%rax), %r9 - mov (+8*2)(%rax), %r10 - mov (+8*3)(%rax), %r11 - mov (+8*4)(%rax), %r12 - mov (+8*5)(%rax), %r13 - mov (+8*6)(%rax), %r14 - mov (+8*7)(%rax), %r15 - - # subtract m - mov (+$pData_offset)(%rsp), %rbx - add \$$M, %rbx - - sub (+8*0)(%rbx), %r8 - sbb (+8*1)(%rbx), %r9 - sbb (+8*2)(%rbx), %r10 - sbb (+8*3)(%rbx), %r11 - sbb (+8*4)(%rbx), %r12 - sbb (+8*5)(%rbx), %r13 - sbb (+8*6)(%rbx), %r14 - sbb (+8*7)(%rbx), %r15 - - # if Carry is clear, replace result with difference - mov (+8*0)(%rax), %rsi - mov (+8*1)(%rax), %rdi - mov (+8*2)(%rax), %rcx - mov (+8*3)(%rax), %rdx - cmovnc %r8, %rsi - cmovnc %r9, %rdi - cmovnc %r10, %rcx - cmovnc %r11, %rdx - mov %rsi, (+8*0)(%rax) - mov %rdi, (+8*1)(%rax) - mov %rcx, (+8*2)(%rax) - mov %rdx, (+8*3)(%rax) - - mov (+8*4)(%rax), %rsi - mov (+8*5)(%rax), %rdi - mov (+8*6)(%rax), %rcx - mov (+8*7)(%rax), %rdx - cmovnc %r12, %rsi - cmovnc %r13, %rdi - cmovnc %r14, %rcx - cmovnc %r15, %rdx - mov %rsi, (+8*4)(%rax) - mov %rdi, (+8*5)(%rax) - mov %rcx, (+8*6)(%rax) - mov %rdx, (+8*7)(%rax) - - mov (+$rsp_offset)(%rsp), %rsi - mov 0(%rsi),%r15 - mov 8(%rsi),%r14 - mov 16(%rsi),%r13 - mov 24(%rsi),%r12 - mov 32(%rsi),%rbx - mov 40(%rsi),%rbp - lea 48(%rsi),%rsp -.Lepilogue: - ret -.size mod_exp_512, . - mod_exp_512 -___ - -if ($win64) { -# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, -# CONTEXT *context,DISPATCHER_CONTEXT *disp) -my $rec="%rcx"; -my $frame="%rdx"; -my $context="%r8"; -my $disp="%r9"; - -$code.=<<___; -.extern __imp_RtlVirtualUnwind -.type mod_exp_512_se_handler,\@abi-omnipotent -.align 16 -mod_exp_512_se_handler: - push %rsi - push %rdi - push %rbx - push %rbp - push %r12 - push %r13 - push %r14 - push %r15 - pushfq - sub \$64,%rsp - - mov 120($context),%rax # pull context->Rax - mov 248($context),%rbx # pull context->Rip - - lea .Lbody(%rip),%r10 - cmp %r10,%rbx # context->RipRsp - - lea .Lepilogue(%rip),%r10 - cmp %r10,%rbx # context->Rip>=epilogue label - jae .Lin_prologue - - mov $rsp_offset(%rax),%rax # pull saved Rsp - - mov 32(%rax),%rbx - mov 40(%rax),%rbp - mov 24(%rax),%r12 - mov 16(%rax),%r13 - mov 8(%rax),%r14 - mov 0(%rax),%r15 - lea 48(%rax),%rax - mov %rbx,144($context) # restore context->Rbx - mov %rbp,160($context) # restore context->Rbp - mov %r12,216($context) # restore context->R12 - mov %r13,224($context) # restore context->R13 - mov %r14,232($context) # restore context->R14 - mov %r15,240($context) # restore context->R15 - -.Lin_prologue: - mov 8(%rax),%rdi - mov 16(%rax),%rsi - mov %rax,152($context) # restore context->Rsp - mov %rsi,168($context) # restore context->Rsi - mov %rdi,176($context) # restore context->Rdi - - mov 40($disp),%rdi # disp->ContextRecord - mov $context,%rsi # context - mov \$154,%ecx # sizeof(CONTEXT) - .long 0xa548f3fc # cld; rep movsq - - mov $disp,%rsi - xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER - mov 8(%rsi),%rdx # arg2, disp->ImageBase - mov 0(%rsi),%r8 # arg3, disp->ControlPc - mov 16(%rsi),%r9 # arg4, disp->FunctionEntry - mov 40(%rsi),%r10 # disp->ContextRecord - lea 56(%rsi),%r11 # &disp->HandlerData - lea 24(%rsi),%r12 # &disp->EstablisherFrame - mov %r10,32(%rsp) # arg5 - mov %r11,40(%rsp) # arg6 - mov %r12,48(%rsp) # arg7 - mov %rcx,56(%rsp) # arg8, (NULL) - call *__imp_RtlVirtualUnwind(%rip) - - mov \$1,%eax # ExceptionContinueSearch - add \$64,%rsp - popfq - pop %r15 - pop %r14 - pop %r13 - pop %r12 - pop %rbp - pop %rbx - pop %rdi - pop %rsi - ret -.size mod_exp_512_se_handler,.-mod_exp_512_se_handler - -.section .pdata -.align 4 - .rva .LSEH_begin_mod_exp_512 - .rva .LSEH_end_mod_exp_512 - .rva .LSEH_info_mod_exp_512 - -.section .xdata -.align 8 -.LSEH_info_mod_exp_512: - .byte 9,0,0,0 - .rva mod_exp_512_se_handler -___ -} - -sub reg_part { -my ($reg,$conv)=@_; - if ($reg =~ /%r[0-9]+/) { $reg .= $conv; } - elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; } - elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; } - elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; } - return $reg; -} - -$code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem; -$code =~ s/\`([^\`]*)\`/eval $1/gem; -$code =~ s/(\(\+[^)]+\))/eval $1/gem; -print $code; -close STDOUT; diff --git a/crypto/bn/asm/ppc-mont.pl b/crypto/bn/asm/ppc-mont.pl index f9b6992ccc82..da69c6aaaf6a 100755 --- a/crypto/bn/asm/ppc-mont.pl +++ b/crypto/bn/asm/ppc-mont.pl @@ -325,6 +325,7 @@ Lcopy: ; copy or in-place refresh .long 0 .byte 0,12,4,0,0x80,12,6,0 .long 0 +.size .bn_mul_mont_int,.-.bn_mul_mont_int .asciz "Montgomery Multiplication for PPC, CRYPTOGAMS by " ___ diff --git a/crypto/bn/asm/ppc.pl b/crypto/bn/asm/ppc.pl index 1249ce229988..04df1fe5cc65 100644 --- a/crypto/bn/asm/ppc.pl +++ b/crypto/bn/asm/ppc.pl @@ -392,6 +392,7 @@ $data=< for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -65,6 +65,14 @@ # others alternative would be to break dependence on upper halves of # GPRs by sticking to 32-bit integer operations... +# December 2012 + +# Remove above mentioned dependence on GPRs' upper halves in 32-bit +# build. No signal masking overhead, but integer instructions are +# *more* numerous... It's still "universally" faster than 32-bit +# ppc-mont.pl, but improvement coefficient is not as impressive +# for longer keys... + $flavour = shift; if ($flavour =~ /32/) { @@ -110,6 +118,9 @@ $tp="r10"; $j="r11"; $i="r12"; # non-volatile registers +$c1="r19"; +$n1="r20"; +$a1="r21"; $nap_d="r22"; # interleaved ap and np in double format $a0="r23"; # ap[0] $t0="r24"; # temporary registers @@ -180,8 +191,8 @@ $T3a="f30"; $T3b="f31"; # . . # +-------------------------------+ # . . -# -12*size_t +-------------------------------+ -# | 10 saved gpr, r22-r31 | +# -13*size_t +-------------------------------+ +# | 13 saved gpr, r19-r31 | # . . # . . # -12*8 +-------------------------------+ @@ -215,6 +226,9 @@ $code=<<___; mr $i,$sp $STUX $sp,$sp,$tp ; alloca + $PUSH r19,`-12*8-13*$SIZE_T`($i) + $PUSH r20,`-12*8-12*$SIZE_T`($i) + $PUSH r21,`-12*8-11*$SIZE_T`($i) $PUSH r22,`-12*8-10*$SIZE_T`($i) $PUSH r23,`-12*8-9*$SIZE_T`($i) $PUSH r24,`-12*8-8*$SIZE_T`($i) @@ -237,40 +251,26 @@ $code=<<___; stfd f29,`-3*8`($i) stfd f30,`-2*8`($i) stfd f31,`-1*8`($i) -___ -$code.=<<___ if ($SIZE_T==8); - ld $a0,0($ap) ; pull ap[0] value - ld $n0,0($n0) ; pull n0[0] value - ld $t3,0($bp) ; bp[0] -___ -$code.=<<___ if ($SIZE_T==4); - mr $t1,$n0 - lwz $a0,0($ap) ; pull ap[0,1] value - lwz $t0,4($ap) - lwz $n0,0($t1) ; pull n0[0,1] value - lwz $t1,4($t1) - lwz $t3,0($bp) ; bp[0,1] - lwz $t2,4($bp) - insrdi $a0,$t0,32,0 - insrdi $n0,$t1,32,0 - insrdi $t3,$t2,32,0 -___ -$code.=<<___; + addi $tp,$sp,`$FRAME+$TRANSFER+8+64` li $i,-64 add $nap_d,$tp,$num and $nap_d,$nap_d,$i ; align to 64 bytes - - mulld $t7,$a0,$t3 ; ap[0]*bp[0] ; nap_d is off by 1, because it's used with stfdu/lfdu addi $nap_d,$nap_d,-8 srwi $j,$num,`3+1` ; counter register, num/2 - mulld $t7,$t7,$n0 ; tp[0]*n0 addi $j,$j,-1 addi $tp,$sp,`$FRAME+$TRANSFER-8` li $carry,0 mtctr $j +___ + +$code.=<<___ if ($SIZE_T==8); + ld $a0,0($ap) ; pull ap[0] value + ld $t3,0($bp) ; bp[0] + ld $n0,0($n0) ; pull n0[0] value + mulld $t7,$a0,$t3 ; ap[0]*bp[0] ; transfer bp[0] to FPU as 4x16-bit values extrdi $t0,$t3,16,48 extrdi $t1,$t3,16,32 @@ -280,6 +280,8 @@ $code.=<<___; std $t1,`$FRAME+8`($sp) std $t2,`$FRAME+16`($sp) std $t3,`$FRAME+24`($sp) + + mulld $t7,$t7,$n0 ; tp[0]*n0 ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values extrdi $t4,$t7,16,48 extrdi $t5,$t7,16,32 @@ -289,21 +291,61 @@ $code.=<<___; std $t5,`$FRAME+40`($sp) std $t6,`$FRAME+48`($sp) std $t7,`$FRAME+56`($sp) -___ -$code.=<<___ if ($SIZE_T==8); - lwz $t0,4($ap) ; load a[j] as 32-bit word pair - lwz $t1,0($ap) - lwz $t2,12($ap) ; load a[j+1] as 32-bit word pair + + extrdi $t0,$a0,32,32 ; lwz $t0,4($ap) + extrdi $t1,$a0,32,0 ; lwz $t1,0($ap) + lwz $t2,12($ap) ; load a[1] as 32-bit word pair lwz $t3,8($ap) - lwz $t4,4($np) ; load n[j] as 32-bit word pair + lwz $t4,4($np) ; load n[0] as 32-bit word pair lwz $t5,0($np) - lwz $t6,12($np) ; load n[j+1] as 32-bit word pair + lwz $t6,12($np) ; load n[1] as 32-bit word pair lwz $t7,8($np) ___ $code.=<<___ if ($SIZE_T==4); - lwz $t0,0($ap) ; load a[j..j+3] as 32-bit word pairs - lwz $t1,4($ap) - lwz $t2,8($ap) + lwz $a0,0($ap) ; pull ap[0,1] value + mr $n1,$n0 + lwz $a1,4($ap) + li $c1,0 + lwz $t1,0($bp) ; bp[0,1] + lwz $t3,4($bp) + lwz $n0,0($n1) ; pull n0[0,1] value + lwz $n1,4($n1) + + mullw $t4,$a0,$t1 ; mulld ap[0]*bp[0] + mulhwu $t5,$a0,$t1 + mullw $t6,$a1,$t1 + mullw $t7,$a0,$t3 + add $t5,$t5,$t6 + add $t5,$t5,$t7 + ; transfer bp[0] to FPU as 4x16-bit values + extrwi $t0,$t1,16,16 + extrwi $t1,$t1,16,0 + extrwi $t2,$t3,16,16 + extrwi $t3,$t3,16,0 + std $t0,`$FRAME+0`($sp) ; yes, std in 32-bit build + std $t1,`$FRAME+8`($sp) + std $t2,`$FRAME+16`($sp) + std $t3,`$FRAME+24`($sp) + + mullw $t0,$t4,$n0 ; mulld tp[0]*n0 + mulhwu $t1,$t4,$n0 + mullw $t2,$t5,$n0 + mullw $t3,$t4,$n1 + add $t1,$t1,$t2 + add $t1,$t1,$t3 + ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values + extrwi $t4,$t0,16,16 + extrwi $t5,$t0,16,0 + extrwi $t6,$t1,16,16 + extrwi $t7,$t1,16,0 + std $t4,`$FRAME+32`($sp) ; yes, std in 32-bit build + std $t5,`$FRAME+40`($sp) + std $t6,`$FRAME+48`($sp) + std $t7,`$FRAME+56`($sp) + + mr $t0,$a0 ; lwz $t0,0($ap) + mr $t1,$a1 ; lwz $t1,4($ap) + lwz $t2,8($ap) ; load a[j..j+3] as 32-bit word pairs lwz $t3,12($ap) lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs lwz $t5,4($np) @@ -319,7 +361,7 @@ $code.=<<___; lfd $nb,`$FRAME+40`($sp) lfd $nc,`$FRAME+48`($sp) lfd $nd,`$FRAME+56`($sp) - std $t0,`$FRAME+64`($sp) + std $t0,`$FRAME+64`($sp) ; yes, std even in 32-bit build std $t1,`$FRAME+72`($sp) std $t2,`$FRAME+80`($sp) std $t3,`$FRAME+88`($sp) @@ -441,7 +483,7 @@ $code.=<<___ if ($SIZE_T==4); lwz $t7,12($np) ___ $code.=<<___; - std $t0,`$FRAME+64`($sp) + std $t0,`$FRAME+64`($sp) ; yes, std even in 32-bit build std $t1,`$FRAME+72`($sp) std $t2,`$FRAME+80`($sp) std $t3,`$FRAME+88`($sp) @@ -449,6 +491,9 @@ $code.=<<___; std $t5,`$FRAME+104`($sp) std $t6,`$FRAME+112`($sp) std $t7,`$FRAME+120`($sp) +___ +if ($SIZE_T==8 or $flavour =~ /osx/) { +$code.=<<___; ld $t0,`$FRAME+0`($sp) ld $t1,`$FRAME+8`($sp) ld $t2,`$FRAME+16`($sp) @@ -457,6 +502,20 @@ $code.=<<___; ld $t5,`$FRAME+40`($sp) ld $t6,`$FRAME+48`($sp) ld $t7,`$FRAME+56`($sp) +___ +} else { +$code.=<<___; + lwz $t1,`$FRAME+0`($sp) + lwz $t0,`$FRAME+4`($sp) + lwz $t3,`$FRAME+8`($sp) + lwz $t2,`$FRAME+12`($sp) + lwz $t5,`$FRAME+16`($sp) + lwz $t4,`$FRAME+20`($sp) + lwz $t7,`$FRAME+24`($sp) + lwz $t6,`$FRAME+28`($sp) +___ +} +$code.=<<___; lfd $A0,`$FRAME+64`($sp) lfd $A1,`$FRAME+72`($sp) lfd $A2,`$FRAME+80`($sp) @@ -488,7 +547,9 @@ $code.=<<___; fmadd $T0b,$A0,$bb,$dotb stfd $A2,24($nap_d) ; save a[j+1] in double format stfd $A3,32($nap_d) - +___ +if ($SIZE_T==8 or $flavour =~ /osx/) { +$code.=<<___; fmadd $T1a,$A0,$bc,$T1a fmadd $T1b,$A0,$bd,$T1b fmadd $T2a,$A1,$bc,$T2a @@ -561,11 +622,123 @@ $code.=<<___; stfd $T3b,`$FRAME+56`($sp) std $t0,8($tp) ; tp[j-1] stdu $t4,16($tp) ; tp[j] +___ +} else { +$code.=<<___; + fmadd $T1a,$A0,$bc,$T1a + fmadd $T1b,$A0,$bd,$T1b + addc $t0,$t0,$carry + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + fmadd $T2a,$A1,$bc,$T2a + fmadd $T2b,$A1,$bd,$T2b + stfd $N0,40($nap_d) ; save n[j] in double format + stfd $N1,48($nap_d) + srwi $c1,$t1,16 + insrwi $carry,$t1,16,0 + fmadd $T3a,$A2,$bc,$T3a + fmadd $T3b,$A2,$bd,$T3b + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + fmul $dota,$A3,$bc + fmul $dotb,$A3,$bd + stfd $N2,56($nap_d) ; save n[j+1] in double format + stfdu $N3,64($nap_d) + insrwi $t0,$t2,16,0 ; 0..31 bits + srwi $c1,$t3,16 + insrwi $carry,$t3,16,0 + + fmadd $T1a,$N1,$na,$T1a + fmadd $T1b,$N1,$nb,$T1b + lwz $t3,`$FRAME+32`($sp) ; permuted $t1 + lwz $t2,`$FRAME+36`($sp) ; permuted $t0 + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + fmadd $T2a,$N2,$na,$T2a + fmadd $T2b,$N2,$nb,$T2b + srwi $c1,$t5,16 + insrwi $carry,$t5,16,0 + fmadd $T3a,$N3,$na,$T3a + fmadd $T3b,$N3,$nb,$T3b + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + fmadd $T0a,$N0,$na,$T0a + fmadd $T0b,$N0,$nb,$T0b + insrwi $t4,$t6,16,0 ; 32..63 bits + srwi $c1,$t7,16 + insrwi $carry,$t7,16,0 + + fmadd $T1a,$N0,$nc,$T1a + fmadd $T1b,$N0,$nd,$T1b + lwz $t7,`$FRAME+40`($sp) ; permuted $t3 + lwz $t6,`$FRAME+44`($sp) ; permuted $t2 + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + fmadd $T2a,$N1,$nc,$T2a + fmadd $T2b,$N1,$nd,$T2b + stw $t0,12($tp) ; tp[j-1] + stw $t4,8($tp) + srwi $c1,$t3,16 + insrwi $carry,$t3,16,0 + fmadd $T3a,$N2,$nc,$T3a + fmadd $T3b,$N2,$nd,$T3b + lwz $t1,`$FRAME+48`($sp) ; permuted $t5 + lwz $t0,`$FRAME+52`($sp) ; permuted $t4 + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + fmadd $dota,$N3,$nc,$dota + fmadd $dotb,$N3,$nd,$dotb + insrwi $t2,$t6,16,0 ; 64..95 bits + srwi $c1,$t7,16 + insrwi $carry,$t7,16,0 + + fctid $T0a,$T0a + fctid $T0b,$T0b + lwz $t5,`$FRAME+56`($sp) ; permuted $t7 + lwz $t4,`$FRAME+60`($sp) ; permuted $t6 + addc $t0,$t0,$carry + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + fctid $T1a,$T1a + fctid $T1b,$T1b + srwi $c1,$t1,16 + insrwi $carry,$t1,16,0 + fctid $T2a,$T2a + fctid $T2b,$T2b + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + fctid $T3a,$T3a + fctid $T3b,$T3b + insrwi $t0,$t4,16,0 ; 96..127 bits + srwi $c1,$t5,16 + insrwi $carry,$t5,16,0 + + stfd $T0a,`$FRAME+0`($sp) + stfd $T0b,`$FRAME+8`($sp) + stfd $T1a,`$FRAME+16`($sp) + stfd $T1b,`$FRAME+24`($sp) + stfd $T2a,`$FRAME+32`($sp) + stfd $T2b,`$FRAME+40`($sp) + stfd $T3a,`$FRAME+48`($sp) + stfd $T3b,`$FRAME+56`($sp) + stw $t2,20($tp) ; tp[j] + stwu $t0,16($tp) +___ +} +$code.=<<___; bdnz- L1st fctid $dota,$dota fctid $dotb,$dotb - +___ +if ($SIZE_T==8 or $flavour =~ /osx/) { +$code.=<<___; ld $t0,`$FRAME+0`($sp) ld $t1,`$FRAME+8`($sp) ld $t2,`$FRAME+16`($sp) @@ -611,33 +784,117 @@ $code.=<<___; insrdi $t6,$t7,48,0 srdi $ovf,$t7,48 std $t6,8($tp) ; tp[num-1] +___ +} else { +$code.=<<___; + lwz $t1,`$FRAME+0`($sp) + lwz $t0,`$FRAME+4`($sp) + lwz $t3,`$FRAME+8`($sp) + lwz $t2,`$FRAME+12`($sp) + lwz $t5,`$FRAME+16`($sp) + lwz $t4,`$FRAME+20`($sp) + lwz $t7,`$FRAME+24`($sp) + lwz $t6,`$FRAME+28`($sp) + stfd $dota,`$FRAME+64`($sp) + stfd $dotb,`$FRAME+72`($sp) + addc $t0,$t0,$carry + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + insrwi $carry,$t1,16,0 + srwi $c1,$t1,16 + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + insrwi $t0,$t2,16,0 ; 0..31 bits + insrwi $carry,$t3,16,0 + srwi $c1,$t3,16 + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + insrwi $carry,$t5,16,0 + srwi $c1,$t5,16 + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + insrwi $t4,$t6,16,0 ; 32..63 bits + insrwi $carry,$t7,16,0 + srwi $c1,$t7,16 + stw $t0,12($tp) ; tp[j-1] + stw $t4,8($tp) + + lwz $t3,`$FRAME+32`($sp) ; permuted $t1 + lwz $t2,`$FRAME+36`($sp) ; permuted $t0 + lwz $t7,`$FRAME+40`($sp) ; permuted $t3 + lwz $t6,`$FRAME+44`($sp) ; permuted $t2 + lwz $t1,`$FRAME+48`($sp) ; permuted $t5 + lwz $t0,`$FRAME+52`($sp) ; permuted $t4 + lwz $t5,`$FRAME+56`($sp) ; permuted $t7 + lwz $t4,`$FRAME+60`($sp) ; permuted $t6 + + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + insrwi $carry,$t3,16,0 + srwi $c1,$t3,16 + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + insrwi $t2,$t6,16,0 ; 64..95 bits + insrwi $carry,$t7,16,0 + srwi $c1,$t7,16 + addc $t0,$t0,$carry + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + insrwi $carry,$t1,16,0 + srwi $c1,$t1,16 + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + insrwi $t0,$t4,16,0 ; 96..127 bits + insrwi $carry,$t5,16,0 + srwi $c1,$t5,16 + stw $t2,20($tp) ; tp[j] + stwu $t0,16($tp) + + lwz $t7,`$FRAME+64`($sp) + lwz $t6,`$FRAME+68`($sp) + lwz $t5,`$FRAME+72`($sp) + lwz $t4,`$FRAME+76`($sp) + + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + insrwi $carry,$t7,16,0 + srwi $c1,$t7,16 + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + + insrwi $t6,$t4,16,0 + srwi $t4,$t4,16 + insrwi $t4,$t5,16,0 + srwi $ovf,$t5,16 + stw $t6,12($tp) ; tp[num-1] + stw $t4,8($tp) +___ +} +$code.=<<___; slwi $t7,$num,2 subf $nap_d,$t7,$nap_d ; rewind pointer li $i,8 ; i=1 .align 5 Louter: -___ -$code.=<<___ if ($SIZE_T==8); - ldx $t3,$bp,$i ; bp[i] -___ -$code.=<<___ if ($SIZE_T==4); - add $t0,$bp,$i - lwz $t3,0($t0) ; bp[i,i+1] - lwz $t0,4($t0) - insrdi $t3,$t0,32,0 -___ -$code.=<<___; - ld $t6,`$FRAME+$TRANSFER+8`($sp) ; tp[0] - mulld $t7,$a0,$t3 ; ap[0]*bp[i] - addi $tp,$sp,`$FRAME+$TRANSFER` - add $t7,$t7,$t6 ; ap[0]*bp[i]+tp[0] li $carry,0 - mulld $t7,$t7,$n0 ; tp[0]*n0 mtctr $j +___ +$code.=<<___ if ($SIZE_T==8); + ldx $t3,$bp,$i ; bp[i] + ld $t6,`$FRAME+$TRANSFER+8`($sp) ; tp[0] + mulld $t7,$a0,$t3 ; ap[0]*bp[i] + add $t7,$t7,$t6 ; ap[0]*bp[i]+tp[0] ; transfer bp[i] to FPU as 4x16-bit values extrdi $t0,$t3,16,48 extrdi $t1,$t3,16,32 @@ -647,6 +904,8 @@ $code.=<<___; std $t1,`$FRAME+8`($sp) std $t2,`$FRAME+16`($sp) std $t3,`$FRAME+24`($sp) + + mulld $t7,$t7,$n0 ; tp[0]*n0 ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values extrdi $t4,$t7,16,48 extrdi $t5,$t7,16,32 @@ -656,7 +915,50 @@ $code.=<<___; std $t5,`$FRAME+40`($sp) std $t6,`$FRAME+48`($sp) std $t7,`$FRAME+56`($sp) +___ +$code.=<<___ if ($SIZE_T==4); + add $t0,$bp,$i + li $c1,0 + lwz $t1,0($t0) ; bp[i,i+1] + lwz $t3,4($t0) + + mullw $t4,$a0,$t1 ; ap[0]*bp[i] + lwz $t0,`$FRAME+$TRANSFER+8+4`($sp) ; tp[0] + mulhwu $t5,$a0,$t1 + lwz $t2,`$FRAME+$TRANSFER+8`($sp) ; tp[0] + mullw $t6,$a1,$t1 + mullw $t7,$a0,$t3 + add $t5,$t5,$t6 + add $t5,$t5,$t7 + addc $t4,$t4,$t0 ; ap[0]*bp[i]+tp[0] + adde $t5,$t5,$t2 + ; transfer bp[i] to FPU as 4x16-bit values + extrwi $t0,$t1,16,16 + extrwi $t1,$t1,16,0 + extrwi $t2,$t3,16,16 + extrwi $t3,$t3,16,0 + std $t0,`$FRAME+0`($sp) ; yes, std in 32-bit build + std $t1,`$FRAME+8`($sp) + std $t2,`$FRAME+16`($sp) + std $t3,`$FRAME+24`($sp) + mullw $t0,$t4,$n0 ; mulld tp[0]*n0 + mulhwu $t1,$t4,$n0 + mullw $t2,$t5,$n0 + mullw $t3,$t4,$n1 + add $t1,$t1,$t2 + add $t1,$t1,$t3 + ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values + extrwi $t4,$t0,16,16 + extrwi $t5,$t0,16,0 + extrwi $t6,$t1,16,16 + extrwi $t7,$t1,16,0 + std $t4,`$FRAME+32`($sp) ; yes, std in 32-bit build + std $t5,`$FRAME+40`($sp) + std $t6,`$FRAME+48`($sp) + std $t7,`$FRAME+56`($sp) +___ +$code.=<<___; lfd $A0,8($nap_d) ; load a[j] in double format lfd $A1,16($nap_d) lfd $A2,24($nap_d) ; load a[j+1] in double format @@ -769,7 +1071,9 @@ Linner: fmul $dotb,$A3,$bd lfd $A2,24($nap_d) ; load a[j+1] in double format lfd $A3,32($nap_d) - +___ +if ($SIZE_T==8 or $flavour =~ /osx/) { +$code.=<<___; fmadd $T1a,$N1,$na,$T1a fmadd $T1b,$N1,$nb,$T1b ld $t0,`$FRAME+0`($sp) @@ -856,10 +1160,131 @@ $code.=<<___; addze $carry,$carry std $t3,-16($tp) ; tp[j-1] std $t5,-8($tp) ; tp[j] +___ +} else { +$code.=<<___; + fmadd $T1a,$N1,$na,$T1a + fmadd $T1b,$N1,$nb,$T1b + lwz $t1,`$FRAME+0`($sp) + lwz $t0,`$FRAME+4`($sp) + fmadd $T2a,$N2,$na,$T2a + fmadd $T2b,$N2,$nb,$T2b + lwz $t3,`$FRAME+8`($sp) + lwz $t2,`$FRAME+12`($sp) + fmadd $T3a,$N3,$na,$T3a + fmadd $T3b,$N3,$nb,$T3b + lwz $t5,`$FRAME+16`($sp) + lwz $t4,`$FRAME+20`($sp) + addc $t0,$t0,$carry + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + fmadd $T0a,$N0,$na,$T0a + fmadd $T0b,$N0,$nb,$T0b + lwz $t7,`$FRAME+24`($sp) + lwz $t6,`$FRAME+28`($sp) + srwi $c1,$t1,16 + insrwi $carry,$t1,16,0 + + fmadd $T1a,$N0,$nc,$T1a + fmadd $T1b,$N0,$nd,$T1b + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + fmadd $T2a,$N1,$nc,$T2a + fmadd $T2b,$N1,$nd,$T2b + insrwi $t0,$t2,16,0 ; 0..31 bits + srwi $c1,$t3,16 + insrwi $carry,$t3,16,0 + fmadd $T3a,$N2,$nc,$T3a + fmadd $T3b,$N2,$nd,$T3b + lwz $t2,12($tp) ; tp[j] + lwz $t3,8($tp) + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + fmadd $dota,$N3,$nc,$dota + fmadd $dotb,$N3,$nd,$dotb + srwi $c1,$t5,16 + insrwi $carry,$t5,16,0 + + fctid $T0a,$T0a + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + fctid $T0b,$T0b + insrwi $t4,$t6,16,0 ; 32..63 bits + srwi $c1,$t7,16 + insrwi $carry,$t7,16,0 + fctid $T1a,$T1a + addc $t0,$t0,$t2 + adde $t4,$t4,$t3 + lwz $t3,`$FRAME+32`($sp) ; permuted $t1 + lwz $t2,`$FRAME+36`($sp) ; permuted $t0 + fctid $T1b,$T1b + addze $carry,$carry + addze $c1,$c1 + stw $t0,4($tp) ; tp[j-1] + stw $t4,0($tp) + fctid $T2a,$T2a + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + lwz $t7,`$FRAME+40`($sp) ; permuted $t3 + lwz $t6,`$FRAME+44`($sp) ; permuted $t2 + fctid $T2b,$T2b + srwi $c1,$t3,16 + insrwi $carry,$t3,16,0 + lwz $t1,`$FRAME+48`($sp) ; permuted $t5 + lwz $t0,`$FRAME+52`($sp) ; permuted $t4 + fctid $T3a,$T3a + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + lwz $t5,`$FRAME+56`($sp) ; permuted $t7 + lwz $t4,`$FRAME+60`($sp) ; permuted $t6 + fctid $T3b,$T3b + + insrwi $t2,$t6,16,0 ; 64..95 bits + insrwi $carry,$t7,16,0 + srwi $c1,$t7,16 + lwz $t6,20($tp) + lwzu $t7,16($tp) + addc $t0,$t0,$carry + stfd $T0a,`$FRAME+0`($sp) + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + stfd $T0b,`$FRAME+8`($sp) + insrwi $carry,$t1,16,0 + srwi $c1,$t1,16 + addc $t4,$t4,$carry + stfd $T1a,`$FRAME+16`($sp) + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + insrwi $t0,$t4,16,0 ; 96..127 bits + stfd $T1b,`$FRAME+24`($sp) + insrwi $carry,$t5,16,0 + srwi $c1,$t5,16 + + addc $t2,$t2,$t6 + stfd $T2a,`$FRAME+32`($sp) + adde $t0,$t0,$t7 + stfd $T2b,`$FRAME+40`($sp) + addze $carry,$carry + stfd $T3a,`$FRAME+48`($sp) + addze $c1,$c1 + stfd $T3b,`$FRAME+56`($sp) + stw $t2,-4($tp) ; tp[j] + stw $t0,-8($tp) +___ +} +$code.=<<___; bdnz- Linner fctid $dota,$dota fctid $dotb,$dotb +___ +if ($SIZE_T==8 or $flavour =~ /osx/) { +$code.=<<___; ld $t0,`$FRAME+0`($sp) ld $t1,`$FRAME+8`($sp) ld $t2,`$FRAME+16`($sp) @@ -926,7 +1351,116 @@ $code.=<<___; insrdi $t6,$t7,48,0 srdi $ovf,$t7,48 std $t6,0($tp) ; tp[num-1] +___ +} else { +$code.=<<___; + lwz $t1,`$FRAME+0`($sp) + lwz $t0,`$FRAME+4`($sp) + lwz $t3,`$FRAME+8`($sp) + lwz $t2,`$FRAME+12`($sp) + lwz $t5,`$FRAME+16`($sp) + lwz $t4,`$FRAME+20`($sp) + lwz $t7,`$FRAME+24`($sp) + lwz $t6,`$FRAME+28`($sp) + stfd $dota,`$FRAME+64`($sp) + stfd $dotb,`$FRAME+72`($sp) + addc $t0,$t0,$carry + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + insrwi $carry,$t1,16,0 + srwi $c1,$t1,16 + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + insrwi $t0,$t2,16,0 ; 0..31 bits + lwz $t2,12($tp) ; tp[j] + insrwi $carry,$t3,16,0 + srwi $c1,$t3,16 + lwz $t3,8($tp) + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + insrwi $carry,$t5,16,0 + srwi $c1,$t5,16 + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + insrwi $t4,$t6,16,0 ; 32..63 bits + insrwi $carry,$t7,16,0 + srwi $c1,$t7,16 + + addc $t0,$t0,$t2 + adde $t4,$t4,$t3 + addze $carry,$carry + addze $c1,$c1 + stw $t0,4($tp) ; tp[j-1] + stw $t4,0($tp) + + lwz $t3,`$FRAME+32`($sp) ; permuted $t1 + lwz $t2,`$FRAME+36`($sp) ; permuted $t0 + lwz $t7,`$FRAME+40`($sp) ; permuted $t3 + lwz $t6,`$FRAME+44`($sp) ; permuted $t2 + lwz $t1,`$FRAME+48`($sp) ; permuted $t5 + lwz $t0,`$FRAME+52`($sp) ; permuted $t4 + lwz $t5,`$FRAME+56`($sp) ; permuted $t7 + lwz $t4,`$FRAME+60`($sp) ; permuted $t6 + + addc $t2,$t2,$carry + adde $t3,$t3,$c1 + srwi $carry,$t2,16 + insrwi $carry,$t3,16,0 + srwi $c1,$t3,16 + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + srwi $carry,$t6,16 + insrwi $t2,$t6,16,0 ; 64..95 bits + lwz $t6,20($tp) + insrwi $carry,$t7,16,0 + srwi $c1,$t7,16 + lwzu $t7,16($tp) + addc $t0,$t0,$carry + adde $t1,$t1,$c1 + srwi $carry,$t0,16 + insrwi $carry,$t1,16,0 + srwi $c1,$t1,16 + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + srwi $carry,$t4,16 + insrwi $t0,$t4,16,0 ; 96..127 bits + insrwi $carry,$t5,16,0 + srwi $c1,$t5,16 + + addc $t2,$t2,$t6 + adde $t0,$t0,$t7 + lwz $t7,`$FRAME+64`($sp) + lwz $t6,`$FRAME+68`($sp) + addze $carry,$carry + addze $c1,$c1 + lwz $t5,`$FRAME+72`($sp) + lwz $t4,`$FRAME+76`($sp) + + addc $t6,$t6,$carry + adde $t7,$t7,$c1 + stw $t2,-4($tp) ; tp[j] + stw $t0,-8($tp) + addc $t6,$t6,$ovf + addze $t7,$t7 + srwi $carry,$t6,16 + insrwi $carry,$t7,16,0 + srwi $c1,$t7,16 + addc $t4,$t4,$carry + adde $t5,$t5,$c1 + + insrwi $t6,$t4,16,0 + srwi $t4,$t4,16 + insrwi $t4,$t5,16,0 + srwi $ovf,$t5,16 + stw $t6,4($tp) ; tp[num-1] + stw $t4,0($tp) +___ +} +$code.=<<___; slwi $t7,$num,2 addi $i,$i,8 subf $nap_d,$t7,$nap_d ; rewind pointer @@ -994,14 +1528,14 @@ $code.=<<___ if ($SIZE_T==4); mtctr $j .align 4 -Lsub: ld $t0,8($tp) ; load tp[j..j+3] in 64-bit word order - ldu $t2,16($tp) +Lsub: lwz $t0,12($tp) ; load tp[j..j+3] in 64-bit word order + lwz $t1,8($tp) + lwz $t2,20($tp) + lwzu $t3,16($tp) lwz $t4,4($np) ; load np[j..j+3] in 32-bit word order lwz $t5,8($np) lwz $t6,12($np) lwzu $t7,16($np) - extrdi $t1,$t0,32,0 - extrdi $t3,$t2,32,0 subfe $t4,$t4,$t0 ; tp[j]-np[j] stw $t0,4($ap) ; save tp[j..j+3] in 32-bit word order subfe $t5,$t5,$t1 ; tp[j+1]-np[j+1] @@ -1052,6 +1586,9 @@ ___ $code.=<<___; $POP $i,0($sp) li r3,1 ; signal "handled" + $POP r19,`-12*8-13*$SIZE_T`($i) + $POP r20,`-12*8-12*$SIZE_T`($i) + $POP r21,`-12*8-11*$SIZE_T`($i) $POP r22,`-12*8-10*$SIZE_T`($i) $POP r23,`-12*8-9*$SIZE_T`($i) $POP r24,`-12*8-8*$SIZE_T`($i) @@ -1077,8 +1614,9 @@ $code.=<<___; mr $sp,$i blr .long 0 - .byte 0,12,4,0,0x8c,10,6,0 + .byte 0,12,4,0,0x8c,13,6,0 .long 0 +.size .$fname,.-.$fname .asciz "Montgomery Multiplication for PPC64, CRYPTOGAMS by " ___ diff --git a/crypto/bn/asm/rsaz-avx2.pl b/crypto/bn/asm/rsaz-avx2.pl new file mode 100755 index 000000000000..3b6ccf83d13e --- /dev/null +++ b/crypto/bn/asm/rsaz-avx2.pl @@ -0,0 +1,1898 @@ +#!/usr/bin/env perl + +############################################################################## +# # +# Copyright (c) 2012, Intel Corporation # +# # +# All rights reserved. # +# # +# Redistribution and use in source and binary forms, with or without # +# modification, are permitted provided that the following conditions are # +# met: # +# # +# * Redistributions of source code must retain the above copyright # +# notice, this list of conditions and the following disclaimer. # +# # +# * Redistributions in binary form must reproduce the above copyright # +# notice, this list of conditions and the following disclaimer in the # +# documentation and/or other materials provided with the # +# distribution. # +# # +# * Neither the name of the Intel Corporation nor the names of its # +# contributors may be used to endorse or promote products derived from # +# this software without specific prior written permission. # +# # +# # +# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY # +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR # +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # +# # +############################################################################## +# Developers and authors: # +# Shay Gueron (1, 2), and Vlad Krasnov (1) # +# (1) Intel Corporation, Israel Development Center, Haifa, Israel # +# (2) University of Haifa, Israel # +############################################################################## +# Reference: # +# [1] S. Gueron, V. Krasnov: "Software Implementation of Modular # +# Exponentiation, Using Advanced Vector Instructions Architectures", # +# F. Ozbudak and F. Rodriguez-Henriquez (Eds.): WAIFI 2012, LNCS 7369, # +# pp. 119?135, 2012. Springer-Verlag Berlin Heidelberg 2012 # +# [2] S. Gueron: "Efficient Software Implementations of Modular # +# Exponentiation", Journal of Cryptographic Engineering 2:31-43 (2012). # +# [3] S. Gueron, V. Krasnov: "Speeding up Big-numbers Squaring",IEEE # +# Proceedings of 9th International Conference on Information Technology: # +# New Generations (ITNG 2012), pp.821-823 (2012) # +# [4] S. Gueron, V. Krasnov: "[PATCH] Efficient and side channel analysis # +# resistant 1024-bit modular exponentiation, for optimizing RSA2048 # +# on AVX2 capable x86_64 platforms", # +# http://rt.openssl.org/Ticket/Display.html?id=2850&user=guest&pass=guest# +############################################################################## +# +# +13% improvement over original submission by +# +# rsa2048 sign/sec OpenSSL 1.0.1 scalar(*) this +# 2.3GHz Haswell 621 765/+23% 1113/+79% +# 2.3GHz Broadwell(**) 688 1200(***)/+74% 1120/+63% +# +# (*) if system doesn't support AVX2, for reference purposes; +# (**) scaled to 2.3GHz to simplify comparison; +# (***) scalar AD*X code is faster than AVX2 and is preferred code +# path for Broadwell; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); + $addx = ($1>=2.23); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); + $addx = ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + $addx = ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { + my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 + $avx = ($ver>=3.0) + ($ver>=3.01); + $addx = ($ver>=3.03); +} + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT = *OUT; + +if ($avx>1) {{{ +{ # void AMS_WW( +my $rp="%rdi"; # BN_ULONG *rp, +my $ap="%rsi"; # const BN_ULONG *ap, +my $np="%rdx"; # const BN_ULONG *np, +my $n0="%ecx"; # const BN_ULONG n0, +my $rep="%r8d"; # int repeat); + +# The registers that hold the accumulated redundant result +# The AMM works on 1024 bit operands, and redundant word size is 29 +# Therefore: ceil(1024/29)/4 = 9 +my $ACC0="%ymm0"; +my $ACC1="%ymm1"; +my $ACC2="%ymm2"; +my $ACC3="%ymm3"; +my $ACC4="%ymm4"; +my $ACC5="%ymm5"; +my $ACC6="%ymm6"; +my $ACC7="%ymm7"; +my $ACC8="%ymm8"; +my $ACC9="%ymm9"; +# Registers that hold the broadcasted words of bp, currently used +my $B1="%ymm10"; +my $B2="%ymm11"; +# Registers that hold the broadcasted words of Y, currently used +my $Y1="%ymm12"; +my $Y2="%ymm13"; +# Helper registers +my $TEMP1="%ymm14"; +my $AND_MASK="%ymm15"; +# alu registers that hold the first words of the ACC +my $r0="%r9"; +my $r1="%r10"; +my $r2="%r11"; +my $r3="%r12"; + +my $i="%r14d"; # loop counter +my $tmp = "%r15"; + +my $FrameSize=32*18+32*8; # place for A^2 and 2*A + +my $aap=$r0; +my $tp0="%rbx"; +my $tp1=$r3; +my $tpa=$tmp; + +$np="%r13"; # reassigned argument + +$code.=<<___; +.text + +.globl rsaz_1024_sqr_avx2 +.type rsaz_1024_sqr_avx2,\@function,5 +.align 64 +rsaz_1024_sqr_avx2: # 702 cycles, 14% faster than rsaz_1024_mul_avx2 + lea (%rsp), %rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + vzeroupper +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + vmovaps %xmm6,-0xd8(%rax) + vmovaps %xmm7,-0xc8(%rax) + vmovaps %xmm8,-0xb8(%rax) + vmovaps %xmm9,-0xa8(%rax) + vmovaps %xmm10,-0x98(%rax) + vmovaps %xmm11,-0x88(%rax) + vmovaps %xmm12,-0x78(%rax) + vmovaps %xmm13,-0x68(%rax) + vmovaps %xmm14,-0x58(%rax) + vmovaps %xmm15,-0x48(%rax) +.Lsqr_1024_body: +___ +$code.=<<___; + mov %rax,%rbp + mov %rdx, $np # reassigned argument + sub \$$FrameSize, %rsp + mov $np, $tmp + sub \$-128, $rp # size optimization + sub \$-128, $ap + sub \$-128, $np + + and \$4095, $tmp # see if $np crosses page + add \$32*10, $tmp + shr \$12, $tmp + vpxor $ACC9,$ACC9,$ACC9 + jz .Lsqr_1024_no_n_copy + + # unaligned 256-bit load that crosses page boundary can + # cause >2x performance degradation here, so if $np does + # cross page boundary, copy it to stack and make sure stack + # frame doesn't... + sub \$32*10,%rsp + vmovdqu 32*0-128($np), $ACC0 + and \$-2048, %rsp + vmovdqu 32*1-128($np), $ACC1 + vmovdqu 32*2-128($np), $ACC2 + vmovdqu 32*3-128($np), $ACC3 + vmovdqu 32*4-128($np), $ACC4 + vmovdqu 32*5-128($np), $ACC5 + vmovdqu 32*6-128($np), $ACC6 + vmovdqu 32*7-128($np), $ACC7 + vmovdqu 32*8-128($np), $ACC8 + lea $FrameSize+128(%rsp),$np + vmovdqu $ACC0, 32*0-128($np) + vmovdqu $ACC1, 32*1-128($np) + vmovdqu $ACC2, 32*2-128($np) + vmovdqu $ACC3, 32*3-128($np) + vmovdqu $ACC4, 32*4-128($np) + vmovdqu $ACC5, 32*5-128($np) + vmovdqu $ACC6, 32*6-128($np) + vmovdqu $ACC7, 32*7-128($np) + vmovdqu $ACC8, 32*8-128($np) + vmovdqu $ACC9, 32*9-128($np) # $ACC9 is zero + +.Lsqr_1024_no_n_copy: + and \$-1024, %rsp + + vmovdqu 32*1-128($ap), $ACC1 + vmovdqu 32*2-128($ap), $ACC2 + vmovdqu 32*3-128($ap), $ACC3 + vmovdqu 32*4-128($ap), $ACC4 + vmovdqu 32*5-128($ap), $ACC5 + vmovdqu 32*6-128($ap), $ACC6 + vmovdqu 32*7-128($ap), $ACC7 + vmovdqu 32*8-128($ap), $ACC8 + + lea 192(%rsp), $tp0 # 64+128=192 + vpbroadcastq .Land_mask(%rip), $AND_MASK + jmp .LOOP_GRANDE_SQR_1024 + +.align 32 +.LOOP_GRANDE_SQR_1024: + lea 32*18+128(%rsp), $aap # size optimization + lea 448(%rsp), $tp1 # 64+128+256=448 + + # the squaring is performed as described in Variant B of + # "Speeding up Big-Number Squaring", so start by calculating + # the A*2=A+A vector + vpaddq $ACC1, $ACC1, $ACC1 + vpbroadcastq 32*0-128($ap), $B1 + vpaddq $ACC2, $ACC2, $ACC2 + vmovdqa $ACC1, 32*0-128($aap) + vpaddq $ACC3, $ACC3, $ACC3 + vmovdqa $ACC2, 32*1-128($aap) + vpaddq $ACC4, $ACC4, $ACC4 + vmovdqa $ACC3, 32*2-128($aap) + vpaddq $ACC5, $ACC5, $ACC5 + vmovdqa $ACC4, 32*3-128($aap) + vpaddq $ACC6, $ACC6, $ACC6 + vmovdqa $ACC5, 32*4-128($aap) + vpaddq $ACC7, $ACC7, $ACC7 + vmovdqa $ACC6, 32*5-128($aap) + vpaddq $ACC8, $ACC8, $ACC8 + vmovdqa $ACC7, 32*6-128($aap) + vpxor $ACC9, $ACC9, $ACC9 + vmovdqa $ACC8, 32*7-128($aap) + + vpmuludq 32*0-128($ap), $B1, $ACC0 + vpbroadcastq 32*1-128($ap), $B2 + vmovdqu $ACC9, 32*9-192($tp0) # zero upper half + vpmuludq $B1, $ACC1, $ACC1 + vmovdqu $ACC9, 32*10-448($tp1) + vpmuludq $B1, $ACC2, $ACC2 + vmovdqu $ACC9, 32*11-448($tp1) + vpmuludq $B1, $ACC3, $ACC3 + vmovdqu $ACC9, 32*12-448($tp1) + vpmuludq $B1, $ACC4, $ACC4 + vmovdqu $ACC9, 32*13-448($tp1) + vpmuludq $B1, $ACC5, $ACC5 + vmovdqu $ACC9, 32*14-448($tp1) + vpmuludq $B1, $ACC6, $ACC6 + vmovdqu $ACC9, 32*15-448($tp1) + vpmuludq $B1, $ACC7, $ACC7 + vmovdqu $ACC9, 32*16-448($tp1) + vpmuludq $B1, $ACC8, $ACC8 + vpbroadcastq 32*2-128($ap), $B1 + vmovdqu $ACC9, 32*17-448($tp1) + + mov $ap, $tpa + mov \$4, $i + jmp .Lsqr_entry_1024 +___ +$TEMP0=$Y1; +$TEMP2=$Y2; +$code.=<<___; +.align 32 +.LOOP_SQR_1024: + vpbroadcastq 32*1-128($tpa), $B2 + vpmuludq 32*0-128($ap), $B1, $ACC0 + vpaddq 32*0-192($tp0), $ACC0, $ACC0 + vpmuludq 32*0-128($aap), $B1, $ACC1 + vpaddq 32*1-192($tp0), $ACC1, $ACC1 + vpmuludq 32*1-128($aap), $B1, $ACC2 + vpaddq 32*2-192($tp0), $ACC2, $ACC2 + vpmuludq 32*2-128($aap), $B1, $ACC3 + vpaddq 32*3-192($tp0), $ACC3, $ACC3 + vpmuludq 32*3-128($aap), $B1, $ACC4 + vpaddq 32*4-192($tp0), $ACC4, $ACC4 + vpmuludq 32*4-128($aap), $B1, $ACC5 + vpaddq 32*5-192($tp0), $ACC5, $ACC5 + vpmuludq 32*5-128($aap), $B1, $ACC6 + vpaddq 32*6-192($tp0), $ACC6, $ACC6 + vpmuludq 32*6-128($aap), $B1, $ACC7 + vpaddq 32*7-192($tp0), $ACC7, $ACC7 + vpmuludq 32*7-128($aap), $B1, $ACC8 + vpbroadcastq 32*2-128($tpa), $B1 + vpaddq 32*8-192($tp0), $ACC8, $ACC8 +.Lsqr_entry_1024: + vmovdqu $ACC0, 32*0-192($tp0) + vmovdqu $ACC1, 32*1-192($tp0) + + vpmuludq 32*1-128($ap), $B2, $TEMP0 + vpaddq $TEMP0, $ACC2, $ACC2 + vpmuludq 32*1-128($aap), $B2, $TEMP1 + vpaddq $TEMP1, $ACC3, $ACC3 + vpmuludq 32*2-128($aap), $B2, $TEMP2 + vpaddq $TEMP2, $ACC4, $ACC4 + vpmuludq 32*3-128($aap), $B2, $TEMP0 + vpaddq $TEMP0, $ACC5, $ACC5 + vpmuludq 32*4-128($aap), $B2, $TEMP1 + vpaddq $TEMP1, $ACC6, $ACC6 + vpmuludq 32*5-128($aap), $B2, $TEMP2 + vpaddq $TEMP2, $ACC7, $ACC7 + vpmuludq 32*6-128($aap), $B2, $TEMP0 + vpaddq $TEMP0, $ACC8, $ACC8 + vpmuludq 32*7-128($aap), $B2, $ACC0 + vpbroadcastq 32*3-128($tpa), $B2 + vpaddq 32*9-192($tp0), $ACC0, $ACC0 + + vmovdqu $ACC2, 32*2-192($tp0) + vmovdqu $ACC3, 32*3-192($tp0) + + vpmuludq 32*2-128($ap), $B1, $TEMP2 + vpaddq $TEMP2, $ACC4, $ACC4 + vpmuludq 32*2-128($aap), $B1, $TEMP0 + vpaddq $TEMP0, $ACC5, $ACC5 + vpmuludq 32*3-128($aap), $B1, $TEMP1 + vpaddq $TEMP1, $ACC6, $ACC6 + vpmuludq 32*4-128($aap), $B1, $TEMP2 + vpaddq $TEMP2, $ACC7, $ACC7 + vpmuludq 32*5-128($aap), $B1, $TEMP0 + vpaddq $TEMP0, $ACC8, $ACC8 + vpmuludq 32*6-128($aap), $B1, $TEMP1 + vpaddq $TEMP1, $ACC0, $ACC0 + vpmuludq 32*7-128($aap), $B1, $ACC1 + vpbroadcastq 32*4-128($tpa), $B1 + vpaddq 32*10-448($tp1), $ACC1, $ACC1 + + vmovdqu $ACC4, 32*4-192($tp0) + vmovdqu $ACC5, 32*5-192($tp0) + + vpmuludq 32*3-128($ap), $B2, $TEMP0 + vpaddq $TEMP0, $ACC6, $ACC6 + vpmuludq 32*3-128($aap), $B2, $TEMP1 + vpaddq $TEMP1, $ACC7, $ACC7 + vpmuludq 32*4-128($aap), $B2, $TEMP2 + vpaddq $TEMP2, $ACC8, $ACC8 + vpmuludq 32*5-128($aap), $B2, $TEMP0 + vpaddq $TEMP0, $ACC0, $ACC0 + vpmuludq 32*6-128($aap), $B2, $TEMP1 + vpaddq $TEMP1, $ACC1, $ACC1 + vpmuludq 32*7-128($aap), $B2, $ACC2 + vpbroadcastq 32*5-128($tpa), $B2 + vpaddq 32*11-448($tp1), $ACC2, $ACC2 + + vmovdqu $ACC6, 32*6-192($tp0) + vmovdqu $ACC7, 32*7-192($tp0) + + vpmuludq 32*4-128($ap), $B1, $TEMP0 + vpaddq $TEMP0, $ACC8, $ACC8 + vpmuludq 32*4-128($aap), $B1, $TEMP1 + vpaddq $TEMP1, $ACC0, $ACC0 + vpmuludq 32*5-128($aap), $B1, $TEMP2 + vpaddq $TEMP2, $ACC1, $ACC1 + vpmuludq 32*6-128($aap), $B1, $TEMP0 + vpaddq $TEMP0, $ACC2, $ACC2 + vpmuludq 32*7-128($aap), $B1, $ACC3 + vpbroadcastq 32*6-128($tpa), $B1 + vpaddq 32*12-448($tp1), $ACC3, $ACC3 + + vmovdqu $ACC8, 32*8-192($tp0) + vmovdqu $ACC0, 32*9-192($tp0) + lea 8($tp0), $tp0 + + vpmuludq 32*5-128($ap), $B2, $TEMP2 + vpaddq $TEMP2, $ACC1, $ACC1 + vpmuludq 32*5-128($aap), $B2, $TEMP0 + vpaddq $TEMP0, $ACC2, $ACC2 + vpmuludq 32*6-128($aap), $B2, $TEMP1 + vpaddq $TEMP1, $ACC3, $ACC3 + vpmuludq 32*7-128($aap), $B2, $ACC4 + vpbroadcastq 32*7-128($tpa), $B2 + vpaddq 32*13-448($tp1), $ACC4, $ACC4 + + vmovdqu $ACC1, 32*10-448($tp1) + vmovdqu $ACC2, 32*11-448($tp1) + + vpmuludq 32*6-128($ap), $B1, $TEMP0 + vpaddq $TEMP0, $ACC3, $ACC3 + vpmuludq 32*6-128($aap), $B1, $TEMP1 + vpbroadcastq 32*8-128($tpa), $ACC0 # borrow $ACC0 for $B1 + vpaddq $TEMP1, $ACC4, $ACC4 + vpmuludq 32*7-128($aap), $B1, $ACC5 + vpbroadcastq 32*0+8-128($tpa), $B1 # for next iteration + vpaddq 32*14-448($tp1), $ACC5, $ACC5 + + vmovdqu $ACC3, 32*12-448($tp1) + vmovdqu $ACC4, 32*13-448($tp1) + lea 8($tpa), $tpa + + vpmuludq 32*7-128($ap), $B2, $TEMP0 + vpaddq $TEMP0, $ACC5, $ACC5 + vpmuludq 32*7-128($aap), $B2, $ACC6 + vpaddq 32*15-448($tp1), $ACC6, $ACC6 + + vpmuludq 32*8-128($ap), $ACC0, $ACC7 + vmovdqu $ACC5, 32*14-448($tp1) + vpaddq 32*16-448($tp1), $ACC7, $ACC7 + vmovdqu $ACC6, 32*15-448($tp1) + vmovdqu $ACC7, 32*16-448($tp1) + lea 8($tp1), $tp1 + + dec $i + jnz .LOOP_SQR_1024 +___ +$ZERO = $ACC9; +$TEMP0 = $B1; +$TEMP2 = $B2; +$TEMP3 = $Y1; +$TEMP4 = $Y2; +$code.=<<___; + #we need to fix indexes 32-39 to avoid overflow + vmovdqu 32*8(%rsp), $ACC8 # 32*8-192($tp0), + vmovdqu 32*9(%rsp), $ACC1 # 32*9-192($tp0) + vmovdqu 32*10(%rsp), $ACC2 # 32*10-192($tp0) + lea 192(%rsp), $tp0 # 64+128=192 + + vpsrlq \$29, $ACC8, $TEMP1 + vpand $AND_MASK, $ACC8, $ACC8 + vpsrlq \$29, $ACC1, $TEMP2 + vpand $AND_MASK, $ACC1, $ACC1 + + vpermq \$0x93, $TEMP1, $TEMP1 + vpxor $ZERO, $ZERO, $ZERO + vpermq \$0x93, $TEMP2, $TEMP2 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC8, $ACC8 + vpblendd \$3, $TEMP2, $ZERO, $TEMP2 + vpaddq $TEMP1, $ACC1, $ACC1 + vpaddq $TEMP2, $ACC2, $ACC2 + vmovdqu $ACC1, 32*9-192($tp0) + vmovdqu $ACC2, 32*10-192($tp0) + + mov (%rsp), %rax + mov 8(%rsp), $r1 + mov 16(%rsp), $r2 + mov 24(%rsp), $r3 + vmovdqu 32*1(%rsp), $ACC1 + vmovdqu 32*2-192($tp0), $ACC2 + vmovdqu 32*3-192($tp0), $ACC3 + vmovdqu 32*4-192($tp0), $ACC4 + vmovdqu 32*5-192($tp0), $ACC5 + vmovdqu 32*6-192($tp0), $ACC6 + vmovdqu 32*7-192($tp0), $ACC7 + + mov %rax, $r0 + imull $n0, %eax + and \$0x1fffffff, %eax + vmovd %eax, $Y1 + + mov %rax, %rdx + imulq -128($np), %rax + vpbroadcastq $Y1, $Y1 + add %rax, $r0 + mov %rdx, %rax + imulq 8-128($np), %rax + shr \$29, $r0 + add %rax, $r1 + mov %rdx, %rax + imulq 16-128($np), %rax + add $r0, $r1 + add %rax, $r2 + imulq 24-128($np), %rdx + add %rdx, $r3 + + mov $r1, %rax + imull $n0, %eax + and \$0x1fffffff, %eax + + mov \$9, $i + jmp .LOOP_REDUCE_1024 + +.align 32 +.LOOP_REDUCE_1024: + vmovd %eax, $Y2 + vpbroadcastq $Y2, $Y2 + + vpmuludq 32*1-128($np), $Y1, $TEMP0 + mov %rax, %rdx + imulq -128($np), %rax + vpaddq $TEMP0, $ACC1, $ACC1 + add %rax, $r1 + vpmuludq 32*2-128($np), $Y1, $TEMP1 + mov %rdx, %rax + imulq 8-128($np), %rax + vpaddq $TEMP1, $ACC2, $ACC2 + vpmuludq 32*3-128($np), $Y1, $TEMP2 + .byte 0x67 + add %rax, $r2 + .byte 0x67 + mov %rdx, %rax + imulq 16-128($np), %rax + shr \$29, $r1 + vpaddq $TEMP2, $ACC3, $ACC3 + vpmuludq 32*4-128($np), $Y1, $TEMP0 + add %rax, $r3 + add $r1, $r2 + vpaddq $TEMP0, $ACC4, $ACC4 + vpmuludq 32*5-128($np), $Y1, $TEMP1 + mov $r2, %rax + imull $n0, %eax + vpaddq $TEMP1, $ACC5, $ACC5 + vpmuludq 32*6-128($np), $Y1, $TEMP2 + and \$0x1fffffff, %eax + vpaddq $TEMP2, $ACC6, $ACC6 + vpmuludq 32*7-128($np), $Y1, $TEMP0 + vpaddq $TEMP0, $ACC7, $ACC7 + vpmuludq 32*8-128($np), $Y1, $TEMP1 + vmovd %eax, $Y1 + #vmovdqu 32*1-8-128($np), $TEMP2 # moved below + vpaddq $TEMP1, $ACC8, $ACC8 + #vmovdqu 32*2-8-128($np), $TEMP0 # moved below + vpbroadcastq $Y1, $Y1 + + vpmuludq 32*1-8-128($np), $Y2, $TEMP2 # see above + vmovdqu 32*3-8-128($np), $TEMP1 + mov %rax, %rdx + imulq -128($np), %rax + vpaddq $TEMP2, $ACC1, $ACC1 + vpmuludq 32*2-8-128($np), $Y2, $TEMP0 # see above + vmovdqu 32*4-8-128($np), $TEMP2 + add %rax, $r2 + mov %rdx, %rax + imulq 8-128($np), %rax + vpaddq $TEMP0, $ACC2, $ACC2 + add $r3, %rax + shr \$29, $r2 + vpmuludq $Y2, $TEMP1, $TEMP1 + vmovdqu 32*5-8-128($np), $TEMP0 + add $r2, %rax + vpaddq $TEMP1, $ACC3, $ACC3 + vpmuludq $Y2, $TEMP2, $TEMP2 + vmovdqu 32*6-8-128($np), $TEMP1 + .byte 0x67 + mov %rax, $r3 + imull $n0, %eax + vpaddq $TEMP2, $ACC4, $ACC4 + vpmuludq $Y2, $TEMP0, $TEMP0 + .byte 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 # vmovdqu 32*7-8-128($np), $TEMP2 + and \$0x1fffffff, %eax + vpaddq $TEMP0, $ACC5, $ACC5 + vpmuludq $Y2, $TEMP1, $TEMP1 + vmovdqu 32*8-8-128($np), $TEMP0 + vpaddq $TEMP1, $ACC6, $ACC6 + vpmuludq $Y2, $TEMP2, $TEMP2 + vmovdqu 32*9-8-128($np), $ACC9 + vmovd %eax, $ACC0 # borrow ACC0 for Y2 + imulq -128($np), %rax + vpaddq $TEMP2, $ACC7, $ACC7 + vpmuludq $Y2, $TEMP0, $TEMP0 + vmovdqu 32*1-16-128($np), $TEMP1 + vpbroadcastq $ACC0, $ACC0 + vpaddq $TEMP0, $ACC8, $ACC8 + vpmuludq $Y2, $ACC9, $ACC9 + vmovdqu 32*2-16-128($np), $TEMP2 + add %rax, $r3 + +___ +($ACC0,$Y2)=($Y2,$ACC0); +$code.=<<___; + vmovdqu 32*1-24-128($np), $ACC0 + vpmuludq $Y1, $TEMP1, $TEMP1 + vmovdqu 32*3-16-128($np), $TEMP0 + vpaddq $TEMP1, $ACC1, $ACC1 + vpmuludq $Y2, $ACC0, $ACC0 + vpmuludq $Y1, $TEMP2, $TEMP2 + .byte 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff # vmovdqu 32*4-16-128($np), $TEMP1 + vpaddq $ACC1, $ACC0, $ACC0 + vpaddq $TEMP2, $ACC2, $ACC2 + vpmuludq $Y1, $TEMP0, $TEMP0 + vmovdqu 32*5-16-128($np), $TEMP2 + .byte 0x67 + vmovq $ACC0, %rax + vmovdqu $ACC0, (%rsp) # transfer $r0-$r3 + vpaddq $TEMP0, $ACC3, $ACC3 + vpmuludq $Y1, $TEMP1, $TEMP1 + vmovdqu 32*6-16-128($np), $TEMP0 + vpaddq $TEMP1, $ACC4, $ACC4 + vpmuludq $Y1, $TEMP2, $TEMP2 + vmovdqu 32*7-16-128($np), $TEMP1 + vpaddq $TEMP2, $ACC5, $ACC5 + vpmuludq $Y1, $TEMP0, $TEMP0 + vmovdqu 32*8-16-128($np), $TEMP2 + vpaddq $TEMP0, $ACC6, $ACC6 + vpmuludq $Y1, $TEMP1, $TEMP1 + shr \$29, $r3 + vmovdqu 32*9-16-128($np), $TEMP0 + add $r3, %rax + vpaddq $TEMP1, $ACC7, $ACC7 + vpmuludq $Y1, $TEMP2, $TEMP2 + #vmovdqu 32*2-24-128($np), $TEMP1 # moved below + mov %rax, $r0 + imull $n0, %eax + vpaddq $TEMP2, $ACC8, $ACC8 + vpmuludq $Y1, $TEMP0, $TEMP0 + and \$0x1fffffff, %eax + vmovd %eax, $Y1 + vmovdqu 32*3-24-128($np), $TEMP2 + .byte 0x67 + vpaddq $TEMP0, $ACC9, $ACC9 + vpbroadcastq $Y1, $Y1 + + vpmuludq 32*2-24-128($np), $Y2, $TEMP1 # see above + vmovdqu 32*4-24-128($np), $TEMP0 + mov %rax, %rdx + imulq -128($np), %rax + mov 8(%rsp), $r1 + vpaddq $TEMP1, $ACC2, $ACC1 + vpmuludq $Y2, $TEMP2, $TEMP2 + vmovdqu 32*5-24-128($np), $TEMP1 + add %rax, $r0 + mov %rdx, %rax + imulq 8-128($np), %rax + .byte 0x67 + shr \$29, $r0 + mov 16(%rsp), $r2 + vpaddq $TEMP2, $ACC3, $ACC2 + vpmuludq $Y2, $TEMP0, $TEMP0 + vmovdqu 32*6-24-128($np), $TEMP2 + add %rax, $r1 + mov %rdx, %rax + imulq 16-128($np), %rax + vpaddq $TEMP0, $ACC4, $ACC3 + vpmuludq $Y2, $TEMP1, $TEMP1 + vmovdqu 32*7-24-128($np), $TEMP0 + imulq 24-128($np), %rdx # future $r3 + add %rax, $r2 + lea ($r0,$r1), %rax + vpaddq $TEMP1, $ACC5, $ACC4 + vpmuludq $Y2, $TEMP2, $TEMP2 + vmovdqu 32*8-24-128($np), $TEMP1 + mov %rax, $r1 + imull $n0, %eax + vpmuludq $Y2, $TEMP0, $TEMP0 + vpaddq $TEMP2, $ACC6, $ACC5 + vmovdqu 32*9-24-128($np), $TEMP2 + and \$0x1fffffff, %eax + vpaddq $TEMP0, $ACC7, $ACC6 + vpmuludq $Y2, $TEMP1, $TEMP1 + add 24(%rsp), %rdx + vpaddq $TEMP1, $ACC8, $ACC7 + vpmuludq $Y2, $TEMP2, $TEMP2 + vpaddq $TEMP2, $ACC9, $ACC8 + vmovq $r3, $ACC9 + mov %rdx, $r3 + + dec $i + jnz .LOOP_REDUCE_1024 +___ +($ACC0,$Y2)=($Y2,$ACC0); +$code.=<<___; + lea 448(%rsp), $tp1 # size optimization + vpaddq $ACC9, $Y2, $ACC0 + vpxor $ZERO, $ZERO, $ZERO + + vpaddq 32*9-192($tp0), $ACC0, $ACC0 + vpaddq 32*10-448($tp1), $ACC1, $ACC1 + vpaddq 32*11-448($tp1), $ACC2, $ACC2 + vpaddq 32*12-448($tp1), $ACC3, $ACC3 + vpaddq 32*13-448($tp1), $ACC4, $ACC4 + vpaddq 32*14-448($tp1), $ACC5, $ACC5 + vpaddq 32*15-448($tp1), $ACC6, $ACC6 + vpaddq 32*16-448($tp1), $ACC7, $ACC7 + vpaddq 32*17-448($tp1), $ACC8, $ACC8 + + vpsrlq \$29, $ACC0, $TEMP1 + vpand $AND_MASK, $ACC0, $ACC0 + vpsrlq \$29, $ACC1, $TEMP2 + vpand $AND_MASK, $ACC1, $ACC1 + vpsrlq \$29, $ACC2, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC2, $ACC2 + vpsrlq \$29, $ACC3, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC3, $ACC3 + vpermq \$0x93, $TEMP3, $TEMP3 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP4, $TEMP4 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC0, $ACC0 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC1, $ACC1 + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC2, $ACC2 + vpblendd \$3, $TEMP4, $ZERO, $TEMP4 + vpaddq $TEMP3, $ACC3, $ACC3 + vpaddq $TEMP4, $ACC4, $ACC4 + + vpsrlq \$29, $ACC0, $TEMP1 + vpand $AND_MASK, $ACC0, $ACC0 + vpsrlq \$29, $ACC1, $TEMP2 + vpand $AND_MASK, $ACC1, $ACC1 + vpsrlq \$29, $ACC2, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC2, $ACC2 + vpsrlq \$29, $ACC3, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC3, $ACC3 + vpermq \$0x93, $TEMP3, $TEMP3 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP4, $TEMP4 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC0, $ACC0 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC1, $ACC1 + vmovdqu $ACC0, 32*0-128($rp) + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC2, $ACC2 + vmovdqu $ACC1, 32*1-128($rp) + vpblendd \$3, $TEMP4, $ZERO, $TEMP4 + vpaddq $TEMP3, $ACC3, $ACC3 + vmovdqu $ACC2, 32*2-128($rp) + vpaddq $TEMP4, $ACC4, $ACC4 + vmovdqu $ACC3, 32*3-128($rp) +___ +$TEMP5=$ACC0; +$code.=<<___; + vpsrlq \$29, $ACC4, $TEMP1 + vpand $AND_MASK, $ACC4, $ACC4 + vpsrlq \$29, $ACC5, $TEMP2 + vpand $AND_MASK, $ACC5, $ACC5 + vpsrlq \$29, $ACC6, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC6, $ACC6 + vpsrlq \$29, $ACC7, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC7, $ACC7 + vpsrlq \$29, $ACC8, $TEMP5 + vpermq \$0x93, $TEMP3, $TEMP3 + vpand $AND_MASK, $ACC8, $ACC8 + vpermq \$0x93, $TEMP4, $TEMP4 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP5, $TEMP5 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC4, $ACC4 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC5, $ACC5 + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC6, $ACC6 + vpblendd \$3, $TEMP4, $TEMP5, $TEMP4 + vpaddq $TEMP3, $ACC7, $ACC7 + vpaddq $TEMP4, $ACC8, $ACC8 + + vpsrlq \$29, $ACC4, $TEMP1 + vpand $AND_MASK, $ACC4, $ACC4 + vpsrlq \$29, $ACC5, $TEMP2 + vpand $AND_MASK, $ACC5, $ACC5 + vpsrlq \$29, $ACC6, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC6, $ACC6 + vpsrlq \$29, $ACC7, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC7, $ACC7 + vpsrlq \$29, $ACC8, $TEMP5 + vpermq \$0x93, $TEMP3, $TEMP3 + vpand $AND_MASK, $ACC8, $ACC8 + vpermq \$0x93, $TEMP4, $TEMP4 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP5, $TEMP5 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC4, $ACC4 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC5, $ACC5 + vmovdqu $ACC4, 32*4-128($rp) + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC6, $ACC6 + vmovdqu $ACC5, 32*5-128($rp) + vpblendd \$3, $TEMP4, $TEMP5, $TEMP4 + vpaddq $TEMP3, $ACC7, $ACC7 + vmovdqu $ACC6, 32*6-128($rp) + vpaddq $TEMP4, $ACC8, $ACC8 + vmovdqu $ACC7, 32*7-128($rp) + vmovdqu $ACC8, 32*8-128($rp) + + mov $rp, $ap + dec $rep + jne .LOOP_GRANDE_SQR_1024 + + vzeroall + mov %rbp, %rax +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp # restore %rsp +.Lsqr_1024_epilogue: + ret +.size rsaz_1024_sqr_avx2,.-rsaz_1024_sqr_avx2 +___ +} + +{ # void AMM_WW( +my $rp="%rdi"; # BN_ULONG *rp, +my $ap="%rsi"; # const BN_ULONG *ap, +my $bp="%rdx"; # const BN_ULONG *bp, +my $np="%rcx"; # const BN_ULONG *np, +my $n0="%r8d"; # unsigned int n0); + +# The registers that hold the accumulated redundant result +# The AMM works on 1024 bit operands, and redundant word size is 29 +# Therefore: ceil(1024/29)/4 = 9 +my $ACC0="%ymm0"; +my $ACC1="%ymm1"; +my $ACC2="%ymm2"; +my $ACC3="%ymm3"; +my $ACC4="%ymm4"; +my $ACC5="%ymm5"; +my $ACC6="%ymm6"; +my $ACC7="%ymm7"; +my $ACC8="%ymm8"; +my $ACC9="%ymm9"; + +# Registers that hold the broadcasted words of multiplier, currently used +my $Bi="%ymm10"; +my $Yi="%ymm11"; + +# Helper registers +my $TEMP0=$ACC0; +my $TEMP1="%ymm12"; +my $TEMP2="%ymm13"; +my $ZERO="%ymm14"; +my $AND_MASK="%ymm15"; + +# alu registers that hold the first words of the ACC +my $r0="%r9"; +my $r1="%r10"; +my $r2="%r11"; +my $r3="%r12"; + +my $i="%r14d"; +my $tmp="%r15"; + +$bp="%r13"; # reassigned argument + +$code.=<<___; +.globl rsaz_1024_mul_avx2 +.type rsaz_1024_mul_avx2,\@function,5 +.align 64 +rsaz_1024_mul_avx2: + lea (%rsp), %rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + vzeroupper + lea -0xa8(%rsp),%rsp + vmovaps %xmm6,-0xd8(%rax) + vmovaps %xmm7,-0xc8(%rax) + vmovaps %xmm8,-0xb8(%rax) + vmovaps %xmm9,-0xa8(%rax) + vmovaps %xmm10,-0x98(%rax) + vmovaps %xmm11,-0x88(%rax) + vmovaps %xmm12,-0x78(%rax) + vmovaps %xmm13,-0x68(%rax) + vmovaps %xmm14,-0x58(%rax) + vmovaps %xmm15,-0x48(%rax) +.Lmul_1024_body: +___ +$code.=<<___; + mov %rax,%rbp + vzeroall + mov %rdx, $bp # reassigned argument + sub \$64,%rsp + + # unaligned 256-bit load that crosses page boundary can + # cause severe performance degradation here, so if $ap does + # cross page boundary, swap it with $bp [meaning that caller + # is advised to lay down $ap and $bp next to each other, so + # that only one can cross page boundary]. + .byte 0x67,0x67 + mov $ap, $tmp + and \$4095, $tmp + add \$32*10, $tmp + shr \$12, $tmp + mov $ap, $tmp + cmovnz $bp, $ap + cmovnz $tmp, $bp + + mov $np, $tmp + sub \$-128,$ap # size optimization + sub \$-128,$np + sub \$-128,$rp + + and \$4095, $tmp # see if $np crosses page + add \$32*10, $tmp + .byte 0x67,0x67 + shr \$12, $tmp + jz .Lmul_1024_no_n_copy + + # unaligned 256-bit load that crosses page boundary can + # cause severe performance degradation here, so if $np does + # cross page boundary, copy it to stack and make sure stack + # frame doesn't... + sub \$32*10,%rsp + vmovdqu 32*0-128($np), $ACC0 + and \$-512, %rsp + vmovdqu 32*1-128($np), $ACC1 + vmovdqu 32*2-128($np), $ACC2 + vmovdqu 32*3-128($np), $ACC3 + vmovdqu 32*4-128($np), $ACC4 + vmovdqu 32*5-128($np), $ACC5 + vmovdqu 32*6-128($np), $ACC6 + vmovdqu 32*7-128($np), $ACC7 + vmovdqu 32*8-128($np), $ACC8 + lea 64+128(%rsp),$np + vmovdqu $ACC0, 32*0-128($np) + vpxor $ACC0, $ACC0, $ACC0 + vmovdqu $ACC1, 32*1-128($np) + vpxor $ACC1, $ACC1, $ACC1 + vmovdqu $ACC2, 32*2-128($np) + vpxor $ACC2, $ACC2, $ACC2 + vmovdqu $ACC3, 32*3-128($np) + vpxor $ACC3, $ACC3, $ACC3 + vmovdqu $ACC4, 32*4-128($np) + vpxor $ACC4, $ACC4, $ACC4 + vmovdqu $ACC5, 32*5-128($np) + vpxor $ACC5, $ACC5, $ACC5 + vmovdqu $ACC6, 32*6-128($np) + vpxor $ACC6, $ACC6, $ACC6 + vmovdqu $ACC7, 32*7-128($np) + vpxor $ACC7, $ACC7, $ACC7 + vmovdqu $ACC8, 32*8-128($np) + vmovdqa $ACC0, $ACC8 + vmovdqu $ACC9, 32*9-128($np) # $ACC9 is zero after vzeroall +.Lmul_1024_no_n_copy: + and \$-64,%rsp + + mov ($bp), %rbx + vpbroadcastq ($bp), $Bi + vmovdqu $ACC0, (%rsp) # clear top of stack + xor $r0, $r0 + .byte 0x67 + xor $r1, $r1 + xor $r2, $r2 + xor $r3, $r3 + + vmovdqu .Land_mask(%rip), $AND_MASK + mov \$9, $i + vmovdqu $ACC9, 32*9-128($rp) # $ACC9 is zero after vzeroall + jmp .Loop_mul_1024 + +.align 32 +.Loop_mul_1024: + vpsrlq \$29, $ACC3, $ACC9 # correct $ACC3(*) + mov %rbx, %rax + imulq -128($ap), %rax + add $r0, %rax + mov %rbx, $r1 + imulq 8-128($ap), $r1 + add 8(%rsp), $r1 + + mov %rax, $r0 + imull $n0, %eax + and \$0x1fffffff, %eax + + mov %rbx, $r2 + imulq 16-128($ap), $r2 + add 16(%rsp), $r2 + + mov %rbx, $r3 + imulq 24-128($ap), $r3 + add 24(%rsp), $r3 + vpmuludq 32*1-128($ap),$Bi,$TEMP0 + vmovd %eax, $Yi + vpaddq $TEMP0,$ACC1,$ACC1 + vpmuludq 32*2-128($ap),$Bi,$TEMP1 + vpbroadcastq $Yi, $Yi + vpaddq $TEMP1,$ACC2,$ACC2 + vpmuludq 32*3-128($ap),$Bi,$TEMP2 + vpand $AND_MASK, $ACC3, $ACC3 # correct $ACC3 + vpaddq $TEMP2,$ACC3,$ACC3 + vpmuludq 32*4-128($ap),$Bi,$TEMP0 + vpaddq $TEMP0,$ACC4,$ACC4 + vpmuludq 32*5-128($ap),$Bi,$TEMP1 + vpaddq $TEMP1,$ACC5,$ACC5 + vpmuludq 32*6-128($ap),$Bi,$TEMP2 + vpaddq $TEMP2,$ACC6,$ACC6 + vpmuludq 32*7-128($ap),$Bi,$TEMP0 + vpermq \$0x93, $ACC9, $ACC9 # correct $ACC3 + vpaddq $TEMP0,$ACC7,$ACC7 + vpmuludq 32*8-128($ap),$Bi,$TEMP1 + vpbroadcastq 8($bp), $Bi + vpaddq $TEMP1,$ACC8,$ACC8 + + mov %rax,%rdx + imulq -128($np),%rax + add %rax,$r0 + mov %rdx,%rax + imulq 8-128($np),%rax + add %rax,$r1 + mov %rdx,%rax + imulq 16-128($np),%rax + add %rax,$r2 + shr \$29, $r0 + imulq 24-128($np),%rdx + add %rdx,$r3 + add $r0, $r1 + + vpmuludq 32*1-128($np),$Yi,$TEMP2 + vmovq $Bi, %rbx + vpaddq $TEMP2,$ACC1,$ACC1 + vpmuludq 32*2-128($np),$Yi,$TEMP0 + vpaddq $TEMP0,$ACC2,$ACC2 + vpmuludq 32*3-128($np),$Yi,$TEMP1 + vpaddq $TEMP1,$ACC3,$ACC3 + vpmuludq 32*4-128($np),$Yi,$TEMP2 + vpaddq $TEMP2,$ACC4,$ACC4 + vpmuludq 32*5-128($np),$Yi,$TEMP0 + vpaddq $TEMP0,$ACC5,$ACC5 + vpmuludq 32*6-128($np),$Yi,$TEMP1 + vpaddq $TEMP1,$ACC6,$ACC6 + vpmuludq 32*7-128($np),$Yi,$TEMP2 + vpblendd \$3, $ZERO, $ACC9, $ACC9 # correct $ACC3 + vpaddq $TEMP2,$ACC7,$ACC7 + vpmuludq 32*8-128($np),$Yi,$TEMP0 + vpaddq $ACC9, $ACC3, $ACC3 # correct $ACC3 + vpaddq $TEMP0,$ACC8,$ACC8 + + mov %rbx, %rax + imulq -128($ap),%rax + add %rax,$r1 + vmovdqu -8+32*1-128($ap),$TEMP1 + mov %rbx, %rax + imulq 8-128($ap),%rax + add %rax,$r2 + vmovdqu -8+32*2-128($ap),$TEMP2 + + mov $r1, %rax + imull $n0, %eax + and \$0x1fffffff, %eax + + imulq 16-128($ap),%rbx + add %rbx,$r3 + vpmuludq $Bi,$TEMP1,$TEMP1 + vmovd %eax, $Yi + vmovdqu -8+32*3-128($ap),$TEMP0 + vpaddq $TEMP1,$ACC1,$ACC1 + vpmuludq $Bi,$TEMP2,$TEMP2 + vpbroadcastq $Yi, $Yi + vmovdqu -8+32*4-128($ap),$TEMP1 + vpaddq $TEMP2,$ACC2,$ACC2 + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovdqu -8+32*5-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC3,$ACC3 + vpmuludq $Bi,$TEMP1,$TEMP1 + vmovdqu -8+32*6-128($ap),$TEMP0 + vpaddq $TEMP1,$ACC4,$ACC4 + vpmuludq $Bi,$TEMP2,$TEMP2 + vmovdqu -8+32*7-128($ap),$TEMP1 + vpaddq $TEMP2,$ACC5,$ACC5 + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovdqu -8+32*8-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC6,$ACC6 + vpmuludq $Bi,$TEMP1,$TEMP1 + vmovdqu -8+32*9-128($ap),$ACC9 + vpaddq $TEMP1,$ACC7,$ACC7 + vpmuludq $Bi,$TEMP2,$TEMP2 + vpaddq $TEMP2,$ACC8,$ACC8 + vpmuludq $Bi,$ACC9,$ACC9 + vpbroadcastq 16($bp), $Bi + + mov %rax,%rdx + imulq -128($np),%rax + add %rax,$r1 + vmovdqu -8+32*1-128($np),$TEMP0 + mov %rdx,%rax + imulq 8-128($np),%rax + add %rax,$r2 + vmovdqu -8+32*2-128($np),$TEMP1 + shr \$29, $r1 + imulq 16-128($np),%rdx + add %rdx,$r3 + add $r1, $r2 + + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovq $Bi, %rbx + vmovdqu -8+32*3-128($np),$TEMP2 + vpaddq $TEMP0,$ACC1,$ACC1 + vpmuludq $Yi,$TEMP1,$TEMP1 + vmovdqu -8+32*4-128($np),$TEMP0 + vpaddq $TEMP1,$ACC2,$ACC2 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovdqu -8+32*5-128($np),$TEMP1 + vpaddq $TEMP2,$ACC3,$ACC3 + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovdqu -8+32*6-128($np),$TEMP2 + vpaddq $TEMP0,$ACC4,$ACC4 + vpmuludq $Yi,$TEMP1,$TEMP1 + vmovdqu -8+32*7-128($np),$TEMP0 + vpaddq $TEMP1,$ACC5,$ACC5 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovdqu -8+32*8-128($np),$TEMP1 + vpaddq $TEMP2,$ACC6,$ACC6 + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovdqu -8+32*9-128($np),$TEMP2 + vpaddq $TEMP0,$ACC7,$ACC7 + vpmuludq $Yi,$TEMP1,$TEMP1 + vpaddq $TEMP1,$ACC8,$ACC8 + vpmuludq $Yi,$TEMP2,$TEMP2 + vpaddq $TEMP2,$ACC9,$ACC9 + + vmovdqu -16+32*1-128($ap),$TEMP0 + mov %rbx,%rax + imulq -128($ap),%rax + add $r2,%rax + + vmovdqu -16+32*2-128($ap),$TEMP1 + mov %rax,$r2 + imull $n0, %eax + and \$0x1fffffff, %eax + + imulq 8-128($ap),%rbx + add %rbx,$r3 + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovd %eax, $Yi + vmovdqu -16+32*3-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC1,$ACC1 + vpmuludq $Bi,$TEMP1,$TEMP1 + vpbroadcastq $Yi, $Yi + vmovdqu -16+32*4-128($ap),$TEMP0 + vpaddq $TEMP1,$ACC2,$ACC2 + vpmuludq $Bi,$TEMP2,$TEMP2 + vmovdqu -16+32*5-128($ap),$TEMP1 + vpaddq $TEMP2,$ACC3,$ACC3 + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovdqu -16+32*6-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC4,$ACC4 + vpmuludq $Bi,$TEMP1,$TEMP1 + vmovdqu -16+32*7-128($ap),$TEMP0 + vpaddq $TEMP1,$ACC5,$ACC5 + vpmuludq $Bi,$TEMP2,$TEMP2 + vmovdqu -16+32*8-128($ap),$TEMP1 + vpaddq $TEMP2,$ACC6,$ACC6 + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovdqu -16+32*9-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC7,$ACC7 + vpmuludq $Bi,$TEMP1,$TEMP1 + vpaddq $TEMP1,$ACC8,$ACC8 + vpmuludq $Bi,$TEMP2,$TEMP2 + vpbroadcastq 24($bp), $Bi + vpaddq $TEMP2,$ACC9,$ACC9 + + vmovdqu -16+32*1-128($np),$TEMP0 + mov %rax,%rdx + imulq -128($np),%rax + add %rax,$r2 + vmovdqu -16+32*2-128($np),$TEMP1 + imulq 8-128($np),%rdx + add %rdx,$r3 + shr \$29, $r2 + + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovq $Bi, %rbx + vmovdqu -16+32*3-128($np),$TEMP2 + vpaddq $TEMP0,$ACC1,$ACC1 + vpmuludq $Yi,$TEMP1,$TEMP1 + vmovdqu -16+32*4-128($np),$TEMP0 + vpaddq $TEMP1,$ACC2,$ACC2 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovdqu -16+32*5-128($np),$TEMP1 + vpaddq $TEMP2,$ACC3,$ACC3 + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovdqu -16+32*6-128($np),$TEMP2 + vpaddq $TEMP0,$ACC4,$ACC4 + vpmuludq $Yi,$TEMP1,$TEMP1 + vmovdqu -16+32*7-128($np),$TEMP0 + vpaddq $TEMP1,$ACC5,$ACC5 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovdqu -16+32*8-128($np),$TEMP1 + vpaddq $TEMP2,$ACC6,$ACC6 + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovdqu -16+32*9-128($np),$TEMP2 + vpaddq $TEMP0,$ACC7,$ACC7 + vpmuludq $Yi,$TEMP1,$TEMP1 + vmovdqu -24+32*1-128($ap),$TEMP0 + vpaddq $TEMP1,$ACC8,$ACC8 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovdqu -24+32*2-128($ap),$TEMP1 + vpaddq $TEMP2,$ACC9,$ACC9 + + add $r2, $r3 + imulq -128($ap),%rbx + add %rbx,$r3 + + mov $r3, %rax + imull $n0, %eax + and \$0x1fffffff, %eax + + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovd %eax, $Yi + vmovdqu -24+32*3-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC1,$ACC1 + vpmuludq $Bi,$TEMP1,$TEMP1 + vpbroadcastq $Yi, $Yi + vmovdqu -24+32*4-128($ap),$TEMP0 + vpaddq $TEMP1,$ACC2,$ACC2 + vpmuludq $Bi,$TEMP2,$TEMP2 + vmovdqu -24+32*5-128($ap),$TEMP1 + vpaddq $TEMP2,$ACC3,$ACC3 + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovdqu -24+32*6-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC4,$ACC4 + vpmuludq $Bi,$TEMP1,$TEMP1 + vmovdqu -24+32*7-128($ap),$TEMP0 + vpaddq $TEMP1,$ACC5,$ACC5 + vpmuludq $Bi,$TEMP2,$TEMP2 + vmovdqu -24+32*8-128($ap),$TEMP1 + vpaddq $TEMP2,$ACC6,$ACC6 + vpmuludq $Bi,$TEMP0,$TEMP0 + vmovdqu -24+32*9-128($ap),$TEMP2 + vpaddq $TEMP0,$ACC7,$ACC7 + vpmuludq $Bi,$TEMP1,$TEMP1 + vpaddq $TEMP1,$ACC8,$ACC8 + vpmuludq $Bi,$TEMP2,$TEMP2 + vpbroadcastq 32($bp), $Bi + vpaddq $TEMP2,$ACC9,$ACC9 + add \$32, $bp # $bp++ + + vmovdqu -24+32*1-128($np),$TEMP0 + imulq -128($np),%rax + add %rax,$r3 + shr \$29, $r3 + + vmovdqu -24+32*2-128($np),$TEMP1 + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovq $Bi, %rbx + vmovdqu -24+32*3-128($np),$TEMP2 + vpaddq $TEMP0,$ACC1,$ACC0 # $ACC0==$TEMP0 + vpmuludq $Yi,$TEMP1,$TEMP1 + vmovdqu $ACC0, (%rsp) # transfer $r0-$r3 + vpaddq $TEMP1,$ACC2,$ACC1 + vmovdqu -24+32*4-128($np),$TEMP0 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovdqu -24+32*5-128($np),$TEMP1 + vpaddq $TEMP2,$ACC3,$ACC2 + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovdqu -24+32*6-128($np),$TEMP2 + vpaddq $TEMP0,$ACC4,$ACC3 + vpmuludq $Yi,$TEMP1,$TEMP1 + vmovdqu -24+32*7-128($np),$TEMP0 + vpaddq $TEMP1,$ACC5,$ACC4 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovdqu -24+32*8-128($np),$TEMP1 + vpaddq $TEMP2,$ACC6,$ACC5 + vpmuludq $Yi,$TEMP0,$TEMP0 + vmovdqu -24+32*9-128($np),$TEMP2 + mov $r3, $r0 + vpaddq $TEMP0,$ACC7,$ACC6 + vpmuludq $Yi,$TEMP1,$TEMP1 + add (%rsp), $r0 + vpaddq $TEMP1,$ACC8,$ACC7 + vpmuludq $Yi,$TEMP2,$TEMP2 + vmovq $r3, $TEMP1 + vpaddq $TEMP2,$ACC9,$ACC8 + + dec $i + jnz .Loop_mul_1024 +___ + +# (*) Original implementation was correcting ACC1-ACC3 for overflow +# after 7 loop runs, or after 28 iterations, or 56 additions. +# But as we underutilize resources, it's possible to correct in +# each iteration with marginal performance loss. But then, as +# we do it in each iteration, we can correct less digits, and +# avoid performance penalties completely. Also note that we +# correct only three digits out of four. This works because +# most significant digit is subjected to less additions. + +$TEMP0 = $ACC9; +$TEMP3 = $Bi; +$TEMP4 = $Yi; +$code.=<<___; + vpermq \$0, $AND_MASK, $AND_MASK + vpaddq (%rsp), $TEMP1, $ACC0 + + vpsrlq \$29, $ACC0, $TEMP1 + vpand $AND_MASK, $ACC0, $ACC0 + vpsrlq \$29, $ACC1, $TEMP2 + vpand $AND_MASK, $ACC1, $ACC1 + vpsrlq \$29, $ACC2, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC2, $ACC2 + vpsrlq \$29, $ACC3, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC3, $ACC3 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP3, $TEMP3 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpermq \$0x93, $TEMP4, $TEMP4 + vpaddq $TEMP0, $ACC0, $ACC0 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC1, $ACC1 + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC2, $ACC2 + vpblendd \$3, $TEMP4, $ZERO, $TEMP4 + vpaddq $TEMP3, $ACC3, $ACC3 + vpaddq $TEMP4, $ACC4, $ACC4 + + vpsrlq \$29, $ACC0, $TEMP1 + vpand $AND_MASK, $ACC0, $ACC0 + vpsrlq \$29, $ACC1, $TEMP2 + vpand $AND_MASK, $ACC1, $ACC1 + vpsrlq \$29, $ACC2, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC2, $ACC2 + vpsrlq \$29, $ACC3, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC3, $ACC3 + vpermq \$0x93, $TEMP3, $TEMP3 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP4, $TEMP4 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC0, $ACC0 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC1, $ACC1 + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC2, $ACC2 + vpblendd \$3, $TEMP4, $ZERO, $TEMP4 + vpaddq $TEMP3, $ACC3, $ACC3 + vpaddq $TEMP4, $ACC4, $ACC4 + + vmovdqu $ACC0, 0-128($rp) + vmovdqu $ACC1, 32-128($rp) + vmovdqu $ACC2, 64-128($rp) + vmovdqu $ACC3, 96-128($rp) +___ + +$TEMP5=$ACC0; +$code.=<<___; + vpsrlq \$29, $ACC4, $TEMP1 + vpand $AND_MASK, $ACC4, $ACC4 + vpsrlq \$29, $ACC5, $TEMP2 + vpand $AND_MASK, $ACC5, $ACC5 + vpsrlq \$29, $ACC6, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC6, $ACC6 + vpsrlq \$29, $ACC7, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC7, $ACC7 + vpsrlq \$29, $ACC8, $TEMP5 + vpermq \$0x93, $TEMP3, $TEMP3 + vpand $AND_MASK, $ACC8, $ACC8 + vpermq \$0x93, $TEMP4, $TEMP4 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP5, $TEMP5 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC4, $ACC4 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC5, $ACC5 + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC6, $ACC6 + vpblendd \$3, $TEMP4, $TEMP5, $TEMP4 + vpaddq $TEMP3, $ACC7, $ACC7 + vpaddq $TEMP4, $ACC8, $ACC8 + + vpsrlq \$29, $ACC4, $TEMP1 + vpand $AND_MASK, $ACC4, $ACC4 + vpsrlq \$29, $ACC5, $TEMP2 + vpand $AND_MASK, $ACC5, $ACC5 + vpsrlq \$29, $ACC6, $TEMP3 + vpermq \$0x93, $TEMP1, $TEMP1 + vpand $AND_MASK, $ACC6, $ACC6 + vpsrlq \$29, $ACC7, $TEMP4 + vpermq \$0x93, $TEMP2, $TEMP2 + vpand $AND_MASK, $ACC7, $ACC7 + vpsrlq \$29, $ACC8, $TEMP5 + vpermq \$0x93, $TEMP3, $TEMP3 + vpand $AND_MASK, $ACC8, $ACC8 + vpermq \$0x93, $TEMP4, $TEMP4 + + vpblendd \$3, $ZERO, $TEMP1, $TEMP0 + vpermq \$0x93, $TEMP5, $TEMP5 + vpblendd \$3, $TEMP1, $TEMP2, $TEMP1 + vpaddq $TEMP0, $ACC4, $ACC4 + vpblendd \$3, $TEMP2, $TEMP3, $TEMP2 + vpaddq $TEMP1, $ACC5, $ACC5 + vpblendd \$3, $TEMP3, $TEMP4, $TEMP3 + vpaddq $TEMP2, $ACC6, $ACC6 + vpblendd \$3, $TEMP4, $TEMP5, $TEMP4 + vpaddq $TEMP3, $ACC7, $ACC7 + vpaddq $TEMP4, $ACC8, $ACC8 + + vmovdqu $ACC4, 128-128($rp) + vmovdqu $ACC5, 160-128($rp) + vmovdqu $ACC6, 192-128($rp) + vmovdqu $ACC7, 224-128($rp) + vmovdqu $ACC8, 256-128($rp) + vzeroupper + + mov %rbp, %rax +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp # restore %rsp +.Lmul_1024_epilogue: + ret +.size rsaz_1024_mul_avx2,.-rsaz_1024_mul_avx2 +___ +} +{ +my ($out,$inp) = $win64 ? ("%rcx","%rdx") : ("%rdi","%rsi"); +my @T = map("%r$_",(8..11)); + +$code.=<<___; +.globl rsaz_1024_red2norm_avx2 +.type rsaz_1024_red2norm_avx2,\@abi-omnipotent +.align 32 +rsaz_1024_red2norm_avx2: + sub \$-128,$inp # size optimization + xor %rax,%rax +___ + +for ($j=0,$i=0; $i<16; $i++) { + my $k=0; + while (29*$j<64*($i+1)) { # load data till boundary + $code.=" mov `8*$j-128`($inp), @T[0]\n"; + $j++; $k++; push(@T,shift(@T)); + } + $l=$k; + while ($k>1) { # shift loaded data but last value + $code.=" shl \$`29*($j-$k)`,@T[-$k]\n"; + $k--; + } + $code.=<<___; # shift last value + mov @T[-1], @T[0] + shl \$`29*($j-1)`, @T[-1] + shr \$`-29*($j-1)`, @T[0] +___ + while ($l) { # accumulate all values + $code.=" add @T[-$l], %rax\n"; + $l--; + } + $code.=<<___; + adc \$0, @T[0] # consume eventual carry + mov %rax, 8*$i($out) + mov @T[0], %rax +___ + push(@T,shift(@T)); +} +$code.=<<___; + ret +.size rsaz_1024_red2norm_avx2,.-rsaz_1024_red2norm_avx2 + +.globl rsaz_1024_norm2red_avx2 +.type rsaz_1024_norm2red_avx2,\@abi-omnipotent +.align 32 +rsaz_1024_norm2red_avx2: + sub \$-128,$out # size optimization + mov ($inp),@T[0] + mov \$0x1fffffff,%eax +___ +for ($j=0,$i=0; $i<16; $i++) { + $code.=" mov `8*($i+1)`($inp),@T[1]\n" if ($i<15); + $code.=" xor @T[1],@T[1]\n" if ($i==15); + my $k=1; + while (29*($j+1)<64*($i+1)) { + $code.=<<___; + mov @T[0],@T[-$k] + shr \$`29*$j`,@T[-$k] + and %rax,@T[-$k] # &0x1fffffff + mov @T[-$k],`8*$j-128`($out) +___ + $j++; $k++; + } + $code.=<<___; + shrd \$`29*$j`,@T[1],@T[0] + and %rax,@T[0] + mov @T[0],`8*$j-128`($out) +___ + $j++; + push(@T,shift(@T)); +} +$code.=<<___; + mov @T[0],`8*$j-128`($out) # zero + mov @T[0],`8*($j+1)-128`($out) + mov @T[0],`8*($j+2)-128`($out) + mov @T[0],`8*($j+3)-128`($out) + ret +.size rsaz_1024_norm2red_avx2,.-rsaz_1024_norm2red_avx2 +___ +} +{ +my ($out,$inp,$power) = $win64 ? ("%rcx","%rdx","%r8d") : ("%rdi","%rsi","%edx"); + +$code.=<<___; +.globl rsaz_1024_scatter5_avx2 +.type rsaz_1024_scatter5_avx2,\@abi-omnipotent +.align 32 +rsaz_1024_scatter5_avx2: + vzeroupper + vmovdqu .Lscatter_permd(%rip),%ymm5 + shl \$4,$power + lea ($out,$power),$out + mov \$9,%eax + jmp .Loop_scatter_1024 + +.align 32 +.Loop_scatter_1024: + vmovdqu ($inp),%ymm0 + lea 32($inp),$inp + vpermd %ymm0,%ymm5,%ymm0 + vmovdqu %xmm0,($out) + lea 16*32($out),$out + dec %eax + jnz .Loop_scatter_1024 + + vzeroupper + ret +.size rsaz_1024_scatter5_avx2,.-rsaz_1024_scatter5_avx2 + +.globl rsaz_1024_gather5_avx2 +.type rsaz_1024_gather5_avx2,\@abi-omnipotent +.align 32 +rsaz_1024_gather5_avx2: +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp),%rax + vzeroupper +.LSEH_begin_rsaz_1024_gather5: + # I can't trust assembler to use specific encoding:-( + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax),%rsp + .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6,-0x20(%rax) + .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7,-0x10(%rax) + .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8,0(%rax) + .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9,0x10(%rax) + .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10,0x20(%rax) + .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11,0x30(%rax) + .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12,0x40(%rax) + .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13,0x50(%rax) + .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14,0x60(%rax) + .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15,0x70(%rax) +___ +$code.=<<___; + lea .Lgather_table(%rip),%r11 + mov $power,%eax + and \$3,$power + shr \$2,%eax # cache line number + shl \$4,$power # offset within cache line + + vmovdqu -32(%r11),%ymm7 # .Lgather_permd + vpbroadcastb 8(%r11,%rax), %xmm8 + vpbroadcastb 7(%r11,%rax), %xmm9 + vpbroadcastb 6(%r11,%rax), %xmm10 + vpbroadcastb 5(%r11,%rax), %xmm11 + vpbroadcastb 4(%r11,%rax), %xmm12 + vpbroadcastb 3(%r11,%rax), %xmm13 + vpbroadcastb 2(%r11,%rax), %xmm14 + vpbroadcastb 1(%r11,%rax), %xmm15 + + lea 64($inp,$power),$inp + mov \$64,%r11 # size optimization + mov \$9,%eax + jmp .Loop_gather_1024 + +.align 32 +.Loop_gather_1024: + vpand -64($inp), %xmm8,%xmm0 + vpand ($inp), %xmm9,%xmm1 + vpand 64($inp), %xmm10,%xmm2 + vpand ($inp,%r11,2), %xmm11,%xmm3 + vpor %xmm0,%xmm1,%xmm1 + vpand 64($inp,%r11,2), %xmm12,%xmm4 + vpor %xmm2,%xmm3,%xmm3 + vpand ($inp,%r11,4), %xmm13,%xmm5 + vpor %xmm1,%xmm3,%xmm3 + vpand 64($inp,%r11,4), %xmm14,%xmm6 + vpor %xmm4,%xmm5,%xmm5 + vpand -128($inp,%r11,8), %xmm15,%xmm2 + lea ($inp,%r11,8),$inp + vpor %xmm3,%xmm5,%xmm5 + vpor %xmm2,%xmm6,%xmm6 + vpor %xmm5,%xmm6,%xmm6 + vpermd %ymm6,%ymm7,%ymm6 + vmovdqu %ymm6,($out) + lea 32($out),$out + dec %eax + jnz .Loop_gather_1024 + + vpxor %ymm0,%ymm0,%ymm0 + vmovdqu %ymm0,($out) + vzeroupper +___ +$code.=<<___ if ($win64); + movaps (%rsp),%xmm6 + movaps 0x10(%rsp),%xmm7 + movaps 0x20(%rsp),%xmm8 + movaps 0x30(%rsp),%xmm9 + movaps 0x40(%rsp),%xmm10 + movaps 0x50(%rsp),%xmm11 + movaps 0x60(%rsp),%xmm12 + movaps 0x70(%rsp),%xmm13 + movaps 0x80(%rsp),%xmm14 + movaps 0x90(%rsp),%xmm15 + lea 0xa8(%rsp),%rsp +.LSEH_end_rsaz_1024_gather5: +___ +$code.=<<___; + ret +.size rsaz_1024_gather5_avx2,.-rsaz_1024_gather5_avx2 +___ +} + +$code.=<<___; +.extern OPENSSL_ia32cap_P +.globl rsaz_avx2_eligible +.type rsaz_avx2_eligible,\@abi-omnipotent +.align 32 +rsaz_avx2_eligible: + mov OPENSSL_ia32cap_P+8(%rip),%eax +___ +$code.=<<___ if ($addx); + mov \$`1<<8|1<<19`,%ecx + mov \$0,%edx + and %eax,%ecx + cmp \$`1<<8|1<<19`,%ecx # check for BMI2+AD*X + cmove %edx,%eax +___ +$code.=<<___; + and \$`1<<5`,%eax + shr \$5,%eax + ret +.size rsaz_avx2_eligible,.-rsaz_avx2_eligible + +.align 64 +.Land_mask: + .quad 0x1fffffff,0x1fffffff,0x1fffffff,-1 +.Lscatter_permd: + .long 0,2,4,6,7,7,7,7 +.Lgather_permd: + .long 0,7,1,7,2,7,3,7 +.Lgather_table: + .byte 0,0,0,0,0,0,0,0, 0xff,0,0,0,0,0,0,0 +.align 64 +___ + +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___ +.extern __imp_RtlVirtualUnwind +.type rsaz_se_handler,\@abi-omnipotent +.align 16 +rsaz_se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + mov 160($context),%rax # pull context->Rbp + + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + mov %r15,240($context) + mov %r14,232($context) + mov %r13,224($context) + mov %r12,216($context) + mov %rbp,160($context) + mov %rbx,144($context) + + lea -0xd8(%rax),%rsi # %xmm save area + lea 512($context),%rdi # & context.Xmm6 + mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) + .long 0xa548f3fc # cld; rep movsq + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size rsaz_se_handler,.-rsaz_se_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_rsaz_1024_sqr_avx2 + .rva .LSEH_end_rsaz_1024_sqr_avx2 + .rva .LSEH_info_rsaz_1024_sqr_avx2 + + .rva .LSEH_begin_rsaz_1024_mul_avx2 + .rva .LSEH_end_rsaz_1024_mul_avx2 + .rva .LSEH_info_rsaz_1024_mul_avx2 + + .rva .LSEH_begin_rsaz_1024_gather5 + .rva .LSEH_end_rsaz_1024_gather5 + .rva .LSEH_info_rsaz_1024_gather5 +.section .xdata +.align 8 +.LSEH_info_rsaz_1024_sqr_avx2: + .byte 9,0,0,0 + .rva rsaz_se_handler + .rva .Lsqr_1024_body,.Lsqr_1024_epilogue +.LSEH_info_rsaz_1024_mul_avx2: + .byte 9,0,0,0 + .rva rsaz_se_handler + .rva .Lmul_1024_body,.Lmul_1024_epilogue +.LSEH_info_rsaz_1024_gather5: + .byte 0x01,0x33,0x16,0x00 + .byte 0x36,0xf8,0x09,0x00 #vmovaps 0x90(rsp),xmm15 + .byte 0x31,0xe8,0x08,0x00 #vmovaps 0x80(rsp),xmm14 + .byte 0x2c,0xd8,0x07,0x00 #vmovaps 0x70(rsp),xmm13 + .byte 0x27,0xc8,0x06,0x00 #vmovaps 0x60(rsp),xmm12 + .byte 0x22,0xb8,0x05,0x00 #vmovaps 0x50(rsp),xmm11 + .byte 0x1d,0xa8,0x04,0x00 #vmovaps 0x40(rsp),xmm10 + .byte 0x18,0x98,0x03,0x00 #vmovaps 0x30(rsp),xmm9 + .byte 0x13,0x88,0x02,0x00 #vmovaps 0x20(rsp),xmm8 + .byte 0x0e,0x78,0x01,0x00 #vmovaps 0x10(rsp),xmm7 + .byte 0x09,0x68,0x00,0x00 #vmovaps 0x00(rsp),xmm6 + .byte 0x04,0x01,0x15,0x00 #sub rsp,0xa8 +___ +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + + s/\b(sh[rl]d?\s+\$)(-?[0-9]+)/$1.$2%64/ge or + + s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go; + print $_,"\n"; +} + +}}} else {{{ +print <<___; # assembler is too old +.text + +.globl rsaz_avx2_eligible +.type rsaz_avx2_eligible,\@abi-omnipotent +rsaz_avx2_eligible: + xor %eax,%eax + ret +.size rsaz_avx2_eligible,.-rsaz_avx2_eligible + +.globl rsaz_1024_sqr_avx2 +.globl rsaz_1024_mul_avx2 +.globl rsaz_1024_norm2red_avx2 +.globl rsaz_1024_red2norm_avx2 +.globl rsaz_1024_scatter5_avx2 +.globl rsaz_1024_gather5_avx2 +.type rsaz_1024_sqr_avx2,\@abi-omnipotent +rsaz_1024_sqr_avx2: +rsaz_1024_mul_avx2: +rsaz_1024_norm2red_avx2: +rsaz_1024_red2norm_avx2: +rsaz_1024_scatter5_avx2: +rsaz_1024_gather5_avx2: + .byte 0x0f,0x0b # ud2 + ret +.size rsaz_1024_sqr_avx2,.-rsaz_1024_sqr_avx2 +___ +}}} + +close STDOUT; diff --git a/crypto/bn/asm/rsaz-x86_64.pl b/crypto/bn/asm/rsaz-x86_64.pl new file mode 100755 index 000000000000..3bd45dbac01d --- /dev/null +++ b/crypto/bn/asm/rsaz-x86_64.pl @@ -0,0 +1,2144 @@ +#!/usr/bin/env perl + +############################################################################## +# # +# Copyright (c) 2012, Intel Corporation # +# # +# All rights reserved. # +# # +# Redistribution and use in source and binary forms, with or without # +# modification, are permitted provided that the following conditions are # +# met: # +# # +# * Redistributions of source code must retain the above copyright # +# notice, this list of conditions and the following disclaimer. # +# # +# * Redistributions in binary form must reproduce the above copyright # +# notice, this list of conditions and the following disclaimer in the # +# documentation and/or other materials provided with the # +# distribution. # +# # +# * Neither the name of the Intel Corporation nor the names of its # +# contributors may be used to endorse or promote products derived from # +# this software without specific prior written permission. # +# # +# # +# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY # +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR # +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # +# # +############################################################################## +# Developers and authors: # +# Shay Gueron (1, 2), and Vlad Krasnov (1) # +# (1) Intel Architecture Group, Microprocessor and Chipset Development, # +# Israel Development Center, Haifa, Israel # +# (2) University of Haifa # +############################################################################## +# Reference: # +# [1] S. Gueron, "Efficient Software Implementations of Modular # +# Exponentiation", http://eprint.iacr.org/2011/239 # +# [2] S. Gueron, V. Krasnov. "Speeding up Big-Numbers Squaring". # +# IEEE Proceedings of 9th International Conference on Information # +# Technology: New Generations (ITNG 2012), 821-823 (2012). # +# [3] S. Gueron, Efficient Software Implementations of Modular Exponentiation# +# Journal of Cryptographic Engineering 2:31-43 (2012). # +# [4] S. Gueron, V. Krasnov: "[PATCH] Efficient and side channel analysis # +# resistant 512-bit and 1024-bit modular exponentiation for optimizing # +# RSA1024 and RSA2048 on x86_64 platforms", # +# http://rt.openssl.org/Ticket/Display.html?id=2582&user=guest&pass=guest# +############################################################################## + +# While original submission covers 512- and 1024-bit exponentiation, +# this module is limited to 512-bit version only (and as such +# accelerates RSA1024 sign). This is because improvement for longer +# keys is not high enough to justify the effort, highest measured +# was ~5% on Westmere. [This is relative to OpenSSL 1.0.2, upcoming +# for the moment of this writing!] Nor does this module implement +# "monolithic" complete exponentiation jumbo-subroutine, but adheres +# to more modular mixture of C and assembly. And it's optimized even +# for processors other than Intel Core family (see table below for +# improvement coefficients). +# +# +# RSA1024 sign/sec this/original |this/rsax(*) this/fips(*) +# ----------------+--------------------------- +# Opteron +13% |+5% +20% +# Bulldozer -0% |-1% +10% +# P4 +11% |+7% +8% +# Westmere +5% |+14% +17% +# Sandy Bridge +2% |+12% +29% +# Ivy Bridge +1% |+11% +35% +# Haswell(**) -0% |+12% +39% +# Atom +13% |+11% +4% +# VIA Nano +70% |+9% +25% +# +# (*) rsax engine and fips numbers are presented for reference +# purposes; +# (**) MULX was attempted, but found to give only marginal improvement; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $addx = ($1>=2.23); +} + +if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $addx = ($1>=2.10); +} + +if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $addx = ($1>=12); +} + +if (!$addx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { + my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 + $addx = ($ver>=3.03); +} + +($out, $inp, $mod) = ("%rdi", "%rsi", "%rbp"); # common internal API +{ +my ($out,$inp,$mod,$n0,$times) = ("%rdi","%rsi","%rdx","%rcx","%r8d"); + +$code.=<<___; +.text + +.extern OPENSSL_ia32cap_P + +.globl rsaz_512_sqr +.type rsaz_512_sqr,\@function,5 +.align 32 +rsaz_512_sqr: # 25-29% faster than rsaz_512_mul + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + + subq \$128+24, %rsp +.Lsqr_body: + movq $mod, %rbp # common argument + movq ($inp), %rdx + movq 8($inp), %rax + movq $n0, 128(%rsp) +___ +$code.=<<___ if ($addx); + movl \$0x80100,%r11d + andl OPENSSL_ia32cap_P+8(%rip),%r11d + cmpl \$0x80100,%r11d # check for MULX and ADO/CX + je .Loop_sqrx +___ +$code.=<<___; + jmp .Loop_sqr + +.align 32 +.Loop_sqr: + movl $times,128+8(%rsp) +#first iteration + movq %rdx, %rbx + mulq %rdx + movq %rax, %r8 + movq 16($inp), %rax + movq %rdx, %r9 + + mulq %rbx + addq %rax, %r9 + movq 24($inp), %rax + movq %rdx, %r10 + adcq \$0, %r10 + + mulq %rbx + addq %rax, %r10 + movq 32($inp), %rax + movq %rdx, %r11 + adcq \$0, %r11 + + mulq %rbx + addq %rax, %r11 + movq 40($inp), %rax + movq %rdx, %r12 + adcq \$0, %r12 + + mulq %rbx + addq %rax, %r12 + movq 48($inp), %rax + movq %rdx, %r13 + adcq \$0, %r13 + + mulq %rbx + addq %rax, %r13 + movq 56($inp), %rax + movq %rdx, %r14 + adcq \$0, %r14 + + mulq %rbx + addq %rax, %r14 + movq %rbx, %rax + movq %rdx, %r15 + adcq \$0, %r15 + + addq %r8, %r8 #shlq \$1, %r8 + movq %r9, %rcx + adcq %r9, %r9 #shld \$1, %r8, %r9 + + mulq %rax + movq %rax, (%rsp) + addq %rdx, %r8 + adcq \$0, %r9 + + movq %r8, 8(%rsp) + shrq \$63, %rcx + +#second iteration + movq 8($inp), %r8 + movq 16($inp), %rax + mulq %r8 + addq %rax, %r10 + movq 24($inp), %rax + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r8 + addq %rax, %r11 + movq 32($inp), %rax + adcq \$0, %rdx + addq %rbx, %r11 + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r8 + addq %rax, %r12 + movq 40($inp), %rax + adcq \$0, %rdx + addq %rbx, %r12 + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r8 + addq %rax, %r13 + movq 48($inp), %rax + adcq \$0, %rdx + addq %rbx, %r13 + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r8 + addq %rax, %r14 + movq 56($inp), %rax + adcq \$0, %rdx + addq %rbx, %r14 + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r8 + addq %rax, %r15 + movq %r8, %rax + adcq \$0, %rdx + addq %rbx, %r15 + movq %rdx, %r8 + movq %r10, %rdx + adcq \$0, %r8 + + add %rdx, %rdx + lea (%rcx,%r10,2), %r10 #shld \$1, %rcx, %r10 + movq %r11, %rbx + adcq %r11, %r11 #shld \$1, %r10, %r11 + + mulq %rax + addq %rax, %r9 + adcq %rdx, %r10 + adcq \$0, %r11 + + movq %r9, 16(%rsp) + movq %r10, 24(%rsp) + shrq \$63, %rbx + +#third iteration + movq 16($inp), %r9 + movq 24($inp), %rax + mulq %r9 + addq %rax, %r12 + movq 32($inp), %rax + movq %rdx, %rcx + adcq \$0, %rcx + + mulq %r9 + addq %rax, %r13 + movq 40($inp), %rax + adcq \$0, %rdx + addq %rcx, %r13 + movq %rdx, %rcx + adcq \$0, %rcx + + mulq %r9 + addq %rax, %r14 + movq 48($inp), %rax + adcq \$0, %rdx + addq %rcx, %r14 + movq %rdx, %rcx + adcq \$0, %rcx + + mulq %r9 + movq %r12, %r10 + lea (%rbx,%r12,2), %r12 #shld \$1, %rbx, %r12 + addq %rax, %r15 + movq 56($inp), %rax + adcq \$0, %rdx + addq %rcx, %r15 + movq %rdx, %rcx + adcq \$0, %rcx + + mulq %r9 + shrq \$63, %r10 + addq %rax, %r8 + movq %r9, %rax + adcq \$0, %rdx + addq %rcx, %r8 + movq %rdx, %r9 + adcq \$0, %r9 + + movq %r13, %rcx + leaq (%r10,%r13,2), %r13 #shld \$1, %r12, %r13 + + mulq %rax + addq %rax, %r11 + adcq %rdx, %r12 + adcq \$0, %r13 + + movq %r11, 32(%rsp) + movq %r12, 40(%rsp) + shrq \$63, %rcx + +#fourth iteration + movq 24($inp), %r10 + movq 32($inp), %rax + mulq %r10 + addq %rax, %r14 + movq 40($inp), %rax + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r10 + addq %rax, %r15 + movq 48($inp), %rax + adcq \$0, %rdx + addq %rbx, %r15 + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r10 + movq %r14, %r12 + leaq (%rcx,%r14,2), %r14 #shld \$1, %rcx, %r14 + addq %rax, %r8 + movq 56($inp), %rax + adcq \$0, %rdx + addq %rbx, %r8 + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r10 + shrq \$63, %r12 + addq %rax, %r9 + movq %r10, %rax + adcq \$0, %rdx + addq %rbx, %r9 + movq %rdx, %r10 + adcq \$0, %r10 + + movq %r15, %rbx + leaq (%r12,%r15,2),%r15 #shld \$1, %r14, %r15 + + mulq %rax + addq %rax, %r13 + adcq %rdx, %r14 + adcq \$0, %r15 + + movq %r13, 48(%rsp) + movq %r14, 56(%rsp) + shrq \$63, %rbx + +#fifth iteration + movq 32($inp), %r11 + movq 40($inp), %rax + mulq %r11 + addq %rax, %r8 + movq 48($inp), %rax + movq %rdx, %rcx + adcq \$0, %rcx + + mulq %r11 + addq %rax, %r9 + movq 56($inp), %rax + adcq \$0, %rdx + movq %r8, %r12 + leaq (%rbx,%r8,2), %r8 #shld \$1, %rbx, %r8 + addq %rcx, %r9 + movq %rdx, %rcx + adcq \$0, %rcx + + mulq %r11 + shrq \$63, %r12 + addq %rax, %r10 + movq %r11, %rax + adcq \$0, %rdx + addq %rcx, %r10 + movq %rdx, %r11 + adcq \$0, %r11 + + movq %r9, %rcx + leaq (%r12,%r9,2), %r9 #shld \$1, %r8, %r9 + + mulq %rax + addq %rax, %r15 + adcq %rdx, %r8 + adcq \$0, %r9 + + movq %r15, 64(%rsp) + movq %r8, 72(%rsp) + shrq \$63, %rcx + +#sixth iteration + movq 40($inp), %r12 + movq 48($inp), %rax + mulq %r12 + addq %rax, %r10 + movq 56($inp), %rax + movq %rdx, %rbx + adcq \$0, %rbx + + mulq %r12 + addq %rax, %r11 + movq %r12, %rax + movq %r10, %r15 + leaq (%rcx,%r10,2), %r10 #shld \$1, %rcx, %r10 + adcq \$0, %rdx + shrq \$63, %r15 + addq %rbx, %r11 + movq %rdx, %r12 + adcq \$0, %r12 + + movq %r11, %rbx + leaq (%r15,%r11,2), %r11 #shld \$1, %r10, %r11 + + mulq %rax + addq %rax, %r9 + adcq %rdx, %r10 + adcq \$0, %r11 + + movq %r9, 80(%rsp) + movq %r10, 88(%rsp) + +#seventh iteration + movq 48($inp), %r13 + movq 56($inp), %rax + mulq %r13 + addq %rax, %r12 + movq %r13, %rax + movq %rdx, %r13 + adcq \$0, %r13 + + xorq %r14, %r14 + shlq \$1, %rbx + adcq %r12, %r12 #shld \$1, %rbx, %r12 + adcq %r13, %r13 #shld \$1, %r12, %r13 + adcq %r14, %r14 #shld \$1, %r13, %r14 + + mulq %rax + addq %rax, %r11 + adcq %rdx, %r12 + adcq \$0, %r13 + + movq %r11, 96(%rsp) + movq %r12, 104(%rsp) + +#eighth iteration + movq 56($inp), %rax + mulq %rax + addq %rax, %r13 + adcq \$0, %rdx + + addq %rdx, %r14 + + movq %r13, 112(%rsp) + movq %r14, 120(%rsp) + + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r13 + movq 48(%rsp), %r14 + movq 56(%rsp), %r15 + + call __rsaz_512_reduce + + addq 64(%rsp), %r8 + adcq 72(%rsp), %r9 + adcq 80(%rsp), %r10 + adcq 88(%rsp), %r11 + adcq 96(%rsp), %r12 + adcq 104(%rsp), %r13 + adcq 112(%rsp), %r14 + adcq 120(%rsp), %r15 + sbbq %rcx, %rcx + + call __rsaz_512_subtract + + movq %r8, %rdx + movq %r9, %rax + movl 128+8(%rsp), $times + movq $out, $inp + + decl $times + jnz .Loop_sqr +___ +if ($addx) { +$code.=<<___; + jmp .Lsqr_tail + +.align 32 +.Loop_sqrx: + movl $times,128+8(%rsp) + movq $out, %xmm0 # off-load + movq %rbp, %xmm1 # off-load +#first iteration + mulx %rax, %r8, %r9 + + mulx 16($inp), %rcx, %r10 + xor %rbp, %rbp # cf=0, of=0 + + mulx 24($inp), %rax, %r11 + adcx %rcx, %r9 + + mulx 32($inp), %rcx, %r12 + adcx %rax, %r10 + + mulx 40($inp), %rax, %r13 + adcx %rcx, %r11 + + .byte 0xc4,0x62,0xf3,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($inp), %rcx, %r14 + adcx %rax, %r12 + adcx %rcx, %r13 + + .byte 0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00 # mulx 56($inp), %rax, %r15 + adcx %rax, %r14 + adcx %rbp, %r15 # %rbp is 0 + + mov %r9, %rcx + shld \$1, %r8, %r9 + shl \$1, %r8 + + xor %ebp, %ebp + mulx %rdx, %rax, %rdx + adcx %rdx, %r8 + mov 8($inp), %rdx + adcx %rbp, %r9 + + mov %rax, (%rsp) + mov %r8, 8(%rsp) + +#second iteration + mulx 16($inp), %rax, %rbx + adox %rax, %r10 + adcx %rbx, %r11 + + .byte 0xc4,0x62,0xc3,0xf6,0x86,0x18,0x00,0x00,0x00 # mulx 24($inp), $out, %r8 + adox $out, %r11 + adcx %r8, %r12 + + mulx 32($inp), %rax, %rbx + adox %rax, %r12 + adcx %rbx, %r13 + + mulx 40($inp), $out, %r8 + adox $out, %r13 + adcx %r8, %r14 + + .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rbx + adox %rax, %r14 + adcx %rbx, %r15 + + .byte 0xc4,0x62,0xc3,0xf6,0x86,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r8 + adox $out, %r15 + adcx %rbp, %r8 + adox %rbp, %r8 + + mov %r11, %rbx + shld \$1, %r10, %r11 + shld \$1, %rcx, %r10 + + xor %ebp,%ebp + mulx %rdx, %rax, %rcx + mov 16($inp), %rdx + adcx %rax, %r9 + adcx %rcx, %r10 + adcx %rbp, %r11 + + mov %r9, 16(%rsp) + .byte 0x4c,0x89,0x94,0x24,0x18,0x00,0x00,0x00 # mov %r10, 24(%rsp) + +#third iteration + .byte 0xc4,0x62,0xc3,0xf6,0x8e,0x18,0x00,0x00,0x00 # mulx 24($inp), $out, %r9 + adox $out, %r12 + adcx %r9, %r13 + + mulx 32($inp), %rax, %rcx + adox %rax, %r13 + adcx %rcx, %r14 + + mulx 40($inp), $out, %r9 + adox $out, %r14 + adcx %r9, %r15 + + .byte 0xc4,0xe2,0xfb,0xf6,0x8e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rcx + adox %rax, %r15 + adcx %rcx, %r8 + + .byte 0xc4,0x62,0xc3,0xf6,0x8e,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r9 + adox $out, %r8 + adcx %rbp, %r9 + adox %rbp, %r9 + + mov %r13, %rcx + shld \$1, %r12, %r13 + shld \$1, %rbx, %r12 + + xor %ebp, %ebp + mulx %rdx, %rax, %rdx + adcx %rax, %r11 + adcx %rdx, %r12 + mov 24($inp), %rdx + adcx %rbp, %r13 + + mov %r11, 32(%rsp) + .byte 0x4c,0x89,0xa4,0x24,0x28,0x00,0x00,0x00 # mov %r12, 40(%rsp) + +#fourth iteration + .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x20,0x00,0x00,0x00 # mulx 32($inp), %rax, %rbx + adox %rax, %r14 + adcx %rbx, %r15 + + mulx 40($inp), $out, %r10 + adox $out, %r15 + adcx %r10, %r8 + + mulx 48($inp), %rax, %rbx + adox %rax, %r8 + adcx %rbx, %r9 + + mulx 56($inp), $out, %r10 + adox $out, %r9 + adcx %rbp, %r10 + adox %rbp, %r10 + + .byte 0x66 + mov %r15, %rbx + shld \$1, %r14, %r15 + shld \$1, %rcx, %r14 + + xor %ebp, %ebp + mulx %rdx, %rax, %rdx + adcx %rax, %r13 + adcx %rdx, %r14 + mov 32($inp), %rdx + adcx %rbp, %r15 + + mov %r13, 48(%rsp) + mov %r14, 56(%rsp) + +#fifth iteration + .byte 0xc4,0x62,0xc3,0xf6,0x9e,0x28,0x00,0x00,0x00 # mulx 40($inp), $out, %r11 + adox $out, %r8 + adcx %r11, %r9 + + mulx 48($inp), %rax, %rcx + adox %rax, %r9 + adcx %rcx, %r10 + + mulx 56($inp), $out, %r11 + adox $out, %r10 + adcx %rbp, %r11 + adox %rbp, %r11 + + mov %r9, %rcx + shld \$1, %r8, %r9 + shld \$1, %rbx, %r8 + + xor %ebp, %ebp + mulx %rdx, %rax, %rdx + adcx %rax, %r15 + adcx %rdx, %r8 + mov 40($inp), %rdx + adcx %rbp, %r9 + + mov %r15, 64(%rsp) + mov %r8, 72(%rsp) + +#sixth iteration + .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rbx + adox %rax, %r10 + adcx %rbx, %r11 + + .byte 0xc4,0x62,0xc3,0xf6,0xa6,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r12 + adox $out, %r11 + adcx %rbp, %r12 + adox %rbp, %r12 + + mov %r11, %rbx + shld \$1, %r10, %r11 + shld \$1, %rcx, %r10 + + xor %ebp, %ebp + mulx %rdx, %rax, %rdx + adcx %rax, %r9 + adcx %rdx, %r10 + mov 48($inp), %rdx + adcx %rbp, %r11 + + mov %r9, 80(%rsp) + mov %r10, 88(%rsp) + +#seventh iteration + .byte 0xc4,0x62,0xfb,0xf6,0xae,0x38,0x00,0x00,0x00 # mulx 56($inp), %rax, %r13 + adox %rax, %r12 + adox %rbp, %r13 + + xor %r14, %r14 + shld \$1, %r13, %r14 + shld \$1, %r12, %r13 + shld \$1, %rbx, %r12 + + xor %ebp, %ebp + mulx %rdx, %rax, %rdx + adcx %rax, %r11 + adcx %rdx, %r12 + mov 56($inp), %rdx + adcx %rbp, %r13 + + .byte 0x4c,0x89,0x9c,0x24,0x60,0x00,0x00,0x00 # mov %r11, 96(%rsp) + .byte 0x4c,0x89,0xa4,0x24,0x68,0x00,0x00,0x00 # mov %r12, 104(%rsp) + +#eighth iteration + mulx %rdx, %rax, %rdx + adox %rax, %r13 + adox %rbp, %rdx + + .byte 0x66 + add %rdx, %r14 + + movq %r13, 112(%rsp) + movq %r14, 120(%rsp) + movq %xmm0, $out + movq %xmm1, %rbp + + movq 128(%rsp), %rdx # pull $n0 + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r13 + movq 48(%rsp), %r14 + movq 56(%rsp), %r15 + + call __rsaz_512_reducex + + addq 64(%rsp), %r8 + adcq 72(%rsp), %r9 + adcq 80(%rsp), %r10 + adcq 88(%rsp), %r11 + adcq 96(%rsp), %r12 + adcq 104(%rsp), %r13 + adcq 112(%rsp), %r14 + adcq 120(%rsp), %r15 + sbbq %rcx, %rcx + + call __rsaz_512_subtract + + movq %r8, %rdx + movq %r9, %rax + movl 128+8(%rsp), $times + movq $out, $inp + + decl $times + jnz .Loop_sqrx + +.Lsqr_tail: +___ +} +$code.=<<___; + + leaq 128+24+48(%rsp), %rax + movq -48(%rax), %r15 + movq -40(%rax), %r14 + movq -32(%rax), %r13 + movq -24(%rax), %r12 + movq -16(%rax), %rbp + movq -8(%rax), %rbx + leaq (%rax), %rsp +.Lsqr_epilogue: + ret +.size rsaz_512_sqr,.-rsaz_512_sqr +___ +} +{ +my ($out,$ap,$bp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx","%r8"); +$code.=<<___; +.globl rsaz_512_mul +.type rsaz_512_mul,\@function,5 +.align 32 +rsaz_512_mul: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + + subq \$128+24, %rsp +.Lmul_body: + movq $out, %xmm0 # off-load arguments + movq $mod, %xmm1 + movq $n0, 128(%rsp) +___ +$code.=<<___ if ($addx); + movl \$0x80100,%r11d + andl OPENSSL_ia32cap_P+8(%rip),%r11d + cmpl \$0x80100,%r11d # check for MULX and ADO/CX + je .Lmulx +___ +$code.=<<___; + movq ($bp), %rbx # pass b[0] + movq $bp, %rbp # pass argument + call __rsaz_512_mul + + movq %xmm0, $out + movq %xmm1, %rbp + + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r13 + movq 48(%rsp), %r14 + movq 56(%rsp), %r15 + + call __rsaz_512_reduce +___ +$code.=<<___ if ($addx); + jmp .Lmul_tail + +.align 32 +.Lmulx: + movq $bp, %rbp # pass argument + movq ($bp), %rdx # pass b[0] + call __rsaz_512_mulx + + movq %xmm0, $out + movq %xmm1, %rbp + + movq 128(%rsp), %rdx # pull $n0 + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r13 + movq 48(%rsp), %r14 + movq 56(%rsp), %r15 + + call __rsaz_512_reducex +.Lmul_tail: +___ +$code.=<<___; + addq 64(%rsp), %r8 + adcq 72(%rsp), %r9 + adcq 80(%rsp), %r10 + adcq 88(%rsp), %r11 + adcq 96(%rsp), %r12 + adcq 104(%rsp), %r13 + adcq 112(%rsp), %r14 + adcq 120(%rsp), %r15 + sbbq %rcx, %rcx + + call __rsaz_512_subtract + + leaq 128+24+48(%rsp), %rax + movq -48(%rax), %r15 + movq -40(%rax), %r14 + movq -32(%rax), %r13 + movq -24(%rax), %r12 + movq -16(%rax), %rbp + movq -8(%rax), %rbx + leaq (%rax), %rsp +.Lmul_epilogue: + ret +.size rsaz_512_mul,.-rsaz_512_mul +___ +} +{ +my ($out,$ap,$bp,$mod,$n0,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d"); +$code.=<<___; +.globl rsaz_512_mul_gather4 +.type rsaz_512_mul_gather4,\@function,6 +.align 32 +rsaz_512_mul_gather4: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + + mov $pwr, $pwr + subq \$128+24, %rsp +.Lmul_gather4_body: +___ +$code.=<<___ if ($addx); + movl \$0x80100,%r11d + andl OPENSSL_ia32cap_P+8(%rip),%r11d + cmpl \$0x80100,%r11d # check for MULX and ADO/CX + je .Lmulx_gather +___ +$code.=<<___; + movl 64($bp,$pwr,4), %eax + movq $out, %xmm0 # off-load arguments + movl ($bp,$pwr,4), %ebx + movq $mod, %xmm1 + movq $n0, 128(%rsp) + + shlq \$32, %rax + or %rax, %rbx + movq ($ap), %rax + movq 8($ap), %rcx + leaq 128($bp,$pwr,4), %rbp + mulq %rbx # 0 iteration + movq %rax, (%rsp) + movq %rcx, %rax + movq %rdx, %r8 + + mulq %rbx + movd (%rbp), %xmm4 + addq %rax, %r8 + movq 16($ap), %rax + movq %rdx, %r9 + adcq \$0, %r9 + + mulq %rbx + movd 64(%rbp), %xmm5 + addq %rax, %r9 + movq 24($ap), %rax + movq %rdx, %r10 + adcq \$0, %r10 + + mulq %rbx + pslldq \$4, %xmm5 + addq %rax, %r10 + movq 32($ap), %rax + movq %rdx, %r11 + adcq \$0, %r11 + + mulq %rbx + por %xmm5, %xmm4 + addq %rax, %r11 + movq 40($ap), %rax + movq %rdx, %r12 + adcq \$0, %r12 + + mulq %rbx + addq %rax, %r12 + movq 48($ap), %rax + movq %rdx, %r13 + adcq \$0, %r13 + + mulq %rbx + leaq 128(%rbp), %rbp + addq %rax, %r13 + movq 56($ap), %rax + movq %rdx, %r14 + adcq \$0, %r14 + + mulq %rbx + movq %xmm4, %rbx + addq %rax, %r14 + movq ($ap), %rax + movq %rdx, %r15 + adcq \$0, %r15 + + leaq 8(%rsp), %rdi + movl \$7, %ecx + jmp .Loop_mul_gather + +.align 32 +.Loop_mul_gather: + mulq %rbx + addq %rax, %r8 + movq 8($ap), %rax + movq %r8, (%rdi) + movq %rdx, %r8 + adcq \$0, %r8 + + mulq %rbx + movd (%rbp), %xmm4 + addq %rax, %r9 + movq 16($ap), %rax + adcq \$0, %rdx + addq %r9, %r8 + movq %rdx, %r9 + adcq \$0, %r9 + + mulq %rbx + movd 64(%rbp), %xmm5 + addq %rax, %r10 + movq 24($ap), %rax + adcq \$0, %rdx + addq %r10, %r9 + movq %rdx, %r10 + adcq \$0, %r10 + + mulq %rbx + pslldq \$4, %xmm5 + addq %rax, %r11 + movq 32($ap), %rax + adcq \$0, %rdx + addq %r11, %r10 + movq %rdx, %r11 + adcq \$0, %r11 + + mulq %rbx + por %xmm5, %xmm4 + addq %rax, %r12 + movq 40($ap), %rax + adcq \$0, %rdx + addq %r12, %r11 + movq %rdx, %r12 + adcq \$0, %r12 + + mulq %rbx + addq %rax, %r13 + movq 48($ap), %rax + adcq \$0, %rdx + addq %r13, %r12 + movq %rdx, %r13 + adcq \$0, %r13 + + mulq %rbx + addq %rax, %r14 + movq 56($ap), %rax + adcq \$0, %rdx + addq %r14, %r13 + movq %rdx, %r14 + adcq \$0, %r14 + + mulq %rbx + movq %xmm4, %rbx + addq %rax, %r15 + movq ($ap), %rax + adcq \$0, %rdx + addq %r15, %r14 + movq %rdx, %r15 + adcq \$0, %r15 + + leaq 128(%rbp), %rbp + leaq 8(%rdi), %rdi + + decl %ecx + jnz .Loop_mul_gather + + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + + movq %xmm0, $out + movq %xmm1, %rbp + + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r13 + movq 48(%rsp), %r14 + movq 56(%rsp), %r15 + + call __rsaz_512_reduce +___ +$code.=<<___ if ($addx); + jmp .Lmul_gather_tail + +.align 32 +.Lmulx_gather: + mov 64($bp,$pwr,4), %eax + movq $out, %xmm0 # off-load arguments + lea 128($bp,$pwr,4), %rbp + mov ($bp,$pwr,4), %edx + movq $mod, %xmm1 + mov $n0, 128(%rsp) + + shl \$32, %rax + or %rax, %rdx + mulx ($ap), %rbx, %r8 # 0 iteration + mov %rbx, (%rsp) + xor %edi, %edi # cf=0, of=0 + + mulx 8($ap), %rax, %r9 + movd (%rbp), %xmm4 + + mulx 16($ap), %rbx, %r10 + movd 64(%rbp), %xmm5 + adcx %rax, %r8 + + mulx 24($ap), %rax, %r11 + pslldq \$4, %xmm5 + adcx %rbx, %r9 + + mulx 32($ap), %rbx, %r12 + por %xmm5, %xmm4 + adcx %rax, %r10 + + mulx 40($ap), %rax, %r13 + adcx %rbx, %r11 + + mulx 48($ap), %rbx, %r14 + lea 128(%rbp), %rbp + adcx %rax, %r12 + + mulx 56($ap), %rax, %r15 + movq %xmm4, %rdx + adcx %rbx, %r13 + adcx %rax, %r14 + mov %r8, %rbx + adcx %rdi, %r15 # %rdi is 0 + + mov \$-7, %rcx + jmp .Loop_mulx_gather + +.align 32 +.Loop_mulx_gather: + mulx ($ap), %rax, %r8 + adcx %rax, %rbx + adox %r9, %r8 + + mulx 8($ap), %rax, %r9 + .byte 0x66,0x0f,0x6e,0xa5,0x00,0x00,0x00,0x00 # movd (%rbp), %xmm4 + adcx %rax, %r8 + adox %r10, %r9 + + mulx 16($ap), %rax, %r10 + movd 64(%rbp), %xmm5 + lea 128(%rbp), %rbp + adcx %rax, %r9 + adox %r11, %r10 + + .byte 0xc4,0x62,0xfb,0xf6,0x9e,0x18,0x00,0x00,0x00 # mulx 24($ap), %rax, %r11 + pslldq \$4, %xmm5 + por %xmm5, %xmm4 + adcx %rax, %r10 + adox %r12, %r11 + + mulx 32($ap), %rax, %r12 + adcx %rax, %r11 + adox %r13, %r12 + + mulx 40($ap), %rax, %r13 + adcx %rax, %r12 + adox %r14, %r13 + + .byte 0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($ap), %rax, %r14 + adcx %rax, %r13 + adox %r15, %r14 + + mulx 56($ap), %rax, %r15 + movq %xmm4, %rdx + mov %rbx, 64(%rsp,%rcx,8) + adcx %rax, %r14 + adox %rdi, %r15 + mov %r8, %rbx + adcx %rdi, %r15 # cf=0 + + inc %rcx # of=0 + jnz .Loop_mulx_gather + + mov %r8, 64(%rsp) + mov %r9, 64+8(%rsp) + mov %r10, 64+16(%rsp) + mov %r11, 64+24(%rsp) + mov %r12, 64+32(%rsp) + mov %r13, 64+40(%rsp) + mov %r14, 64+48(%rsp) + mov %r15, 64+56(%rsp) + + movq %xmm0, $out + movq %xmm1, %rbp + + mov 128(%rsp), %rdx # pull $n0 + mov (%rsp), %r8 + mov 8(%rsp), %r9 + mov 16(%rsp), %r10 + mov 24(%rsp), %r11 + mov 32(%rsp), %r12 + mov 40(%rsp), %r13 + mov 48(%rsp), %r14 + mov 56(%rsp), %r15 + + call __rsaz_512_reducex + +.Lmul_gather_tail: +___ +$code.=<<___; + addq 64(%rsp), %r8 + adcq 72(%rsp), %r9 + adcq 80(%rsp), %r10 + adcq 88(%rsp), %r11 + adcq 96(%rsp), %r12 + adcq 104(%rsp), %r13 + adcq 112(%rsp), %r14 + adcq 120(%rsp), %r15 + sbbq %rcx, %rcx + + call __rsaz_512_subtract + + leaq 128+24+48(%rsp), %rax + movq -48(%rax), %r15 + movq -40(%rax), %r14 + movq -32(%rax), %r13 + movq -24(%rax), %r12 + movq -16(%rax), %rbp + movq -8(%rax), %rbx + leaq (%rax), %rsp +.Lmul_gather4_epilogue: + ret +.size rsaz_512_mul_gather4,.-rsaz_512_mul_gather4 +___ +} +{ +my ($out,$ap,$mod,$n0,$tbl,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d"); +$code.=<<___; +.globl rsaz_512_mul_scatter4 +.type rsaz_512_mul_scatter4,\@function,6 +.align 32 +rsaz_512_mul_scatter4: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + + mov $pwr, $pwr + subq \$128+24, %rsp +.Lmul_scatter4_body: + leaq ($tbl,$pwr,4), $tbl + movq $out, %xmm0 # off-load arguments + movq $mod, %xmm1 + movq $tbl, %xmm2 + movq $n0, 128(%rsp) + + movq $out, %rbp +___ +$code.=<<___ if ($addx); + movl \$0x80100,%r11d + andl OPENSSL_ia32cap_P+8(%rip),%r11d + cmpl \$0x80100,%r11d # check for MULX and ADO/CX + je .Lmulx_scatter +___ +$code.=<<___; + movq ($out),%rbx # pass b[0] + call __rsaz_512_mul + + movq %xmm0, $out + movq %xmm1, %rbp + + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r13 + movq 48(%rsp), %r14 + movq 56(%rsp), %r15 + + call __rsaz_512_reduce +___ +$code.=<<___ if ($addx); + jmp .Lmul_scatter_tail + +.align 32 +.Lmulx_scatter: + movq ($out), %rdx # pass b[0] + call __rsaz_512_mulx + + movq %xmm0, $out + movq %xmm1, %rbp + + movq 128(%rsp), %rdx # pull $n0 + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r13 + movq 48(%rsp), %r14 + movq 56(%rsp), %r15 + + call __rsaz_512_reducex + +.Lmul_scatter_tail: +___ +$code.=<<___; + addq 64(%rsp), %r8 + adcq 72(%rsp), %r9 + adcq 80(%rsp), %r10 + adcq 88(%rsp), %r11 + adcq 96(%rsp), %r12 + adcq 104(%rsp), %r13 + adcq 112(%rsp), %r14 + adcq 120(%rsp), %r15 + movq %xmm2, $inp + sbbq %rcx, %rcx + + call __rsaz_512_subtract + + movl %r8d, 64*0($inp) # scatter + shrq \$32, %r8 + movl %r9d, 64*2($inp) + shrq \$32, %r9 + movl %r10d, 64*4($inp) + shrq \$32, %r10 + movl %r11d, 64*6($inp) + shrq \$32, %r11 + movl %r12d, 64*8($inp) + shrq \$32, %r12 + movl %r13d, 64*10($inp) + shrq \$32, %r13 + movl %r14d, 64*12($inp) + shrq \$32, %r14 + movl %r15d, 64*14($inp) + shrq \$32, %r15 + movl %r8d, 64*1($inp) + movl %r9d, 64*3($inp) + movl %r10d, 64*5($inp) + movl %r11d, 64*7($inp) + movl %r12d, 64*9($inp) + movl %r13d, 64*11($inp) + movl %r14d, 64*13($inp) + movl %r15d, 64*15($inp) + + leaq 128+24+48(%rsp), %rax + movq -48(%rax), %r15 + movq -40(%rax), %r14 + movq -32(%rax), %r13 + movq -24(%rax), %r12 + movq -16(%rax), %rbp + movq -8(%rax), %rbx + leaq (%rax), %rsp +.Lmul_scatter4_epilogue: + ret +.size rsaz_512_mul_scatter4,.-rsaz_512_mul_scatter4 +___ +} +{ +my ($out,$inp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx"); +$code.=<<___; +.globl rsaz_512_mul_by_one +.type rsaz_512_mul_by_one,\@function,4 +.align 32 +rsaz_512_mul_by_one: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + + subq \$128+24, %rsp +.Lmul_by_one_body: +___ +$code.=<<___ if ($addx); + movl OPENSSL_ia32cap_P+8(%rip),%eax +___ +$code.=<<___; + movq $mod, %rbp # reassign argument + movq $n0, 128(%rsp) + + movq ($inp), %r8 + pxor %xmm0, %xmm0 + movq 8($inp), %r9 + movq 16($inp), %r10 + movq 24($inp), %r11 + movq 32($inp), %r12 + movq 40($inp), %r13 + movq 48($inp), %r14 + movq 56($inp), %r15 + + movdqa %xmm0, (%rsp) + movdqa %xmm0, 16(%rsp) + movdqa %xmm0, 32(%rsp) + movdqa %xmm0, 48(%rsp) + movdqa %xmm0, 64(%rsp) + movdqa %xmm0, 80(%rsp) + movdqa %xmm0, 96(%rsp) +___ +$code.=<<___ if ($addx); + andl \$0x80100,%eax + cmpl \$0x80100,%eax # check for MULX and ADO/CX + je .Lby_one_callx +___ +$code.=<<___; + call __rsaz_512_reduce +___ +$code.=<<___ if ($addx); + jmp .Lby_one_tail +.align 32 +.Lby_one_callx: + movq 128(%rsp), %rdx # pull $n0 + call __rsaz_512_reducex +.Lby_one_tail: +___ +$code.=<<___; + movq %r8, ($out) + movq %r9, 8($out) + movq %r10, 16($out) + movq %r11, 24($out) + movq %r12, 32($out) + movq %r13, 40($out) + movq %r14, 48($out) + movq %r15, 56($out) + + leaq 128+24+48(%rsp), %rax + movq -48(%rax), %r15 + movq -40(%rax), %r14 + movq -32(%rax), %r13 + movq -24(%rax), %r12 + movq -16(%rax), %rbp + movq -8(%rax), %rbx + leaq (%rax), %rsp +.Lmul_by_one_epilogue: + ret +.size rsaz_512_mul_by_one,.-rsaz_512_mul_by_one +___ +} +{ # __rsaz_512_reduce + # + # input: %r8-%r15, %rbp - mod, 128(%rsp) - n0 + # output: %r8-%r15 + # clobbers: everything except %rbp and %rdi +$code.=<<___; +.type __rsaz_512_reduce,\@abi-omnipotent +.align 32 +__rsaz_512_reduce: + movq %r8, %rbx + imulq 128+8(%rsp), %rbx + movq 0(%rbp), %rax + movl \$8, %ecx + jmp .Lreduction_loop + +.align 32 +.Lreduction_loop: + mulq %rbx + movq 8(%rbp), %rax + negq %r8 + movq %rdx, %r8 + adcq \$0, %r8 + + mulq %rbx + addq %rax, %r9 + movq 16(%rbp), %rax + adcq \$0, %rdx + addq %r9, %r8 + movq %rdx, %r9 + adcq \$0, %r9 + + mulq %rbx + addq %rax, %r10 + movq 24(%rbp), %rax + adcq \$0, %rdx + addq %r10, %r9 + movq %rdx, %r10 + adcq \$0, %r10 + + mulq %rbx + addq %rax, %r11 + movq 32(%rbp), %rax + adcq \$0, %rdx + addq %r11, %r10 + movq 128+8(%rsp), %rsi + #movq %rdx, %r11 + #adcq \$0, %r11 + adcq \$0, %rdx + movq %rdx, %r11 + + mulq %rbx + addq %rax, %r12 + movq 40(%rbp), %rax + adcq \$0, %rdx + imulq %r8, %rsi + addq %r12, %r11 + movq %rdx, %r12 + adcq \$0, %r12 + + mulq %rbx + addq %rax, %r13 + movq 48(%rbp), %rax + adcq \$0, %rdx + addq %r13, %r12 + movq %rdx, %r13 + adcq \$0, %r13 + + mulq %rbx + addq %rax, %r14 + movq 56(%rbp), %rax + adcq \$0, %rdx + addq %r14, %r13 + movq %rdx, %r14 + adcq \$0, %r14 + + mulq %rbx + movq %rsi, %rbx + addq %rax, %r15 + movq 0(%rbp), %rax + adcq \$0, %rdx + addq %r15, %r14 + movq %rdx, %r15 + adcq \$0, %r15 + + decl %ecx + jne .Lreduction_loop + + ret +.size __rsaz_512_reduce,.-__rsaz_512_reduce +___ +} +if ($addx) { + # __rsaz_512_reducex + # + # input: %r8-%r15, %rbp - mod, 128(%rsp) - n0 + # output: %r8-%r15 + # clobbers: everything except %rbp and %rdi +$code.=<<___; +.type __rsaz_512_reducex,\@abi-omnipotent +.align 32 +__rsaz_512_reducex: + #movq 128+8(%rsp), %rdx # pull $n0 + imulq %r8, %rdx + xorq %rsi, %rsi # cf=0,of=0 + movl \$8, %ecx + jmp .Lreduction_loopx + +.align 32 +.Lreduction_loopx: + mov %r8, %rbx + mulx 0(%rbp), %rax, %r8 + adcx %rbx, %rax + adox %r9, %r8 + + mulx 8(%rbp), %rax, %r9 + adcx %rax, %r8 + adox %r10, %r9 + + mulx 16(%rbp), %rbx, %r10 + adcx %rbx, %r9 + adox %r11, %r10 + + mulx 24(%rbp), %rbx, %r11 + adcx %rbx, %r10 + adox %r12, %r11 + + .byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 32(%rbp), %rbx, %r12 + mov %rdx, %rax + mov %r8, %rdx + adcx %rbx, %r11 + adox %r13, %r12 + + mulx 128+8(%rsp), %rbx, %rdx + mov %rax, %rdx + + mulx 40(%rbp), %rax, %r13 + adcx %rax, %r12 + adox %r14, %r13 + + .byte 0xc4,0x62,0xfb,0xf6,0xb5,0x30,0x00,0x00,0x00 # mulx 48(%rbp), %rax, %r14 + adcx %rax, %r13 + adox %r15, %r14 + + mulx 56(%rbp), %rax, %r15 + mov %rbx, %rdx + adcx %rax, %r14 + adox %rsi, %r15 # %rsi is 0 + adcx %rsi, %r15 # cf=0 + + decl %ecx # of=0 + jne .Lreduction_loopx + + ret +.size __rsaz_512_reducex,.-__rsaz_512_reducex +___ +} +{ # __rsaz_512_subtract + # input: %r8-%r15, %rdi - $out, %rbp - $mod, %rcx - mask + # output: + # clobbers: everything but %rdi, %rsi and %rbp +$code.=<<___; +.type __rsaz_512_subtract,\@abi-omnipotent +.align 32 +__rsaz_512_subtract: + movq %r8, ($out) + movq %r9, 8($out) + movq %r10, 16($out) + movq %r11, 24($out) + movq %r12, 32($out) + movq %r13, 40($out) + movq %r14, 48($out) + movq %r15, 56($out) + + movq 0($mod), %r8 + movq 8($mod), %r9 + negq %r8 + notq %r9 + andq %rcx, %r8 + movq 16($mod), %r10 + andq %rcx, %r9 + notq %r10 + movq 24($mod), %r11 + andq %rcx, %r10 + notq %r11 + movq 32($mod), %r12 + andq %rcx, %r11 + notq %r12 + movq 40($mod), %r13 + andq %rcx, %r12 + notq %r13 + movq 48($mod), %r14 + andq %rcx, %r13 + notq %r14 + movq 56($mod), %r15 + andq %rcx, %r14 + notq %r15 + andq %rcx, %r15 + + addq ($out), %r8 + adcq 8($out), %r9 + adcq 16($out), %r10 + adcq 24($out), %r11 + adcq 32($out), %r12 + adcq 40($out), %r13 + adcq 48($out), %r14 + adcq 56($out), %r15 + + movq %r8, ($out) + movq %r9, 8($out) + movq %r10, 16($out) + movq %r11, 24($out) + movq %r12, 32($out) + movq %r13, 40($out) + movq %r14, 48($out) + movq %r15, 56($out) + + ret +.size __rsaz_512_subtract,.-__rsaz_512_subtract +___ +} +{ # __rsaz_512_mul + # + # input: %rsi - ap, %rbp - bp + # ouput: + # clobbers: everything +my ($ap,$bp) = ("%rsi","%rbp"); +$code.=<<___; +.type __rsaz_512_mul,\@abi-omnipotent +.align 32 +__rsaz_512_mul: + leaq 8(%rsp), %rdi + + movq ($ap), %rax + mulq %rbx + movq %rax, (%rdi) + movq 8($ap), %rax + movq %rdx, %r8 + + mulq %rbx + addq %rax, %r8 + movq 16($ap), %rax + movq %rdx, %r9 + adcq \$0, %r9 + + mulq %rbx + addq %rax, %r9 + movq 24($ap), %rax + movq %rdx, %r10 + adcq \$0, %r10 + + mulq %rbx + addq %rax, %r10 + movq 32($ap), %rax + movq %rdx, %r11 + adcq \$0, %r11 + + mulq %rbx + addq %rax, %r11 + movq 40($ap), %rax + movq %rdx, %r12 + adcq \$0, %r12 + + mulq %rbx + addq %rax, %r12 + movq 48($ap), %rax + movq %rdx, %r13 + adcq \$0, %r13 + + mulq %rbx + addq %rax, %r13 + movq 56($ap), %rax + movq %rdx, %r14 + adcq \$0, %r14 + + mulq %rbx + addq %rax, %r14 + movq ($ap), %rax + movq %rdx, %r15 + adcq \$0, %r15 + + leaq 8($bp), $bp + leaq 8(%rdi), %rdi + + movl \$7, %ecx + jmp .Loop_mul + +.align 32 +.Loop_mul: + movq ($bp), %rbx + mulq %rbx + addq %rax, %r8 + movq 8($ap), %rax + movq %r8, (%rdi) + movq %rdx, %r8 + adcq \$0, %r8 + + mulq %rbx + addq %rax, %r9 + movq 16($ap), %rax + adcq \$0, %rdx + addq %r9, %r8 + movq %rdx, %r9 + adcq \$0, %r9 + + mulq %rbx + addq %rax, %r10 + movq 24($ap), %rax + adcq \$0, %rdx + addq %r10, %r9 + movq %rdx, %r10 + adcq \$0, %r10 + + mulq %rbx + addq %rax, %r11 + movq 32($ap), %rax + adcq \$0, %rdx + addq %r11, %r10 + movq %rdx, %r11 + adcq \$0, %r11 + + mulq %rbx + addq %rax, %r12 + movq 40($ap), %rax + adcq \$0, %rdx + addq %r12, %r11 + movq %rdx, %r12 + adcq \$0, %r12 + + mulq %rbx + addq %rax, %r13 + movq 48($ap), %rax + adcq \$0, %rdx + addq %r13, %r12 + movq %rdx, %r13 + adcq \$0, %r13 + + mulq %rbx + addq %rax, %r14 + movq 56($ap), %rax + adcq \$0, %rdx + addq %r14, %r13 + movq %rdx, %r14 + leaq 8($bp), $bp + adcq \$0, %r14 + + mulq %rbx + addq %rax, %r15 + movq ($ap), %rax + adcq \$0, %rdx + addq %r15, %r14 + movq %rdx, %r15 + adcq \$0, %r15 + + leaq 8(%rdi), %rdi + + decl %ecx + jnz .Loop_mul + + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + + ret +.size __rsaz_512_mul,.-__rsaz_512_mul +___ +} +if ($addx) { + # __rsaz_512_mulx + # + # input: %rsi - ap, %rbp - bp + # ouput: + # clobbers: everything +my ($ap,$bp,$zero) = ("%rsi","%rbp","%rdi"); +$code.=<<___; +.type __rsaz_512_mulx,\@abi-omnipotent +.align 32 +__rsaz_512_mulx: + mulx ($ap), %rbx, %r8 # initial %rdx preloaded by caller + mov \$-6, %rcx + + mulx 8($ap), %rax, %r9 + movq %rbx, 8(%rsp) + + mulx 16($ap), %rbx, %r10 + adc %rax, %r8 + + mulx 24($ap), %rax, %r11 + adc %rbx, %r9 + + mulx 32($ap), %rbx, %r12 + adc %rax, %r10 + + mulx 40($ap), %rax, %r13 + adc %rbx, %r11 + + mulx 48($ap), %rbx, %r14 + adc %rax, %r12 + + mulx 56($ap), %rax, %r15 + mov 8($bp), %rdx + adc %rbx, %r13 + adc %rax, %r14 + adc \$0, %r15 + + xor $zero, $zero # cf=0,of=0 + jmp .Loop_mulx + +.align 32 +.Loop_mulx: + movq %r8, %rbx + mulx ($ap), %rax, %r8 + adcx %rax, %rbx + adox %r9, %r8 + + mulx 8($ap), %rax, %r9 + adcx %rax, %r8 + adox %r10, %r9 + + mulx 16($ap), %rax, %r10 + adcx %rax, %r9 + adox %r11, %r10 + + mulx 24($ap), %rax, %r11 + adcx %rax, %r10 + adox %r12, %r11 + + .byte 0x3e,0xc4,0x62,0xfb,0xf6,0xa6,0x20,0x00,0x00,0x00 # mulx 32($ap), %rax, %r12 + adcx %rax, %r11 + adox %r13, %r12 + + mulx 40($ap), %rax, %r13 + adcx %rax, %r12 + adox %r14, %r13 + + mulx 48($ap), %rax, %r14 + adcx %rax, %r13 + adox %r15, %r14 + + mulx 56($ap), %rax, %r15 + movq 64($bp,%rcx,8), %rdx + movq %rbx, 8+64-8(%rsp,%rcx,8) + adcx %rax, %r14 + adox $zero, %r15 + adcx $zero, %r15 # cf=0 + + inc %rcx # of=0 + jnz .Loop_mulx + + movq %r8, %rbx + mulx ($ap), %rax, %r8 + adcx %rax, %rbx + adox %r9, %r8 + + .byte 0xc4,0x62,0xfb,0xf6,0x8e,0x08,0x00,0x00,0x00 # mulx 8($ap), %rax, %r9 + adcx %rax, %r8 + adox %r10, %r9 + + .byte 0xc4,0x62,0xfb,0xf6,0x96,0x10,0x00,0x00,0x00 # mulx 16($ap), %rax, %r10 + adcx %rax, %r9 + adox %r11, %r10 + + mulx 24($ap), %rax, %r11 + adcx %rax, %r10 + adox %r12, %r11 + + mulx 32($ap), %rax, %r12 + adcx %rax, %r11 + adox %r13, %r12 + + mulx 40($ap), %rax, %r13 + adcx %rax, %r12 + adox %r14, %r13 + + .byte 0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($ap), %rax, %r14 + adcx %rax, %r13 + adox %r15, %r14 + + .byte 0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00 # mulx 56($ap), %rax, %r15 + adcx %rax, %r14 + adox $zero, %r15 + adcx $zero, %r15 + + mov %rbx, 8+64-8(%rsp) + mov %r8, 8+64(%rsp) + mov %r9, 8+64+8(%rsp) + mov %r10, 8+64+16(%rsp) + mov %r11, 8+64+24(%rsp) + mov %r12, 8+64+32(%rsp) + mov %r13, 8+64+40(%rsp) + mov %r14, 8+64+48(%rsp) + mov %r15, 8+64+56(%rsp) + + ret +.size __rsaz_512_mulx,.-__rsaz_512_mulx +___ +} +{ +my ($out,$inp,$power)= $win64 ? ("%rcx","%rdx","%r8d") : ("%rdi","%rsi","%edx"); +$code.=<<___; +.globl rsaz_512_scatter4 +.type rsaz_512_scatter4,\@abi-omnipotent +.align 16 +rsaz_512_scatter4: + leaq ($out,$power,4), $out + movl \$8, %r9d + jmp .Loop_scatter +.align 16 +.Loop_scatter: + movq ($inp), %rax + leaq 8($inp), $inp + movl %eax, ($out) + shrq \$32, %rax + movl %eax, 64($out) + leaq 128($out), $out + decl %r9d + jnz .Loop_scatter + ret +.size rsaz_512_scatter4,.-rsaz_512_scatter4 + +.globl rsaz_512_gather4 +.type rsaz_512_gather4,\@abi-omnipotent +.align 16 +rsaz_512_gather4: + leaq ($inp,$power,4), $inp + movl \$8, %r9d + jmp .Loop_gather +.align 16 +.Loop_gather: + movl ($inp), %eax + movl 64($inp), %r8d + leaq 128($inp), $inp + shlq \$32, %r8 + or %r8, %rax + movq %rax, ($out) + leaq 8($out), $out + decl %r9d + jnz .Loop_gather + ret +.size rsaz_512_gather4,.-rsaz_512_gather4 +___ +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # end of prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + lea 128+24+48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R15 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size sqr_handler,.-sqr_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_rsaz_512_sqr + .rva .LSEH_end_rsaz_512_sqr + .rva .LSEH_info_rsaz_512_sqr + + .rva .LSEH_begin_rsaz_512_mul + .rva .LSEH_end_rsaz_512_mul + .rva .LSEH_info_rsaz_512_mul + + .rva .LSEH_begin_rsaz_512_mul_gather4 + .rva .LSEH_end_rsaz_512_mul_gather4 + .rva .LSEH_info_rsaz_512_mul_gather4 + + .rva .LSEH_begin_rsaz_512_mul_scatter4 + .rva .LSEH_end_rsaz_512_mul_scatter4 + .rva .LSEH_info_rsaz_512_mul_scatter4 + + .rva .LSEH_begin_rsaz_512_mul_by_one + .rva .LSEH_end_rsaz_512_mul_by_one + .rva .LSEH_info_rsaz_512_mul_by_one + +.section .xdata +.align 8 +.LSEH_info_rsaz_512_sqr: + .byte 9,0,0,0 + .rva se_handler + .rva .Lsqr_body,.Lsqr_epilogue # HandlerData[] +.LSEH_info_rsaz_512_mul: + .byte 9,0,0,0 + .rva se_handler + .rva .Lmul_body,.Lmul_epilogue # HandlerData[] +.LSEH_info_rsaz_512_mul_gather4: + .byte 9,0,0,0 + .rva se_handler + .rva .Lmul_gather4_body,.Lmul_gather4_epilogue # HandlerData[] +.LSEH_info_rsaz_512_mul_scatter4: + .byte 9,0,0,0 + .rva se_handler + .rva .Lmul_scatter4_body,.Lmul_scatter4_epilogue # HandlerData[] +.LSEH_info_rsaz_512_mul_by_one: + .byte 9,0,0,0 + .rva se_handler + .rva .Lmul_by_one_body,.Lmul_by_one_epilogue # HandlerData[] +___ +} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/bn/asm/sparct4-mont.pl b/crypto/bn/asm/sparct4-mont.pl new file mode 100755 index 000000000000..71b45002a42f --- /dev/null +++ b/crypto/bn/asm/sparct4-mont.pl @@ -0,0 +1,1222 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by David S. Miller and Andy Polyakov +# . The module is licensed under 2-clause BSD +# license. November 2012. All rights reserved. +# ==================================================================== + +###################################################################### +# Montgomery squaring-n-multiplication module for SPARC T4. +# +# The module consists of three parts: +# +# 1) collection of "single-op" subroutines that perform single +# operation, Montgomery squaring or multiplication, on 512-, +# 1024-, 1536- and 2048-bit operands; +# 2) collection of "multi-op" subroutines that perform 5 squaring and +# 1 multiplication operations on operands of above lengths; +# 3) fall-back and helper VIS3 subroutines. +# +# RSA sign is dominated by multi-op subroutine, while RSA verify and +# DSA - by single-op. Special note about 4096-bit RSA verify result. +# Operands are too long for dedicated hardware and it's handled by +# VIS3 code, which is why you don't see any improvement. It's surely +# possible to improve it [by deploying 'mpmul' instruction], maybe in +# the future... +# +# Performance improvement. +# +# 64-bit process, VIS3: +# sign verify sign/s verify/s +# rsa 1024 bits 0.000628s 0.000028s 1592.4 35434.4 +# rsa 2048 bits 0.003282s 0.000106s 304.7 9438.3 +# rsa 4096 bits 0.025866s 0.000340s 38.7 2940.9 +# dsa 1024 bits 0.000301s 0.000332s 3323.7 3013.9 +# dsa 2048 bits 0.001056s 0.001233s 946.9 810.8 +# +# 64-bit process, this module: +# sign verify sign/s verify/s +# rsa 1024 bits 0.000256s 0.000016s 3904.4 61411.9 +# rsa 2048 bits 0.000946s 0.000029s 1056.8 34292.7 +# rsa 4096 bits 0.005061s 0.000340s 197.6 2940.5 +# dsa 1024 bits 0.000176s 0.000195s 5674.7 5130.5 +# dsa 2048 bits 0.000296s 0.000354s 3383.2 2827.6 +# +###################################################################### +# 32-bit process, VIS3: +# sign verify sign/s verify/s +# rsa 1024 bits 0.000665s 0.000028s 1504.8 35233.3 +# rsa 2048 bits 0.003349s 0.000106s 298.6 9433.4 +# rsa 4096 bits 0.025959s 0.000341s 38.5 2934.8 +# dsa 1024 bits 0.000320s 0.000341s 3123.3 2929.6 +# dsa 2048 bits 0.001101s 0.001260s 908.2 793.4 +# +# 32-bit process, this module: +# sign verify sign/s verify/s +# rsa 1024 bits 0.000301s 0.000017s 3317.1 60240.0 +# rsa 2048 bits 0.001034s 0.000030s 966.9 33812.7 +# rsa 4096 bits 0.005244s 0.000341s 190.7 2935.4 +# dsa 1024 bits 0.000201s 0.000205s 4976.1 4879.2 +# dsa 2048 bits 0.000328s 0.000360s 3051.1 2774.2 +# +# 32-bit code is prone to performance degradation as interrupt rate +# dispatched to CPU executing the code grows. This is because in +# standard process of handling interrupt in 32-bit process context +# upper halves of most integer registers used as input or output are +# zeroed. This renders result invalid, and operation has to be re-run. +# If CPU is "bothered" with timer interrupts only, the penalty is +# hardly measurable. But in order to mitigate this problem for higher +# interrupt rates contemporary Linux kernel recognizes biased stack +# even in 32-bit process context and preserves full register contents. +# See http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=517ffce4e1a03aea979fe3a18a3dd1761a24fafb +# for details. + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +push(@INC,"${dir}","${dir}../../perlasm"); +require "sparcv9_modes.pl"; + +$code.=<<___; +#include "sparc_arch.h" + +#ifdef __arch64__ +.register %g2,#scratch +.register %g3,#scratch +#endif + +.section ".text",#alloc,#execinstr + +#ifdef __PIC__ +SPARC_PIC_THUNK(%g1) +#endif +___ + +######################################################################## +# Register layout for mont[mul|sqr] instructions. +# For details see "Oracle SPARC Architecture 2011" manual at +# http://www.oracle.com/technetwork/server-storage/sun-sparc-enterprise/documentation/. +# +my @R=map("%f".2*$_,(0..11,30,31,12..29)); +my @N=(map("%l$_",(0..7)),map("%o$_",(0..5))); @N=(@N,@N,@N[0..3]); +my @A=(@N[0..13],@R[14..31]); +my @B=(map("%i$_",(0..5)),map("%l$_",(0..7))); @B=(@B,@B,map("%o$_",(0..3))); + +######################################################################## +# int bn_mul_mont_t4_$NUM(u64 *rp,const u64 *ap,const u64 *bp, +# const u64 *np,const BN_ULONG *n0); +# +sub generate_bn_mul_mont_t4() { +my $NUM=shift; +my ($rp,$ap,$bp,$np,$sentinel)=map("%g$_",(1..5)); + +$code.=<<___; +.globl bn_mul_mont_t4_$NUM +.align 32 +bn_mul_mont_t4_$NUM: +#ifdef __arch64__ + mov 0,$sentinel + mov -128,%g4 +#elif defined(SPARCV9_64BIT_STACK) + SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5) + ld [%g1+0],%g1 ! OPENSSL_sparcv9_P[0] + mov -2047,%g4 + and %g1,SPARCV9_64BIT_STACK,%g1 + movrz %g1,0,%g4 + mov -1,$sentinel + add %g4,-128,%g4 +#else + mov -1,$sentinel + mov -128,%g4 +#endif + sllx $sentinel,32,$sentinel + save %sp,%g4,%sp +#ifndef __arch64__ + save %sp,-128,%sp ! warm it up + save %sp,-128,%sp + save %sp,-128,%sp + save %sp,-128,%sp + save %sp,-128,%sp + save %sp,-128,%sp + restore + restore + restore + restore + restore + restore +#endif + and %sp,1,%g4 + or $sentinel,%fp,%fp + or %g4,$sentinel,$sentinel + + ! copy arguments to global registers + mov %i0,$rp + mov %i1,$ap + mov %i2,$bp + mov %i3,$np + ld [%i4+0],%f1 ! load *n0 + ld [%i4+4],%f0 + fsrc2 %f0,%f60 +___ + +# load ap[$NUM] ######################################################## +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for($i=0; $i<14 && $i<$NUM; $i++) { +my $lo=$i<13?@A[$i+1]:"%o7"; +$code.=<<___; + ld [$ap+$i*8+0],$lo + ld [$ap+$i*8+4],@A[$i] + sllx @A[$i],32,@A[$i] + or $lo,@A[$i],@A[$i] +___ +} +for(; $i<$NUM; $i++) { +my ($hi,$lo)=("%f".2*($i%4),"%f".(2*($i%4)+1)); +$code.=<<___; + ld [$ap+$i*8+0],$lo + ld [$ap+$i*8+4],$hi + fsrc2 $hi,@A[$i] +___ +} +# load np[$NUM] ######################################################## +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for($i=0; $i<14 && $i<$NUM; $i++) { +my $lo=$i<13?@N[$i+1]:"%o7"; +$code.=<<___; + ld [$np+$i*8+0],$lo + ld [$np+$i*8+4],@N[$i] + sllx @N[$i],32,@N[$i] + or $lo,@N[$i],@N[$i] +___ +} +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for(; $i<28 && $i<$NUM; $i++) { +my $lo=$i<27?@N[$i+1]:"%o7"; +$code.=<<___; + ld [$np+$i*8+0],$lo + ld [$np+$i*8+4],@N[$i] + sllx @N[$i],32,@N[$i] + or $lo,@N[$i],@N[$i] +___ +} +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for(; $i<$NUM; $i++) { +my $lo=($i<$NUM-1)?@N[$i+1]:"%o7"; +$code.=<<___; + ld [$np+$i*8+0],$lo + ld [$np+$i*8+4],@N[$i] + sllx @N[$i],32,@N[$i] + or $lo,@N[$i],@N[$i] +___ +} +$code.=<<___; + cmp $ap,$bp + be SIZE_T_CC,.Lmsquare_$NUM + nop +___ + +# load bp[$NUM] ######################################################## +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for($i=0; $i<14 && $i<$NUM; $i++) { +my $lo=$i<13?@B[$i+1]:"%o7"; +$code.=<<___; + ld [$bp+$i*8+0],$lo + ld [$bp+$i*8+4],@B[$i] + sllx @B[$i],32,@B[$i] + or $lo,@B[$i],@B[$i] +___ +} +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for(; $i<$NUM; $i++) { +my $lo=($i<$NUM-1)?@B[$i+1]:"%o7"; +$code.=<<___; + ld [$bp+$i*8+0],$lo + ld [$bp+$i*8+4],@B[$i] + sllx @B[$i],32,@B[$i] + or $lo,@B[$i],@B[$i] +___ +} +# magic ################################################################ +$code.=<<___; + .word 0x81b02920+$NUM-1 ! montmul $NUM-1 +.Lmresume_$NUM: + fbu,pn %fcc3,.Lmabort_$NUM +#ifndef __arch64__ + and %fp,$sentinel,$sentinel + brz,pn $sentinel,.Lmabort_$NUM +#endif + nop +#ifdef __arch64__ + restore + restore + restore + restore + restore +#else + restore; and %fp,$sentinel,$sentinel + restore; and %fp,$sentinel,$sentinel + restore; and %fp,$sentinel,$sentinel + restore; and %fp,$sentinel,$sentinel + brz,pn $sentinel,.Lmabort1_$NUM + restore +#endif +___ + +# save tp[$NUM] ######################################################## +for($i=0; $i<14 && $i<$NUM; $i++) { +$code.=<<___; + movxtod @A[$i],@R[$i] +___ +} +$code.=<<___; +#ifdef __arch64__ + restore +#else + and %fp,$sentinel,$sentinel + restore + and $sentinel,1,%o7 + and %fp,$sentinel,$sentinel + srl %fp,0,%fp ! just in case? + or %o7,$sentinel,$sentinel + brz,a,pn $sentinel,.Lmdone_$NUM + mov 0,%i0 ! return failure +#endif +___ +for($i=0; $i<12 && $i<$NUM; $i++) { +@R[$i] =~ /%f([0-9]+)/; +my $lo = "%f".($1+1); +$code.=<<___; + st $lo,[$rp+$i*8+0] + st @R[$i],[$rp+$i*8+4] +___ +} +for(; $i<$NUM; $i++) { +my ($hi,$lo)=("%f".2*($i%4),"%f".(2*($i%4)+1)); +$code.=<<___; + fsrc2 @R[$i],$hi + st $lo,[$rp+$i*8+0] + st $hi,[$rp+$i*8+4] +___ +} +$code.=<<___; + mov 1,%i0 ! return success +.Lmdone_$NUM: + ret + restore + +.Lmabort_$NUM: + restore + restore + restore + restore + restore +.Lmabort1_$NUM: + restore + + mov 0,%i0 ! return failure + ret + restore + +.align 32 +.Lmsquare_$NUM: + save %sp,-128,%sp; or $sentinel,%fp,%fp + save %sp,-128,%sp; or $sentinel,%fp,%fp + .word 0x81b02940+$NUM-1 ! montsqr $NUM-1 + ba .Lmresume_$NUM + nop +.type bn_mul_mont_t4_$NUM, #function +.size bn_mul_mont_t4_$NUM, .-bn_mul_mont_t4_$NUM +___ +} + +for ($i=8;$i<=32;$i+=8) { + &generate_bn_mul_mont_t4($i); +} + +######################################################################## +# +sub load_ccr { +my ($ptbl,$pwr,$ccr,$skip_wr)=@_; +$code.=<<___; + srl $pwr, 2, %o4 + and $pwr, 3, %o5 + and %o4, 7, %o4 + sll %o5, 3, %o5 ! offset within first cache line + add %o5, $ptbl, $ptbl ! of the pwrtbl + or %g0, 1, %o5 + sll %o5, %o4, $ccr +___ +$code.=<<___ if (!$skip_wr); + wr $ccr, %g0, %ccr +___ +} +sub load_b_pair { +my ($pwrtbl,$B0,$B1)=@_; + +$code.=<<___; + ldx [$pwrtbl+0*32], $B0 + ldx [$pwrtbl+8*32], $B1 + ldx [$pwrtbl+1*32], %o4 + ldx [$pwrtbl+9*32], %o5 + movvs %icc, %o4, $B0 + ldx [$pwrtbl+2*32], %o4 + movvs %icc, %o5, $B1 + ldx [$pwrtbl+10*32],%o5 + move %icc, %o4, $B0 + ldx [$pwrtbl+3*32], %o4 + move %icc, %o5, $B1 + ldx [$pwrtbl+11*32],%o5 + movneg %icc, %o4, $B0 + ldx [$pwrtbl+4*32], %o4 + movneg %icc, %o5, $B1 + ldx [$pwrtbl+12*32],%o5 + movcs %xcc, %o4, $B0 + ldx [$pwrtbl+5*32],%o4 + movcs %xcc, %o5, $B1 + ldx [$pwrtbl+13*32],%o5 + movvs %xcc, %o4, $B0 + ldx [$pwrtbl+6*32], %o4 + movvs %xcc, %o5, $B1 + ldx [$pwrtbl+14*32],%o5 + move %xcc, %o4, $B0 + ldx [$pwrtbl+7*32], %o4 + move %xcc, %o5, $B1 + ldx [$pwrtbl+15*32],%o5 + movneg %xcc, %o4, $B0 + add $pwrtbl,16*32, $pwrtbl + movneg %xcc, %o5, $B1 +___ +} +sub load_b { +my ($pwrtbl,$Bi)=@_; + +$code.=<<___; + ldx [$pwrtbl+0*32], $Bi + ldx [$pwrtbl+1*32], %o4 + ldx [$pwrtbl+2*32], %o5 + movvs %icc, %o4, $Bi + ldx [$pwrtbl+3*32], %o4 + move %icc, %o5, $Bi + ldx [$pwrtbl+4*32], %o5 + movneg %icc, %o4, $Bi + ldx [$pwrtbl+5*32], %o4 + movcs %xcc, %o5, $Bi + ldx [$pwrtbl+6*32], %o5 + movvs %xcc, %o4, $Bi + ldx [$pwrtbl+7*32], %o4 + move %xcc, %o5, $Bi + add $pwrtbl,8*32, $pwrtbl + movneg %xcc, %o4, $Bi +___ +} + +######################################################################## +# int bn_pwr5_mont_t4_$NUM(u64 *tp,const u64 *np,const BN_ULONG *n0, +# const u64 *pwrtbl,int pwr,int stride); +# +sub generate_bn_pwr5_mont_t4() { +my $NUM=shift; +my ($tp,$np,$pwrtbl,$pwr,$sentinel)=map("%g$_",(1..5)); + +$code.=<<___; +.globl bn_pwr5_mont_t4_$NUM +.align 32 +bn_pwr5_mont_t4_$NUM: +#ifdef __arch64__ + mov 0,$sentinel + mov -128,%g4 +#elif defined(SPARCV9_64BIT_STACK) + SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5) + ld [%g1+0],%g1 ! OPENSSL_sparcv9_P[0] + mov -2047,%g4 + and %g1,SPARCV9_64BIT_STACK,%g1 + movrz %g1,0,%g4 + mov -1,$sentinel + add %g4,-128,%g4 +#else + mov -1,$sentinel + mov -128,%g4 +#endif + sllx $sentinel,32,$sentinel + save %sp,%g4,%sp +#ifndef __arch64__ + save %sp,-128,%sp ! warm it up + save %sp,-128,%sp + save %sp,-128,%sp + save %sp,-128,%sp + save %sp,-128,%sp + save %sp,-128,%sp + restore + restore + restore + restore + restore + restore +#endif + and %sp,1,%g4 + or $sentinel,%fp,%fp + or %g4,$sentinel,$sentinel + + ! copy arguments to global registers + mov %i0,$tp + mov %i1,$np + ld [%i2+0],%f1 ! load *n0 + ld [%i2+4],%f0 + mov %i3,$pwrtbl + srl %i4,%g0,%i4 ! pack last arguments + sllx %i5,32,$pwr + or %i4,$pwr,$pwr + fsrc2 %f0,%f60 +___ + +# load tp[$NUM] ######################################################## +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for($i=0; $i<14 && $i<$NUM; $i++) { +$code.=<<___; + ldx [$tp+$i*8],@A[$i] +___ +} +for(; $i<$NUM; $i++) { +$code.=<<___; + ldd [$tp+$i*8],@A[$i] +___ +} +# load np[$NUM] ######################################################## +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for($i=0; $i<14 && $i<$NUM; $i++) { +$code.=<<___; + ldx [$np+$i*8],@N[$i] +___ +} +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for(; $i<28 && $i<$NUM; $i++) { +$code.=<<___; + ldx [$np+$i*8],@N[$i] +___ +} +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for(; $i<$NUM; $i++) { +$code.=<<___; + ldx [$np+$i*8],@N[$i] +___ +} +# load pwrtbl[pwr] ######################################################## +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp + + srlx $pwr, 32, %o4 ! unpack $pwr + srl $pwr, %g0, %o5 + sub %o4, 5, %o4 + mov $pwrtbl, %o7 + sllx %o4, 32, $pwr ! re-pack $pwr + or %o5, $pwr, $pwr + srl %o5, %o4, %o5 +___ + &load_ccr("%o7","%o5","%o4"); +$code.=<<___; + b .Lstride_$NUM + nop +.align 16 +.Lstride_$NUM: +___ +for($i=0; $i<14 && $i<$NUM; $i+=2) { + &load_b_pair("%o7",@B[$i],@B[$i+1]); +} +$code.=<<___; + save %sp,-128,%sp; or $sentinel,%fp,%fp +___ +for(; $i<$NUM; $i+=2) { + &load_b_pair("%i7",@B[$i],@B[$i+1]); +} +$code.=<<___; + srax $pwr, 32, %o4 ! unpack $pwr + srl $pwr, %g0, %o5 + sub %o4, 5, %o4 + mov $pwrtbl, %i7 + sllx %o4, 32, $pwr ! re-pack $pwr + or %o5, $pwr, $pwr + srl %o5, %o4, %o5 +___ + &load_ccr("%i7","%o5","%o4",1); + +# magic ################################################################ +for($i=0; $i<5; $i++) { +$code.=<<___; + .word 0x81b02940+$NUM-1 ! montsqr $NUM-1 + fbu,pn %fcc3,.Labort_$NUM +#ifndef __arch64__ + and %fp,$sentinel,$sentinel + brz,pn $sentinel,.Labort_$NUM +#endif + nop +___ +} +$code.=<<___; + wr %o4, %g0, %ccr + .word 0x81b02920+$NUM-1 ! montmul $NUM-1 + fbu,pn %fcc3,.Labort_$NUM +#ifndef __arch64__ + and %fp,$sentinel,$sentinel + brz,pn $sentinel,.Labort_$NUM +#endif + + srax $pwr, 32, %o4 +#ifdef __arch64__ + brgez %o4,.Lstride_$NUM + restore + restore + restore + restore + restore +#else + brgez %o4,.Lstride_$NUM + restore; and %fp,$sentinel,$sentinel + restore; and %fp,$sentinel,$sentinel + restore; and %fp,$sentinel,$sentinel + restore; and %fp,$sentinel,$sentinel + brz,pn $sentinel,.Labort1_$NUM + restore +#endif +___ + +# save tp[$NUM] ######################################################## +for($i=0; $i<14 && $i<$NUM; $i++) { +$code.=<<___; + movxtod @A[$i],@R[$i] +___ +} +$code.=<<___; +#ifdef __arch64__ + restore +#else + and %fp,$sentinel,$sentinel + restore + and $sentinel,1,%o7 + and %fp,$sentinel,$sentinel + srl %fp,0,%fp ! just in case? + or %o7,$sentinel,$sentinel + brz,a,pn $sentinel,.Ldone_$NUM + mov 0,%i0 ! return failure +#endif +___ +for($i=0; $i<$NUM; $i++) { +$code.=<<___; + std @R[$i],[$tp+$i*8] +___ +} +$code.=<<___; + mov 1,%i0 ! return success +.Ldone_$NUM: + ret + restore + +.Labort_$NUM: + restore + restore + restore + restore + restore +.Labort1_$NUM: + restore + + mov 0,%i0 ! return failure + ret + restore +.type bn_pwr5_mont_t4_$NUM, #function +.size bn_pwr5_mont_t4_$NUM, .-bn_pwr5_mont_t4_$NUM +___ +} + +for ($i=8;$i<=32;$i+=8) { + &generate_bn_pwr5_mont_t4($i); +} + +{ +######################################################################## +# Fall-back subroutines +# +# copy of bn_mul_mont_vis3 adjusted for vectors of 64-bit values +# +($n0,$m0,$m1,$lo0,$hi0, $lo1,$hi1,$aj,$alo,$nj,$nlo,$tj)= + (map("%g$_",(1..5)),map("%o$_",(0..5,7))); + +# int bn_mul_mont( +$rp="%o0"; # u64 *rp, +$ap="%o1"; # const u64 *ap, +$bp="%o2"; # const u64 *bp, +$np="%o3"; # const u64 *np, +$n0p="%o4"; # const BN_ULONG *n0, +$num="%o5"; # int num); # caller ensures that num is >=3 +$code.=<<___; +.globl bn_mul_mont_t4 +.align 32 +bn_mul_mont_t4: + add %sp, STACK_BIAS, %g4 ! real top of stack + sll $num, 3, $num ! size in bytes + add $num, 63, %g1 + andn %g1, 63, %g1 ! buffer size rounded up to 64 bytes + sub %g4, %g1, %g1 + andn %g1, 63, %g1 ! align at 64 byte + sub %g1, STACK_FRAME, %g1 ! new top of stack + sub %g1, %g4, %g1 + + save %sp, %g1, %sp +___ +# +-------------------------------+<----- %sp +# . . +# +-------------------------------+<----- aligned at 64 bytes +# | __int64 tmp[0] | +# +-------------------------------+ +# . . +# . . +# +-------------------------------+<----- aligned at 64 bytes +# . . +($rp,$ap,$bp,$np,$n0p,$num)=map("%i$_",(0..5)); +($t0,$t1,$t2,$t3,$cnt,$tp,$bufsz)=map("%l$_",(0..7)); +($ovf,$i)=($t0,$t1); +$code.=<<___; + ld [$n0p+0], $t0 ! pull n0[0..1] value + ld [$n0p+4], $t1 + add %sp, STACK_BIAS+STACK_FRAME, $tp + ldx [$bp+0], $m0 ! m0=bp[0] + sllx $t1, 32, $n0 + add $bp, 8, $bp + or $t0, $n0, $n0 + + ldx [$ap+0], $aj ! ap[0] + + mulx $aj, $m0, $lo0 ! ap[0]*bp[0] + umulxhi $aj, $m0, $hi0 + + ldx [$ap+8], $aj ! ap[1] + add $ap, 16, $ap + ldx [$np+0], $nj ! np[0] + + mulx $lo0, $n0, $m1 ! "tp[0]"*n0 + + mulx $aj, $m0, $alo ! ap[1]*bp[0] + umulxhi $aj, $m0, $aj ! ahi=aj + + mulx $nj, $m1, $lo1 ! np[0]*m1 + umulxhi $nj, $m1, $hi1 + + ldx [$np+8], $nj ! np[1] + + addcc $lo0, $lo1, $lo1 + add $np, 16, $np + addxc %g0, $hi1, $hi1 + + mulx $nj, $m1, $nlo ! np[1]*m1 + umulxhi $nj, $m1, $nj ! nhi=nj + + ba .L1st + sub $num, 24, $cnt ! cnt=num-3 + +.align 16 +.L1st: + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 + + ldx [$ap+0], $aj ! ap[j] + addcc $nlo, $hi1, $lo1 + add $ap, 8, $ap + addxc $nj, %g0, $hi1 ! nhi=nj + + ldx [$np+0], $nj ! np[j] + mulx $aj, $m0, $alo ! ap[j]*bp[0] + add $np, 8, $np + umulxhi $aj, $m0, $aj ! ahi=aj + + mulx $nj, $m1, $nlo ! np[j]*m1 + addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] + umulxhi $nj, $m1, $nj ! nhi=nj + addxc %g0, $hi1, $hi1 + stxa $lo1, [$tp]0xe2 ! tp[j-1] + add $tp, 8, $tp ! tp++ + + brnz,pt $cnt, .L1st + sub $cnt, 8, $cnt ! j-- +!.L1st + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 ! ahi=aj + + addcc $nlo, $hi1, $lo1 + addxc $nj, %g0, $hi1 + addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] + addxc %g0, $hi1, $hi1 + stxa $lo1, [$tp]0xe2 ! tp[j-1] + add $tp, 8, $tp + + addcc $hi0, $hi1, $hi1 + addxc %g0, %g0, $ovf ! upmost overflow bit + stxa $hi1, [$tp]0xe2 + add $tp, 8, $tp + + ba .Louter + sub $num, 16, $i ! i=num-2 + +.align 16 +.Louter: + ldx [$bp+0], $m0 ! m0=bp[i] + add $bp, 8, $bp + + sub $ap, $num, $ap ! rewind + sub $np, $num, $np + sub $tp, $num, $tp + + ldx [$ap+0], $aj ! ap[0] + ldx [$np+0], $nj ! np[0] + + mulx $aj, $m0, $lo0 ! ap[0]*bp[i] + ldx [$tp], $tj ! tp[0] + umulxhi $aj, $m0, $hi0 + ldx [$ap+8], $aj ! ap[1] + addcc $lo0, $tj, $lo0 ! ap[0]*bp[i]+tp[0] + mulx $aj, $m0, $alo ! ap[1]*bp[i] + addxc %g0, $hi0, $hi0 + mulx $lo0, $n0, $m1 ! tp[0]*n0 + umulxhi $aj, $m0, $aj ! ahi=aj + mulx $nj, $m1, $lo1 ! np[0]*m1 + add $ap, 16, $ap + umulxhi $nj, $m1, $hi1 + ldx [$np+8], $nj ! np[1] + add $np, 16, $np + addcc $lo1, $lo0, $lo1 + mulx $nj, $m1, $nlo ! np[1]*m1 + addxc %g0, $hi1, $hi1 + umulxhi $nj, $m1, $nj ! nhi=nj + + ba .Linner + sub $num, 24, $cnt ! cnt=num-3 +.align 16 +.Linner: + addcc $alo, $hi0, $lo0 + ldx [$tp+8], $tj ! tp[j] + addxc $aj, %g0, $hi0 ! ahi=aj + ldx [$ap+0], $aj ! ap[j] + add $ap, 8, $ap + addcc $nlo, $hi1, $lo1 + mulx $aj, $m0, $alo ! ap[j]*bp[i] + addxc $nj, %g0, $hi1 ! nhi=nj + ldx [$np+0], $nj ! np[j] + add $np, 8, $np + umulxhi $aj, $m0, $aj ! ahi=aj + addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] + mulx $nj, $m1, $nlo ! np[j]*m1 + addxc %g0, $hi0, $hi0 + umulxhi $nj, $m1, $nj ! nhi=nj + addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + add $tp, 8, $tp + brnz,pt $cnt, .Linner + sub $cnt, 8, $cnt +!.Linner + ldx [$tp+8], $tj ! tp[j] + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 ! ahi=aj + addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] + addxc %g0, $hi0, $hi0 + + addcc $nlo, $hi1, $lo1 + addxc $nj, %g0, $hi1 ! nhi=nj + addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + + subcc %g0, $ovf, %g0 ! move upmost overflow to CCR.xcc + addxccc $hi1, $hi0, $hi1 + addxc %g0, %g0, $ovf + stx $hi1, [$tp+8] + add $tp, 16, $tp + + brnz,pt $i, .Louter + sub $i, 8, $i + + sub $ap, $num, $ap ! rewind + sub $np, $num, $np + sub $tp, $num, $tp + ba .Lsub + subcc $num, 8, $cnt ! cnt=num-1 and clear CCR.xcc + +.align 16 +.Lsub: + ldx [$tp], $tj + add $tp, 8, $tp + ldx [$np+0], $nj + add $np, 8, $np + subccc $tj, $nj, $t2 ! tp[j]-np[j] + srlx $tj, 32, $tj + srlx $nj, 32, $nj + subccc $tj, $nj, $t3 + add $rp, 8, $rp + st $t2, [$rp-4] ! reverse order + st $t3, [$rp-8] + brnz,pt $cnt, .Lsub + sub $cnt, 8, $cnt + + sub $np, $num, $np ! rewind + sub $tp, $num, $tp + sub $rp, $num, $rp + + subc $ovf, %g0, $ovf ! handle upmost overflow bit + and $tp, $ovf, $ap + andn $rp, $ovf, $np + or $np, $ap, $ap ! ap=borrow?tp:rp + ba .Lcopy + sub $num, 8, $cnt + +.align 16 +.Lcopy: ! copy or in-place refresh + ldx [$ap+0], $t2 + add $ap, 8, $ap + stx %g0, [$tp] ! zap + add $tp, 8, $tp + stx $t2, [$rp+0] + add $rp, 8, $rp + brnz $cnt, .Lcopy + sub $cnt, 8, $cnt + + mov 1, %o0 + ret + restore +.type bn_mul_mont_t4, #function +.size bn_mul_mont_t4, .-bn_mul_mont_t4 +___ + +# int bn_mul_mont_gather5( +$rp="%o0"; # u64 *rp, +$ap="%o1"; # const u64 *ap, +$bp="%o2"; # const u64 *pwrtbl, +$np="%o3"; # const u64 *np, +$n0p="%o4"; # const BN_ULONG *n0, +$num="%o5"; # int num, # caller ensures that num is >=3 + # int power); +$code.=<<___; +.globl bn_mul_mont_gather5_t4 +.align 32 +bn_mul_mont_gather5_t4: + add %sp, STACK_BIAS, %g4 ! real top of stack + sll $num, 3, $num ! size in bytes + add $num, 63, %g1 + andn %g1, 63, %g1 ! buffer size rounded up to 64 bytes + sub %g4, %g1, %g1 + andn %g1, 63, %g1 ! align at 64 byte + sub %g1, STACK_FRAME, %g1 ! new top of stack + sub %g1, %g4, %g1 + LDPTR [%sp+STACK_7thARG], %g4 ! load power, 7th argument + + save %sp, %g1, %sp +___ +# +-------------------------------+<----- %sp +# . . +# +-------------------------------+<----- aligned at 64 bytes +# | __int64 tmp[0] | +# +-------------------------------+ +# . . +# . . +# +-------------------------------+<----- aligned at 64 bytes +# . . +($rp,$ap,$bp,$np,$n0p,$num)=map("%i$_",(0..5)); +($t0,$t1,$t2,$t3,$cnt,$tp,$bufsz,$ccr)=map("%l$_",(0..7)); +($ovf,$i)=($t0,$t1); + &load_ccr($bp,"%g4",$ccr); + &load_b($bp,$m0,"%o7"); # m0=bp[0] + +$code.=<<___; + ld [$n0p+0], $t0 ! pull n0[0..1] value + ld [$n0p+4], $t1 + add %sp, STACK_BIAS+STACK_FRAME, $tp + sllx $t1, 32, $n0 + or $t0, $n0, $n0 + + ldx [$ap+0], $aj ! ap[0] + + mulx $aj, $m0, $lo0 ! ap[0]*bp[0] + umulxhi $aj, $m0, $hi0 + + ldx [$ap+8], $aj ! ap[1] + add $ap, 16, $ap + ldx [$np+0], $nj ! np[0] + + mulx $lo0, $n0, $m1 ! "tp[0]"*n0 + + mulx $aj, $m0, $alo ! ap[1]*bp[0] + umulxhi $aj, $m0, $aj ! ahi=aj + + mulx $nj, $m1, $lo1 ! np[0]*m1 + umulxhi $nj, $m1, $hi1 + + ldx [$np+8], $nj ! np[1] + + addcc $lo0, $lo1, $lo1 + add $np, 16, $np + addxc %g0, $hi1, $hi1 + + mulx $nj, $m1, $nlo ! np[1]*m1 + umulxhi $nj, $m1, $nj ! nhi=nj + + ba .L1st_g5 + sub $num, 24, $cnt ! cnt=num-3 + +.align 16 +.L1st_g5: + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 + + ldx [$ap+0], $aj ! ap[j] + addcc $nlo, $hi1, $lo1 + add $ap, 8, $ap + addxc $nj, %g0, $hi1 ! nhi=nj + + ldx [$np+0], $nj ! np[j] + mulx $aj, $m0, $alo ! ap[j]*bp[0] + add $np, 8, $np + umulxhi $aj, $m0, $aj ! ahi=aj + + mulx $nj, $m1, $nlo ! np[j]*m1 + addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] + umulxhi $nj, $m1, $nj ! nhi=nj + addxc %g0, $hi1, $hi1 + stxa $lo1, [$tp]0xe2 ! tp[j-1] + add $tp, 8, $tp ! tp++ + + brnz,pt $cnt, .L1st_g5 + sub $cnt, 8, $cnt ! j-- +!.L1st_g5 + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 ! ahi=aj + + addcc $nlo, $hi1, $lo1 + addxc $nj, %g0, $hi1 + addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] + addxc %g0, $hi1, $hi1 + stxa $lo1, [$tp]0xe2 ! tp[j-1] + add $tp, 8, $tp + + addcc $hi0, $hi1, $hi1 + addxc %g0, %g0, $ovf ! upmost overflow bit + stxa $hi1, [$tp]0xe2 + add $tp, 8, $tp + + ba .Louter_g5 + sub $num, 16, $i ! i=num-2 + +.align 16 +.Louter_g5: + wr $ccr, %g0, %ccr +___ + &load_b($bp,$m0); # m0=bp[i] +$code.=<<___; + sub $ap, $num, $ap ! rewind + sub $np, $num, $np + sub $tp, $num, $tp + + ldx [$ap+0], $aj ! ap[0] + ldx [$np+0], $nj ! np[0] + + mulx $aj, $m0, $lo0 ! ap[0]*bp[i] + ldx [$tp], $tj ! tp[0] + umulxhi $aj, $m0, $hi0 + ldx [$ap+8], $aj ! ap[1] + addcc $lo0, $tj, $lo0 ! ap[0]*bp[i]+tp[0] + mulx $aj, $m0, $alo ! ap[1]*bp[i] + addxc %g0, $hi0, $hi0 + mulx $lo0, $n0, $m1 ! tp[0]*n0 + umulxhi $aj, $m0, $aj ! ahi=aj + mulx $nj, $m1, $lo1 ! np[0]*m1 + add $ap, 16, $ap + umulxhi $nj, $m1, $hi1 + ldx [$np+8], $nj ! np[1] + add $np, 16, $np + addcc $lo1, $lo0, $lo1 + mulx $nj, $m1, $nlo ! np[1]*m1 + addxc %g0, $hi1, $hi1 + umulxhi $nj, $m1, $nj ! nhi=nj + + ba .Linner_g5 + sub $num, 24, $cnt ! cnt=num-3 +.align 16 +.Linner_g5: + addcc $alo, $hi0, $lo0 + ldx [$tp+8], $tj ! tp[j] + addxc $aj, %g0, $hi0 ! ahi=aj + ldx [$ap+0], $aj ! ap[j] + add $ap, 8, $ap + addcc $nlo, $hi1, $lo1 + mulx $aj, $m0, $alo ! ap[j]*bp[i] + addxc $nj, %g0, $hi1 ! nhi=nj + ldx [$np+0], $nj ! np[j] + add $np, 8, $np + umulxhi $aj, $m0, $aj ! ahi=aj + addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] + mulx $nj, $m1, $nlo ! np[j]*m1 + addxc %g0, $hi0, $hi0 + umulxhi $nj, $m1, $nj ! nhi=nj + addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + add $tp, 8, $tp + brnz,pt $cnt, .Linner_g5 + sub $cnt, 8, $cnt +!.Linner_g5 + ldx [$tp+8], $tj ! tp[j] + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 ! ahi=aj + addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] + addxc %g0, $hi0, $hi0 + + addcc $nlo, $hi1, $lo1 + addxc $nj, %g0, $hi1 ! nhi=nj + addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + + subcc %g0, $ovf, %g0 ! move upmost overflow to CCR.xcc + addxccc $hi1, $hi0, $hi1 + addxc %g0, %g0, $ovf + stx $hi1, [$tp+8] + add $tp, 16, $tp + + brnz,pt $i, .Louter_g5 + sub $i, 8, $i + + sub $ap, $num, $ap ! rewind + sub $np, $num, $np + sub $tp, $num, $tp + ba .Lsub_g5 + subcc $num, 8, $cnt ! cnt=num-1 and clear CCR.xcc + +.align 16 +.Lsub_g5: + ldx [$tp], $tj + add $tp, 8, $tp + ldx [$np+0], $nj + add $np, 8, $np + subccc $tj, $nj, $t2 ! tp[j]-np[j] + srlx $tj, 32, $tj + srlx $nj, 32, $nj + subccc $tj, $nj, $t3 + add $rp, 8, $rp + st $t2, [$rp-4] ! reverse order + st $t3, [$rp-8] + brnz,pt $cnt, .Lsub_g5 + sub $cnt, 8, $cnt + + sub $np, $num, $np ! rewind + sub $tp, $num, $tp + sub $rp, $num, $rp + + subc $ovf, %g0, $ovf ! handle upmost overflow bit + and $tp, $ovf, $ap + andn $rp, $ovf, $np + or $np, $ap, $ap ! ap=borrow?tp:rp + ba .Lcopy_g5 + sub $num, 8, $cnt + +.align 16 +.Lcopy_g5: ! copy or in-place refresh + ldx [$ap+0], $t2 + add $ap, 8, $ap + stx %g0, [$tp] ! zap + add $tp, 8, $tp + stx $t2, [$rp+0] + add $rp, 8, $rp + brnz $cnt, .Lcopy_g5 + sub $cnt, 8, $cnt + + mov 1, %o0 + ret + restore +.type bn_mul_mont_gather5_t4, #function +.size bn_mul_mont_gather5_t4, .-bn_mul_mont_gather5_t4 +___ +} + +$code.=<<___; +.globl bn_flip_t4 +.align 32 +bn_flip_t4: +.Loop_flip: + ld [%o1+0], %o4 + sub %o2, 1, %o2 + ld [%o1+4], %o5 + add %o1, 8, %o1 + st %o5, [%o0+0] + st %o4, [%o0+4] + brnz %o2, .Loop_flip + add %o0, 8, %o0 + retl + nop +.type bn_flip_t4, #function +.size bn_flip_t4, .-bn_flip_t4 + +.globl bn_flip_n_scatter5_t4 +.align 32 +bn_flip_n_scatter5_t4: + sll %o3, 3, %o3 + srl %o1, 1, %o1 + add %o3, %o2, %o2 ! &pwrtbl[pwr] + sub %o1, 1, %o1 +.Loop_flip_n_scatter5: + ld [%o0+0], %o4 ! inp[i] + ld [%o0+4], %o5 + add %o0, 8, %o0 + sllx %o5, 32, %o5 + or %o4, %o5, %o5 + stx %o5, [%o2] + add %o2, 32*8, %o2 + brnz %o1, .Loop_flip_n_scatter5 + sub %o1, 1, %o1 + retl + nop +.type bn_flip_n_scatter5_t4, #function +.size bn_flip_n_scatter5_t4, .-bn_flip_n_scatter5_t4 + +.globl bn_gather5_t4 +.align 32 +bn_gather5_t4: +___ + &load_ccr("%o2","%o3","%g1"); +$code.=<<___; + sub %o1, 1, %o1 +.Loop_gather5: +___ + &load_b("%o2","%g1"); +$code.=<<___; + stx %g1, [%o0] + add %o0, 8, %o0 + brnz %o1, .Loop_gather5 + sub %o1, 1, %o1 + + retl + nop +.type bn_gather5_t4, #function +.size bn_gather5_t4, .-bn_gather5_t4 + +.asciz "Montgomery Multiplication for SPARC T4, David S. Miller, Andy Polyakov" +.align 4 +___ + +&emit_assembler(); + +close STDOUT; diff --git a/crypto/bn/asm/sparcv9-gf2m.pl b/crypto/bn/asm/sparcv9-gf2m.pl new file mode 100755 index 000000000000..ab94cd917c41 --- /dev/null +++ b/crypto/bn/asm/sparcv9-gf2m.pl @@ -0,0 +1,190 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# October 2012 +# +# The module implements bn_GF2m_mul_2x2 polynomial multiplication used +# in bn_gf2m.c. It's kind of low-hanging mechanical port from C for +# the time being... Except that it has two code paths: one suitable +# for all SPARCv9 processors and one for VIS3-capable ones. Former +# delivers ~25-45% more, more for longer keys, heaviest DH and DSA +# verify operations on venerable UltraSPARC II. On T4 VIS3 code is +# ~100-230% faster than gcc-generated code and ~35-90% faster than +# the pure SPARCv9 code path. + +$locals=16*8; + +$tab="%l0"; + +@T=("%g2","%g3"); +@i=("%g4","%g5"); + +($a1,$a2,$a4,$a8,$a12,$a48)=map("%o$_",(0..5)); +($lo,$hi,$b)=("%g1",$a8,"%o7"); $a=$lo; + +$code.=<<___; +#include + +#ifdef __arch64__ +.register %g2,#scratch +.register %g3,#scratch +#endif + +#ifdef __PIC__ +SPARC_PIC_THUNK(%g1) +#endif + +.globl bn_GF2m_mul_2x2 +.align 16 +bn_GF2m_mul_2x2: + SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5) + ld [%g1+0],%g1 ! OPENSSL_sparcv9cap_P[0] + + andcc %g1, SPARCV9_VIS3, %g0 + bz,pn %icc,.Lsoftware + nop + + sllx %o1, 32, %o1 + sllx %o3, 32, %o3 + or %o2, %o1, %o1 + or %o4, %o3, %o3 + .word 0x95b262ab ! xmulx %o1, %o3, %o2 + .word 0x99b262cb ! xmulxhi %o1, %o3, %o4 + srlx %o2, 32, %o1 ! 13 cycles later + st %o2, [%o0+0] + st %o1, [%o0+4] + srlx %o4, 32, %o3 + st %o4, [%o0+8] + retl + st %o3, [%o0+12] + +.align 16 +.Lsoftware: + save %sp,-STACK_FRAME-$locals,%sp + + sllx %i1,32,$a + mov -1,$a12 + sllx %i3,32,$b + or %i2,$a,$a + srlx $a12,1,$a48 ! 0x7fff... + or %i4,$b,$b + srlx $a12,2,$a12 ! 0x3fff... + add %sp,STACK_BIAS+STACK_FRAME,$tab + + sllx $a,2,$a4 + mov $a,$a1 + sllx $a,1,$a2 + + srax $a4,63,@i[1] ! broadcast 61st bit + and $a48,$a4,$a4 ! (a<<2)&0x7fff... + srlx $a48,2,$a48 + srax $a2,63,@i[0] ! broadcast 62nd bit + and $a12,$a2,$a2 ! (a<<1)&0x3fff... + srax $a1,63,$lo ! broadcast 63rd bit + and $a48,$a1,$a1 ! (a<<0)&0x1fff... + + sllx $a1,3,$a8 + and $b,$lo,$lo + and $b,@i[0],@i[0] + and $b,@i[1],@i[1] + + stx %g0,[$tab+0*8] ! tab[0]=0 + xor $a1,$a2,$a12 + stx $a1,[$tab+1*8] ! tab[1]=a1 + stx $a2,[$tab+2*8] ! tab[2]=a2 + xor $a4,$a8,$a48 + stx $a12,[$tab+3*8] ! tab[3]=a1^a2 + xor $a4,$a1,$a1 + + stx $a4,[$tab+4*8] ! tab[4]=a4 + xor $a4,$a2,$a2 + stx $a1,[$tab+5*8] ! tab[5]=a1^a4 + xor $a4,$a12,$a12 + stx $a2,[$tab+6*8] ! tab[6]=a2^a4 + xor $a48,$a1,$a1 + stx $a12,[$tab+7*8] ! tab[7]=a1^a2^a4 + xor $a48,$a2,$a2 + + stx $a8,[$tab+8*8] ! tab[8]=a8 + xor $a48,$a12,$a12 + stx $a1,[$tab+9*8] ! tab[9]=a1^a8 + xor $a4,$a1,$a1 + stx $a2,[$tab+10*8] ! tab[10]=a2^a8 + xor $a4,$a2,$a2 + stx $a12,[$tab+11*8] ! tab[11]=a1^a2^a8 + + xor $a4,$a12,$a12 + stx $a48,[$tab+12*8] ! tab[12]=a4^a8 + srlx $lo,1,$hi + stx $a1,[$tab+13*8] ! tab[13]=a1^a4^a8 + sllx $lo,63,$lo + stx $a2,[$tab+14*8] ! tab[14]=a2^a4^a8 + srlx @i[0],2,@T[0] + stx $a12,[$tab+15*8] ! tab[15]=a1^a2^a4^a8 + + sllx @i[0],62,$a1 + sllx $b,3,@i[0] + srlx @i[1],3,@T[1] + and @i[0],`0xf<<3`,@i[0] + sllx @i[1],61,$a2 + ldx [$tab+@i[0]],@i[0] + srlx $b,4-3,@i[1] + xor @T[0],$hi,$hi + and @i[1],`0xf<<3`,@i[1] + xor $a1,$lo,$lo + ldx [$tab+@i[1]],@i[1] + xor @T[1],$hi,$hi + + xor @i[0],$lo,$lo + srlx $b,8-3,@i[0] + xor $a2,$lo,$lo + and @i[0],`0xf<<3`,@i[0] +___ +for($n=1;$n<14;$n++) { +$code.=<<___; + sllx @i[1],`$n*4`,@T[0] + ldx [$tab+@i[0]],@i[0] + srlx @i[1],`64-$n*4`,@T[1] + xor @T[0],$lo,$lo + srlx $b,`($n+2)*4`-3,@i[1] + xor @T[1],$hi,$hi + and @i[1],`0xf<<3`,@i[1] +___ + push(@i,shift(@i)); push(@T,shift(@T)); +} +$code.=<<___; + sllx @i[1],`$n*4`,@T[0] + ldx [$tab+@i[0]],@i[0] + srlx @i[1],`64-$n*4`,@T[1] + xor @T[0],$lo,$lo + + sllx @i[0],`($n+1)*4`,@T[0] + xor @T[1],$hi,$hi + srlx @i[0],`64-($n+1)*4`,@T[1] + xor @T[0],$lo,$lo + xor @T[1],$hi,$hi + + srlx $lo,32,%i1 + st $lo,[%i0+0] + st %i1,[%i0+4] + srlx $hi,32,%i2 + st $hi,[%i0+8] + st %i2,[%i0+12] + + ret + restore +.type bn_GF2m_mul_2x2,#function +.size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2 +.asciz "GF(2^m) Multiplication for SPARCv9, CRYPTOGAMS by " +.align 4 +___ + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; +print $code; +close STDOUT; diff --git a/crypto/bn/asm/vis3-mont.pl b/crypto/bn/asm/vis3-mont.pl new file mode 100755 index 000000000000..263ac02b6f45 --- /dev/null +++ b/crypto/bn/asm/vis3-mont.pl @@ -0,0 +1,373 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# October 2012. +# +# SPARCv9 VIS3 Montgomery multiplicaion procedure suitable for T3 and +# onward. There are three new instructions used here: umulxhi, +# addxc[cc] and initializing store. On T3 RSA private key operations +# are 1.54/1.87/2.11/2.26 times faster for 512/1024/2048/4096-bit key +# lengths. This is without dedicated squaring procedure. On T4 +# corresponding coefficients are 1.47/2.10/2.80/2.90x, which is mostly +# for reference purposes, because T4 has dedicated Montgomery +# multiplication and squaring *instructions* that deliver even more. + +$bits=32; +for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } +if ($bits==64) { $bias=2047; $frame=192; } +else { $bias=0; $frame=112; } + +$code.=<<___ if ($bits==64); +.register %g2,#scratch +.register %g3,#scratch +___ +$code.=<<___; +.section ".text",#alloc,#execinstr +___ + +($n0,$m0,$m1,$lo0,$hi0, $lo1,$hi1,$aj,$alo,$nj,$nlo,$tj)= + (map("%g$_",(1..5)),map("%o$_",(0..5,7))); + +# int bn_mul_mont( +$rp="%o0"; # BN_ULONG *rp, +$ap="%o1"; # const BN_ULONG *ap, +$bp="%o2"; # const BN_ULONG *bp, +$np="%o3"; # const BN_ULONG *np, +$n0p="%o4"; # const BN_ULONG *n0, +$num="%o5"; # int num); # caller ensures that num is even + # and >=6 +$code.=<<___; +.globl bn_mul_mont_vis3 +.align 32 +bn_mul_mont_vis3: + add %sp, $bias, %g4 ! real top of stack + sll $num, 2, $num ! size in bytes + add $num, 63, %g5 + andn %g5, 63, %g5 ! buffer size rounded up to 64 bytes + add %g5, %g5, %g1 + add %g5, %g1, %g1 ! 3*buffer size + sub %g4, %g1, %g1 + andn %g1, 63, %g1 ! align at 64 byte + sub %g1, $frame, %g1 ! new top of stack + sub %g1, %g4, %g1 + + save %sp, %g1, %sp +___ + +# +-------------------------------+<----- %sp +# . . +# +-------------------------------+<----- aligned at 64 bytes +# | __int64 tmp[0] | +# +-------------------------------+ +# . . +# . . +# +-------------------------------+<----- aligned at 64 bytes +# | __int64 ap[1..0] | converted ap[] +# +-------------------------------+ +# | __int64 np[1..0] | converted np[] +# +-------------------------------+ +# | __int64 ap[3..2] | +# . . +# . . +# +-------------------------------+ +($rp,$ap,$bp,$np,$n0p,$num)=map("%i$_",(0..5)); +($t0,$t1,$t2,$t3,$cnt,$tp,$bufsz,$anp)=map("%l$_",(0..7)); +($ovf,$i)=($t0,$t1); +$code.=<<___; + ld [$n0p+0], $t0 ! pull n0[0..1] value + add %sp, $bias+$frame, $tp + ld [$n0p+4], $t1 + add $tp, %g5, $anp + ld [$bp+0], $t2 ! m0=bp[0] + sllx $t1, 32, $n0 + ld [$bp+4], $t3 + or $t0, $n0, $n0 + add $bp, 8, $bp + + ld [$ap+0], $t0 ! ap[0] + sllx $t3, 32, $m0 + ld [$ap+4], $t1 + or $t2, $m0, $m0 + + ld [$ap+8], $t2 ! ap[1] + sllx $t1, 32, $aj + ld [$ap+12], $t3 + or $t0, $aj, $aj + add $ap, 16, $ap + stx $aj, [$anp] ! converted ap[0] + + mulx $aj, $m0, $lo0 ! ap[0]*bp[0] + umulxhi $aj, $m0, $hi0 + + ld [$np+0], $t0 ! np[0] + sllx $t3, 32, $aj + ld [$np+4], $t1 + or $t2, $aj, $aj + + ld [$np+8], $t2 ! np[1] + sllx $t1, 32, $nj + ld [$np+12], $t3 + or $t0, $nj, $nj + add $np, 16, $np + stx $nj, [$anp+8] ! converted np[0] + + mulx $lo0, $n0, $m1 ! "tp[0]"*n0 + stx $aj, [$anp+16] ! converted ap[1] + + mulx $aj, $m0, $alo ! ap[1]*bp[0] + umulxhi $aj, $m0, $aj ! ahi=aj + + mulx $nj, $m1, $lo1 ! np[0]*m1 + umulxhi $nj, $m1, $hi1 + + sllx $t3, 32, $nj + or $t2, $nj, $nj + stx $nj, [$anp+24] ! converted np[1] + add $anp, 32, $anp + + addcc $lo0, $lo1, $lo1 + addxc %g0, $hi1, $hi1 + + mulx $nj, $m1, $nlo ! np[1]*m1 + umulxhi $nj, $m1, $nj ! nhi=nj + + ba .L1st + sub $num, 24, $cnt ! cnt=num-3 + +.align 16 +.L1st: + ld [$ap+0], $t0 ! ap[j] + addcc $alo, $hi0, $lo0 + ld [$ap+4], $t1 + addxc $aj, %g0, $hi0 + + sllx $t1, 32, $aj + add $ap, 8, $ap + or $t0, $aj, $aj + stx $aj, [$anp] ! converted ap[j] + + ld [$np+0], $t2 ! np[j] + addcc $nlo, $hi1, $lo1 + ld [$np+4], $t3 + addxc $nj, %g0, $hi1 ! nhi=nj + + sllx $t3, 32, $nj + add $np, 8, $np + mulx $aj, $m0, $alo ! ap[j]*bp[0] + or $t2, $nj, $nj + umulxhi $aj, $m0, $aj ! ahi=aj + stx $nj, [$anp+8] ! converted np[j] + add $anp, 16, $anp ! anp++ + + mulx $nj, $m1, $nlo ! np[j]*m1 + addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] + umulxhi $nj, $m1, $nj ! nhi=nj + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + add $tp, 8, $tp ! tp++ + + brnz,pt $cnt, .L1st + sub $cnt, 8, $cnt ! j-- +!.L1st + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 ! ahi=aj + + addcc $nlo, $hi1, $lo1 + addxc $nj, %g0, $hi1 + addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + add $tp, 8, $tp + + addcc $hi0, $hi1, $hi1 + addxc %g0, %g0, $ovf ! upmost overflow bit + stx $hi1, [$tp] + add $tp, 8, $tp + + ba .Louter + sub $num, 16, $i ! i=num-2 + +.align 16 +.Louter: + ld [$bp+0], $t2 ! m0=bp[i] + ld [$bp+4], $t3 + + sub $anp, $num, $anp ! rewind + sub $tp, $num, $tp + sub $anp, $num, $anp + + add $bp, 8, $bp + sllx $t3, 32, $m0 + ldx [$anp+0], $aj ! ap[0] + or $t2, $m0, $m0 + ldx [$anp+8], $nj ! np[0] + + mulx $aj, $m0, $lo0 ! ap[0]*bp[i] + ldx [$tp], $tj ! tp[0] + umulxhi $aj, $m0, $hi0 + ldx [$anp+16], $aj ! ap[1] + addcc $lo0, $tj, $lo0 ! ap[0]*bp[i]+tp[0] + mulx $aj, $m0, $alo ! ap[1]*bp[i] + addxc %g0, $hi0, $hi0 + mulx $lo0, $n0, $m1 ! tp[0]*n0 + umulxhi $aj, $m0, $aj ! ahi=aj + mulx $nj, $m1, $lo1 ! np[0]*m1 + umulxhi $nj, $m1, $hi1 + ldx [$anp+24], $nj ! np[1] + add $anp, 32, $anp + addcc $lo1, $lo0, $lo1 + mulx $nj, $m1, $nlo ! np[1]*m1 + addxc %g0, $hi1, $hi1 + umulxhi $nj, $m1, $nj ! nhi=nj + + ba .Linner + sub $num, 24, $cnt ! cnt=num-3 +.align 16 +.Linner: + addcc $alo, $hi0, $lo0 + ldx [$tp+8], $tj ! tp[j] + addxc $aj, %g0, $hi0 ! ahi=aj + ldx [$anp+0], $aj ! ap[j] + addcc $nlo, $hi1, $lo1 + mulx $aj, $m0, $alo ! ap[j]*bp[i] + addxc $nj, %g0, $hi1 ! nhi=nj + ldx [$anp+8], $nj ! np[j] + add $anp, 16, $anp + umulxhi $aj, $m0, $aj ! ahi=aj + addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] + mulx $nj, $m1, $nlo ! np[j]*m1 + addxc %g0, $hi0, $hi0 + umulxhi $nj, $m1, $nj ! nhi=nj + addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + add $tp, 8, $tp + brnz,pt $cnt, .Linner + sub $cnt, 8, $cnt +!.Linner + ldx [$tp+8], $tj ! tp[j] + addcc $alo, $hi0, $lo0 + addxc $aj, %g0, $hi0 ! ahi=aj + addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] + addxc %g0, $hi0, $hi0 + + addcc $nlo, $hi1, $lo1 + addxc $nj, %g0, $hi1 ! nhi=nj + addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] + addxc %g0, $hi1, $hi1 + stx $lo1, [$tp] ! tp[j-1] + + subcc %g0, $ovf, %g0 ! move upmost overflow to CCR.xcc + addxccc $hi1, $hi0, $hi1 + addxc %g0, %g0, $ovf + stx $hi1, [$tp+8] + add $tp, 16, $tp + + brnz,pt $i, .Louter + sub $i, 8, $i + + sub $anp, $num, $anp ! rewind + sub $tp, $num, $tp + sub $anp, $num, $anp + ba .Lsub + subcc $num, 8, $cnt ! cnt=num-1 and clear CCR.xcc + +.align 16 +.Lsub: + ldx [$tp], $tj + add $tp, 8, $tp + ldx [$anp+8], $nj + add $anp, 16, $anp + subccc $tj, $nj, $t2 ! tp[j]-np[j] + srlx $tj, 32, $tj + srlx $nj, 32, $nj + subccc $tj, $nj, $t3 + add $rp, 8, $rp + st $t2, [$rp-4] ! reverse order + st $t3, [$rp-8] + brnz,pt $cnt, .Lsub + sub $cnt, 8, $cnt + + sub $anp, $num, $anp ! rewind + sub $tp, $num, $tp + sub $anp, $num, $anp + sub $rp, $num, $rp + + subc $ovf, %g0, $ovf ! handle upmost overflow bit + and $tp, $ovf, $ap + andn $rp, $ovf, $np + or $np, $ap, $ap ! ap=borrow?tp:rp + ba .Lcopy + sub $num, 8, $cnt + +.align 16 +.Lcopy: ! copy or in-place refresh + ld [$ap+0], $t2 + ld [$ap+4], $t3 + add $ap, 8, $ap + stx %g0, [$tp] ! zap + add $tp, 8, $tp + stx %g0, [$anp] ! zap + stx %g0, [$anp+8] + add $anp, 16, $anp + st $t3, [$rp+0] ! flip order + st $t2, [$rp+4] + add $rp, 8, $rp + brnz $cnt, .Lcopy + sub $cnt, 8, $cnt + + mov 1, %o0 + ret + restore +.type bn_mul_mont_vis3, #function +.size bn_mul_mont_vis3, .-bn_mul_mont_vis3 +.asciz "Montgomery Multiplication for SPARCv9 VIS3, CRYPTOGAMS by " +.align 4 +___ + +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis3 { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my ($ref,$opf); +my %visopf = ( "addxc" => 0x011, + "addxccc" => 0x013, + "umulxhi" => 0x016 ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%([goli])([0-9])/); + $_=$bias{$1}+$2; + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/\b(umulxhi|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ + &unvis3($1,$2,$3,$4) + /ge; + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/bn/asm/x86_64-gcc.c b/crypto/bn/asm/x86_64-gcc.c index 9c5074b30858..d5488866e0cf 100644 --- a/crypto/bn/asm/x86_64-gcc.c +++ b/crypto/bn/asm/x86_64-gcc.c @@ -55,7 +55,7 @@ * machine. */ -# ifdef _WIN64 +# if defined(_WIN64) || !defined(__LP64__) # define BN_ULONG unsigned long long # else # define BN_ULONG unsigned long @@ -63,7 +63,6 @@ # undef mul # undef mul_add -# undef sqr /*- * "m"(a), "+m"(r) is the way to favor DirectPath µ-code; @@ -99,8 +98,8 @@ : "cc"); \ (r)=carry, carry=high; \ } while (0) - -# define sqr(r0,r1,a) \ +# undef sqr +# define sqr(r0,r1,a) \ asm ("mulq %2" \ : "=a"(r0),"=d"(r1) \ : "a"(a) \ @@ -204,20 +203,22 @@ BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d) BN_ULONG bn_add_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, int n) { - BN_ULONG ret = 0, i = 0; + BN_ULONG ret; + size_t i = 0; if (n <= 0) return 0; - asm volatile (" subq %2,%2 \n" + asm volatile (" subq %0,%0 \n" /* clear carry */ + " jmp 1f \n" ".p2align 4 \n" "1: movq (%4,%2,8),%0 \n" " adcq (%5,%2,8),%0 \n" " movq %0,(%3,%2,8) \n" - " leaq 1(%2),%2 \n" + " lea 1(%2),%2 \n" " loop 1b \n" - " sbbq %0,%0 \n":"=&a" (ret), "+c"(n), - "=&r"(i) + " sbbq %0,%0 \n":"=&r" (ret), "+c"(n), + "+r"(i) :"r"(rp), "r"(ap), "r"(bp) :"cc", "memory"); @@ -228,20 +229,22 @@ BN_ULONG bn_add_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, BN_ULONG bn_sub_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, int n) { - BN_ULONG ret = 0, i = 0; + BN_ULONG ret; + size_t i = 0; if (n <= 0) return 0; - asm volatile (" subq %2,%2 \n" + asm volatile (" subq %0,%0 \n" /* clear borrow */ + " jmp 1f \n" ".p2align 4 \n" "1: movq (%4,%2,8),%0 \n" " sbbq (%5,%2,8),%0 \n" " movq %0,(%3,%2,8) \n" - " leaq 1(%2),%2 \n" + " lea 1(%2),%2 \n" " loop 1b \n" - " sbbq %0,%0 \n":"=&a" (ret), "+c"(n), - "=&r"(i) + " sbbq %0,%0 \n":"=&r" (ret), "+c"(n), + "+r"(i) :"r"(rp), "r"(ap), "r"(bp) :"cc", "memory"); @@ -313,55 +316,58 @@ BN_ULONG bn_sub_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n) */ # if 0 /* original macros are kept for reference purposes */ -# define mul_add_c(a,b,c0,c1,c2) { \ - BN_ULONG ta=(a),tb=(b); \ - t1 = ta * tb; \ - t2 = BN_UMULT_HIGH(ta,tb); \ - c0 += t1; t2 += (c0&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $addx = ($1>=2.23); +} + +if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $addx = ($1>=2.10); +} + +if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $addx = ($1>=12); +} + # int bn_mul_mont( $rp="%rdi"; # BN_ULONG *rp, $ap="%rsi"; # const BN_ULONG *ap, @@ -61,6 +86,8 @@ $m1="%rbp"; $code=<<___; .text +.extern OPENSSL_ia32cap_P + .globl bn_mul_mont .type bn_mul_mont,\@function,6 .align 16 @@ -69,9 +96,16 @@ bn_mul_mont: jnz .Lmul_enter cmp \$8,${num}d jb .Lmul_enter +___ +$code.=<<___ if ($addx); + mov OPENSSL_ia32cap_P+8(%rip),%r11d +___ +$code.=<<___; cmp $ap,$bp jne .Lmul4x_enter - jmp .Lsqr4x_enter + test \$7,${num}d + jz .Lsqr8x_enter + jmp .Lmul4x_enter .align 16 .Lmul_enter: @@ -227,7 +261,7 @@ $code.=<<___; lea 1($i),$i # i++ cmp $num,$i - jl .Louter + jb .Louter xor $i,$i # i=0 and clear CF! mov (%rsp),%rax # tp[0] @@ -280,6 +314,13 @@ $code.=<<___; .align 16 bn_mul4x_mont: .Lmul4x_enter: +___ +$code.=<<___ if ($addx); + and \$0x80100,%r11d + cmp \$0x80100,%r11d + je .Lmulx4x_enter +___ +$code.=<<___; push %rbx push %rbp push %r12 @@ -401,7 +442,7 @@ $code.=<<___; mov $N[1],-32(%rsp,$j,8) # tp[j-1] mov %rdx,$N[0] cmp $num,$j - jl .L1st4x + jb .L1st4x mulq $m0 # ap[j]*bp[0] add %rax,$A[0] @@ -549,7 +590,7 @@ $code.=<<___; mov $N[1],-32(%rsp,$j,8) # tp[j-1] mov %rdx,$N[0] cmp $num,$j - jl .Linner4x + jb .Linner4x mulq $m0 # ap[j]*bp[i] add %rax,$A[0] @@ -595,7 +636,7 @@ $code.=<<___; mov $N[1],(%rsp,$j,8) # store upmost overflow bit cmp $num,$i - jl .Louter4x + jb .Louter4x ___ { my @ri=("%rax","%rdx",$m0,$m1); @@ -688,25 +729,30 @@ ___ }}} {{{ ###################################################################### -# void bn_sqr4x_mont( +# void bn_sqr8x_mont( my $rptr="%rdi"; # const BN_ULONG *rptr, my $aptr="%rsi"; # const BN_ULONG *aptr, my $bptr="%rdx"; # not used my $nptr="%rcx"; # const BN_ULONG *nptr, my $n0 ="%r8"; # const BN_ULONG *n0); -my $num ="%r9"; # int num, has to be divisible by 4 and - # not less than 8 +my $num ="%r9"; # int num, has to be divisible by 8 my ($i,$j,$tptr)=("%rbp","%rcx",$rptr); my @A0=("%r10","%r11"); my @A1=("%r12","%r13"); my ($a0,$a1,$ai)=("%r14","%r15","%rbx"); +$code.=<<___ if ($addx); +.extern bn_sqrx8x_internal # see x86_64-mont5 module +___ $code.=<<___; -.type bn_sqr4x_mont,\@function,6 -.align 16 -bn_sqr4x_mont: -.Lsqr4x_enter: +.extern bn_sqr8x_internal # see x86_64-mont5 module + +.type bn_sqr8x_mont,\@function,6 +.align 32 +bn_sqr8x_mont: +.Lsqr8x_enter: + mov %rsp,%rax push %rbx push %rbp push %r12 @@ -714,787 +760,445 @@ bn_sqr4x_mont: push %r14 push %r15 + mov ${num}d,%r10d shl \$3,${num}d # convert $num to bytes - xor %r10,%r10 - mov %rsp,%r11 # put aside %rsp - sub $num,%r10 # -$num - mov ($n0),$n0 # *n0 - lea -72(%rsp,%r10,2),%rsp # alloca(frame+2*$num) - and \$-1024,%rsp # minimize TLB usage - ############################################################## - # Stack layout - # - # +0 saved $num, used in reduction section - # +8 &t[2*$num], used in reduction section - # +32 saved $rptr - # +40 saved $nptr - # +48 saved *n0 - # +56 saved %rsp - # +64 t[2*$num] - # - mov $rptr,32(%rsp) # save $rptr - mov $nptr,40(%rsp) - mov $n0, 48(%rsp) - mov %r11, 56(%rsp) # save original %rsp -.Lsqr4x_body: + shl \$3+2,%r10 # 4*$num + neg $num + ############################################################## - # Squaring part: - # - # a) multiply-n-add everything but a[i]*a[i]; - # b) shift result of a) by 1 to the left and accumulate - # a[i]*a[i] products; + # ensure that stack frame doesn't alias with $aptr modulo + # 4096. this is done to allow memory disambiguation logic + # do its job. # - lea 32(%r10),$i # $i=-($num-32) - lea ($aptr,$num),$aptr # end of a[] buffer, ($aptr,$i)=&ap[2] - - mov $num,$j # $j=$num - - # comments apply to $num==8 case - mov -32($aptr,$i),$a0 # a[0] - lea 64(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] - mov -24($aptr,$i),%rax # a[1] - lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] - mov -16($aptr,$i),$ai # a[2] - mov %rax,$a1 - - mul $a0 # a[1]*a[0] - mov %rax,$A0[0] # a[1]*a[0] - mov $ai,%rax # a[2] - mov %rdx,$A0[1] - mov $A0[0],-24($tptr,$i) # t[1] - - xor $A0[0],$A0[0] - mul $a0 # a[2]*a[0] - add %rax,$A0[1] - mov $ai,%rax - adc %rdx,$A0[0] - mov $A0[1],-16($tptr,$i) # t[2] - - lea -16($i),$j # j=-16 - - - mov 8($aptr,$j),$ai # a[3] - mul $a1 # a[2]*a[1] - mov %rax,$A1[0] # a[2]*a[1]+t[3] - mov $ai,%rax - mov %rdx,$A1[1] - - xor $A0[1],$A0[1] - add $A1[0],$A0[0] - lea 16($j),$j - adc \$0,$A0[1] - mul $a0 # a[3]*a[0] - add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] - mov $ai,%rax - adc %rdx,$A0[1] - mov $A0[0],-8($tptr,$j) # t[3] - jmp .Lsqr4x_1st + lea -64(%rsp,$num,4),%r11 + mov ($n0),$n0 # *n0 + sub $aptr,%r11 + and \$4095,%r11 + cmp %r11,%r10 + jb .Lsqr8x_sp_alt + sub %r11,%rsp # align with $aptr + lea -64(%rsp,$num,4),%rsp # alloca(frame+4*$num) + jmp .Lsqr8x_sp_done + +.align 32 +.Lsqr8x_sp_alt: + lea 4096-64(,$num,4),%r10 # 4096-frame-4*$num + lea -64(%rsp,$num,4),%rsp # alloca(frame+4*$num) + sub %r10,%r11 + mov \$0,%r10 + cmovc %r10,%r11 + sub %r11,%rsp +.Lsqr8x_sp_done: + and \$-64,%rsp + mov $num,%r10 + neg $num + + lea 64(%rsp,$num,2),%r11 # copy of modulus + mov $n0, 32(%rsp) + mov %rax, 40(%rsp) # save original %rsp +.Lsqr8x_body: + + mov $num,$i + movq %r11, %xmm2 # save pointer to modulus copy + shr \$3+2,$i + mov OPENSSL_ia32cap_P+8(%rip),%eax + jmp .Lsqr8x_copy_n + +.align 32 +.Lsqr8x_copy_n: + movq 8*0($nptr),%xmm0 + movq 8*1($nptr),%xmm1 + movq 8*2($nptr),%xmm3 + movq 8*3($nptr),%xmm4 + lea 8*4($nptr),$nptr + movdqa %xmm0,16*0(%r11) + movdqa %xmm1,16*1(%r11) + movdqa %xmm3,16*2(%r11) + movdqa %xmm4,16*3(%r11) + lea 16*4(%r11),%r11 + dec $i + jnz .Lsqr8x_copy_n -.align 16 -.Lsqr4x_1st: - mov ($aptr,$j),$ai # a[4] - xor $A1[0],$A1[0] - mul $a1 # a[3]*a[1] - add %rax,$A1[1] # a[3]*a[1]+t[4] - mov $ai,%rax - adc %rdx,$A1[0] - - xor $A0[0],$A0[0] - add $A1[1],$A0[1] - adc \$0,$A0[0] - mul $a0 # a[4]*a[0] - add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4] - mov $ai,%rax # a[3] - adc %rdx,$A0[0] - mov $A0[1],($tptr,$j) # t[4] - - - mov 8($aptr,$j),$ai # a[5] - xor $A1[1],$A1[1] - mul $a1 # a[4]*a[3] - add %rax,$A1[0] # a[4]*a[3]+t[5] - mov $ai,%rax - adc %rdx,$A1[1] - - xor $A0[1],$A0[1] - add $A1[0],$A0[0] - adc \$0,$A0[1] - mul $a0 # a[5]*a[2] - add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5] - mov $ai,%rax - adc %rdx,$A0[1] - mov $A0[0],8($tptr,$j) # t[5] - - mov 16($aptr,$j),$ai # a[6] - xor $A1[0],$A1[0] - mul $a1 # a[5]*a[3] - add %rax,$A1[1] # a[5]*a[3]+t[6] - mov $ai,%rax - adc %rdx,$A1[0] - - xor $A0[0],$A0[0] - add $A1[1],$A0[1] - adc \$0,$A0[0] - mul $a0 # a[6]*a[2] - add %rax,$A0[1] # a[6]*a[2]+a[5]*a[3]+t[6] - mov $ai,%rax # a[3] - adc %rdx,$A0[0] - mov $A0[1],16($tptr,$j) # t[6] - - - mov 24($aptr,$j),$ai # a[7] - xor $A1[1],$A1[1] - mul $a1 # a[6]*a[5] - add %rax,$A1[0] # a[6]*a[5]+t[7] - mov $ai,%rax - adc %rdx,$A1[1] - - xor $A0[1],$A0[1] - add $A1[0],$A0[0] - lea 32($j),$j - adc \$0,$A0[1] - mul $a0 # a[7]*a[4] - add %rax,$A0[0] # a[7]*a[4]+a[6]*a[5]+t[6] - mov $ai,%rax - adc %rdx,$A0[1] - mov $A0[0],-8($tptr,$j) # t[7] - - cmp \$0,$j - jne .Lsqr4x_1st - - xor $A1[0],$A1[0] - add $A0[1],$A1[1] - adc \$0,$A1[0] - mul $a1 # a[7]*a[5] - add %rax,$A1[1] - adc %rdx,$A1[0] - - mov $A1[1],($tptr) # t[8] - lea 16($i),$i - mov $A1[0],8($tptr) # t[9] - jmp .Lsqr4x_outer + pxor %xmm0,%xmm0 + movq $rptr,%xmm1 # save $rptr + movq %r10, %xmm3 # -$num +___ +$code.=<<___ if ($addx); + and \$0x80100,%eax + cmp \$0x80100,%eax + jne .Lsqr8x_nox -.align 16 -.Lsqr4x_outer: # comments apply to $num==6 case - mov -32($aptr,$i),$a0 # a[0] - lea 64(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] - mov -24($aptr,$i),%rax # a[1] - lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] - mov -16($aptr,$i),$ai # a[2] - mov %rax,$a1 - - mov -24($tptr,$i),$A0[0] # t[1] - xor $A0[1],$A0[1] - mul $a0 # a[1]*a[0] - add %rax,$A0[0] # a[1]*a[0]+t[1] - mov $ai,%rax # a[2] - adc %rdx,$A0[1] - mov $A0[0],-24($tptr,$i) # t[1] - - xor $A0[0],$A0[0] - add -16($tptr,$i),$A0[1] # a[2]*a[0]+t[2] - adc \$0,$A0[0] - mul $a0 # a[2]*a[0] - add %rax,$A0[1] - mov $ai,%rax - adc %rdx,$A0[0] - mov $A0[1],-16($tptr,$i) # t[2] - - lea -16($i),$j # j=-16 - xor $A1[0],$A1[0] - - - mov 8($aptr,$j),$ai # a[3] - xor $A1[1],$A1[1] - add 8($tptr,$j),$A1[0] - adc \$0,$A1[1] - mul $a1 # a[2]*a[1] - add %rax,$A1[0] # a[2]*a[1]+t[3] - mov $ai,%rax - adc %rdx,$A1[1] - - xor $A0[1],$A0[1] - add $A1[0],$A0[0] - adc \$0,$A0[1] - mul $a0 # a[3]*a[0] - add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] - mov $ai,%rax - adc %rdx,$A0[1] - mov $A0[0],8($tptr,$j) # t[3] - - lea 16($j),$j - jmp .Lsqr4x_inner + call bn_sqrx8x_internal # see x86_64-mont5 module -.align 16 -.Lsqr4x_inner: - mov ($aptr,$j),$ai # a[4] - xor $A1[0],$A1[0] - add ($tptr,$j),$A1[1] - adc \$0,$A1[0] - mul $a1 # a[3]*a[1] - add %rax,$A1[1] # a[3]*a[1]+t[4] - mov $ai,%rax - adc %rdx,$A1[0] - - xor $A0[0],$A0[0] - add $A1[1],$A0[1] - adc \$0,$A0[0] - mul $a0 # a[4]*a[0] - add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4] - mov $ai,%rax # a[3] - adc %rdx,$A0[0] - mov $A0[1],($tptr,$j) # t[4] - - mov 8($aptr,$j),$ai # a[5] - xor $A1[1],$A1[1] - add 8($tptr,$j),$A1[0] - adc \$0,$A1[1] - mul $a1 # a[4]*a[3] - add %rax,$A1[0] # a[4]*a[3]+t[5] - mov $ai,%rax - adc %rdx,$A1[1] - - xor $A0[1],$A0[1] - add $A1[0],$A0[0] - lea 16($j),$j # j++ - adc \$0,$A0[1] - mul $a0 # a[5]*a[2] - add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5] - mov $ai,%rax - adc %rdx,$A0[1] - mov $A0[0],-8($tptr,$j) # t[5], "preloaded t[1]" below - - cmp \$0,$j - jne .Lsqr4x_inner - - xor $A1[0],$A1[0] - add $A0[1],$A1[1] - adc \$0,$A1[0] - mul $a1 # a[5]*a[3] - add %rax,$A1[1] - adc %rdx,$A1[0] - - mov $A1[1],($tptr) # t[6], "preloaded t[2]" below - mov $A1[0],8($tptr) # t[7], "preloaded t[3]" below - - add \$16,$i - jnz .Lsqr4x_outer - - # comments apply to $num==4 case - mov -32($aptr),$a0 # a[0] - lea 64(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] - mov -24($aptr),%rax # a[1] - lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] - mov -16($aptr),$ai # a[2] - mov %rax,$a1 - - xor $A0[1],$A0[1] - mul $a0 # a[1]*a[0] - add %rax,$A0[0] # a[1]*a[0]+t[1], preloaded t[1] - mov $ai,%rax # a[2] - adc %rdx,$A0[1] - mov $A0[0],-24($tptr) # t[1] - - xor $A0[0],$A0[0] - add $A1[1],$A0[1] # a[2]*a[0]+t[2], preloaded t[2] - adc \$0,$A0[0] - mul $a0 # a[2]*a[0] - add %rax,$A0[1] - mov $ai,%rax - adc %rdx,$A0[0] - mov $A0[1],-16($tptr) # t[2] - - mov -8($aptr),$ai # a[3] - mul $a1 # a[2]*a[1] - add %rax,$A1[0] # a[2]*a[1]+t[3], preloaded t[3] - mov $ai,%rax - adc \$0,%rdx - - xor $A0[1],$A0[1] - add $A1[0],$A0[0] - mov %rdx,$A1[1] - adc \$0,$A0[1] - mul $a0 # a[3]*a[0] - add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] - mov $ai,%rax - adc %rdx,$A0[1] - mov $A0[0],-8($tptr) # t[3] - - xor $A1[0],$A1[0] - add $A0[1],$A1[1] - adc \$0,$A1[0] - mul $a1 # a[3]*a[1] - add %rax,$A1[1] - mov -16($aptr),%rax # a[2] - adc %rdx,$A1[0] - - mov $A1[1],($tptr) # t[4] - mov $A1[0],8($tptr) # t[5] - - mul $ai # a[2]*a[3] + pxor %xmm0,%xmm0 + lea 48(%rsp),%rax + lea 64(%rsp,$num,2),%rdx + shr \$3+2,$num + mov 40(%rsp),%rsi # restore %rsp + jmp .Lsqr8x_zero + +.align 32 +.Lsqr8x_nox: ___ -{ -my ($shift,$carry)=($a0,$a1); -my @S=(@A1,$ai,$n0); $code.=<<___; - add \$16,$i - xor $shift,$shift - sub $num,$i # $i=16-$num - xor $carry,$carry - - add $A1[0],%rax # t[5] - adc \$0,%rdx - mov %rax,8($tptr) # t[5] - mov %rdx,16($tptr) # t[6] - mov $carry,24($tptr) # t[7] - - mov -16($aptr,$i),%rax # a[0] - lea 64(%rsp,$num,2),$tptr - xor $A0[0],$A0[0] # t[0] - mov -24($tptr,$i,2),$A0[1] # t[1] - - lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[1] # | t[2*i]>>63 - mov -16($tptr,$i,2),$A0[0] # t[2*i+2] # prefetch - mov $A0[1],$shift # shift=t[2*i+1]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - mov -8($tptr,$i,2),$A0[1] # t[2*i+2+1] # prefetch - adc %rax,$S[0] - mov -8($aptr,$i),%rax # a[i+1] # prefetch - mov $S[0],-32($tptr,$i,2) - adc %rdx,$S[1] - - lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift - mov $S[1],-24($tptr,$i,2) - sbb $carry,$carry # mov cf,$carry - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[3] # | t[2*i]>>63 - mov 0($tptr,$i,2),$A0[0] # t[2*i+2] # prefetch - mov $A0[1],$shift # shift=t[2*i+1]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - mov 8($tptr,$i,2),$A0[1] # t[2*i+2+1] # prefetch - adc %rax,$S[2] - mov 0($aptr,$i),%rax # a[i+1] # prefetch - mov $S[2],-16($tptr,$i,2) - adc %rdx,$S[3] - lea 16($i),$i - mov $S[3],-40($tptr,$i,2) - sbb $carry,$carry # mov cf,$carry - jmp .Lsqr4x_shift_n_add + call bn_sqr8x_internal # see x86_64-mont5 module -.align 16 -.Lsqr4x_shift_n_add: - lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[1] # | t[2*i]>>63 - mov -16($tptr,$i,2),$A0[0] # t[2*i+2] # prefetch - mov $A0[1],$shift # shift=t[2*i+1]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - mov -8($tptr,$i,2),$A0[1] # t[2*i+2+1] # prefetch - adc %rax,$S[0] - mov -8($aptr,$i),%rax # a[i+1] # prefetch - mov $S[0],-32($tptr,$i,2) - adc %rdx,$S[1] - - lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift - mov $S[1],-24($tptr,$i,2) - sbb $carry,$carry # mov cf,$carry - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[3] # | t[2*i]>>63 - mov 0($tptr,$i,2),$A0[0] # t[2*i+2] # prefetch - mov $A0[1],$shift # shift=t[2*i+1]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - mov 8($tptr,$i,2),$A0[1] # t[2*i+2+1] # prefetch - adc %rax,$S[2] - mov 0($aptr,$i),%rax # a[i+1] # prefetch - mov $S[2],-16($tptr,$i,2) - adc %rdx,$S[3] - - lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift - mov $S[3],-8($tptr,$i,2) - sbb $carry,$carry # mov cf,$carry - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[1] # | t[2*i]>>63 - mov 16($tptr,$i,2),$A0[0] # t[2*i+2] # prefetch - mov $A0[1],$shift # shift=t[2*i+1]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - mov 24($tptr,$i,2),$A0[1] # t[2*i+2+1] # prefetch - adc %rax,$S[0] - mov 8($aptr,$i),%rax # a[i+1] # prefetch - mov $S[0],0($tptr,$i,2) - adc %rdx,$S[1] - - lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift - mov $S[1],8($tptr,$i,2) - sbb $carry,$carry # mov cf,$carry - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[3] # | t[2*i]>>63 - mov 32($tptr,$i,2),$A0[0] # t[2*i+2] # prefetch - mov $A0[1],$shift # shift=t[2*i+1]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - mov 40($tptr,$i,2),$A0[1] # t[2*i+2+1] # prefetch - adc %rax,$S[2] - mov 16($aptr,$i),%rax # a[i+1] # prefetch - mov $S[2],16($tptr,$i,2) - adc %rdx,$S[3] - mov $S[3],24($tptr,$i,2) - sbb $carry,$carry # mov cf,$carry - add \$32,$i - jnz .Lsqr4x_shift_n_add - - lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[1] # | t[2*i]>>63 - mov -16($tptr),$A0[0] # t[2*i+2] # prefetch - mov $A0[1],$shift # shift=t[2*i+1]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - mov -8($tptr),$A0[1] # t[2*i+2+1] # prefetch - adc %rax,$S[0] - mov -8($aptr),%rax # a[i+1] # prefetch - mov $S[0],-32($tptr) - adc %rdx,$S[1] - - lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1|shift - mov $S[1],-24($tptr) - sbb $carry,$carry # mov cf,$carry - shr \$63,$A0[0] - lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | - shr \$63,$A0[1] - or $A0[0],$S[3] # | t[2*i]>>63 - mul %rax # a[i]*a[i] - neg $carry # mov $carry,cf - adc %rax,$S[2] - adc %rdx,$S[3] - mov $S[2],-16($tptr) - mov $S[3],-8($tptr) -___ -} -############################################################## -# Montgomery reduction part, "word-by-word" algorithm. -# -{ -my ($topbit,$nptr)=("%rbp",$aptr); -my ($m0,$m1)=($a0,$a1); -my @Ni=("%rbx","%r9"); -$code.=<<___; - mov 40(%rsp),$nptr # restore $nptr - mov 48(%rsp),$n0 # restore *n0 - xor $j,$j - mov $num,0(%rsp) # save $num - sub $num,$j # $j=-$num - mov 64(%rsp),$A0[0] # t[0] # modsched # - mov $n0,$m0 # # modsched # - lea 64(%rsp,$num,2),%rax # end of t[] buffer - lea 64(%rsp,$num),$tptr # end of t[] window - mov %rax,8(%rsp) # save end of t[] buffer - lea ($nptr,$num),$nptr # end of n[] buffer - xor $topbit,$topbit # $topbit=0 - - mov 0($nptr,$j),%rax # n[0] # modsched # - mov 8($nptr,$j),$Ni[1] # n[1] # modsched # - imulq $A0[0],$m0 # m0=t[0]*n0 # modsched # - mov %rax,$Ni[0] # # modsched # - jmp .Lsqr4x_mont_outer + pxor %xmm0,%xmm0 + lea 48(%rsp),%rax + lea 64(%rsp,$num,2),%rdx + shr \$3+2,$num + mov 40(%rsp),%rsi # restore %rsp + jmp .Lsqr8x_zero + +.align 32 +.Lsqr8x_zero: + movdqa %xmm0,16*0(%rax) # wipe t + movdqa %xmm0,16*1(%rax) + movdqa %xmm0,16*2(%rax) + movdqa %xmm0,16*3(%rax) + lea 16*4(%rax),%rax + movdqa %xmm0,16*0(%rdx) # wipe n + movdqa %xmm0,16*1(%rdx) + movdqa %xmm0,16*2(%rdx) + movdqa %xmm0,16*3(%rdx) + lea 16*4(%rdx),%rdx + dec $num + jnz .Lsqr8x_zero -.align 16 -.Lsqr4x_mont_outer: - xor $A0[1],$A0[1] - mul $m0 # n[0]*m0 - add %rax,$A0[0] # n[0]*m0+t[0] - mov $Ni[1],%rax - adc %rdx,$A0[1] - mov $n0,$m1 + mov \$1,%rax + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lsqr8x_epilogue: + ret +.size bn_sqr8x_mont,.-bn_sqr8x_mont +___ +}}} + +if ($addx) {{{ +my $bp="%rdx"; # original value - xor $A0[0],$A0[0] - add 8($tptr,$j),$A0[1] - adc \$0,$A0[0] - mul $m0 # n[1]*m0 - add %rax,$A0[1] # n[1]*m0+t[1] - mov $Ni[0],%rax - adc %rdx,$A0[0] - - imulq $A0[1],$m1 - - mov 16($nptr,$j),$Ni[0] # n[2] - xor $A1[1],$A1[1] - add $A0[1],$A1[0] - adc \$0,$A1[1] - mul $m1 # n[0]*m1 - add %rax,$A1[0] # n[0]*m1+"t[1]" - mov $Ni[0],%rax - adc %rdx,$A1[1] - mov $A1[0],8($tptr,$j) # "t[1]" - - xor $A0[1],$A0[1] - add 16($tptr,$j),$A0[0] - adc \$0,$A0[1] - mul $m0 # n[2]*m0 - add %rax,$A0[0] # n[2]*m0+t[2] - mov $Ni[1],%rax - adc %rdx,$A0[1] - - mov 24($nptr,$j),$Ni[1] # n[3] - xor $A1[0],$A1[0] - add $A0[0],$A1[1] - adc \$0,$A1[0] - mul $m1 # n[1]*m1 - add %rax,$A1[1] # n[1]*m1+"t[2]" - mov $Ni[1],%rax - adc %rdx,$A1[0] - mov $A1[1],16($tptr,$j) # "t[2]" - - xor $A0[0],$A0[0] - add 24($tptr,$j),$A0[1] - lea 32($j),$j - adc \$0,$A0[0] - mul $m0 # n[3]*m0 - add %rax,$A0[1] # n[3]*m0+t[3] - mov $Ni[0],%rax - adc %rdx,$A0[0] - jmp .Lsqr4x_mont_inner +$code.=<<___; +.type bn_mulx4x_mont,\@function,6 +.align 32 +bn_mulx4x_mont: +.Lmulx4x_enter: + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 -.align 16 -.Lsqr4x_mont_inner: - mov ($nptr,$j),$Ni[0] # n[4] - xor $A1[1],$A1[1] - add $A0[1],$A1[0] - adc \$0,$A1[1] - mul $m1 # n[2]*m1 - add %rax,$A1[0] # n[2]*m1+"t[3]" - mov $Ni[0],%rax - adc %rdx,$A1[1] - mov $A1[0],-8($tptr,$j) # "t[3]" - - xor $A0[1],$A0[1] - add ($tptr,$j),$A0[0] - adc \$0,$A0[1] - mul $m0 # n[4]*m0 - add %rax,$A0[0] # n[4]*m0+t[4] - mov $Ni[1],%rax - adc %rdx,$A0[1] - - mov 8($nptr,$j),$Ni[1] # n[5] - xor $A1[0],$A1[0] - add $A0[0],$A1[1] - adc \$0,$A1[0] - mul $m1 # n[3]*m1 - add %rax,$A1[1] # n[3]*m1+"t[4]" - mov $Ni[1],%rax - adc %rdx,$A1[0] - mov $A1[1],($tptr,$j) # "t[4]" - - xor $A0[0],$A0[0] - add 8($tptr,$j),$A0[1] - adc \$0,$A0[0] - mul $m0 # n[5]*m0 - add %rax,$A0[1] # n[5]*m0+t[5] - mov $Ni[0],%rax - adc %rdx,$A0[0] - - - mov 16($nptr,$j),$Ni[0] # n[6] - xor $A1[1],$A1[1] - add $A0[1],$A1[0] - adc \$0,$A1[1] - mul $m1 # n[4]*m1 - add %rax,$A1[0] # n[4]*m1+"t[5]" - mov $Ni[0],%rax - adc %rdx,$A1[1] - mov $A1[0],8($tptr,$j) # "t[5]" - - xor $A0[1],$A0[1] - add 16($tptr,$j),$A0[0] - adc \$0,$A0[1] - mul $m0 # n[6]*m0 - add %rax,$A0[0] # n[6]*m0+t[6] - mov $Ni[1],%rax - adc %rdx,$A0[1] - - mov 24($nptr,$j),$Ni[1] # n[7] - xor $A1[0],$A1[0] - add $A0[0],$A1[1] - adc \$0,$A1[0] - mul $m1 # n[5]*m1 - add %rax,$A1[1] # n[5]*m1+"t[6]" - mov $Ni[1],%rax - adc %rdx,$A1[0] - mov $A1[1],16($tptr,$j) # "t[6]" - - xor $A0[0],$A0[0] - add 24($tptr,$j),$A0[1] - lea 32($j),$j - adc \$0,$A0[0] - mul $m0 # n[7]*m0 - add %rax,$A0[1] # n[7]*m0+t[7] - mov $Ni[0],%rax - adc %rdx,$A0[0] - cmp \$0,$j - jne .Lsqr4x_mont_inner - - sub 0(%rsp),$j # $j=-$num # modsched # - mov $n0,$m0 # # modsched # - - xor $A1[1],$A1[1] - add $A0[1],$A1[0] - adc \$0,$A1[1] - mul $m1 # n[6]*m1 - add %rax,$A1[0] # n[6]*m1+"t[7]" - mov $Ni[1],%rax - adc %rdx,$A1[1] - mov $A1[0],-8($tptr) # "t[7]" - - xor $A0[1],$A0[1] - add ($tptr),$A0[0] # +t[8] - adc \$0,$A0[1] - mov 0($nptr,$j),$Ni[0] # n[0] # modsched # - add $topbit,$A0[0] - adc \$0,$A0[1] - - imulq 16($tptr,$j),$m0 # m0=t[0]*n0 # modsched # - xor $A1[0],$A1[0] - mov 8($nptr,$j),$Ni[1] # n[1] # modsched # - add $A0[0],$A1[1] - mov 16($tptr,$j),$A0[0] # t[0] # modsched # - adc \$0,$A1[0] - mul $m1 # n[7]*m1 - add %rax,$A1[1] # n[7]*m1+"t[8]" - mov $Ni[0],%rax # # modsched # - adc %rdx,$A1[0] - mov $A1[1],($tptr) # "t[8]" - - xor $topbit,$topbit - add 8($tptr),$A1[0] # +t[9] - adc $topbit,$topbit - add $A0[1],$A1[0] - lea 16($tptr),$tptr # "t[$num]>>128" - adc \$0,$topbit - mov $A1[0],-8($tptr) # "t[9]" - cmp 8(%rsp),$tptr # are we done? - jb .Lsqr4x_mont_outer - - mov 0(%rsp),$num # restore $num - mov $topbit,($tptr) # save $topbit + shl \$3,${num}d # convert $num to bytes + .byte 0x67 + xor %r10,%r10 + sub $num,%r10 # -$num + mov ($n0),$n0 # *n0 + lea -72(%rsp,%r10),%rsp # alloca(frame+$num+8) + lea ($bp,$num),%r10 + and \$-128,%rsp + ############################################################## + # Stack layout + # +0 num + # +8 off-loaded &b[i] + # +16 end of b[num] + # +24 saved n0 + # +32 saved rp + # +40 saved %rsp + # +48 inner counter + # +56 + # +64 tmp[num+1] + # + mov $num,0(%rsp) # save $num + shr \$5,$num + mov %r10,16(%rsp) # end of b[num] + sub \$1,$num + mov $n0, 24(%rsp) # save *n0 + mov $rp, 32(%rsp) # save $rp + mov %rax,40(%rsp) # save original %rsp + mov $num,48(%rsp) # inner counter + jmp .Lmulx4x_body + +.align 32 +.Lmulx4x_body: ___ -} -############################################################## -# Post-condition, 4x unrolled copy from bn_mul_mont -# -{ -my ($tptr,$nptr)=("%rbx",$aptr); -my @ri=("%rax","%rdx","%r10","%r11"); +my ($aptr, $bptr, $nptr, $tptr, $mi, $bi, $zero, $num)= + ("%rsi","%rdi","%rcx","%rbx","%r8","%r9","%rbp","%rax"); +my $rptr=$bptr; $code.=<<___; - mov 64(%rsp,$num),@ri[0] # tp[0] - lea 64(%rsp,$num),$tptr # upper half of t[2*$num] holds result - mov 40(%rsp),$nptr # restore $nptr - shr \$5,$num # num/4 - mov 8($tptr),@ri[1] # t[1] - xor $i,$i # i=0 and clear CF! - - mov 32(%rsp),$rptr # restore $rptr - sub 0($nptr),@ri[0] - mov 16($tptr),@ri[2] # t[2] - mov 24($tptr),@ri[3] # t[3] - sbb 8($nptr),@ri[1] - lea -1($num),$j # j=num/4-1 - jmp .Lsqr4x_sub -.align 16 -.Lsqr4x_sub: - mov @ri[0],0($rptr,$i,8) # rp[i]=tp[i]-np[i] - mov @ri[1],8($rptr,$i,8) # rp[i]=tp[i]-np[i] - sbb 16($nptr,$i,8),@ri[2] - mov 32($tptr,$i,8),@ri[0] # tp[i+1] - mov 40($tptr,$i,8),@ri[1] - sbb 24($nptr,$i,8),@ri[3] - mov @ri[2],16($rptr,$i,8) # rp[i]=tp[i]-np[i] - mov @ri[3],24($rptr,$i,8) # rp[i]=tp[i]-np[i] - sbb 32($nptr,$i,8),@ri[0] - mov 48($tptr,$i,8),@ri[2] - mov 56($tptr,$i,8),@ri[3] - sbb 40($nptr,$i,8),@ri[1] - lea 4($i),$i # i++ - dec $j # doesn't affect CF! - jnz .Lsqr4x_sub - - mov @ri[0],0($rptr,$i,8) # rp[i]=tp[i]-np[i] - mov 32($tptr,$i,8),@ri[0] # load overflow bit - sbb 16($nptr,$i,8),@ri[2] - mov @ri[1],8($rptr,$i,8) # rp[i]=tp[i]-np[i] - sbb 24($nptr,$i,8),@ri[3] - mov @ri[2],16($rptr,$i,8) # rp[i]=tp[i]-np[i] - - sbb \$0,@ri[0] # handle upmost overflow bit - mov @ri[3],24($rptr,$i,8) # rp[i]=tp[i]-np[i] - xor $i,$i # i=0 - and @ri[0],$tptr - not @ri[0] - mov $rptr,$nptr - and @ri[0],$nptr - lea -1($num),$j - or $nptr,$tptr # tp=borrow?tp:rp + lea 8($bp),$bptr + mov ($bp),%rdx # b[0], $bp==%rdx actually + lea 64+32(%rsp),$tptr + mov %rdx,$bi + + mulx 0*8($aptr),$mi,%rax # a[0]*b[0] + mulx 1*8($aptr),%r11,%r14 # a[1]*b[0] + add %rax,%r11 + mov $bptr,8(%rsp) # off-load &b[i] + mulx 2*8($aptr),%r12,%r13 # ... + adc %r14,%r12 + adc \$0,%r13 + + mov $mi,$bptr # borrow $bptr + imulq 24(%rsp),$mi # "t[0]"*n0 + xor $zero,$zero # cf=0, of=0 + + mulx 3*8($aptr),%rax,%r14 + mov $mi,%rdx + lea 4*8($aptr),$aptr + adcx %rax,%r13 + adcx $zero,%r14 # cf=0 + + mulx 0*8($nptr),%rax,%r10 + adcx %rax,$bptr # discarded + adox %r11,%r10 + mulx 1*8($nptr),%rax,%r11 + adcx %rax,%r10 + adox %r12,%r11 + .byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 # mulx 2*8($nptr),%rax,%r12 + mov 48(%rsp),$bptr # counter value + mov %r10,-4*8($tptr) + adcx %rax,%r11 + adox %r13,%r12 + mulx 3*8($nptr),%rax,%r15 + mov $bi,%rdx + mov %r11,-3*8($tptr) + adcx %rax,%r12 + adox $zero,%r15 # of=0 + lea 4*8($nptr),$nptr + mov %r12,-2*8($tptr) + + jmp .Lmulx4x_1st + +.align 32 +.Lmulx4x_1st: + adcx $zero,%r15 # cf=0, modulo-scheduled + mulx 0*8($aptr),%r10,%rax # a[4]*b[0] + adcx %r14,%r10 + mulx 1*8($aptr),%r11,%r14 # a[5]*b[0] + adcx %rax,%r11 + mulx 2*8($aptr),%r12,%rax # ... + adcx %r14,%r12 + mulx 3*8($aptr),%r13,%r14 + .byte 0x67,0x67 + mov $mi,%rdx + adcx %rax,%r13 + adcx $zero,%r14 # cf=0 + lea 4*8($aptr),$aptr + lea 4*8($tptr),$tptr + + adox %r15,%r10 + mulx 0*8($nptr),%rax,%r15 + adcx %rax,%r10 + adox %r15,%r11 + mulx 1*8($nptr),%rax,%r15 + adcx %rax,%r11 + adox %r15,%r12 + mulx 2*8($nptr),%rax,%r15 + mov %r10,-5*8($tptr) + adcx %rax,%r12 + mov %r11,-4*8($tptr) + adox %r15,%r13 + mulx 3*8($nptr),%rax,%r15 + mov $bi,%rdx + mov %r12,-3*8($tptr) + adcx %rax,%r13 + adox $zero,%r15 + lea 4*8($nptr),$nptr + mov %r13,-2*8($tptr) + + dec $bptr # of=0, pass cf + jnz .Lmulx4x_1st + + mov 0(%rsp),$num # load num + mov 8(%rsp),$bptr # re-load &b[i] + adc $zero,%r15 # modulo-scheduled + add %r15,%r14 + sbb %r15,%r15 # top-most carry + mov %r14,-1*8($tptr) + jmp .Lmulx4x_outer + +.align 32 +.Lmulx4x_outer: + mov ($bptr),%rdx # b[i] + lea 8($bptr),$bptr # b++ + sub $num,$aptr # rewind $aptr + mov %r15,($tptr) # save top-most carry + lea 64+4*8(%rsp),$tptr + sub $num,$nptr # rewind $nptr + + mulx 0*8($aptr),$mi,%r11 # a[0]*b[i] + xor %ebp,%ebp # xor $zero,$zero # cf=0, of=0 + mov %rdx,$bi + mulx 1*8($aptr),%r14,%r12 # a[1]*b[i] + adox -4*8($tptr),$mi + adcx %r14,%r11 + mulx 2*8($aptr),%r15,%r13 # ... + adox -3*8($tptr),%r11 + adcx %r15,%r12 + adox $zero,%r12 + adcx $zero,%r13 + + mov $bptr,8(%rsp) # off-load &b[i] + .byte 0x67 + mov $mi,%r15 + imulq 24(%rsp),$mi # "t[0]"*n0 + xor %ebp,%ebp # xor $zero,$zero # cf=0, of=0 + + mulx 3*8($aptr),%rax,%r14 + mov $mi,%rdx + adox -2*8($tptr),%r12 + adcx %rax,%r13 + adox -1*8($tptr),%r13 + adcx $zero,%r14 + lea 4*8($aptr),$aptr + adox $zero,%r14 + + mulx 0*8($nptr),%rax,%r10 + adcx %rax,%r15 # discarded + adox %r11,%r10 + mulx 1*8($nptr),%rax,%r11 + adcx %rax,%r10 + adox %r12,%r11 + mulx 2*8($nptr),%rax,%r12 + mov %r10,-4*8($tptr) + adcx %rax,%r11 + adox %r13,%r12 + mulx 3*8($nptr),%rax,%r15 + mov $bi,%rdx + mov %r11,-3*8($tptr) + lea 4*8($nptr),$nptr + adcx %rax,%r12 + adox $zero,%r15 # of=0 + mov 48(%rsp),$bptr # counter value + mov %r12,-2*8($tptr) + + jmp .Lmulx4x_inner + +.align 32 +.Lmulx4x_inner: + mulx 0*8($aptr),%r10,%rax # a[4]*b[i] + adcx $zero,%r15 # cf=0, modulo-scheduled + adox %r14,%r10 + mulx 1*8($aptr),%r11,%r14 # a[5]*b[i] + adcx 0*8($tptr),%r10 + adox %rax,%r11 + mulx 2*8($aptr),%r12,%rax # ... + adcx 1*8($tptr),%r11 + adox %r14,%r12 + mulx 3*8($aptr),%r13,%r14 + mov $mi,%rdx + adcx 2*8($tptr),%r12 + adox %rax,%r13 + adcx 3*8($tptr),%r13 + adox $zero,%r14 # of=0 + lea 4*8($aptr),$aptr + lea 4*8($tptr),$tptr + adcx $zero,%r14 # cf=0 + + adox %r15,%r10 + mulx 0*8($nptr),%rax,%r15 + adcx %rax,%r10 + adox %r15,%r11 + mulx 1*8($nptr),%rax,%r15 + adcx %rax,%r11 + adox %r15,%r12 + mulx 2*8($nptr),%rax,%r15 + mov %r10,-5*8($tptr) + adcx %rax,%r12 + adox %r15,%r13 + mulx 3*8($nptr),%rax,%r15 + mov $bi,%rdx + mov %r11,-4*8($tptr) + mov %r12,-3*8($tptr) + adcx %rax,%r13 + adox $zero,%r15 + lea 4*8($nptr),$nptr + mov %r13,-2*8($tptr) + + dec $bptr # of=0, pass cf + jnz .Lmulx4x_inner + + mov 0(%rsp),$num # load num + mov 8(%rsp),$bptr # re-load &b[i] + adc $zero,%r15 # modulo-scheduled + sub 0*8($tptr),$zero # pull top-most carry + adc %r15,%r14 + mov -8($nptr),$mi + sbb %r15,%r15 # top-most carry + mov %r14,-1*8($tptr) + + cmp 16(%rsp),$bptr + jne .Lmulx4x_outer + + sub %r14,$mi # compare top-most words + sbb $mi,$mi + or $mi,%r15 + + neg $num + xor %rdx,%rdx + mov 32(%rsp),$rptr # restore rp + lea 64(%rsp),$tptr pxor %xmm0,%xmm0 - lea 64(%rsp,$num,8),$nptr - movdqu ($tptr),%xmm1 - lea ($nptr,$num,8),$nptr - movdqa %xmm0,64(%rsp) # zap lower half of temporary vector - movdqa %xmm0,($nptr) # zap upper half of temporary vector - movdqu %xmm1,($rptr) - jmp .Lsqr4x_copy -.align 16 -.Lsqr4x_copy: # copy or in-place refresh - movdqu 16($tptr,$i),%xmm2 - movdqu 32($tptr,$i),%xmm1 - movdqa %xmm0,80(%rsp,$i) # zap lower half of temporary vector - movdqa %xmm0,96(%rsp,$i) # zap lower half of temporary vector - movdqa %xmm0,16($nptr,$i) # zap upper half of temporary vector - movdqa %xmm0,32($nptr,$i) # zap upper half of temporary vector - movdqu %xmm2,16($rptr,$i) - movdqu %xmm1,32($rptr,$i) - lea 32($i),$i - dec $j - jnz .Lsqr4x_copy - - movdqu 16($tptr,$i),%xmm2 - movdqa %xmm0,80(%rsp,$i) # zap lower half of temporary vector - movdqa %xmm0,16($nptr,$i) # zap upper half of temporary vector - movdqu %xmm2,16($rptr,$i) -___ -} -$code.=<<___; - mov 56(%rsp),%rsi # restore %rsp + mov 0*8($nptr,$num),%r8 + mov 1*8($nptr,$num),%r9 + neg %r8 + jmp .Lmulx4x_sub_entry + +.align 32 +.Lmulx4x_sub: + mov 0*8($nptr,$num),%r8 + mov 1*8($nptr,$num),%r9 + not %r8 +.Lmulx4x_sub_entry: + mov 2*8($nptr,$num),%r10 + not %r9 + and %r15,%r8 + mov 3*8($nptr,$num),%r11 + not %r10 + and %r15,%r9 + not %r11 + and %r15,%r10 + and %r15,%r11 + + neg %rdx # mov %rdx,%cf + adc 0*8($tptr),%r8 + adc 1*8($tptr),%r9 + movdqa %xmm0,($tptr) + adc 2*8($tptr),%r10 + adc 3*8($tptr),%r11 + movdqa %xmm0,16($tptr) + lea 4*8($tptr),$tptr + sbb %rdx,%rdx # mov %cf,%rdx + + mov %r8,0*8($rptr) + mov %r9,1*8($rptr) + mov %r10,2*8($rptr) + mov %r11,3*8($rptr) + lea 4*8($rptr),$rptr + + add \$32,$num + jnz .Lmulx4x_sub + + mov 40(%rsp),%rsi # restore %rsp mov \$1,%rax - mov 0(%rsi),%r15 - mov 8(%rsi),%r14 - mov 16(%rsi),%r13 - mov 24(%rsi),%r12 - mov 32(%rsi),%rbp - mov 40(%rsi),%rbx - lea 48(%rsi),%rsp -.Lsqr4x_epilogue: + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lmulx4x_epilogue: ret -.size bn_sqr4x_mont,.-bn_sqr4x_mont +.size bn_mulx4x_mont,.-bn_mulx4x_mont ___ }}} $code.=<<___; @@ -1581,18 +1285,22 @@ sqr_handler: mov 120($context),%rax # pull context->Rax mov 248($context),%rbx # pull context->Rip - lea .Lsqr4x_body(%rip),%r10 + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # end of prologue label cmp %r10,%rbx # context->Rip<.Lsqr_body jb .Lcommon_seh_tail mov 152($context),%rax # pull context->Rsp - lea .Lsqr4x_epilogue(%rip),%r10 + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label cmp %r10,%rbx # context->Rip>=.Lsqr_epilogue jae .Lcommon_seh_tail - mov 56(%rax),%rax # pull saved stack pointer - lea 48(%rax),%rax + mov 40(%rax),%rax # pull saved stack pointer mov -8(%rax),%rbx mov -16(%rax),%rbp @@ -1657,10 +1365,16 @@ sqr_handler: .rva .LSEH_end_bn_mul4x_mont .rva .LSEH_info_bn_mul4x_mont - .rva .LSEH_begin_bn_sqr4x_mont - .rva .LSEH_end_bn_sqr4x_mont - .rva .LSEH_info_bn_sqr4x_mont - + .rva .LSEH_begin_bn_sqr8x_mont + .rva .LSEH_end_bn_sqr8x_mont + .rva .LSEH_info_bn_sqr8x_mont +___ +$code.=<<___ if ($addx); + .rva .LSEH_begin_bn_mulx4x_mont + .rva .LSEH_end_bn_mulx4x_mont + .rva .LSEH_info_bn_mulx4x_mont +___ +$code.=<<___; .section .xdata .align 8 .LSEH_info_bn_mul_mont: @@ -1671,9 +1385,16 @@ sqr_handler: .byte 9,0,0,0 .rva mul_handler .rva .Lmul4x_body,.Lmul4x_epilogue # HandlerData[] -.LSEH_info_bn_sqr4x_mont: +.LSEH_info_bn_sqr8x_mont: + .byte 9,0,0,0 + .rva sqr_handler + .rva .Lsqr8x_body,.Lsqr8x_epilogue # HandlerData[] +___ +$code.=<<___ if ($addx); +.LSEH_info_bn_mulx4x_mont: .byte 9,0,0,0 .rva sqr_handler + .rva .Lmulx4x_body,.Lmulx4x_epilogue # HandlerData[] ___ } diff --git a/crypto/bn/asm/x86_64-mont5.pl b/crypto/bn/asm/x86_64-mont5.pl index dae0fe245318..820de3d6f627 100755 --- a/crypto/bn/asm/x86_64-mont5.pl +++ b/crypto/bn/asm/x86_64-mont5.pl @@ -17,6 +17,13 @@ # is implemented, so that scatter-/gathering can be tuned without # bn_exp.c modifications. +# August 2013. +# +# Add MULX/AD*X code paths and additional interfaces to optimize for +# branch prediction unit. For input lengths that are multiples of 8 +# the np argument is not just modulus value, but one interleaved +# with 0. This is to optimize post-condition... + $flavour = shift; $output = shift; if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } @@ -31,6 +38,21 @@ die "can't locate x86_64-xlate.pl"; open OUT,"| \"$^X\" $xlate $flavour $output"; *STDOUT=*OUT; +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $addx = ($1>=2.23); +} + +if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $addx = ($1>=2.10); +} + +if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $addx = ($1>=12); +} + # int bn_mul_mont_gather5( $rp="%rdi"; # BN_ULONG *rp, $ap="%rsi"; # const BN_ULONG *ap, @@ -53,19 +75,25 @@ $m1="%rbp"; $code=<<___; .text +.extern OPENSSL_ia32cap_P + .globl bn_mul_mont_gather5 .type bn_mul_mont_gather5,\@function,6 .align 64 bn_mul_mont_gather5: - test \$3,${num}d + test \$7,${num}d jnz .Lmul_enter - cmp \$8,${num}d - jb .Lmul_enter +___ +$code.=<<___ if ($addx); + mov OPENSSL_ia32cap_P+8(%rip),%r11d +___ +$code.=<<___; jmp .Lmul4x_enter .align 16 .Lmul_enter: mov ${num}d,${num}d + mov %rsp,%rax mov `($win64?56:8)`(%rsp),%r10d # load 7th argument push %rbx push %rbp @@ -78,10 +106,8 @@ $code.=<<___ if ($win64); lea -0x28(%rsp),%rsp movaps %xmm6,(%rsp) movaps %xmm7,0x10(%rsp) -.Lmul_alloca: ___ $code.=<<___; - mov %rsp,%rax lea 2($num),%r11 neg %r11 lea (%rsp,%r11,8),%rsp # tp=alloca(8*(num+2)) @@ -287,7 +313,7 @@ $code.=<<___; lea 1($i),$i # i++ cmp $num,$i - jl .Louter + jb .Louter xor $i,$i # i=0 and clear CF! mov (%rsp),%rax # tp[0] @@ -323,18 +349,17 @@ $code.=<<___; mov \$1,%rax ___ $code.=<<___ if ($win64); - movaps (%rsi),%xmm6 - movaps 0x10(%rsi),%xmm7 - lea 0x28(%rsi),%rsi + movaps -88(%rsi),%xmm6 + movaps -72(%rsi),%xmm7 ___ $code.=<<___; - mov (%rsi),%r15 - mov 8(%rsi),%r14 - mov 16(%rsi),%r13 - mov 24(%rsi),%r12 - mov 32(%rsi),%rbp - mov 40(%rsi),%rbx - lea 48(%rsi),%rsp + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp .Lmul_epilogue: ret .size bn_mul_mont_gather5,.-bn_mul_mont_gather5 @@ -344,11 +369,18 @@ my @A=("%r10","%r11"); my @N=("%r13","%rdi"); $code.=<<___; .type bn_mul4x_mont_gather5,\@function,6 -.align 16 +.align 32 bn_mul4x_mont_gather5: .Lmul4x_enter: - mov ${num}d,${num}d - mov `($win64?56:8)`(%rsp),%r10d # load 7th argument +___ +$code.=<<___ if ($addx); + and \$0x80100,%r11d + cmp \$0x80100,%r11d + je .Lmulx4x_enter +___ +$code.=<<___; + .byte 0x67 + mov %rsp,%rax push %rbx push %rbp push %r12 @@ -360,23 +392,78 @@ $code.=<<___ if ($win64); lea -0x28(%rsp),%rsp movaps %xmm6,(%rsp) movaps %xmm7,0x10(%rsp) -.Lmul4x_alloca: ___ $code.=<<___; - mov %rsp,%rax - lea 4($num),%r11 - neg %r11 - lea (%rsp,%r11,8),%rsp # tp=alloca(8*(num+4)) - and \$-1024,%rsp # minimize TLB usage - - mov %rax,8(%rsp,$num,8) # tp[num+1]=%rsp + .byte 0x67 + mov ${num}d,%r10d + shl \$3,${num}d + shl \$3+2,%r10d # 4*$num + neg $num # -$num + + ############################################################## + # ensure that stack frame doesn't alias with $aptr+4*$num + # modulo 4096, which covers ret[num], am[num] and n[2*num] + # (see bn_exp.c). this is done to allow memory disambiguation + # logic do its magic. [excessive frame is allocated in order + # to allow bn_from_mont8x to clear it.] + # + lea -64(%rsp,$num,2),%r11 + sub $ap,%r11 + and \$4095,%r11 + cmp %r11,%r10 + jb .Lmul4xsp_alt + sub %r11,%rsp # align with $ap + lea -64(%rsp,$num,2),%rsp # alloca(128+num*8) + jmp .Lmul4xsp_done + +.align 32 +.Lmul4xsp_alt: + lea 4096-64(,$num,2),%r10 + lea -64(%rsp,$num,2),%rsp # alloca(128+num*8) + sub %r10,%r11 + mov \$0,%r10 + cmovc %r10,%r11 + sub %r11,%rsp +.Lmul4xsp_done: + and \$-64,%rsp + neg $num + + mov %rax,40(%rsp) .Lmul4x_body: - mov $rp,16(%rsp,$num,8) # tp[num+2]=$rp - mov %rdx,%r12 # reassign $bp + + call mul4x_internal + + mov 40(%rsp),%rsi # restore %rsp + mov \$1,%rax +___ +$code.=<<___ if ($win64); + movaps -88(%rsi),%xmm6 + movaps -72(%rsi),%xmm7 +___ +$code.=<<___; + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lmul4x_epilogue: + ret +.size bn_mul4x_mont_gather5,.-bn_mul4x_mont_gather5 + +.type mul4x_internal,\@abi-omnipotent +.align 32 +mul4x_internal: + shl \$5,$num + mov `($win64?56:8)`(%rax),%r10d # load 7th argument + lea 256(%rdx,$num),%r13 + shr \$5,$num # restore $num ___ $bp="%r12"; $STRIDE=2**5*8; # 5 is "window size" $N=$STRIDE/4; # should match cache line size + $tp=$i; $code.=<<___; mov %r10,%r11 shr \$`log($N/8)/log(2)`,%r10 @@ -384,459 +471,2776 @@ $code.=<<___; not %r10 lea .Lmagic_masks(%rip),%rax and \$`2**5/($N/8)-1`,%r10 # 5 is "window size" - lea 96($bp,%r11,8),$bp # pointer within 1st cache line + lea 96(%rdx,%r11,8),$bp # pointer within 1st cache line movq 0(%rax,%r10,8),%xmm4 # set of masks denoting which movq 8(%rax,%r10,8),%xmm5 # cache line contains element + add \$7,%r11 movq 16(%rax,%r10,8),%xmm6 # denoted by 7th argument movq 24(%rax,%r10,8),%xmm7 + and \$7,%r11 movq `0*$STRIDE/4-96`($bp),%xmm0 + lea $STRIDE($bp),$tp # borrow $tp movq `1*$STRIDE/4-96`($bp),%xmm1 pand %xmm4,%xmm0 movq `2*$STRIDE/4-96`($bp),%xmm2 pand %xmm5,%xmm1 movq `3*$STRIDE/4-96`($bp),%xmm3 pand %xmm6,%xmm2 + .byte 0x67 por %xmm1,%xmm0 + movq `0*$STRIDE/4-96`($tp),%xmm1 + .byte 0x67 pand %xmm7,%xmm3 + .byte 0x67 por %xmm2,%xmm0 - lea $STRIDE($bp),$bp + movq `1*$STRIDE/4-96`($tp),%xmm2 + .byte 0x67 + pand %xmm4,%xmm1 + .byte 0x67 por %xmm3,%xmm0 + movq `2*$STRIDE/4-96`($tp),%xmm3 movq %xmm0,$m0 # m0=bp[0] + movq `3*$STRIDE/4-96`($tp),%xmm0 + mov %r13,16+8(%rsp) # save end of b[num] + mov $rp, 56+8(%rsp) # save $rp + mov ($n0),$n0 # pull n0[0] value mov ($ap),%rax - - xor $i,$i # i=0 - xor $j,$j # j=0 - - movq `0*$STRIDE/4-96`($bp),%xmm0 - movq `1*$STRIDE/4-96`($bp),%xmm1 - pand %xmm4,%xmm0 - movq `2*$STRIDE/4-96`($bp),%xmm2 - pand %xmm5,%xmm1 + lea ($ap,$num),$ap # end of a[num] + neg $num mov $n0,$m1 mulq $m0 # ap[0]*bp[0] mov %rax,$A[0] mov ($np),%rax - movq `3*$STRIDE/4-96`($bp),%xmm3 - pand %xmm6,%xmm2 - por %xmm1,%xmm0 - pand %xmm7,%xmm3 + pand %xmm5,%xmm2 + pand %xmm6,%xmm3 + por %xmm2,%xmm1 imulq $A[0],$m1 # "tp[0]"*n0 + ############################################################## + # $tp is chosen so that writing to top-most element of the + # vector occurs just "above" references to powers table, + # "above" modulo cache-line size, which effectively precludes + # possibility of memory disambiguation logic failure when + # accessing the table. + # + lea 64+8(%rsp,%r11,8),$tp mov %rdx,$A[1] - por %xmm2,%xmm0 - lea $STRIDE($bp),$bp - por %xmm3,%xmm0 + pand %xmm7,%xmm0 + por %xmm3,%xmm1 + lea 2*$STRIDE($bp),$bp + por %xmm1,%xmm0 mulq $m1 # np[0]*m1 add %rax,$A[0] # discarded - mov 8($ap),%rax + mov 8($ap,$num),%rax adc \$0,%rdx mov %rdx,$N[1] mulq $m0 add %rax,$A[1] - mov 8($np),%rax + mov 16*1($np),%rax # interleaved with 0, therefore 16*n adc \$0,%rdx mov %rdx,$A[0] mulq $m1 add %rax,$N[1] - mov 16($ap),%rax + mov 16($ap,$num),%rax adc \$0,%rdx add $A[1],$N[1] - lea 4($j),$j # j++ + lea 4*8($num),$j # j=4 + lea 16*4($np),$np adc \$0,%rdx - mov $N[1],(%rsp) + mov $N[1],($tp) mov %rdx,$N[0] jmp .L1st4x -.align 16 + +.align 32 .L1st4x: mulq $m0 # ap[j]*bp[0] add %rax,$A[0] - mov -16($np,$j,8),%rax + mov -16*2($np),%rax + lea 32($tp),$tp adc \$0,%rdx mov %rdx,$A[1] mulq $m1 # np[j]*m1 add %rax,$N[0] - mov -8($ap,$j,8),%rax + mov -8($ap,$j),%rax adc \$0,%rdx add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] adc \$0,%rdx - mov $N[0],-24(%rsp,$j,8) # tp[j-1] + mov $N[0],-24($tp) # tp[j-1] mov %rdx,$N[1] mulq $m0 # ap[j]*bp[0] add %rax,$A[1] - mov -8($np,$j,8),%rax + mov -16*1($np),%rax adc \$0,%rdx mov %rdx,$A[0] mulq $m1 # np[j]*m1 add %rax,$N[1] - mov ($ap,$j,8),%rax + mov ($ap,$j),%rax adc \$0,%rdx add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] adc \$0,%rdx - mov $N[1],-16(%rsp,$j,8) # tp[j-1] + mov $N[1],-16($tp) # tp[j-1] mov %rdx,$N[0] mulq $m0 # ap[j]*bp[0] add %rax,$A[0] - mov ($np,$j,8),%rax + mov 16*0($np),%rax adc \$0,%rdx mov %rdx,$A[1] mulq $m1 # np[j]*m1 add %rax,$N[0] - mov 8($ap,$j,8),%rax + mov 8($ap,$j),%rax adc \$0,%rdx add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] adc \$0,%rdx - mov $N[0],-8(%rsp,$j,8) # tp[j-1] + mov $N[0],-8($tp) # tp[j-1] mov %rdx,$N[1] mulq $m0 # ap[j]*bp[0] add %rax,$A[1] - mov 8($np,$j,8),%rax + mov 16*1($np),%rax adc \$0,%rdx - lea 4($j),$j # j++ mov %rdx,$A[0] mulq $m1 # np[j]*m1 add %rax,$N[1] - mov -16($ap,$j,8),%rax + mov 16($ap,$j),%rax adc \$0,%rdx add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] + lea 16*4($np),$np adc \$0,%rdx - mov $N[1],-32(%rsp,$j,8) # tp[j-1] + mov $N[1],($tp) # tp[j-1] mov %rdx,$N[0] - cmp $num,$j - jl .L1st4x + + add \$32,$j # j+=4 + jnz .L1st4x mulq $m0 # ap[j]*bp[0] add %rax,$A[0] - mov -16($np,$j,8),%rax + mov -16*2($np),%rax + lea 32($tp),$tp adc \$0,%rdx mov %rdx,$A[1] mulq $m1 # np[j]*m1 add %rax,$N[0] - mov -8($ap,$j,8),%rax + mov -8($ap),%rax adc \$0,%rdx add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] adc \$0,%rdx - mov $N[0],-24(%rsp,$j,8) # tp[j-1] + mov $N[0],-24($tp) # tp[j-1] mov %rdx,$N[1] mulq $m0 # ap[j]*bp[0] add %rax,$A[1] - mov -8($np,$j,8),%rax + mov -16*1($np),%rax adc \$0,%rdx mov %rdx,$A[0] mulq $m1 # np[j]*m1 add %rax,$N[1] - mov ($ap),%rax # ap[0] + mov ($ap,$num),%rax # ap[0] adc \$0,%rdx add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] adc \$0,%rdx - mov $N[1],-16(%rsp,$j,8) # tp[j-1] + mov $N[1],-16($tp) # tp[j-1] mov %rdx,$N[0] movq %xmm0,$m0 # bp[1] + lea ($np,$num,2),$np # rewind $np xor $N[1],$N[1] add $A[0],$N[0] adc \$0,$N[1] - mov $N[0],-8(%rsp,$j,8) - mov $N[1],(%rsp,$j,8) # store upmost overflow bit + mov $N[0],-8($tp) - lea 1($i),$i # i++ -.align 4 -.Louter4x: - xor $j,$j # j=0 - movq `0*$STRIDE/4-96`($bp),%xmm0 - movq `1*$STRIDE/4-96`($bp),%xmm1 - pand %xmm4,%xmm0 - movq `2*$STRIDE/4-96`($bp),%xmm2 - pand %xmm5,%xmm1 + jmp .Louter4x - mov (%rsp),$A[0] +.align 32 +.Louter4x: + mov ($tp,$num),$A[0] mov $n0,$m1 mulq $m0 # ap[0]*bp[i] add %rax,$A[0] # ap[0]*bp[i]+tp[0] mov ($np),%rax adc \$0,%rdx + movq `0*$STRIDE/4-96`($bp),%xmm0 + movq `1*$STRIDE/4-96`($bp),%xmm1 + pand %xmm4,%xmm0 + movq `2*$STRIDE/4-96`($bp),%xmm2 + pand %xmm5,%xmm1 movq `3*$STRIDE/4-96`($bp),%xmm3 - pand %xmm6,%xmm2 - por %xmm1,%xmm0 - pand %xmm7,%xmm3 imulq $A[0],$m1 # tp[0]*n0 + .byte 0x67 mov %rdx,$A[1] + mov $N[1],($tp) # store upmost overflow bit + pand %xmm6,%xmm2 + por %xmm1,%xmm0 + pand %xmm7,%xmm3 por %xmm2,%xmm0 + lea ($tp,$num),$tp # rewind $tp lea $STRIDE($bp),$bp por %xmm3,%xmm0 mulq $m1 # np[0]*m1 add %rax,$A[0] # "$N[0]", discarded - mov 8($ap),%rax + mov 8($ap,$num),%rax adc \$0,%rdx mov %rdx,$N[1] mulq $m0 # ap[j]*bp[i] add %rax,$A[1] - mov 8($np),%rax + mov 16*1($np),%rax # interleaved with 0, therefore 16*n adc \$0,%rdx - add 8(%rsp),$A[1] # +tp[1] + add 8($tp),$A[1] # +tp[1] adc \$0,%rdx mov %rdx,$A[0] mulq $m1 # np[j]*m1 add %rax,$N[1] - mov 16($ap),%rax + mov 16($ap,$num),%rax adc \$0,%rdx add $A[1],$N[1] # np[j]*m1+ap[j]*bp[i]+tp[j] - lea 4($j),$j # j+=2 + lea 4*8($num),$j # j=4 + lea 16*4($np),$np adc \$0,%rdx mov %rdx,$N[0] jmp .Linner4x -.align 16 + +.align 32 .Linner4x: mulq $m0 # ap[j]*bp[i] add %rax,$A[0] - mov -16($np,$j,8),%rax + mov -16*2($np),%rax adc \$0,%rdx - add -16(%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j] + add 16($tp),$A[0] # ap[j]*bp[i]+tp[j] + lea 32($tp),$tp adc \$0,%rdx mov %rdx,$A[1] mulq $m1 # np[j]*m1 add %rax,$N[0] - mov -8($ap,$j,8),%rax + mov -8($ap,$j),%rax adc \$0,%rdx add $A[0],$N[0] adc \$0,%rdx - mov $N[1],-32(%rsp,$j,8) # tp[j-1] + mov $N[1],-32($tp) # tp[j-1] mov %rdx,$N[1] mulq $m0 # ap[j]*bp[i] add %rax,$A[1] - mov -8($np,$j,8),%rax + mov -16*1($np),%rax adc \$0,%rdx - add -8(%rsp,$j,8),$A[1] + add -8($tp),$A[1] adc \$0,%rdx mov %rdx,$A[0] mulq $m1 # np[j]*m1 add %rax,$N[1] - mov ($ap,$j,8),%rax + mov ($ap,$j),%rax adc \$0,%rdx add $A[1],$N[1] adc \$0,%rdx - mov $N[0],-24(%rsp,$j,8) # tp[j-1] + mov $N[0],-24($tp) # tp[j-1] mov %rdx,$N[0] mulq $m0 # ap[j]*bp[i] add %rax,$A[0] - mov ($np,$j,8),%rax + mov 16*0($np),%rax adc \$0,%rdx - add (%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j] + add ($tp),$A[0] # ap[j]*bp[i]+tp[j] adc \$0,%rdx mov %rdx,$A[1] mulq $m1 # np[j]*m1 add %rax,$N[0] - mov 8($ap,$j,8),%rax + mov 8($ap,$j),%rax adc \$0,%rdx add $A[0],$N[0] adc \$0,%rdx - mov $N[1],-16(%rsp,$j,8) # tp[j-1] + mov $N[1],-16($tp) # tp[j-1] mov %rdx,$N[1] mulq $m0 # ap[j]*bp[i] add %rax,$A[1] - mov 8($np,$j,8),%rax + mov 16*1($np),%rax adc \$0,%rdx - add 8(%rsp,$j,8),$A[1] + add 8($tp),$A[1] adc \$0,%rdx - lea 4($j),$j # j++ mov %rdx,$A[0] mulq $m1 # np[j]*m1 add %rax,$N[1] - mov -16($ap,$j,8),%rax + mov 16($ap,$j),%rax adc \$0,%rdx add $A[1],$N[1] + lea 16*4($np),$np adc \$0,%rdx - mov $N[0],-40(%rsp,$j,8) # tp[j-1] + mov $N[0],-8($tp) # tp[j-1] mov %rdx,$N[0] - cmp $num,$j - jl .Linner4x + + add \$32,$j # j+=4 + jnz .Linner4x mulq $m0 # ap[j]*bp[i] add %rax,$A[0] - mov -16($np,$j,8),%rax + mov -16*2($np),%rax adc \$0,%rdx - add -16(%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j] + add 16($tp),$A[0] # ap[j]*bp[i]+tp[j] + lea 32($tp),$tp adc \$0,%rdx mov %rdx,$A[1] mulq $m1 # np[j]*m1 add %rax,$N[0] - mov -8($ap,$j,8),%rax + mov -8($ap),%rax adc \$0,%rdx add $A[0],$N[0] adc \$0,%rdx - mov $N[1],-32(%rsp,$j,8) # tp[j-1] + mov $N[1],-32($tp) # tp[j-1] mov %rdx,$N[1] mulq $m0 # ap[j]*bp[i] add %rax,$A[1] - mov -8($np,$j,8),%rax + mov $m1,%rax + mov -16*1($np),$m1 adc \$0,%rdx - add -8(%rsp,$j,8),$A[1] + add -8($tp),$A[1] adc \$0,%rdx - lea 1($i),$i # i++ mov %rdx,$A[0] mulq $m1 # np[j]*m1 add %rax,$N[1] - mov ($ap),%rax # ap[0] + mov ($ap,$num),%rax # ap[0] adc \$0,%rdx add $A[1],$N[1] adc \$0,%rdx - mov $N[0],-24(%rsp,$j,8) # tp[j-1] + mov $N[0],-24($tp) # tp[j-1] mov %rdx,$N[0] movq %xmm0,$m0 # bp[i+1] - mov $N[1],-16(%rsp,$j,8) # tp[j-1] + mov $N[1],-16($tp) # tp[j-1] + lea ($np,$num,2),$np # rewind $np xor $N[1],$N[1] add $A[0],$N[0] adc \$0,$N[1] - add (%rsp,$num,8),$N[0] # pull upmost overflow bit - adc \$0,$N[1] - mov $N[0],-8(%rsp,$j,8) - mov $N[1],(%rsp,$j,8) # store upmost overflow bit + add ($tp),$N[0] # pull upmost overflow bit + adc \$0,$N[1] # upmost overflow bit + mov $N[0],-8($tp) - cmp $num,$i - jl .Louter4x + cmp 16+8(%rsp),$bp + jb .Louter4x ___ -{ -my @ri=("%rax","%rdx",$m0,$m1); +if (1) { $code.=<<___; - mov 16(%rsp,$num,8),$rp # restore $rp - mov 0(%rsp),@ri[0] # tp[0] - pxor %xmm0,%xmm0 - mov 8(%rsp),@ri[1] # tp[1] - shr \$2,$num # num/=4 - lea (%rsp),$ap # borrow ap for tp - xor $i,$i # i=0 and clear CF! - - sub 0($np),@ri[0] - mov 16($ap),@ri[2] # tp[2] - mov 24($ap),@ri[3] # tp[3] - sbb 8($np),@ri[1] - lea -1($num),$j # j=num/4-1 + sub $N[0],$m1 # compare top-most words + adc $j,$j # $j is zero + or $j,$N[1] + xor \$1,$N[1] + lea ($tp,$num),%rbx # tptr in .sqr4x_sub + lea ($np,$N[1],8),%rbp # nptr in .sqr4x_sub + mov %r9,%rcx + sar \$3+2,%rcx # cf=0 + mov 56+8(%rsp),%rdi # rptr in .sqr4x_sub + jmp .Lsqr4x_sub +___ +} else { +my @ri=("%rax",$bp,$m0,$m1); +my $rp="%rdx"; +$code.=<<___ + xor \$1,$N[1] + lea ($tp,$num),$tp # rewind $tp + sar \$5,$num # cf=0 + lea ($np,$N[1],8),$np + mov 56+8(%rsp),$rp # restore $rp jmp .Lsub4x -.align 16 + +.align 32 .Lsub4x: - mov @ri[0],0($rp,$i,8) # rp[i]=tp[i]-np[i] - mov @ri[1],8($rp,$i,8) # rp[i]=tp[i]-np[i] - sbb 16($np,$i,8),@ri[2] - mov 32($ap,$i,8),@ri[0] # tp[i+1] - mov 40($ap,$i,8),@ri[1] - sbb 24($np,$i,8),@ri[3] - mov @ri[2],16($rp,$i,8) # rp[i]=tp[i]-np[i] - mov @ri[3],24($rp,$i,8) # rp[i]=tp[i]-np[i] - sbb 32($np,$i,8),@ri[0] - mov 48($ap,$i,8),@ri[2] - mov 56($ap,$i,8),@ri[3] - sbb 40($np,$i,8),@ri[1] - lea 4($i),$i # i++ - dec $j # doesnn't affect CF! + .byte 0x66 + mov 8*0($tp),@ri[0] + mov 8*1($tp),@ri[1] + .byte 0x66 + sbb 16*0($np),@ri[0] + mov 8*2($tp),@ri[2] + sbb 16*1($np),@ri[1] + mov 3*8($tp),@ri[3] + lea 4*8($tp),$tp + sbb 16*2($np),@ri[2] + mov @ri[0],8*0($rp) + sbb 16*3($np),@ri[3] + lea 16*4($np),$np + mov @ri[1],8*1($rp) + mov @ri[2],8*2($rp) + mov @ri[3],8*3($rp) + lea 8*4($rp),$rp + + inc $num jnz .Lsub4x - mov @ri[0],0($rp,$i,8) # rp[i]=tp[i]-np[i] - mov 32($ap,$i,8),@ri[0] # load overflow bit - sbb 16($np,$i,8),@ri[2] - mov @ri[1],8($rp,$i,8) # rp[i]=tp[i]-np[i] - sbb 24($np,$i,8),@ri[3] - mov @ri[2],16($rp,$i,8) # rp[i]=tp[i]-np[i] + ret +___ +} +$code.=<<___; +.size mul4x_internal,.-mul4x_internal +___ +}}} + {{{ +###################################################################### +# void bn_power5( +my $rptr="%rdi"; # BN_ULONG *rptr, +my $aptr="%rsi"; # const BN_ULONG *aptr, +my $bptr="%rdx"; # const void *table, +my $nptr="%rcx"; # const BN_ULONG *nptr, +my $n0 ="%r8"; # const BN_ULONG *n0); +my $num ="%r9"; # int num, has to be divisible by 8 + # int pwr + +my ($i,$j,$tptr)=("%rbp","%rcx",$rptr); +my @A0=("%r10","%r11"); +my @A1=("%r12","%r13"); +my ($a0,$a1,$ai)=("%r14","%r15","%rbx"); - sbb \$0,@ri[0] # handle upmost overflow bit - mov @ri[3],24($rp,$i,8) # rp[i]=tp[i]-np[i] - xor $i,$i # i=0 - and @ri[0],$ap - not @ri[0] - mov $rp,$np - and @ri[0],$np - lea -1($num),$j - or $np,$ap # ap=borrow?tp:rp +$code.=<<___; +.globl bn_power5 +.type bn_power5,\@function,6 +.align 32 +bn_power5: +___ +$code.=<<___ if ($addx); + mov OPENSSL_ia32cap_P+8(%rip),%r11d + and \$0x80100,%r11d + cmp \$0x80100,%r11d + je .Lpowerx5_enter +___ +$code.=<<___; + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0x28(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) +___ +$code.=<<___; + mov ${num}d,%r10d + shl \$3,${num}d # convert $num to bytes + shl \$3+2,%r10d # 4*$num + neg $num + mov ($n0),$n0 # *n0 + + ############################################################## + # ensure that stack frame doesn't alias with $aptr+4*$num + # modulo 4096, which covers ret[num], am[num] and n[2*num] + # (see bn_exp.c). this is done to allow memory disambiguation + # logic do its magic. + # + lea -64(%rsp,$num,2),%r11 + sub $aptr,%r11 + and \$4095,%r11 + cmp %r11,%r10 + jb .Lpwr_sp_alt + sub %r11,%rsp # align with $aptr + lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) + jmp .Lpwr_sp_done + +.align 32 +.Lpwr_sp_alt: + lea 4096-64(,$num,2),%r10 # 4096-frame-2*$num + lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) + sub %r10,%r11 + mov \$0,%r10 + cmovc %r10,%r11 + sub %r11,%rsp +.Lpwr_sp_done: + and \$-64,%rsp + mov $num,%r10 + neg $num + + ############################################################## + # Stack layout + # + # +0 saved $num, used in reduction section + # +8 &t[2*$num], used in reduction section + # +32 saved *n0 + # +40 saved %rsp + # +48 t[2*$num] + # + mov $n0, 32(%rsp) + mov %rax, 40(%rsp) # save original %rsp +.Lpower5_body: + movq $rptr,%xmm1 # save $rptr + movq $nptr,%xmm2 # save $nptr + movq %r10, %xmm3 # -$num + movq $bptr,%xmm4 + + call __bn_sqr8x_internal + call __bn_sqr8x_internal + call __bn_sqr8x_internal + call __bn_sqr8x_internal + call __bn_sqr8x_internal + + movq %xmm2,$nptr + movq %xmm4,$bptr + mov $aptr,$rptr + mov 40(%rsp),%rax + lea 32(%rsp),$n0 + + call mul4x_internal + + mov 40(%rsp),%rsi # restore %rsp + mov \$1,%rax + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lpower5_epilogue: + ret +.size bn_power5,.-bn_power5 + +.globl bn_sqr8x_internal +.hidden bn_sqr8x_internal +.type bn_sqr8x_internal,\@abi-omnipotent +.align 32 +bn_sqr8x_internal: +__bn_sqr8x_internal: + ############################################################## + # Squaring part: + # + # a) multiply-n-add everything but a[i]*a[i]; + # b) shift result of a) by 1 to the left and accumulate + # a[i]*a[i] products; + # + ############################################################## + # a[1]a[0] + # a[2]a[0] + # a[3]a[0] + # a[2]a[1] + # a[4]a[0] + # a[3]a[1] + # a[5]a[0] + # a[4]a[1] + # a[3]a[2] + # a[6]a[0] + # a[5]a[1] + # a[4]a[2] + # a[7]a[0] + # a[6]a[1] + # a[5]a[2] + # a[4]a[3] + # a[7]a[1] + # a[6]a[2] + # a[5]a[3] + # a[7]a[2] + # a[6]a[3] + # a[5]a[4] + # a[7]a[3] + # a[6]a[4] + # a[7]a[4] + # a[6]a[5] + # a[7]a[5] + # a[7]a[6] + # a[1]a[0] + # a[2]a[0] + # a[3]a[0] + # a[4]a[0] + # a[5]a[0] + # a[6]a[0] + # a[7]a[0] + # a[2]a[1] + # a[3]a[1] + # a[4]a[1] + # a[5]a[1] + # a[6]a[1] + # a[7]a[1] + # a[3]a[2] + # a[4]a[2] + # a[5]a[2] + # a[6]a[2] + # a[7]a[2] + # a[4]a[3] + # a[5]a[3] + # a[6]a[3] + # a[7]a[3] + # a[5]a[4] + # a[6]a[4] + # a[7]a[4] + # a[6]a[5] + # a[7]a[5] + # a[7]a[6] + # a[0]a[0] + # a[1]a[1] + # a[2]a[2] + # a[3]a[3] + # a[4]a[4] + # a[5]a[5] + # a[6]a[6] + # a[7]a[7] + + lea 32(%r10),$i # $i=-($num-32) + lea ($aptr,$num),$aptr # end of a[] buffer, ($aptr,$i)=&ap[2] + + mov $num,$j # $j=$num + + # comments apply to $num==8 case + mov -32($aptr,$i),$a0 # a[0] + lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] + mov -24($aptr,$i),%rax # a[1] + lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] + mov -16($aptr,$i),$ai # a[2] + mov %rax,$a1 + + mul $a0 # a[1]*a[0] + mov %rax,$A0[0] # a[1]*a[0] + mov $ai,%rax # a[2] + mov %rdx,$A0[1] + mov $A0[0],-24($tptr,$i) # t[1] + + mul $a0 # a[2]*a[0] + add %rax,$A0[1] + mov $ai,%rax + adc \$0,%rdx + mov $A0[1],-16($tptr,$i) # t[2] + mov %rdx,$A0[0] + + + mov -8($aptr,$i),$ai # a[3] + mul $a1 # a[2]*a[1] + mov %rax,$A1[0] # a[2]*a[1]+t[3] + mov $ai,%rax + mov %rdx,$A1[1] + + lea ($i),$j + mul $a0 # a[3]*a[0] + add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] + mov $ai,%rax + mov %rdx,$A0[1] + adc \$0,$A0[1] + add $A1[0],$A0[0] + adc \$0,$A0[1] + mov $A0[0],-8($tptr,$j) # t[3] + jmp .Lsqr4x_1st + +.align 32 +.Lsqr4x_1st: + mov ($aptr,$j),$ai # a[4] + mul $a1 # a[3]*a[1] + add %rax,$A1[1] # a[3]*a[1]+t[4] + mov $ai,%rax + mov %rdx,$A1[0] + adc \$0,$A1[0] + + mul $a0 # a[4]*a[0] + add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4] + mov $ai,%rax # a[3] + mov 8($aptr,$j),$ai # a[5] + mov %rdx,$A0[0] + adc \$0,$A0[0] + add $A1[1],$A0[1] + adc \$0,$A0[0] + + + mul $a1 # a[4]*a[3] + add %rax,$A1[0] # a[4]*a[3]+t[5] + mov $ai,%rax + mov $A0[1],($tptr,$j) # t[4] + mov %rdx,$A1[1] + adc \$0,$A1[1] + + mul $a0 # a[5]*a[2] + add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5] + mov $ai,%rax + mov 16($aptr,$j),$ai # a[6] + mov %rdx,$A0[1] + adc \$0,$A0[1] + add $A1[0],$A0[0] + adc \$0,$A0[1] + + mul $a1 # a[5]*a[3] + add %rax,$A1[1] # a[5]*a[3]+t[6] + mov $ai,%rax + mov $A0[0],8($tptr,$j) # t[5] + mov %rdx,$A1[0] + adc \$0,$A1[0] + + mul $a0 # a[6]*a[2] + add %rax,$A0[1] # a[6]*a[2]+a[5]*a[3]+t[6] + mov $ai,%rax # a[3] + mov 24($aptr,$j),$ai # a[7] + mov %rdx,$A0[0] + adc \$0,$A0[0] + add $A1[1],$A0[1] + adc \$0,$A0[0] + + + mul $a1 # a[6]*a[5] + add %rax,$A1[0] # a[6]*a[5]+t[7] + mov $ai,%rax + mov $A0[1],16($tptr,$j) # t[6] + mov %rdx,$A1[1] + adc \$0,$A1[1] + lea 32($j),$j + + mul $a0 # a[7]*a[4] + add %rax,$A0[0] # a[7]*a[4]+a[6]*a[5]+t[6] + mov $ai,%rax + mov %rdx,$A0[1] + adc \$0,$A0[1] + add $A1[0],$A0[0] + adc \$0,$A0[1] + mov $A0[0],-8($tptr,$j) # t[7] + + cmp \$0,$j + jne .Lsqr4x_1st + + mul $a1 # a[7]*a[5] + add %rax,$A1[1] + lea 16($i),$i + adc \$0,%rdx + add $A0[1],$A1[1] + adc \$0,%rdx + + mov $A1[1],($tptr) # t[8] + mov %rdx,$A1[0] + mov %rdx,8($tptr) # t[9] + jmp .Lsqr4x_outer + +.align 32 +.Lsqr4x_outer: # comments apply to $num==6 case + mov -32($aptr,$i),$a0 # a[0] + lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] + mov -24($aptr,$i),%rax # a[1] + lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] + mov -16($aptr,$i),$ai # a[2] + mov %rax,$a1 + + mul $a0 # a[1]*a[0] + mov -24($tptr,$i),$A0[0] # t[1] + add %rax,$A0[0] # a[1]*a[0]+t[1] + mov $ai,%rax # a[2] + adc \$0,%rdx + mov $A0[0],-24($tptr,$i) # t[1] + mov %rdx,$A0[1] + + mul $a0 # a[2]*a[0] + add %rax,$A0[1] + mov $ai,%rax + adc \$0,%rdx + add -16($tptr,$i),$A0[1] # a[2]*a[0]+t[2] + mov %rdx,$A0[0] + adc \$0,$A0[0] + mov $A0[1],-16($tptr,$i) # t[2] + + xor $A1[0],$A1[0] + + mov -8($aptr,$i),$ai # a[3] + mul $a1 # a[2]*a[1] + add %rax,$A1[0] # a[2]*a[1]+t[3] + mov $ai,%rax + adc \$0,%rdx + add -8($tptr,$i),$A1[0] + mov %rdx,$A1[1] + adc \$0,$A1[1] + + mul $a0 # a[3]*a[0] + add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] + mov $ai,%rax + adc \$0,%rdx + add $A1[0],$A0[0] + mov %rdx,$A0[1] + adc \$0,$A0[1] + mov $A0[0],-8($tptr,$i) # t[3] + + lea ($i),$j + jmp .Lsqr4x_inner + +.align 32 +.Lsqr4x_inner: + mov ($aptr,$j),$ai # a[4] + mul $a1 # a[3]*a[1] + add %rax,$A1[1] # a[3]*a[1]+t[4] + mov $ai,%rax + mov %rdx,$A1[0] + adc \$0,$A1[0] + add ($tptr,$j),$A1[1] + adc \$0,$A1[0] + + .byte 0x67 + mul $a0 # a[4]*a[0] + add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4] + mov $ai,%rax # a[3] + mov 8($aptr,$j),$ai # a[5] + mov %rdx,$A0[0] + adc \$0,$A0[0] + add $A1[1],$A0[1] + adc \$0,$A0[0] + + mul $a1 # a[4]*a[3] + add %rax,$A1[0] # a[4]*a[3]+t[5] + mov $A0[1],($tptr,$j) # t[4] + mov $ai,%rax + mov %rdx,$A1[1] + adc \$0,$A1[1] + add 8($tptr,$j),$A1[0] + lea 16($j),$j # j++ + adc \$0,$A1[1] + + mul $a0 # a[5]*a[2] + add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5] + mov $ai,%rax + adc \$0,%rdx + add $A1[0],$A0[0] + mov %rdx,$A0[1] + adc \$0,$A0[1] + mov $A0[0],-8($tptr,$j) # t[5], "preloaded t[1]" below + + cmp \$0,$j + jne .Lsqr4x_inner + + .byte 0x67 + mul $a1 # a[5]*a[3] + add %rax,$A1[1] + adc \$0,%rdx + add $A0[1],$A1[1] + adc \$0,%rdx + + mov $A1[1],($tptr) # t[6], "preloaded t[2]" below + mov %rdx,$A1[0] + mov %rdx,8($tptr) # t[7], "preloaded t[3]" below + + add \$16,$i + jnz .Lsqr4x_outer + + # comments apply to $num==4 case + mov -32($aptr),$a0 # a[0] + lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] + mov -24($aptr),%rax # a[1] + lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] + mov -16($aptr),$ai # a[2] + mov %rax,$a1 + + mul $a0 # a[1]*a[0] + add %rax,$A0[0] # a[1]*a[0]+t[1], preloaded t[1] + mov $ai,%rax # a[2] + mov %rdx,$A0[1] + adc \$0,$A0[1] + + mul $a0 # a[2]*a[0] + add %rax,$A0[1] + mov $ai,%rax + mov $A0[0],-24($tptr) # t[1] + mov %rdx,$A0[0] + adc \$0,$A0[0] + add $A1[1],$A0[1] # a[2]*a[0]+t[2], preloaded t[2] + mov -8($aptr),$ai # a[3] + adc \$0,$A0[0] + + mul $a1 # a[2]*a[1] + add %rax,$A1[0] # a[2]*a[1]+t[3], preloaded t[3] + mov $ai,%rax + mov $A0[1],-16($tptr) # t[2] + mov %rdx,$A1[1] + adc \$0,$A1[1] + + mul $a0 # a[3]*a[0] + add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] + mov $ai,%rax + mov %rdx,$A0[1] + adc \$0,$A0[1] + add $A1[0],$A0[0] + adc \$0,$A0[1] + mov $A0[0],-8($tptr) # t[3] + + mul $a1 # a[3]*a[1] + add %rax,$A1[1] + mov -16($aptr),%rax # a[2] + adc \$0,%rdx + add $A0[1],$A1[1] + adc \$0,%rdx + + mov $A1[1],($tptr) # t[4] + mov %rdx,$A1[0] + mov %rdx,8($tptr) # t[5] + + mul $ai # a[2]*a[3] +___ +{ +my ($shift,$carry)=($a0,$a1); +my @S=(@A1,$ai,$n0); +$code.=<<___; + add \$16,$i + xor $shift,$shift + sub $num,$i # $i=16-$num + xor $carry,$carry + + add $A1[0],%rax # t[5] + adc \$0,%rdx + mov %rax,8($tptr) # t[5] + mov %rdx,16($tptr) # t[6] + mov $carry,24($tptr) # t[7] + + mov -16($aptr,$i),%rax # a[0] + lea 48+8(%rsp),$tptr + xor $A0[0],$A0[0] # t[0] + mov 8($tptr),$A0[1] # t[1] + + lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[1] # | t[2*i]>>63 + mov 16($tptr),$A0[0] # t[2*i+2] # prefetch + mov $A0[1],$shift # shift=t[2*i+1]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + mov 24($tptr),$A0[1] # t[2*i+2+1] # prefetch + adc %rax,$S[0] + mov -8($aptr,$i),%rax # a[i+1] # prefetch + mov $S[0],($tptr) + adc %rdx,$S[1] + + lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift + mov $S[1],8($tptr) + sbb $carry,$carry # mov cf,$carry + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[3] # | t[2*i]>>63 + mov 32($tptr),$A0[0] # t[2*i+2] # prefetch + mov $A0[1],$shift # shift=t[2*i+1]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + mov 40($tptr),$A0[1] # t[2*i+2+1] # prefetch + adc %rax,$S[2] + mov 0($aptr,$i),%rax # a[i+1] # prefetch + mov $S[2],16($tptr) + adc %rdx,$S[3] + lea 16($i),$i + mov $S[3],24($tptr) + sbb $carry,$carry # mov cf,$carry + lea 64($tptr),$tptr + jmp .Lsqr4x_shift_n_add + +.align 32 +.Lsqr4x_shift_n_add: + lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[1] # | t[2*i]>>63 + mov -16($tptr),$A0[0] # t[2*i+2] # prefetch + mov $A0[1],$shift # shift=t[2*i+1]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + mov -8($tptr),$A0[1] # t[2*i+2+1] # prefetch + adc %rax,$S[0] + mov -8($aptr,$i),%rax # a[i+1] # prefetch + mov $S[0],-32($tptr) + adc %rdx,$S[1] + + lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift + mov $S[1],-24($tptr) + sbb $carry,$carry # mov cf,$carry + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[3] # | t[2*i]>>63 + mov 0($tptr),$A0[0] # t[2*i+2] # prefetch + mov $A0[1],$shift # shift=t[2*i+1]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + mov 8($tptr),$A0[1] # t[2*i+2+1] # prefetch + adc %rax,$S[2] + mov 0($aptr,$i),%rax # a[i+1] # prefetch + mov $S[2],-16($tptr) + adc %rdx,$S[3] + + lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift + mov $S[3],-8($tptr) + sbb $carry,$carry # mov cf,$carry + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[1] # | t[2*i]>>63 + mov 16($tptr),$A0[0] # t[2*i+2] # prefetch + mov $A0[1],$shift # shift=t[2*i+1]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + mov 24($tptr),$A0[1] # t[2*i+2+1] # prefetch + adc %rax,$S[0] + mov 8($aptr,$i),%rax # a[i+1] # prefetch + mov $S[0],0($tptr) + adc %rdx,$S[1] + + lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift + mov $S[1],8($tptr) + sbb $carry,$carry # mov cf,$carry + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[3] # | t[2*i]>>63 + mov 32($tptr),$A0[0] # t[2*i+2] # prefetch + mov $A0[1],$shift # shift=t[2*i+1]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + mov 40($tptr),$A0[1] # t[2*i+2+1] # prefetch + adc %rax,$S[2] + mov 16($aptr,$i),%rax # a[i+1] # prefetch + mov $S[2],16($tptr) + adc %rdx,$S[3] + mov $S[3],24($tptr) + sbb $carry,$carry # mov cf,$carry + lea 64($tptr),$tptr + add \$32,$i + jnz .Lsqr4x_shift_n_add + + lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift + .byte 0x67 + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[1] # | t[2*i]>>63 + mov -16($tptr),$A0[0] # t[2*i+2] # prefetch + mov $A0[1],$shift # shift=t[2*i+1]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + mov -8($tptr),$A0[1] # t[2*i+2+1] # prefetch + adc %rax,$S[0] + mov -8($aptr),%rax # a[i+1] # prefetch + mov $S[0],-32($tptr) + adc %rdx,$S[1] + + lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1|shift + mov $S[1],-24($tptr) + sbb $carry,$carry # mov cf,$carry + shr \$63,$A0[0] + lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | + shr \$63,$A0[1] + or $A0[0],$S[3] # | t[2*i]>>63 + mul %rax # a[i]*a[i] + neg $carry # mov $carry,cf + adc %rax,$S[2] + adc %rdx,$S[3] + mov $S[2],-16($tptr) + mov $S[3],-8($tptr) +___ +} +###################################################################### +# Montgomery reduction part, "word-by-word" algorithm. +# +# This new path is inspired by multiple submissions from Intel, by +# Shay Gueron, Vlad Krasnov, Erdinc Ozturk, James Guilford, +# Vinodh Gopal... +{ +my ($nptr,$tptr,$carry,$m0)=("%rbp","%rdi","%rsi","%rbx"); - movdqu ($ap),%xmm1 - movdqa %xmm0,(%rsp) - movdqu %xmm1,($rp) - jmp .Lcopy4x -.align 16 -.Lcopy4x: # copy or in-place refresh - movdqu 16($ap,$i),%xmm2 - movdqu 32($ap,$i),%xmm1 - movdqa %xmm0,16(%rsp,$i) - movdqu %xmm2,16($rp,$i) - movdqa %xmm0,32(%rsp,$i) - movdqu %xmm1,32($rp,$i) - lea 32($i),$i - dec $j - jnz .Lcopy4x - - shl \$2,$num - movdqu 16($ap,$i),%xmm2 - movdqa %xmm0,16(%rsp,$i) - movdqu %xmm2,16($rp,$i) +$code.=<<___; + movq %xmm2,$nptr +sqr8x_reduction: + xor %rax,%rax + lea ($nptr,$num,2),%rcx # end of n[] + lea 48+8(%rsp,$num,2),%rdx # end of t[] buffer + mov %rcx,0+8(%rsp) + lea 48+8(%rsp,$num),$tptr # end of initial t[] window + mov %rdx,8+8(%rsp) + neg $num + jmp .L8x_reduction_loop + +.align 32 +.L8x_reduction_loop: + lea ($tptr,$num),$tptr # start of current t[] window + .byte 0x66 + mov 8*0($tptr),$m0 + mov 8*1($tptr),%r9 + mov 8*2($tptr),%r10 + mov 8*3($tptr),%r11 + mov 8*4($tptr),%r12 + mov 8*5($tptr),%r13 + mov 8*6($tptr),%r14 + mov 8*7($tptr),%r15 + mov %rax,(%rdx) # store top-most carry bit + lea 8*8($tptr),$tptr + + .byte 0x67 + mov $m0,%r8 + imulq 32+8(%rsp),$m0 # n0*a[0] + mov 16*0($nptr),%rax # n[0] + mov \$8,%ecx + jmp .L8x_reduce + +.align 32 +.L8x_reduce: + mulq $m0 + mov 16*1($nptr),%rax # n[1] + neg %r8 + mov %rdx,%r8 + adc \$0,%r8 + + mulq $m0 + add %rax,%r9 + mov 16*2($nptr),%rax + adc \$0,%rdx + add %r9,%r8 + mov $m0,48-8+8(%rsp,%rcx,8) # put aside n0*a[i] + mov %rdx,%r9 + adc \$0,%r9 + + mulq $m0 + add %rax,%r10 + mov 16*3($nptr),%rax + adc \$0,%rdx + add %r10,%r9 + mov 32+8(%rsp),$carry # pull n0, borrow $carry + mov %rdx,%r10 + adc \$0,%r10 + + mulq $m0 + add %rax,%r11 + mov 16*4($nptr),%rax + adc \$0,%rdx + imulq %r8,$carry # modulo-scheduled + add %r11,%r10 + mov %rdx,%r11 + adc \$0,%r11 + + mulq $m0 + add %rax,%r12 + mov 16*5($nptr),%rax + adc \$0,%rdx + add %r12,%r11 + mov %rdx,%r12 + adc \$0,%r12 + + mulq $m0 + add %rax,%r13 + mov 16*6($nptr),%rax + adc \$0,%rdx + add %r13,%r12 + mov %rdx,%r13 + adc \$0,%r13 + + mulq $m0 + add %rax,%r14 + mov 16*7($nptr),%rax + adc \$0,%rdx + add %r14,%r13 + mov %rdx,%r14 + adc \$0,%r14 + + mulq $m0 + mov $carry,$m0 # n0*a[i] + add %rax,%r15 + mov 16*0($nptr),%rax # n[0] + adc \$0,%rdx + add %r15,%r14 + mov %rdx,%r15 + adc \$0,%r15 + + dec %ecx + jnz .L8x_reduce + + lea 16*8($nptr),$nptr + xor %rax,%rax + mov 8+8(%rsp),%rdx # pull end of t[] + cmp 0+8(%rsp),$nptr # end of n[]? + jae .L8x_no_tail + + .byte 0x66 + add 8*0($tptr),%r8 + adc 8*1($tptr),%r9 + adc 8*2($tptr),%r10 + adc 8*3($tptr),%r11 + adc 8*4($tptr),%r12 + adc 8*5($tptr),%r13 + adc 8*6($tptr),%r14 + adc 8*7($tptr),%r15 + sbb $carry,$carry # top carry + + mov 48+56+8(%rsp),$m0 # pull n0*a[0] + mov \$8,%ecx + mov 16*0($nptr),%rax + jmp .L8x_tail + +.align 32 +.L8x_tail: + mulq $m0 + add %rax,%r8 + mov 16*1($nptr),%rax + mov %r8,($tptr) # save result + mov %rdx,%r8 + adc \$0,%r8 + + mulq $m0 + add %rax,%r9 + mov 16*2($nptr),%rax + adc \$0,%rdx + add %r9,%r8 + lea 8($tptr),$tptr # $tptr++ + mov %rdx,%r9 + adc \$0,%r9 + + mulq $m0 + add %rax,%r10 + mov 16*3($nptr),%rax + adc \$0,%rdx + add %r10,%r9 + mov %rdx,%r10 + adc \$0,%r10 + + mulq $m0 + add %rax,%r11 + mov 16*4($nptr),%rax + adc \$0,%rdx + add %r11,%r10 + mov %rdx,%r11 + adc \$0,%r11 + + mulq $m0 + add %rax,%r12 + mov 16*5($nptr),%rax + adc \$0,%rdx + add %r12,%r11 + mov %rdx,%r12 + adc \$0,%r12 + + mulq $m0 + add %rax,%r13 + mov 16*6($nptr),%rax + adc \$0,%rdx + add %r13,%r12 + mov %rdx,%r13 + adc \$0,%r13 + + mulq $m0 + add %rax,%r14 + mov 16*7($nptr),%rax + adc \$0,%rdx + add %r14,%r13 + mov %rdx,%r14 + adc \$0,%r14 + + mulq $m0 + mov 48-16+8(%rsp,%rcx,8),$m0# pull n0*a[i] + add %rax,%r15 + adc \$0,%rdx + add %r15,%r14 + mov 16*0($nptr),%rax # pull n[0] + mov %rdx,%r15 + adc \$0,%r15 + + dec %ecx + jnz .L8x_tail + + lea 16*8($nptr),$nptr + mov 8+8(%rsp),%rdx # pull end of t[] + cmp 0+8(%rsp),$nptr # end of n[]? + jae .L8x_tail_done # break out of loop + + mov 48+56+8(%rsp),$m0 # pull n0*a[0] + neg $carry + mov 8*0($nptr),%rax # pull n[0] + adc 8*0($tptr),%r8 + adc 8*1($tptr),%r9 + adc 8*2($tptr),%r10 + adc 8*3($tptr),%r11 + adc 8*4($tptr),%r12 + adc 8*5($tptr),%r13 + adc 8*6($tptr),%r14 + adc 8*7($tptr),%r15 + sbb $carry,$carry # top carry + + mov \$8,%ecx + jmp .L8x_tail + +.align 32 +.L8x_tail_done: + add (%rdx),%r8 # can this overflow? + xor %rax,%rax + + neg $carry +.L8x_no_tail: + adc 8*0($tptr),%r8 + adc 8*1($tptr),%r9 + adc 8*2($tptr),%r10 + adc 8*3($tptr),%r11 + adc 8*4($tptr),%r12 + adc 8*5($tptr),%r13 + adc 8*6($tptr),%r14 + adc 8*7($tptr),%r15 + adc \$0,%rax # top-most carry + mov -16($nptr),%rcx # np[num-1] + xor $carry,$carry + + movq %xmm2,$nptr # restore $nptr + + mov %r8,8*0($tptr) # store top 512 bits + mov %r9,8*1($tptr) + movq %xmm3,$num # $num is %r9, can't be moved upwards + mov %r10,8*2($tptr) + mov %r11,8*3($tptr) + mov %r12,8*4($tptr) + mov %r13,8*5($tptr) + mov %r14,8*6($tptr) + mov %r15,8*7($tptr) + lea 8*8($tptr),$tptr + + cmp %rdx,$tptr # end of t[]? + jb .L8x_reduction_loop +___ +} +############################################################## +# Post-condition, 4x unrolled +# +{ +my ($tptr,$nptr)=("%rbx","%rbp"); +$code.=<<___; + #xor %rsi,%rsi # %rsi was $carry above + sub %r15,%rcx # compare top-most words + lea (%rdi,$num),$tptr # %rdi was $tptr above + adc %rsi,%rsi + mov $num,%rcx + or %rsi,%rax + movq %xmm1,$rptr # restore $rptr + xor \$1,%rax + movq %xmm1,$aptr # prepare for back-to-back call + lea ($nptr,%rax,8),$nptr + sar \$3+2,%rcx # cf=0 + jmp .Lsqr4x_sub + +.align 32 +.Lsqr4x_sub: + .byte 0x66 + mov 8*0($tptr),%r12 + mov 8*1($tptr),%r13 + sbb 16*0($nptr),%r12 + mov 8*2($tptr),%r14 + sbb 16*1($nptr),%r13 + mov 8*3($tptr),%r15 + lea 8*4($tptr),$tptr + sbb 16*2($nptr),%r14 + mov %r12,8*0($rptr) + sbb 16*3($nptr),%r15 + lea 16*4($nptr),$nptr + mov %r13,8*1($rptr) + mov %r14,8*2($rptr) + mov %r15,8*3($rptr) + lea 8*4($rptr),$rptr + + inc %rcx # pass %cf + jnz .Lsqr4x_sub ___ } $code.=<<___; - mov 8(%rsp,$num,8),%rsi # restore %rsp - mov \$1,%rax + mov $num,%r10 # prepare for back-to-back call + neg $num # restore $num + ret +.size bn_sqr8x_internal,.-bn_sqr8x_internal +___ +{ +$code.=<<___; +.globl bn_from_montgomery +.type bn_from_montgomery,\@abi-omnipotent +.align 32 +bn_from_montgomery: + testl \$7,`($win64?"48(%rsp)":"%r9d")` + jz bn_from_mont8x + xor %eax,%eax + ret +.size bn_from_montgomery,.-bn_from_montgomery + +.type bn_from_mont8x,\@function,6 +.align 32 +bn_from_mont8x: + .byte 0x67 + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 ___ $code.=<<___ if ($win64); - movaps (%rsi),%xmm6 - movaps 0x10(%rsi),%xmm7 - lea 0x28(%rsi),%rsi + lea -0x28(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) ___ $code.=<<___; - mov (%rsi),%r15 - mov 8(%rsi),%r14 - mov 16(%rsi),%r13 - mov 24(%rsi),%r12 - mov 32(%rsi),%rbp - mov 40(%rsi),%rbx - lea 48(%rsi),%rsp -.Lmul4x_epilogue: + .byte 0x67 + mov ${num}d,%r10d + shl \$3,${num}d # convert $num to bytes + shl \$3+2,%r10d # 4*$num + neg $num + mov ($n0),$n0 # *n0 + + ############################################################## + # ensure that stack frame doesn't alias with $aptr+4*$num + # modulo 4096, which covers ret[num], am[num] and n[2*num] + # (see bn_exp.c). this is done to allow memory disambiguation + # logic do its magic. + # + lea -64(%rsp,$num,2),%r11 + sub $aptr,%r11 + and \$4095,%r11 + cmp %r11,%r10 + jb .Lfrom_sp_alt + sub %r11,%rsp # align with $aptr + lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) + jmp .Lfrom_sp_done + +.align 32 +.Lfrom_sp_alt: + lea 4096-64(,$num,2),%r10 # 4096-frame-2*$num + lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) + sub %r10,%r11 + mov \$0,%r10 + cmovc %r10,%r11 + sub %r11,%rsp +.Lfrom_sp_done: + and \$-64,%rsp + mov $num,%r10 + neg $num + + ############################################################## + # Stack layout + # + # +0 saved $num, used in reduction section + # +8 &t[2*$num], used in reduction section + # +32 saved *n0 + # +40 saved %rsp + # +48 t[2*$num] + # + mov $n0, 32(%rsp) + mov %rax, 40(%rsp) # save original %rsp +.Lfrom_body: + mov $num,%r11 + lea 48(%rsp),%rax + pxor %xmm0,%xmm0 + jmp .Lmul_by_1 + +.align 32 +.Lmul_by_1: + movdqu ($aptr),%xmm1 + movdqu 16($aptr),%xmm2 + movdqu 32($aptr),%xmm3 + movdqa %xmm0,(%rax,$num) + movdqu 48($aptr),%xmm4 + movdqa %xmm0,16(%rax,$num) + .byte 0x48,0x8d,0xb6,0x40,0x00,0x00,0x00 # lea 64($aptr),$aptr + movdqa %xmm1,(%rax) + movdqa %xmm0,32(%rax,$num) + movdqa %xmm2,16(%rax) + movdqa %xmm0,48(%rax,$num) + movdqa %xmm3,32(%rax) + movdqa %xmm4,48(%rax) + lea 64(%rax),%rax + sub \$64,%r11 + jnz .Lmul_by_1 + + movq $rptr,%xmm1 + movq $nptr,%xmm2 + .byte 0x67 + mov $nptr,%rbp + movq %r10, %xmm3 # -num +___ +$code.=<<___ if ($addx); + mov OPENSSL_ia32cap_P+8(%rip),%r11d + and \$0x80100,%r11d + cmp \$0x80100,%r11d + jne .Lfrom_mont_nox + + lea (%rax,$num),$rptr + call sqrx8x_reduction + + pxor %xmm0,%xmm0 + lea 48(%rsp),%rax + mov 40(%rsp),%rsi # restore %rsp + jmp .Lfrom_mont_zero + +.align 32 +.Lfrom_mont_nox: +___ +$code.=<<___; + call sqr8x_reduction + + pxor %xmm0,%xmm0 + lea 48(%rsp),%rax + mov 40(%rsp),%rsi # restore %rsp + jmp .Lfrom_mont_zero + +.align 32 +.Lfrom_mont_zero: + movdqa %xmm0,16*0(%rax) + movdqa %xmm0,16*1(%rax) + movdqa %xmm0,16*2(%rax) + movdqa %xmm0,16*3(%rax) + lea 16*4(%rax),%rax + sub \$32,$num + jnz .Lfrom_mont_zero + + mov \$1,%rax + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lfrom_epilogue: ret -.size bn_mul4x_mont_gather5,.-bn_mul4x_mont_gather5 +.size bn_from_mont8x,.-bn_from_mont8x ___ +} }}} + +if ($addx) {{{ +my $bp="%rdx"; # restore original value + +$code.=<<___; +.type bn_mulx4x_mont_gather5,\@function,6 +.align 32 +bn_mulx4x_mont_gather5: +.Lmulx4x_enter: + .byte 0x67 + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0x28(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) +___ +$code.=<<___; + .byte 0x67 + mov ${num}d,%r10d + shl \$3,${num}d # convert $num to bytes + shl \$3+2,%r10d # 4*$num + neg $num # -$num + mov ($n0),$n0 # *n0 + + ############################################################## + # ensure that stack frame doesn't alias with $aptr+4*$num + # modulo 4096, which covers a[num], ret[num] and n[2*num] + # (see bn_exp.c). this is done to allow memory disambiguation + # logic do its magic. [excessive frame is allocated in order + # to allow bn_from_mont8x to clear it.] + # + lea -64(%rsp,$num,2),%r11 + sub $ap,%r11 + and \$4095,%r11 + cmp %r11,%r10 + jb .Lmulx4xsp_alt + sub %r11,%rsp # align with $aptr + lea -64(%rsp,$num,2),%rsp # alloca(frame+$num) + jmp .Lmulx4xsp_done + +.align 32 +.Lmulx4xsp_alt: + lea 4096-64(,$num,2),%r10 # 4096-frame-$num + lea -64(%rsp,$num,2),%rsp # alloca(frame+$num) + sub %r10,%r11 + mov \$0,%r10 + cmovc %r10,%r11 + sub %r11,%rsp +.Lmulx4xsp_done: + and \$-64,%rsp # ensure alignment + ############################################################## + # Stack layout + # +0 -num + # +8 off-loaded &b[i] + # +16 end of b[num] + # +24 inner counter + # +32 saved n0 + # +40 saved %rsp + # +48 + # +56 saved rp + # +64 tmp[num+1] + # + mov $n0, 32(%rsp) # save *n0 + mov %rax,40(%rsp) # save original %rsp +.Lmulx4x_body: + call mulx4x_internal + + mov 40(%rsp),%rsi # restore %rsp + mov \$1,%rax +___ +$code.=<<___ if ($win64); + movaps -88(%rsi),%xmm6 + movaps -72(%rsi),%xmm7 +___ +$code.=<<___; + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lmulx4x_epilogue: + ret +.size bn_mulx4x_mont_gather5,.-bn_mulx4x_mont_gather5 + +.type mulx4x_internal,\@abi-omnipotent +.align 32 +mulx4x_internal: + .byte 0x4c,0x89,0x8c,0x24,0x08,0x00,0x00,0x00 # mov $num,8(%rsp) # save -$num + .byte 0x67 + neg $num # restore $num + shl \$5,$num + lea 256($bp,$num),%r13 + shr \$5+5,$num + mov `($win64?56:8)`(%rax),%r10d # load 7th argument + sub \$1,$num + mov %r13,16+8(%rsp) # end of b[num] + mov $num,24+8(%rsp) # inner counter + mov $rp, 56+8(%rsp) # save $rp +___ +my ($aptr, $bptr, $nptr, $tptr, $mi, $bi, $zero, $num)= + ("%rsi","%rdi","%rcx","%rbx","%r8","%r9","%rbp","%rax"); +my $rptr=$bptr; +my $STRIDE=2**5*8; # 5 is "window size" +my $N=$STRIDE/4; # should match cache line size +$code.=<<___; + mov %r10,%r11 + shr \$`log($N/8)/log(2)`,%r10 + and \$`$N/8-1`,%r11 + not %r10 + lea .Lmagic_masks(%rip),%rax + and \$`2**5/($N/8)-1`,%r10 # 5 is "window size" + lea 96($bp,%r11,8),$bptr # pointer within 1st cache line + movq 0(%rax,%r10,8),%xmm4 # set of masks denoting which + movq 8(%rax,%r10,8),%xmm5 # cache line contains element + add \$7,%r11 + movq 16(%rax,%r10,8),%xmm6 # denoted by 7th argument + movq 24(%rax,%r10,8),%xmm7 + and \$7,%r11 + + movq `0*$STRIDE/4-96`($bptr),%xmm0 + lea $STRIDE($bptr),$tptr # borrow $tptr + movq `1*$STRIDE/4-96`($bptr),%xmm1 + pand %xmm4,%xmm0 + movq `2*$STRIDE/4-96`($bptr),%xmm2 + pand %xmm5,%xmm1 + movq `3*$STRIDE/4-96`($bptr),%xmm3 + pand %xmm6,%xmm2 + por %xmm1,%xmm0 + movq `0*$STRIDE/4-96`($tptr),%xmm1 + pand %xmm7,%xmm3 + por %xmm2,%xmm0 + movq `1*$STRIDE/4-96`($tptr),%xmm2 + por %xmm3,%xmm0 + .byte 0x67,0x67 + pand %xmm4,%xmm1 + movq `2*$STRIDE/4-96`($tptr),%xmm3 + + movq %xmm0,%rdx # bp[0] + movq `3*$STRIDE/4-96`($tptr),%xmm0 + lea 2*$STRIDE($bptr),$bptr # next &b[i] + pand %xmm5,%xmm2 + .byte 0x67,0x67 + pand %xmm6,%xmm3 + ############################################################## + # $tptr is chosen so that writing to top-most element of the + # vector occurs just "above" references to powers table, + # "above" modulo cache-line size, which effectively precludes + # possibility of memory disambiguation logic failure when + # accessing the table. + # + lea 64+8*4+8(%rsp,%r11,8),$tptr + + mov %rdx,$bi + mulx 0*8($aptr),$mi,%rax # a[0]*b[0] + mulx 1*8($aptr),%r11,%r12 # a[1]*b[0] + add %rax,%r11 + mulx 2*8($aptr),%rax,%r13 # ... + adc %rax,%r12 + adc \$0,%r13 + mulx 3*8($aptr),%rax,%r14 + + mov $mi,%r15 + imulq 32+8(%rsp),$mi # "t[0]"*n0 + xor $zero,$zero # cf=0, of=0 + mov $mi,%rdx + + por %xmm2,%xmm1 + pand %xmm7,%xmm0 + por %xmm3,%xmm1 + mov $bptr,8+8(%rsp) # off-load &b[i] + por %xmm1,%xmm0 + + .byte 0x48,0x8d,0xb6,0x20,0x00,0x00,0x00 # lea 4*8($aptr),$aptr + adcx %rax,%r13 + adcx $zero,%r14 # cf=0 + + mulx 0*16($nptr),%rax,%r10 + adcx %rax,%r15 # discarded + adox %r11,%r10 + mulx 1*16($nptr),%rax,%r11 + adcx %rax,%r10 + adox %r12,%r11 + mulx 2*16($nptr),%rax,%r12 + mov 24+8(%rsp),$bptr # counter value + .byte 0x66 + mov %r10,-8*4($tptr) + adcx %rax,%r11 + adox %r13,%r12 + mulx 3*16($nptr),%rax,%r15 + .byte 0x67,0x67 + mov $bi,%rdx + mov %r11,-8*3($tptr) + adcx %rax,%r12 + adox $zero,%r15 # of=0 + .byte 0x48,0x8d,0x89,0x40,0x00,0x00,0x00 # lea 4*16($nptr),$nptr + mov %r12,-8*2($tptr) + #jmp .Lmulx4x_1st + +.align 32 +.Lmulx4x_1st: + adcx $zero,%r15 # cf=0, modulo-scheduled + mulx 0*8($aptr),%r10,%rax # a[4]*b[0] + adcx %r14,%r10 + mulx 1*8($aptr),%r11,%r14 # a[5]*b[0] + adcx %rax,%r11 + mulx 2*8($aptr),%r12,%rax # ... + adcx %r14,%r12 + mulx 3*8($aptr),%r13,%r14 + .byte 0x67,0x67 + mov $mi,%rdx + adcx %rax,%r13 + adcx $zero,%r14 # cf=0 + lea 4*8($aptr),$aptr + lea 4*8($tptr),$tptr + + adox %r15,%r10 + mulx 0*16($nptr),%rax,%r15 + adcx %rax,%r10 + adox %r15,%r11 + mulx 1*16($nptr),%rax,%r15 + adcx %rax,%r11 + adox %r15,%r12 + mulx 2*16($nptr),%rax,%r15 + mov %r10,-5*8($tptr) + adcx %rax,%r12 + mov %r11,-4*8($tptr) + adox %r15,%r13 + mulx 3*16($nptr),%rax,%r15 + mov $bi,%rdx + mov %r12,-3*8($tptr) + adcx %rax,%r13 + adox $zero,%r15 + lea 4*16($nptr),$nptr + mov %r13,-2*8($tptr) + + dec $bptr # of=0, pass cf + jnz .Lmulx4x_1st + + mov 8(%rsp),$num # load -num + movq %xmm0,%rdx # bp[1] + adc $zero,%r15 # modulo-scheduled + lea ($aptr,$num),$aptr # rewind $aptr + add %r15,%r14 + mov 8+8(%rsp),$bptr # re-load &b[i] + adc $zero,$zero # top-most carry + mov %r14,-1*8($tptr) + jmp .Lmulx4x_outer + +.align 32 +.Lmulx4x_outer: + mov $zero,($tptr) # save top-most carry + lea 4*8($tptr,$num),$tptr # rewind $tptr + mulx 0*8($aptr),$mi,%r11 # a[0]*b[i] + xor $zero,$zero # cf=0, of=0 + mov %rdx,$bi + mulx 1*8($aptr),%r14,%r12 # a[1]*b[i] + adox -4*8($tptr),$mi # +t[0] + adcx %r14,%r11 + mulx 2*8($aptr),%r15,%r13 # ... + adox -3*8($tptr),%r11 + adcx %r15,%r12 + mulx 3*8($aptr),%rdx,%r14 + adox -2*8($tptr),%r12 + adcx %rdx,%r13 + lea ($nptr,$num,2),$nptr # rewind $nptr + lea 4*8($aptr),$aptr + adox -1*8($tptr),%r13 + adcx $zero,%r14 + adox $zero,%r14 + + .byte 0x67 + mov $mi,%r15 + imulq 32+8(%rsp),$mi # "t[0]"*n0 + + movq `0*$STRIDE/4-96`($bptr),%xmm0 + .byte 0x67,0x67 + mov $mi,%rdx + movq `1*$STRIDE/4-96`($bptr),%xmm1 + .byte 0x67 + pand %xmm4,%xmm0 + movq `2*$STRIDE/4-96`($bptr),%xmm2 + .byte 0x67 + pand %xmm5,%xmm1 + movq `3*$STRIDE/4-96`($bptr),%xmm3 + add \$$STRIDE,$bptr # next &b[i] + .byte 0x67 + pand %xmm6,%xmm2 + por %xmm1,%xmm0 + pand %xmm7,%xmm3 + xor $zero,$zero # cf=0, of=0 + mov $bptr,8+8(%rsp) # off-load &b[i] + + mulx 0*16($nptr),%rax,%r10 + adcx %rax,%r15 # discarded + adox %r11,%r10 + mulx 1*16($nptr),%rax,%r11 + adcx %rax,%r10 + adox %r12,%r11 + mulx 2*16($nptr),%rax,%r12 + adcx %rax,%r11 + adox %r13,%r12 + mulx 3*16($nptr),%rax,%r15 + mov $bi,%rdx + por %xmm2,%xmm0 + mov 24+8(%rsp),$bptr # counter value + mov %r10,-8*4($tptr) + por %xmm3,%xmm0 + adcx %rax,%r12 + mov %r11,-8*3($tptr) + adox $zero,%r15 # of=0 + mov %r12,-8*2($tptr) + lea 4*16($nptr),$nptr + jmp .Lmulx4x_inner + +.align 32 +.Lmulx4x_inner: + mulx 0*8($aptr),%r10,%rax # a[4]*b[i] + adcx $zero,%r15 # cf=0, modulo-scheduled + adox %r14,%r10 + mulx 1*8($aptr),%r11,%r14 # a[5]*b[i] + adcx 0*8($tptr),%r10 + adox %rax,%r11 + mulx 2*8($aptr),%r12,%rax # ... + adcx 1*8($tptr),%r11 + adox %r14,%r12 + mulx 3*8($aptr),%r13,%r14 + mov $mi,%rdx + adcx 2*8($tptr),%r12 + adox %rax,%r13 + adcx 3*8($tptr),%r13 + adox $zero,%r14 # of=0 + lea 4*8($aptr),$aptr + lea 4*8($tptr),$tptr + adcx $zero,%r14 # cf=0 + + adox %r15,%r10 + mulx 0*16($nptr),%rax,%r15 + adcx %rax,%r10 + adox %r15,%r11 + mulx 1*16($nptr),%rax,%r15 + adcx %rax,%r11 + adox %r15,%r12 + mulx 2*16($nptr),%rax,%r15 + mov %r10,-5*8($tptr) + adcx %rax,%r12 + adox %r15,%r13 + mov %r11,-4*8($tptr) + mulx 3*16($nptr),%rax,%r15 + mov $bi,%rdx + lea 4*16($nptr),$nptr + mov %r12,-3*8($tptr) + adcx %rax,%r13 + adox $zero,%r15 + mov %r13,-2*8($tptr) + + dec $bptr # of=0, pass cf + jnz .Lmulx4x_inner + + mov 0+8(%rsp),$num # load -num + movq %xmm0,%rdx # bp[i+1] + adc $zero,%r15 # modulo-scheduled + sub 0*8($tptr),$bptr # pull top-most carry to %cf + mov 8+8(%rsp),$bptr # re-load &b[i] + mov 16+8(%rsp),%r10 + adc %r15,%r14 + lea ($aptr,$num),$aptr # rewind $aptr + adc $zero,$zero # top-most carry + mov %r14,-1*8($tptr) + + cmp %r10,$bptr + jb .Lmulx4x_outer + + mov -16($nptr),%r10 + xor %r15,%r15 + sub %r14,%r10 # compare top-most words + adc %r15,%r15 + or %r15,$zero + xor \$1,$zero + lea ($tptr,$num),%rdi # rewind $tptr + lea ($nptr,$num,2),$nptr # rewind $nptr + .byte 0x67,0x67 + sar \$3+2,$num # cf=0 + lea ($nptr,$zero,8),%rbp + mov 56+8(%rsp),%rdx # restore rp + mov $num,%rcx + jmp .Lsqrx4x_sub # common post-condition +.size mulx4x_internal,.-mulx4x_internal +___ +} { +###################################################################### +# void bn_power5( +my $rptr="%rdi"; # BN_ULONG *rptr, +my $aptr="%rsi"; # const BN_ULONG *aptr, +my $bptr="%rdx"; # const void *table, +my $nptr="%rcx"; # const BN_ULONG *nptr, +my $n0 ="%r8"; # const BN_ULONG *n0); +my $num ="%r9"; # int num, has to be divisible by 8 + # int pwr); + +my ($i,$j,$tptr)=("%rbp","%rcx",$rptr); +my @A0=("%r10","%r11"); +my @A1=("%r12","%r13"); +my ($a0,$a1,$ai)=("%r14","%r15","%rbx"); + +$code.=<<___; +.type bn_powerx5,\@function,6 +.align 32 +bn_powerx5: +.Lpowerx5_enter: + .byte 0x67 + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0x28(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) +___ +$code.=<<___; + .byte 0x67 + mov ${num}d,%r10d + shl \$3,${num}d # convert $num to bytes + shl \$3+2,%r10d # 4*$num + neg $num + mov ($n0),$n0 # *n0 + + ############################################################## + # ensure that stack frame doesn't alias with $aptr+4*$num + # modulo 4096, which covers ret[num], am[num] and n[2*num] + # (see bn_exp.c). this is done to allow memory disambiguation + # logic do its magic. + # + lea -64(%rsp,$num,2),%r11 + sub $aptr,%r11 + and \$4095,%r11 + cmp %r11,%r10 + jb .Lpwrx_sp_alt + sub %r11,%rsp # align with $aptr + lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) + jmp .Lpwrx_sp_done + +.align 32 +.Lpwrx_sp_alt: + lea 4096-64(,$num,2),%r10 # 4096-frame-2*$num + lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) + sub %r10,%r11 + mov \$0,%r10 + cmovc %r10,%r11 + sub %r11,%rsp +.Lpwrx_sp_done: + and \$-64,%rsp + mov $num,%r10 + neg $num + + ############################################################## + # Stack layout + # + # +0 saved $num, used in reduction section + # +8 &t[2*$num], used in reduction section + # +16 intermediate carry bit + # +24 top-most carry bit, used in reduction section + # +32 saved *n0 + # +40 saved %rsp + # +48 t[2*$num] + # + pxor %xmm0,%xmm0 + movq $rptr,%xmm1 # save $rptr + movq $nptr,%xmm2 # save $nptr + movq %r10, %xmm3 # -$num + movq $bptr,%xmm4 + mov $n0, 32(%rsp) + mov %rax, 40(%rsp) # save original %rsp +.Lpowerx5_body: + + call __bn_sqrx8x_internal + call __bn_sqrx8x_internal + call __bn_sqrx8x_internal + call __bn_sqrx8x_internal + call __bn_sqrx8x_internal + + mov %r10,$num # -num + mov $aptr,$rptr + movq %xmm2,$nptr + movq %xmm4,$bptr + mov 40(%rsp),%rax + + call mulx4x_internal + + mov 40(%rsp),%rsi # restore %rsp + mov \$1,%rax +___ +$code.=<<___ if ($win64); + movaps -88(%rsi),%xmm6 + movaps -72(%rsi),%xmm7 +___ +$code.=<<___; + mov -48(%rsi),%r15 + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lpowerx5_epilogue: + ret +.size bn_powerx5,.-bn_powerx5 + +.globl bn_sqrx8x_internal +.hidden bn_sqrx8x_internal +.type bn_sqrx8x_internal,\@abi-omnipotent +.align 32 +bn_sqrx8x_internal: +__bn_sqrx8x_internal: + ################################################################## + # Squaring part: + # + # a) multiply-n-add everything but a[i]*a[i]; + # b) shift result of a) by 1 to the left and accumulate + # a[i]*a[i] products; + # + ################################################################## + # a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] + # a[1]a[0] + # a[2]a[0] + # a[3]a[0] + # a[2]a[1] + # a[3]a[1] + # a[3]a[2] + # + # a[4]a[0] + # a[5]a[0] + # a[6]a[0] + # a[7]a[0] + # a[4]a[1] + # a[5]a[1] + # a[6]a[1] + # a[7]a[1] + # a[4]a[2] + # a[5]a[2] + # a[6]a[2] + # a[7]a[2] + # a[4]a[3] + # a[5]a[3] + # a[6]a[3] + # a[7]a[3] + # + # a[5]a[4] + # a[6]a[4] + # a[7]a[4] + # a[6]a[5] + # a[7]a[5] + # a[7]a[6] + # a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] +___ +{ +my ($zero,$carry)=("%rbp","%rcx"); +my $aaptr=$zero; +$code.=<<___; + lea 48+8(%rsp),$tptr + lea ($aptr,$num),$aaptr + mov $num,0+8(%rsp) # save $num + mov $aaptr,8+8(%rsp) # save end of $aptr + jmp .Lsqr8x_zero_start + +.align 32 +.byte 0x66,0x66,0x66,0x2e,0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00 +.Lsqrx8x_zero: + .byte 0x3e + movdqa %xmm0,0*8($tptr) + movdqa %xmm0,2*8($tptr) + movdqa %xmm0,4*8($tptr) + movdqa %xmm0,6*8($tptr) +.Lsqr8x_zero_start: # aligned at 32 + movdqa %xmm0,8*8($tptr) + movdqa %xmm0,10*8($tptr) + movdqa %xmm0,12*8($tptr) + movdqa %xmm0,14*8($tptr) + lea 16*8($tptr),$tptr + sub \$64,$num + jnz .Lsqrx8x_zero + + mov 0*8($aptr),%rdx # a[0], modulo-scheduled + #xor %r9,%r9 # t[1], ex-$num, zero already + xor %r10,%r10 + xor %r11,%r11 + xor %r12,%r12 + xor %r13,%r13 + xor %r14,%r14 + xor %r15,%r15 + lea 48+8(%rsp),$tptr + xor $zero,$zero # cf=0, cf=0 + jmp .Lsqrx8x_outer_loop + +.align 32 +.Lsqrx8x_outer_loop: + mulx 1*8($aptr),%r8,%rax # a[1]*a[0] + adcx %r9,%r8 # a[1]*a[0]+=t[1] + adox %rax,%r10 + mulx 2*8($aptr),%r9,%rax # a[2]*a[0] + adcx %r10,%r9 + adox %rax,%r11 + .byte 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 # mulx 3*8($aptr),%r10,%rax # ... + adcx %r11,%r10 + adox %rax,%r12 + .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 # mulx 4*8($aptr),%r11,%rax + adcx %r12,%r11 + adox %rax,%r13 + mulx 5*8($aptr),%r12,%rax + adcx %r13,%r12 + adox %rax,%r14 + mulx 6*8($aptr),%r13,%rax + adcx %r14,%r13 + adox %r15,%rax + mulx 7*8($aptr),%r14,%r15 + mov 1*8($aptr),%rdx # a[1] + adcx %rax,%r14 + adox $zero,%r15 + adc 8*8($tptr),%r15 + mov %r8,1*8($tptr) # t[1] + mov %r9,2*8($tptr) # t[2] + sbb $carry,$carry # mov %cf,$carry + xor $zero,$zero # cf=0, of=0 + + + mulx 2*8($aptr),%r8,%rbx # a[2]*a[1] + mulx 3*8($aptr),%r9,%rax # a[3]*a[1] + adcx %r10,%r8 + adox %rbx,%r9 + mulx 4*8($aptr),%r10,%rbx # ... + adcx %r11,%r9 + adox %rax,%r10 + .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 # mulx 5*8($aptr),%r11,%rax + adcx %r12,%r10 + adox %rbx,%r11 + .byte 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 6*8($aptr),%r12,%rbx + adcx %r13,%r11 + adox %r14,%r12 + .byte 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 # mulx 7*8($aptr),%r13,%r14 + mov 2*8($aptr),%rdx # a[2] + adcx %rax,%r12 + adox %rbx,%r13 + adcx %r15,%r13 + adox $zero,%r14 # of=0 + adcx $zero,%r14 # cf=0 + + mov %r8,3*8($tptr) # t[3] + mov %r9,4*8($tptr) # t[4] + + mulx 3*8($aptr),%r8,%rbx # a[3]*a[2] + mulx 4*8($aptr),%r9,%rax # a[4]*a[2] + adcx %r10,%r8 + adox %rbx,%r9 + mulx 5*8($aptr),%r10,%rbx # ... + adcx %r11,%r9 + adox %rax,%r10 + .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 # mulx 6*8($aptr),%r11,%rax + adcx %r12,%r10 + adox %r13,%r11 + .byte 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 # mulx 7*8($aptr),%r12,%r13 + .byte 0x3e + mov 3*8($aptr),%rdx # a[3] + adcx %rbx,%r11 + adox %rax,%r12 + adcx %r14,%r12 + mov %r8,5*8($tptr) # t[5] + mov %r9,6*8($tptr) # t[6] + mulx 4*8($aptr),%r8,%rax # a[4]*a[3] + adox $zero,%r13 # of=0 + adcx $zero,%r13 # cf=0 + + mulx 5*8($aptr),%r9,%rbx # a[5]*a[3] + adcx %r10,%r8 + adox %rax,%r9 + mulx 6*8($aptr),%r10,%rax # ... + adcx %r11,%r9 + adox %r12,%r10 + mulx 7*8($aptr),%r11,%r12 + mov 4*8($aptr),%rdx # a[4] + mov 5*8($aptr),%r14 # a[5] + adcx %rbx,%r10 + adox %rax,%r11 + mov 6*8($aptr),%r15 # a[6] + adcx %r13,%r11 + adox $zero,%r12 # of=0 + adcx $zero,%r12 # cf=0 + + mov %r8,7*8($tptr) # t[7] + mov %r9,8*8($tptr) # t[8] + + mulx %r14,%r9,%rax # a[5]*a[4] + mov 7*8($aptr),%r8 # a[7] + adcx %r10,%r9 + mulx %r15,%r10,%rbx # a[6]*a[4] + adox %rax,%r10 + adcx %r11,%r10 + mulx %r8,%r11,%rax # a[7]*a[4] + mov %r14,%rdx # a[5] + adox %rbx,%r11 + adcx %r12,%r11 + #adox $zero,%rax # of=0 + adcx $zero,%rax # cf=0 + + mulx %r15,%r14,%rbx # a[6]*a[5] + mulx %r8,%r12,%r13 # a[7]*a[5] + mov %r15,%rdx # a[6] + lea 8*8($aptr),$aptr + adcx %r14,%r11 + adox %rbx,%r12 + adcx %rax,%r12 + adox $zero,%r13 + + .byte 0x67,0x67 + mulx %r8,%r8,%r14 # a[7]*a[6] + adcx %r8,%r13 + adcx $zero,%r14 + + cmp 8+8(%rsp),$aptr + je .Lsqrx8x_outer_break + + neg $carry # mov $carry,%cf + mov \$-8,%rcx + mov $zero,%r15 + mov 8*8($tptr),%r8 + adcx 9*8($tptr),%r9 # +=t[9] + adcx 10*8($tptr),%r10 # ... + adcx 11*8($tptr),%r11 + adc 12*8($tptr),%r12 + adc 13*8($tptr),%r13 + adc 14*8($tptr),%r14 + adc 15*8($tptr),%r15 + lea ($aptr),$aaptr + lea 2*64($tptr),$tptr + sbb %rax,%rax # mov %cf,$carry + + mov -64($aptr),%rdx # a[0] + mov %rax,16+8(%rsp) # offload $carry + mov $tptr,24+8(%rsp) + + #lea 8*8($tptr),$tptr # see 2*8*8($tptr) above + xor %eax,%eax # cf=0, of=0 + jmp .Lsqrx8x_loop + +.align 32 +.Lsqrx8x_loop: + mov %r8,%rbx + mulx 0*8($aaptr),%rax,%r8 # a[8]*a[i] + adcx %rax,%rbx # +=t[8] + adox %r9,%r8 + + mulx 1*8($aaptr),%rax,%r9 # ... + adcx %rax,%r8 + adox %r10,%r9 + + mulx 2*8($aaptr),%rax,%r10 + adcx %rax,%r9 + adox %r11,%r10 + + mulx 3*8($aaptr),%rax,%r11 + adcx %rax,%r10 + adox %r12,%r11 + + .byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 4*8($aaptr),%rax,%r12 + adcx %rax,%r11 + adox %r13,%r12 + + mulx 5*8($aaptr),%rax,%r13 + adcx %rax,%r12 + adox %r14,%r13 + + mulx 6*8($aaptr),%rax,%r14 + mov %rbx,($tptr,%rcx,8) # store t[8+i] + mov \$0,%ebx + adcx %rax,%r13 + adox %r15,%r14 + + .byte 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 # mulx 7*8($aaptr),%rax,%r15 + mov 8($aptr,%rcx,8),%rdx # a[i] + adcx %rax,%r14 + adox %rbx,%r15 # %rbx is 0, of=0 + adcx %rbx,%r15 # cf=0 + + .byte 0x67 + inc %rcx # of=0 + jnz .Lsqrx8x_loop + + lea 8*8($aaptr),$aaptr + mov \$-8,%rcx + cmp 8+8(%rsp),$aaptr # done? + je .Lsqrx8x_break + + sub 16+8(%rsp),%rbx # mov 16(%rsp),%cf + .byte 0x66 + mov -64($aptr),%rdx + adcx 0*8($tptr),%r8 + adcx 1*8($tptr),%r9 + adc 2*8($tptr),%r10 + adc 3*8($tptr),%r11 + adc 4*8($tptr),%r12 + adc 5*8($tptr),%r13 + adc 6*8($tptr),%r14 + adc 7*8($tptr),%r15 + lea 8*8($tptr),$tptr + .byte 0x67 + sbb %rax,%rax # mov %cf,%rax + xor %ebx,%ebx # cf=0, of=0 + mov %rax,16+8(%rsp) # offload carry + jmp .Lsqrx8x_loop + +.align 32 +.Lsqrx8x_break: + sub 16+8(%rsp),%r8 # consume last carry + mov 24+8(%rsp),$carry # initial $tptr, borrow $carry + mov 0*8($aptr),%rdx # a[8], modulo-scheduled + xor %ebp,%ebp # xor $zero,$zero + mov %r8,0*8($tptr) + cmp $carry,$tptr # cf=0, of=0 + je .Lsqrx8x_outer_loop + + mov %r9,1*8($tptr) + mov 1*8($carry),%r9 + mov %r10,2*8($tptr) + mov 2*8($carry),%r10 + mov %r11,3*8($tptr) + mov 3*8($carry),%r11 + mov %r12,4*8($tptr) + mov 4*8($carry),%r12 + mov %r13,5*8($tptr) + mov 5*8($carry),%r13 + mov %r14,6*8($tptr) + mov 6*8($carry),%r14 + mov %r15,7*8($tptr) + mov 7*8($carry),%r15 + mov $carry,$tptr + jmp .Lsqrx8x_outer_loop + +.align 32 +.Lsqrx8x_outer_break: + mov %r9,9*8($tptr) # t[9] + movq %xmm3,%rcx # -$num + mov %r10,10*8($tptr) # ... + mov %r11,11*8($tptr) + mov %r12,12*8($tptr) + mov %r13,13*8($tptr) + mov %r14,14*8($tptr) +___ +} { +my $i="%rcx"; +$code.=<<___; + lea 48+8(%rsp),$tptr + mov ($aptr,$i),%rdx # a[0] + + mov 8($tptr),$A0[1] # t[1] + xor $A0[0],$A0[0] # t[0], of=0, cf=0 + mov 0+8(%rsp),$num # restore $num + adox $A0[1],$A0[1] + mov 16($tptr),$A1[0] # t[2] # prefetch + mov 24($tptr),$A1[1] # t[3] # prefetch + #jmp .Lsqrx4x_shift_n_add # happens to be aligned + +.align 32 +.Lsqrx4x_shift_n_add: + mulx %rdx,%rax,%rbx + adox $A1[0],$A1[0] + adcx $A0[0],%rax + .byte 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 # mov 8($aptr,$i),%rdx # a[i+1] # prefetch + .byte 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 # mov 32($tptr),$A0[0] # t[2*i+4] # prefetch + adox $A1[1],$A1[1] + adcx $A0[1],%rbx + mov 40($tptr),$A0[1] # t[2*i+4+1] # prefetch + mov %rax,0($tptr) + mov %rbx,8($tptr) + + mulx %rdx,%rax,%rbx + adox $A0[0],$A0[0] + adcx $A1[0],%rax + mov 16($aptr,$i),%rdx # a[i+2] # prefetch + mov 48($tptr),$A1[0] # t[2*i+6] # prefetch + adox $A0[1],$A0[1] + adcx $A1[1],%rbx + mov 56($tptr),$A1[1] # t[2*i+6+1] # prefetch + mov %rax,16($tptr) + mov %rbx,24($tptr) + + mulx %rdx,%rax,%rbx + adox $A1[0],$A1[0] + adcx $A0[0],%rax + mov 24($aptr,$i),%rdx # a[i+3] # prefetch + lea 32($i),$i + mov 64($tptr),$A0[0] # t[2*i+8] # prefetch + adox $A1[1],$A1[1] + adcx $A0[1],%rbx + mov 72($tptr),$A0[1] # t[2*i+8+1] # prefetch + mov %rax,32($tptr) + mov %rbx,40($tptr) + + mulx %rdx,%rax,%rbx + adox $A0[0],$A0[0] + adcx $A1[0],%rax + jrcxz .Lsqrx4x_shift_n_add_break + .byte 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 # mov 0($aptr,$i),%rdx # a[i+4] # prefetch + adox $A0[1],$A0[1] + adcx $A1[1],%rbx + mov 80($tptr),$A1[0] # t[2*i+10] # prefetch + mov 88($tptr),$A1[1] # t[2*i+10+1] # prefetch + mov %rax,48($tptr) + mov %rbx,56($tptr) + lea 64($tptr),$tptr + nop + jmp .Lsqrx4x_shift_n_add + +.align 32 +.Lsqrx4x_shift_n_add_break: + adcx $A1[1],%rbx + mov %rax,48($tptr) + mov %rbx,56($tptr) + lea 64($tptr),$tptr # end of t[] buffer +___ +} +###################################################################### +# Montgomery reduction part, "word-by-word" algorithm. +# +# This new path is inspired by multiple submissions from Intel, by +# Shay Gueron, Vlad Krasnov, Erdinc Ozturk, James Guilford, +# Vinodh Gopal... +{ +my ($nptr,$carry,$m0)=("%rbp","%rsi","%rdx"); + +$code.=<<___; + movq %xmm2,$nptr +sqrx8x_reduction: + xor %eax,%eax # initial top-most carry bit + mov 32+8(%rsp),%rbx # n0 + mov 48+8(%rsp),%rdx # "%r8", 8*0($tptr) + lea -128($nptr,$num,2),%rcx # end of n[] + #lea 48+8(%rsp,$num,2),$tptr # end of t[] buffer + mov %rcx, 0+8(%rsp) # save end of n[] + mov $tptr,8+8(%rsp) # save end of t[] + + lea 48+8(%rsp),$tptr # initial t[] window + jmp .Lsqrx8x_reduction_loop + +.align 32 +.Lsqrx8x_reduction_loop: + mov 8*1($tptr),%r9 + mov 8*2($tptr),%r10 + mov 8*3($tptr),%r11 + mov 8*4($tptr),%r12 + mov %rdx,%r8 + imulq %rbx,%rdx # n0*a[i] + mov 8*5($tptr),%r13 + mov 8*6($tptr),%r14 + mov 8*7($tptr),%r15 + mov %rax,24+8(%rsp) # store top-most carry bit + + lea 8*8($tptr),$tptr + xor $carry,$carry # cf=0,of=0 + mov \$-8,%rcx + jmp .Lsqrx8x_reduce + +.align 32 +.Lsqrx8x_reduce: + mov %r8, %rbx + mulx 16*0($nptr),%rax,%r8 # n[0] + adcx %rbx,%rax # discarded + adox %r9,%r8 + + mulx 16*1($nptr),%rbx,%r9 # n[1] + adcx %rbx,%r8 + adox %r10,%r9 + + mulx 16*2($nptr),%rbx,%r10 + adcx %rbx,%r9 + adox %r11,%r10 + + mulx 16*3($nptr),%rbx,%r11 + adcx %rbx,%r10 + adox %r12,%r11 + + .byte 0xc4,0x62,0xe3,0xf6,0xa5,0x40,0x00,0x00,0x00 # mulx 16*4($nptr),%rbx,%r12 + mov %rdx,%rax + mov %r8,%rdx + adcx %rbx,%r11 + adox %r13,%r12 + + mulx 32+8(%rsp),%rbx,%rdx # %rdx discarded + mov %rax,%rdx + mov %rax,64+48+8(%rsp,%rcx,8) # put aside n0*a[i] + + mulx 16*5($nptr),%rax,%r13 + adcx %rax,%r12 + adox %r14,%r13 + + mulx 16*6($nptr),%rax,%r14 + adcx %rax,%r13 + adox %r15,%r14 + + mulx 16*7($nptr),%rax,%r15 + mov %rbx,%rdx + adcx %rax,%r14 + adox $carry,%r15 # $carry is 0 + adcx $carry,%r15 # cf=0 + + .byte 0x67,0x67,0x67 + inc %rcx # of=0 + jnz .Lsqrx8x_reduce + + mov $carry,%rax # xor %rax,%rax + cmp 0+8(%rsp),$nptr # end of n[]? + jae .Lsqrx8x_no_tail + + mov 48+8(%rsp),%rdx # pull n0*a[0] + add 8*0($tptr),%r8 + lea 16*8($nptr),$nptr + mov \$-8,%rcx + adcx 8*1($tptr),%r9 + adcx 8*2($tptr),%r10 + adc 8*3($tptr),%r11 + adc 8*4($tptr),%r12 + adc 8*5($tptr),%r13 + adc 8*6($tptr),%r14 + adc 8*7($tptr),%r15 + lea 8*8($tptr),$tptr + sbb %rax,%rax # top carry + + xor $carry,$carry # of=0, cf=0 + mov %rax,16+8(%rsp) + jmp .Lsqrx8x_tail + +.align 32 +.Lsqrx8x_tail: + mov %r8,%rbx + mulx 16*0($nptr),%rax,%r8 + adcx %rax,%rbx + adox %r9,%r8 + + mulx 16*1($nptr),%rax,%r9 + adcx %rax,%r8 + adox %r10,%r9 + + mulx 16*2($nptr),%rax,%r10 + adcx %rax,%r9 + adox %r11,%r10 + + mulx 16*3($nptr),%rax,%r11 + adcx %rax,%r10 + adox %r12,%r11 + + .byte 0xc4,0x62,0xfb,0xf6,0xa5,0x40,0x00,0x00,0x00 # mulx 16*4($nptr),%rax,%r12 + adcx %rax,%r11 + adox %r13,%r12 + + mulx 16*5($nptr),%rax,%r13 + adcx %rax,%r12 + adox %r14,%r13 + + mulx 16*6($nptr),%rax,%r14 + adcx %rax,%r13 + adox %r15,%r14 + + mulx 16*7($nptr),%rax,%r15 + mov 72+48+8(%rsp,%rcx,8),%rdx # pull n0*a[i] + adcx %rax,%r14 + adox $carry,%r15 + mov %rbx,($tptr,%rcx,8) # save result + mov %r8,%rbx + adcx $carry,%r15 # cf=0 + + inc %rcx # of=0 + jnz .Lsqrx8x_tail + + cmp 0+8(%rsp),$nptr # end of n[]? + jae .Lsqrx8x_tail_done # break out of loop + + sub 16+8(%rsp),$carry # mov 16(%rsp),%cf + mov 48+8(%rsp),%rdx # pull n0*a[0] + lea 16*8($nptr),$nptr + adc 8*0($tptr),%r8 + adc 8*1($tptr),%r9 + adc 8*2($tptr),%r10 + adc 8*3($tptr),%r11 + adc 8*4($tptr),%r12 + adc 8*5($tptr),%r13 + adc 8*6($tptr),%r14 + adc 8*7($tptr),%r15 + lea 8*8($tptr),$tptr + sbb %rax,%rax + sub \$8,%rcx # mov \$-8,%rcx + + xor $carry,$carry # of=0, cf=0 + mov %rax,16+8(%rsp) + jmp .Lsqrx8x_tail + +.align 32 +.Lsqrx8x_tail_done: + add 24+8(%rsp),%r8 # can this overflow? + mov $carry,%rax # xor %rax,%rax + + sub 16+8(%rsp),$carry # mov 16(%rsp),%cf +.Lsqrx8x_no_tail: # %cf is 0 if jumped here + adc 8*0($tptr),%r8 + movq %xmm3,%rcx + adc 8*1($tptr),%r9 + mov 16*7($nptr),$carry + movq %xmm2,$nptr # restore $nptr + adc 8*2($tptr),%r10 + adc 8*3($tptr),%r11 + adc 8*4($tptr),%r12 + adc 8*5($tptr),%r13 + adc 8*6($tptr),%r14 + adc 8*7($tptr),%r15 + adc %rax,%rax # top-most carry + + mov 32+8(%rsp),%rbx # n0 + mov 8*8($tptr,%rcx),%rdx # modulo-scheduled "%r8" + + mov %r8,8*0($tptr) # store top 512 bits + lea 8*8($tptr),%r8 # borrow %r8 + mov %r9,8*1($tptr) + mov %r10,8*2($tptr) + mov %r11,8*3($tptr) + mov %r12,8*4($tptr) + mov %r13,8*5($tptr) + mov %r14,8*6($tptr) + mov %r15,8*7($tptr) + + lea 8*8($tptr,%rcx),$tptr # start of current t[] window + cmp 8+8(%rsp),%r8 # end of t[]? + jb .Lsqrx8x_reduction_loop +___ +} +############################################################## +# Post-condition, 4x unrolled +# +{ +my ($rptr,$nptr)=("%rdx","%rbp"); +my @ri=map("%r$_",(10..13)); +my @ni=map("%r$_",(14..15)); +$code.=<<___; + xor %rbx,%rbx + sub %r15,%rsi # compare top-most words + adc %rbx,%rbx + mov %rcx,%r10 # -$num + .byte 0x67 + or %rbx,%rax + .byte 0x67 + mov %rcx,%r9 # -$num + xor \$1,%rax + sar \$3+2,%rcx # cf=0 + #lea 48+8(%rsp,%r9),$tptr + lea ($nptr,%rax,8),$nptr + movq %xmm1,$rptr # restore $rptr + movq %xmm1,$aptr # prepare for back-to-back call + jmp .Lsqrx4x_sub + +.align 32 +.Lsqrx4x_sub: + .byte 0x66 + mov 8*0($tptr),%r12 + mov 8*1($tptr),%r13 + sbb 16*0($nptr),%r12 + mov 8*2($tptr),%r14 + sbb 16*1($nptr),%r13 + mov 8*3($tptr),%r15 + lea 8*4($tptr),$tptr + sbb 16*2($nptr),%r14 + mov %r12,8*0($rptr) + sbb 16*3($nptr),%r15 + lea 16*4($nptr),$nptr + mov %r13,8*1($rptr) + mov %r14,8*2($rptr) + mov %r15,8*3($rptr) + lea 8*4($rptr),$rptr + + inc %rcx + jnz .Lsqrx4x_sub +___ +} +$code.=<<___; + neg %r9 # restore $num + ret +.size bn_sqrx8x_internal,.-bn_sqrx8x_internal +___ +}}} { -my ($inp,$num,$tbl,$idx)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order - ("%rdi","%rsi","%rdx","%rcx"); # Unix order +my ($inp,$num,$tbl,$idx)=$win64?("%rcx","%edx","%r8", "%r9d") : # Win64 order + ("%rdi","%esi","%rdx","%ecx"); # Unix order my $out=$inp; my $STRIDE=2**5*8; my $N=$STRIDE/4; $code.=<<___; +.globl bn_get_bits5 +.type bn_get_bits5,\@abi-omnipotent +.align 16 +bn_get_bits5: + lea 0($inp),%r10 + lea 1($inp),%r11 + mov $num,%ecx + shr \$4,$num + and \$15,%ecx + lea -8(%ecx),%eax + cmp \$11,%ecx + cmova %r11,%r10 + cmova %eax,%ecx + movzw (%r10,$num,2),%eax + shrl %cl,%eax + and \$31,%eax + ret +.size bn_get_bits5,.-bn_get_bits5 + .globl bn_scatter5 .type bn_scatter5,\@abi-omnipotent .align 16 @@ -868,13 +3272,13 @@ $code.=<<___ if ($win64); .byte 0x0f,0x29,0x7c,0x24,0x10 #movdqa %xmm7,0x10(%rsp) ___ $code.=<<___; - mov $idx,%r11 + mov $idx,%r11d shr \$`log($N/8)/log(2)`,$idx and \$`$N/8-1`,%r11 not $idx lea .Lmagic_masks(%rip),%rax and \$`2**5/($N/8)-1`,$idx # 5 is "window size" - lea 96($tbl,%r11,8),$tbl # pointer within 1st cache line + lea 128($tbl,%r11,8),$tbl # pointer within 1st cache line movq 0(%rax,$idx,8),%xmm4 # set of masks denoting which movq 8(%rax,$idx,8),%xmm5 # cache line contains element movq 16(%rax,$idx,8),%xmm6 # denoted by 7th argument @@ -882,15 +3286,16 @@ $code.=<<___; jmp .Lgather .align 16 .Lgather: - movq `0*$STRIDE/4-96`($tbl),%xmm0 - movq `1*$STRIDE/4-96`($tbl),%xmm1 + movq `0*$STRIDE/4-128`($tbl),%xmm0 + movq `1*$STRIDE/4-128`($tbl),%xmm1 pand %xmm4,%xmm0 - movq `2*$STRIDE/4-96`($tbl),%xmm2 + movq `2*$STRIDE/4-128`($tbl),%xmm2 pand %xmm5,%xmm1 - movq `3*$STRIDE/4-96`($tbl),%xmm3 + movq `3*$STRIDE/4-128`($tbl),%xmm3 pand %xmm6,%xmm2 por %xmm1,%xmm0 pand %xmm7,%xmm3 + .byte 0x67,0x67 por %xmm2,%xmm0 lea $STRIDE($tbl),$tbl por %xmm3,%xmm0 @@ -954,26 +3359,27 @@ mul_handler: cmp %r10,%rbx # context->RipRipRsp - mov 8(%r11),%r10d # HandlerData[2] + mov 4(%r11),%r10d # HandlerData[1] lea (%rsi,%r10),%r10 # epilogue label cmp %r10,%rbx # context->Rip>=epilogue label jae .Lcommon_seh_tail + lea .Lmul_epilogue(%rip),%r10 + cmp %r10,%rbx + jb .Lbody_40 + mov 192($context),%r10 # pull $num mov 8(%rax,%r10,8),%rax # pull saved stack pointer + jmp .Lbody_proceed + +.Lbody_40: + mov 40(%rax),%rax # pull saved stack pointer +.Lbody_proceed: - movaps (%rax),%xmm0 - movaps 16(%rax),%xmm1 - lea `40+48`(%rax),%rax + movaps -88(%rax),%xmm0 + movaps -72(%rax),%xmm1 mov -8(%rax),%rbx mov -16(%rax),%rbp @@ -1040,6 +3446,24 @@ mul_handler: .rva .LSEH_end_bn_mul4x_mont_gather5 .rva .LSEH_info_bn_mul4x_mont_gather5 + .rva .LSEH_begin_bn_power5 + .rva .LSEH_end_bn_power5 + .rva .LSEH_info_bn_power5 + + .rva .LSEH_begin_bn_from_mont8x + .rva .LSEH_end_bn_from_mont8x + .rva .LSEH_info_bn_from_mont8x +___ +$code.=<<___ if ($addx); + .rva .LSEH_begin_bn_mulx4x_mont_gather5 + .rva .LSEH_end_bn_mulx4x_mont_gather5 + .rva .LSEH_info_bn_mulx4x_mont_gather5 + + .rva .LSEH_begin_bn_powerx5 + .rva .LSEH_end_bn_powerx5 + .rva .LSEH_info_bn_powerx5 +___ +$code.=<<___; .rva .LSEH_begin_bn_gather5 .rva .LSEH_end_bn_gather5 .rva .LSEH_info_bn_gather5 @@ -1049,12 +3473,36 @@ mul_handler: .LSEH_info_bn_mul_mont_gather5: .byte 9,0,0,0 .rva mul_handler - .rva .Lmul_alloca,.Lmul_body,.Lmul_epilogue # HandlerData[] + .rva .Lmul_body,.Lmul_epilogue # HandlerData[] .align 8 .LSEH_info_bn_mul4x_mont_gather5: .byte 9,0,0,0 .rva mul_handler - .rva .Lmul4x_alloca,.Lmul4x_body,.Lmul4x_epilogue # HandlerData[] + .rva .Lmul4x_body,.Lmul4x_epilogue # HandlerData[] +.align 8 +.LSEH_info_bn_power5: + .byte 9,0,0,0 + .rva mul_handler + .rva .Lpower5_body,.Lpower5_epilogue # HandlerData[] +.align 8 +.LSEH_info_bn_from_mont8x: + .byte 9,0,0,0 + .rva mul_handler + .rva .Lfrom_body,.Lfrom_epilogue # HandlerData[] +___ +$code.=<<___ if ($addx); +.align 8 +.LSEH_info_bn_mulx4x_mont_gather5: + .byte 9,0,0,0 + .rva mul_handler + .rva .Lmulx4x_body,.Lmulx4x_epilogue # HandlerData[] +.align 8 +.LSEH_info_bn_powerx5: + .byte 9,0,0,0 + .rva mul_handler + .rva .Lpowerx5_body,.Lpowerx5_epilogue # HandlerData[] +___ +$code.=<<___; .align 8 .LSEH_info_bn_gather5: .byte 0x01,0x0d,0x05,0x00 diff --git a/crypto/bn/bn.h b/crypto/bn/bn.h index 47d8c71d9ed1..5696965e9a09 100644 --- a/crypto/bn/bn.h +++ b/crypto/bn/bn.h @@ -256,24 +256,6 @@ extern "C" { # define BN_HEX_FMT2 "%08X" # endif -/* - * 2011-02-22 SMS. In various places, a size_t variable or a type cast to - * size_t was used to perform integer-only operations on pointers. This - * failed on VMS with 64-bit pointers (CC /POINTER_SIZE = 64) because size_t - * is still only 32 bits. What's needed in these cases is an integer type - * with the same size as a pointer, which size_t is not certain to be. The - * only fix here is VMS-specific. - */ -# if defined(OPENSSL_SYS_VMS) -# if __INITIAL_POINTER_SIZE == 64 -# define PTR_SIZE_INT long long -# else /* __INITIAL_POINTER_SIZE == 64 */ -# define PTR_SIZE_INT int -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ -# else /* defined(OPENSSL_SYS_VMS) */ -# define PTR_SIZE_INT size_t -# endif /* defined(OPENSSL_SYS_VMS) [else] */ - # define BN_DEFAULT_BITS 1280 # define BN_FLG_MALLOCED 0x01 diff --git a/crypto/bn/bn_asm.c b/crypto/bn/bn_asm.c index 114acf3dc248..03a33cffe5ee 100644 --- a/crypto/bn/bn_asm.c +++ b/crypto/bn/bn_asm.c @@ -489,121 +489,144 @@ BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, * c=(c2,c1,c0) */ +# ifdef BN_LLONG /* - * Keep in mind that carrying into high part of multiplication result - * can not overflow, because it cannot be all-ones. + * Keep in mind that additions to multiplication result can not + * overflow, because its high half cannot be all-ones. */ -# ifdef BN_LLONG -# define mul_add_c(a,b,c0,c1,c2) \ - t=(BN_ULLONG)a*b; \ - t1=(BN_ULONG)Lw(t); \ - t2=(BN_ULONG)Hw(t); \ - c0=(c0+t1)&BN_MASK2; if ((c0) < t1) t2++; \ - c1=(c1+t2)&BN_MASK2; if ((c1) < t2) c2++; - -# define mul_add_c2(a,b,c0,c1,c2) \ - t=(BN_ULLONG)a*b; \ - tt=(t+t)&BN_MASK; \ - if (tt < t) c2++; \ - t1=(BN_ULONG)Lw(tt); \ - t2=(BN_ULONG)Hw(tt); \ - c0=(c0+t1)&BN_MASK2; \ - if ((c0 < t1) && (((++t2)&BN_MASK2) == 0)) c2++; \ - c1=(c1+t2)&BN_MASK2; if ((c1) < t2) c2++; - -# define sqr_add_c(a,i,c0,c1,c2) \ - t=(BN_ULLONG)a[i]*a[i]; \ - t1=(BN_ULONG)Lw(t); \ - t2=(BN_ULONG)Hw(t); \ - c0=(c0+t1)&BN_MASK2; if ((c0) < t1) t2++; \ - c1=(c1+t2)&BN_MASK2; if ((c1) < t2) c2++; +# define mul_add_c(a,b,c0,c1,c2) do { \ + BN_ULONG hi; \ + BN_ULLONG t = (BN_ULLONG)(a)*(b); \ + t += c0; /* no carry */ \ + c0 = (BN_ULONG)Lw(t); \ + hi = (BN_ULONG)Hw(t); \ + c1 = (c1+hi)&BN_MASK2; if (c1 +#endif + +#include "rsaz_exp.h" + +#undef SPARC_T4_MONT +#if defined(OPENSSL_BN_ASM_MONT) && (defined(__sparc__) || defined(__sparc)) +# include "sparc_arch.h" +extern unsigned int OPENSSL_sparcv9cap_P[]; +# define SPARC_T4_MONT #endif /* maximum precomputation table size for *variable* sliding windows */ @@ -464,6 +475,23 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, wstart = bits - 1; /* The top bit of the window */ wend = 0; /* The bottom bit of the window */ +#if 1 /* by Shay Gueron's suggestion */ + j = m->top; /* borrow j */ + if (m->d[j - 1] & (((BN_ULONG)1) << (BN_BITS2 - 1))) { + if (bn_wexpand(r, j) == NULL) + goto err; + /* 2^(top*BN_BITS2) - m */ + r->d[0] = (0 - m->d[0]) & BN_MASK2; + for (i = 1; i < j; i++) + r->d[i] = (~m->d[i]) & BN_MASK2; + r->top = j; + /* + * Upper words will be zero if the corresponding words of 'm' were + * 0xfff[...], so decrement r->top accordingly. + */ + bn_correct_top(r); + } else +#endif if (!BN_to_montgomery(r, BN_value_one(), mont, ctx)) goto err; for (;;) { @@ -515,6 +543,17 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, if (wstart < 0) break; } +#if defined(SPARC_T4_MONT) + if (OPENSSL_sparcv9cap_P[0] & (SPARCV9_VIS3 | SPARCV9_PREFER_FPU)) { + j = mont->N.top; /* borrow j */ + val[0]->d[0] = 1; /* borrow val[0] */ + for (i = 1; i < j; i++) + val[0]->d[i] = 0; + val[0]->top = j; + if (!BN_mod_mul_montgomery(rr, r, val[0], mont, ctx)) + goto err; + } else +#endif if (!BN_from_montgomery(rr, r, mont, ctx)) goto err; ret = 1; @@ -526,6 +565,27 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, return (ret); } +#if defined(SPARC_T4_MONT) +static BN_ULONG bn_get_bits(const BIGNUM *a, int bitpos) +{ + BN_ULONG ret = 0; + int wordpos; + + wordpos = bitpos / BN_BITS2; + bitpos %= BN_BITS2; + if (wordpos >= 0 && wordpos < a->top) { + ret = a->d[wordpos] & BN_MASK2; + if (bitpos) { + ret >>= bitpos; + if (++wordpos < a->top) + ret |= a->d[wordpos] << (BN_BITS2 - bitpos); + } + } + + return ret & BN_MASK2; +} +#endif + /* * BN_mod_exp_mont_consttime() stores the precomputed powers in a specific * layout so that accessing any of these table values shows the same access @@ -594,6 +654,9 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, int powerbufLen = 0; unsigned char *powerbuf = NULL; BIGNUM tmp, am; +#if defined(SPARC_T4_MONT) + unsigned int t4 = 0; +#endif bn_check_top(a); bn_check_top(p); @@ -626,21 +689,62 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, goto err; } +#ifdef RSAZ_ENABLED + /* + * If the size of the operands allow it, perform the optimized + * RSAZ exponentiation. For further information see + * crypto/bn/rsaz_exp.c and accompanying assembly modules. + */ + if ((16 == a->top) && (16 == p->top) && (BN_num_bits(m) == 1024) + && rsaz_avx2_eligible()) { + if (NULL == bn_wexpand(rr, 16)) + goto err; + RSAZ_1024_mod_exp_avx2(rr->d, a->d, p->d, m->d, mont->RR.d, + mont->n0[0]); + rr->top = 16; + rr->neg = 0; + bn_correct_top(rr); + ret = 1; + goto err; + } else if ((8 == a->top) && (8 == p->top) && (BN_num_bits(m) == 512)) { + if (NULL == bn_wexpand(rr, 8)) + goto err; + RSAZ_512_mod_exp(rr->d, a->d, p->d, m->d, mont->n0[0], mont->RR.d); + rr->top = 8; + rr->neg = 0; + bn_correct_top(rr); + ret = 1; + goto err; + } +#endif + /* Get the window size to use with size of p. */ window = BN_window_bits_for_ctime_exponent_size(bits); +#if defined(SPARC_T4_MONT) + if (window >= 5 && (top & 15) == 0 && top <= 64 && + (OPENSSL_sparcv9cap_P[1] & (CFR_MONTMUL | CFR_MONTSQR)) == + (CFR_MONTMUL | CFR_MONTSQR) && (t4 = OPENSSL_sparcv9cap_P[0])) + window = 5; + else +#endif #if defined(OPENSSL_BN_ASM_MONT5) - if (window == 6 && bits <= 1024) - window = 5; /* ~5% improvement of 2048-bit RSA sign */ + if (window >= 5) { + window = 5; /* ~5% improvement for RSA2048 sign, and even + * for RSA4096 */ + if ((top & 7) == 0) + powerbufLen += 2 * top * sizeof(m->d[0]); + } #endif + (void)0; /* * Allocate a buffer large enough to hold all of the pre-computed powers * of am, am itself and tmp. */ numPowers = 1 << window; - powerbufLen = sizeof(m->d[0]) * (top * numPowers + - ((2 * top) > - numPowers ? (2 * top) : numPowers)); + powerbufLen += sizeof(m->d[0]) * (top * numPowers + + ((2 * top) > + numPowers ? (2 * top) : numPowers)); #ifdef alloca if (powerbufLen < 3072) powerbufFree = @@ -670,15 +774,17 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, tmp.flags = am.flags = BN_FLG_STATIC_DATA; /* prepare a^0 in Montgomery domain */ -#if 1 +#if 1 /* by Shay Gueron's suggestion */ + if (m->d[top - 1] & (((BN_ULONG)1) << (BN_BITS2 - 1))) { + /* 2^(top*BN_BITS2) - m */ + tmp.d[0] = (0 - m->d[0]) & BN_MASK2; + for (i = 1; i < top; i++) + tmp.d[i] = (~m->d[i]) & BN_MASK2; + tmp.top = top; + } else +#endif if (!BN_to_montgomery(&tmp, BN_value_one(), mont, ctx)) goto err; -#else - tmp.d[0] = (0 - m->d[0]) & BN_MASK2; /* 2^(top*BN_BITS2) - m */ - for (i = 1; i < top; i++) - tmp.d[i] = (~m->d[i]) & BN_MASK2; - tmp.top = top; -#endif /* prepare a^1 in Montgomery domain */ if (a->neg || BN_ucmp(a, m) >= 0) { @@ -689,6 +795,138 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, } else if (!BN_to_montgomery(&am, a, mont, ctx)) goto err; +#if defined(SPARC_T4_MONT) + if (t4) { + typedef int (*bn_pwr5_mont_f) (BN_ULONG *tp, const BN_ULONG *np, + const BN_ULONG *n0, const void *table, + int power, int bits); + int bn_pwr5_mont_t4_8(BN_ULONG *tp, const BN_ULONG *np, + const BN_ULONG *n0, const void *table, + int power, int bits); + int bn_pwr5_mont_t4_16(BN_ULONG *tp, const BN_ULONG *np, + const BN_ULONG *n0, const void *table, + int power, int bits); + int bn_pwr5_mont_t4_24(BN_ULONG *tp, const BN_ULONG *np, + const BN_ULONG *n0, const void *table, + int power, int bits); + int bn_pwr5_mont_t4_32(BN_ULONG *tp, const BN_ULONG *np, + const BN_ULONG *n0, const void *table, + int power, int bits); + static const bn_pwr5_mont_f pwr5_funcs[4] = { + bn_pwr5_mont_t4_8, bn_pwr5_mont_t4_16, + bn_pwr5_mont_t4_24, bn_pwr5_mont_t4_32 + }; + bn_pwr5_mont_f pwr5_worker = pwr5_funcs[top / 16 - 1]; + + typedef int (*bn_mul_mont_f) (BN_ULONG *rp, const BN_ULONG *ap, + const void *bp, const BN_ULONG *np, + const BN_ULONG *n0); + int bn_mul_mont_t4_8(BN_ULONG *rp, const BN_ULONG *ap, const void *bp, + const BN_ULONG *np, const BN_ULONG *n0); + int bn_mul_mont_t4_16(BN_ULONG *rp, const BN_ULONG *ap, + const void *bp, const BN_ULONG *np, + const BN_ULONG *n0); + int bn_mul_mont_t4_24(BN_ULONG *rp, const BN_ULONG *ap, + const void *bp, const BN_ULONG *np, + const BN_ULONG *n0); + int bn_mul_mont_t4_32(BN_ULONG *rp, const BN_ULONG *ap, + const void *bp, const BN_ULONG *np, + const BN_ULONG *n0); + static const bn_mul_mont_f mul_funcs[4] = { + bn_mul_mont_t4_8, bn_mul_mont_t4_16, + bn_mul_mont_t4_24, bn_mul_mont_t4_32 + }; + bn_mul_mont_f mul_worker = mul_funcs[top / 16 - 1]; + + void bn_mul_mont_vis3(BN_ULONG *rp, const BN_ULONG *ap, + const void *bp, const BN_ULONG *np, + const BN_ULONG *n0, int num); + void bn_mul_mont_t4(BN_ULONG *rp, const BN_ULONG *ap, + const void *bp, const BN_ULONG *np, + const BN_ULONG *n0, int num); + void bn_mul_mont_gather5_t4(BN_ULONG *rp, const BN_ULONG *ap, + const void *table, const BN_ULONG *np, + const BN_ULONG *n0, int num, int power); + void bn_flip_n_scatter5_t4(const BN_ULONG *inp, size_t num, + void *table, size_t power); + void bn_gather5_t4(BN_ULONG *out, size_t num, + void *table, size_t power); + void bn_flip_t4(BN_ULONG *dst, BN_ULONG *src, size_t num); + + BN_ULONG *np = mont->N.d, *n0 = mont->n0; + int stride = 5 * (6 - (top / 16 - 1)); /* multiple of 5, but less + * than 32 */ + + /* + * BN_to_montgomery can contaminate words above .top [in + * BN_DEBUG[_DEBUG] build]... + */ + for (i = am.top; i < top; i++) + am.d[i] = 0; + for (i = tmp.top; i < top; i++) + tmp.d[i] = 0; + + bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, 0); + bn_flip_n_scatter5_t4(am.d, top, powerbuf, 1); + if (!(*mul_worker) (tmp.d, am.d, am.d, np, n0) && + !(*mul_worker) (tmp.d, am.d, am.d, np, n0)) + bn_mul_mont_vis3(tmp.d, am.d, am.d, np, n0, top); + bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, 2); + + for (i = 3; i < 32; i++) { + /* Calculate a^i = a^(i-1) * a */ + if (!(*mul_worker) (tmp.d, tmp.d, am.d, np, n0) && + !(*mul_worker) (tmp.d, tmp.d, am.d, np, n0)) + bn_mul_mont_vis3(tmp.d, tmp.d, am.d, np, n0, top); + bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, i); + } + + /* switch to 64-bit domain */ + np = alloca(top * sizeof(BN_ULONG)); + top /= 2; + bn_flip_t4(np, mont->N.d, top); + + bits--; + for (wvalue = 0, i = bits % 5; i >= 0; i--, bits--) + wvalue = (wvalue << 1) + BN_is_bit_set(p, bits); + bn_gather5_t4(tmp.d, top, powerbuf, wvalue); + + /* + * Scan the exponent one window at a time starting from the most + * significant bits. + */ + while (bits >= 0) { + if (bits < stride) + stride = bits + 1; + bits -= stride; + wvalue = bn_get_bits(p, bits + 1); + + if ((*pwr5_worker) (tmp.d, np, n0, powerbuf, wvalue, stride)) + continue; + /* retry once and fall back */ + if ((*pwr5_worker) (tmp.d, np, n0, powerbuf, wvalue, stride)) + continue; + + bits += stride - 5; + wvalue >>= stride - 5; + wvalue &= 31; + bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont_gather5_t4(tmp.d, tmp.d, powerbuf, np, n0, top, + wvalue); + } + + bn_flip_t4(tmp.d, tmp.d, top); + top *= 2; + /* back to 32-bit domain */ + tmp.top = top; + bn_correct_top(&tmp); + OPENSSL_cleanse(np, top * sizeof(BN_ULONG)); + } else +#endif #if defined(OPENSSL_BN_ASM_MONT5) if (window == 5 && top > 1) { /* @@ -707,8 +945,15 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, void bn_scatter5(const BN_ULONG *inp, size_t num, void *table, size_t power); void bn_gather5(BN_ULONG *out, size_t num, void *table, size_t power); + void bn_power5(BN_ULONG *rp, const BN_ULONG *ap, + const void *table, const BN_ULONG *np, + const BN_ULONG *n0, int num, int power); + int bn_get_bits5(const BN_ULONG *ap, int off); + int bn_from_montgomery(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *not_used, const BN_ULONG *np, + const BN_ULONG *n0, int num); - BN_ULONG *np = mont->N.d, *n0 = mont->n0; + BN_ULONG *np = mont->N.d, *n0 = mont->n0, *np2; /* * BN_to_montgomery can contaminate words above .top [in @@ -719,6 +964,12 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, for (i = tmp.top; i < top; i++) tmp.d[i] = 0; + if (top & 7) + np2 = np; + else + for (np2 = am.d + top, i = 0; i < top; i++) + np2[2 * i] = np[i]; + bn_scatter5(tmp.d, top, powerbuf, 0); bn_scatter5(am.d, am.top, powerbuf, 1); bn_mul_mont(tmp.d, am.d, am.d, np, n0, top); @@ -727,7 +978,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, # if 0 for (i = 3; i < 32; i++) { /* Calculate a^i = a^(i-1) * a */ - bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np, n0, top, i - 1); + bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np2, n0, top, i - 1); bn_scatter5(tmp.d, top, powerbuf, i); } # else @@ -738,7 +989,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, } for (i = 3; i < 8; i += 2) { int j; - bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np, n0, top, i - 1); + bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np2, n0, top, i - 1); bn_scatter5(tmp.d, top, powerbuf, i); for (j = 2 * i; j < 32; j *= 2) { bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); @@ -746,13 +997,13 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, } } for (; i < 16; i += 2) { - bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np, n0, top, i - 1); + bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np2, n0, top, i - 1); bn_scatter5(tmp.d, top, powerbuf, i); bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); bn_scatter5(tmp.d, top, powerbuf, 2 * i); } for (; i < 32; i += 2) { - bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np, n0, top, i - 1); + bn_mul_mont_gather5(tmp.d, am.d, powerbuf, np2, n0, top, i - 1); bn_scatter5(tmp.d, top, powerbuf, i); } # endif @@ -765,20 +1016,34 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, * Scan the exponent one window at a time starting from the most * significant bits. */ - while (bits >= 0) { - for (wvalue = 0, i = 0; i < 5; i++, bits--) - wvalue = (wvalue << 1) + BN_is_bit_set(p, bits); + if (top & 7) + while (bits >= 0) { + for (wvalue = 0, i = 0; i < 5; i++, bits--) + wvalue = (wvalue << 1) + BN_is_bit_set(p, bits); - bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont_gather5(tmp.d, tmp.d, powerbuf, np, n0, top, wvalue); + bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont(tmp.d, tmp.d, tmp.d, np, n0, top); + bn_mul_mont_gather5(tmp.d, tmp.d, powerbuf, np, n0, top, + wvalue); + } else { + while (bits >= 0) { + wvalue = bn_get_bits5(p->d, bits - 4); + bits -= 5; + bn_power5(tmp.d, tmp.d, powerbuf, np2, n0, top, wvalue); + } } + ret = bn_from_montgomery(tmp.d, tmp.d, NULL, np2, n0, top); tmp.top = top; bn_correct_top(&tmp); + if (ret) { + if (!BN_copy(rr, &tmp)) + ret = 0; + goto err; /* non-zero ret means it's not error */ + } } else #endif { @@ -844,6 +1109,15 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, } /* Convert the final result from montgomery to standard format */ +#if defined(SPARC_T4_MONT) + if (OPENSSL_sparcv9cap_P[0] & (SPARCV9_VIS3 | SPARCV9_PREFER_FPU)) { + am.d[0] = 1; /* borrow am */ + for (i = 1; i < top; i++) + am.d[i] = 0; + if (!BN_mod_mul_montgomery(rr, &tmp, &am, mont, ctx)) + goto err; + } else +#endif if (!BN_from_montgomery(rr, &tmp, mont, ctx)) goto err; ret = 1; diff --git a/crypto/bn/bn_gf2m.c b/crypto/bn/bn_gf2m.c index a0ba8de31ad4..cfa1c7ce1499 100644 --- a/crypto/bn/bn_gf2m.c +++ b/crypto/bn/bn_gf2m.c @@ -450,8 +450,7 @@ int BN_GF2m_mod_arr(BIGNUM *r, const BIGNUM *a, const int p[]) d0 = p[k] % BN_BITS2; d1 = BN_BITS2 - d0; z[n] ^= (zz << d0); - tmp_ulong = zz >> d1; - if (d0 && tmp_ulong) + if (d0 && (tmp_ulong = zz >> d1)) z[n + 1] ^= tmp_ulong; } diff --git a/crypto/bn/bn_lcl.h b/crypto/bn/bn_lcl.h index 904a723497d1..00f4f09945b3 100644 --- a/crypto/bn/bn_lcl.h +++ b/crypto/bn/bn_lcl.h @@ -204,6 +204,24 @@ extern "C" { # define BN_MUL_LOW_RECURSIVE_SIZE_NORMAL (32)/* 32 */ # define BN_MONT_CTX_SET_SIZE_WORD (64)/* 32 */ +/* + * 2011-02-22 SMS. In various places, a size_t variable or a type cast to + * size_t was used to perform integer-only operations on pointers. This + * failed on VMS with 64-bit pointers (CC /POINTER_SIZE = 64) because size_t + * is still only 32 bits. What's needed in these cases is an integer type + * with the same size as a pointer, which size_t is not certain to be. The + * only fix here is VMS-specific. + */ +# if defined(OPENSSL_SYS_VMS) +# if __INITIAL_POINTER_SIZE == 64 +# define PTR_SIZE_INT long long +# else /* __INITIAL_POINTER_SIZE == 64 */ +# define PTR_SIZE_INT int +# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ +# elif !defined(PTR_SIZE_INT) /* defined(OPENSSL_SYS_VMS) */ +# define PTR_SIZE_INT size_t +# endif /* defined(OPENSSL_SYS_VMS) [else] */ + # if !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) && !defined(PEDANTIC) /* * BN_UMULT_HIGH section. @@ -295,6 +313,15 @@ unsigned __int64 _umul128(unsigned __int64 a, unsigned __int64 b, : "r"(a), "r"(b)); # endif # endif +# elif defined(__aarch64__) && defined(SIXTY_FOUR_BIT_LONG) +# if defined(__GNUC__) && __GNUC__>=2 +# define BN_UMULT_HIGH(a,b) ({ \ + register BN_ULONG ret; \ + asm ("umulh %0,%1,%2" \ + : "=r"(ret) \ + : "r"(a), "r"(b)); \ + ret; }) +# endif # endif /* cpu */ # endif /* OPENSSL_NO_ASM */ diff --git a/crypto/bn/bntest.c b/crypto/bn/bntest.c index 06662c58b32f..470d5dabf1ec 100644 --- a/crypto/bn/bntest.c +++ b/crypto/bn/bntest.c @@ -1042,7 +1042,6 @@ int test_mod_exp_mont_consttime(BIO *bp, BN_CTX *ctx) int test_mod_exp_mont5(BIO *bp, BN_CTX *ctx) { BIGNUM *a, *p, *m, *d, *e; - BN_MONT_CTX *mont; a = BN_new(); @@ -1050,7 +1049,6 @@ int test_mod_exp_mont5(BIO *bp, BN_CTX *ctx) m = BN_new(); d = BN_new(); e = BN_new(); - mont = BN_MONT_CTX_new(); BN_bntest_rand(m, 1024, 0, 1); /* must be odd for montgomery */ @@ -1099,6 +1097,7 @@ int test_mod_exp_mont5(BIO *bp, BN_CTX *ctx) fprintf(stderr, "Modular exponentiation test failed!\n"); return 0; } + BN_MONT_CTX_free(mont); BN_free(a); BN_free(p); BN_free(m); diff --git a/crypto/bn/rsaz_exp.c b/crypto/bn/rsaz_exp.c new file mode 100644 index 000000000000..c54c6feb51b5 --- /dev/null +++ b/crypto/bn/rsaz_exp.c @@ -0,0 +1,346 @@ +/***************************************************************************** +* * +* Copyright (c) 2012, Intel Corporation * +* * +* All rights reserved. * +* * +* Redistribution and use in source and binary forms, with or without * +* modification, are permitted provided that the following conditions are * +* met: * +* * +* * Redistributions of source code must retain the above copyright * +* notice, this list of conditions and the following disclaimer. * +* * +* * Redistributions in binary form must reproduce the above copyright * +* notice, this list of conditions and the following disclaimer in the * +* documentation and/or other materials provided with the * +* distribution. * +* * +* * Neither the name of the Intel Corporation nor the names of its * +* contributors may be used to endorse or promote products derived from * +* this software without specific prior written permission. * +* * +* * +* THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY * +* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * +* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * +* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR * +* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * +* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * +* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * +* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * +* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * +* * +****************************************************************************** +* Developers and authors: * +* Shay Gueron (1, 2), and Vlad Krasnov (1) * +* (1) Intel Corporation, Israel Development Center, Haifa, Israel * +* (2) University of Haifa, Israel * +*****************************************************************************/ + +#include "rsaz_exp.h" + +#ifdef RSAZ_ENABLED + +/* + * See crypto/bn/asm/rsaz-avx2.pl for further details. + */ +void rsaz_1024_norm2red_avx2(void *red, const void *norm); +void rsaz_1024_mul_avx2(void *ret, const void *a, const void *b, + const void *n, BN_ULONG k); +void rsaz_1024_sqr_avx2(void *ret, const void *a, const void *n, BN_ULONG k, + int cnt); +void rsaz_1024_scatter5_avx2(void *tbl, const void *val, int i); +void rsaz_1024_gather5_avx2(void *val, const void *tbl, int i); +void rsaz_1024_red2norm_avx2(void *norm, const void *red); + +#if defined(__GNUC__) +# define ALIGN64 __attribute__((aligned(64))) +#elif defined(_MSC_VER) +# define ALIGN64 __declspec(align(64)) +#elif defined(__SUNPRO_C) +# define ALIGN64 +# pragma align 64(one,two80) +#else +/* not fatal, might hurt performance a little */ +# define ALIGN64 +#endif + +ALIGN64 static const BN_ULONG one[40] = { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +ALIGN64 static const BN_ULONG two80[40] = { + 0, 0, 1 << 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +void RSAZ_1024_mod_exp_avx2(BN_ULONG result_norm[16], + const BN_ULONG base_norm[16], + const BN_ULONG exponent[16], + const BN_ULONG m_norm[16], const BN_ULONG RR[16], + BN_ULONG k0) +{ + unsigned char storage[320 * 3 + 32 * 9 * 16 + 64]; /* 5.5KB */ + unsigned char *p_str = storage + (64 - ((size_t)storage % 64)); + unsigned char *a_inv, *m, *result; + unsigned char *table_s = p_str + 320 * 3; + unsigned char *R2 = table_s; /* borrow */ + int index; + int wvalue; + + if ((((size_t)p_str & 4095) + 320) >> 12) { + result = p_str; + a_inv = p_str + 320; + m = p_str + 320 * 2; /* should not cross page */ + } else { + m = p_str; /* should not cross page */ + result = p_str + 320; + a_inv = p_str + 320 * 2; + } + + rsaz_1024_norm2red_avx2(m, m_norm); + rsaz_1024_norm2red_avx2(a_inv, base_norm); + rsaz_1024_norm2red_avx2(R2, RR); + + rsaz_1024_mul_avx2(R2, R2, R2, m, k0); + rsaz_1024_mul_avx2(R2, R2, two80, m, k0); + + /* table[0] = 1 */ + rsaz_1024_mul_avx2(result, R2, one, m, k0); + /* table[1] = a_inv^1 */ + rsaz_1024_mul_avx2(a_inv, a_inv, R2, m, k0); + + rsaz_1024_scatter5_avx2(table_s, result, 0); + rsaz_1024_scatter5_avx2(table_s, a_inv, 1); + + /* table[2] = a_inv^2 */ + rsaz_1024_sqr_avx2(result, a_inv, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 2); +#if 0 + /* this is almost 2x smaller and less than 1% slower */ + for (index = 3; index < 32; index++) { + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, index); + } +#else + /* table[4] = a_inv^4 */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 4); + /* table[8] = a_inv^8 */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 8); + /* table[16] = a_inv^16 */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 16); + /* table[17] = a_inv^17 */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 17); + + /* table[3] */ + rsaz_1024_gather5_avx2(result, table_s, 2); + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 3); + /* table[6] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 6); + /* table[12] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 12); + /* table[24] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 24); + /* table[25] */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 25); + + /* table[5] */ + rsaz_1024_gather5_avx2(result, table_s, 4); + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 5); + /* table[10] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 10); + /* table[20] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 20); + /* table[21] */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 21); + + /* table[7] */ + rsaz_1024_gather5_avx2(result, table_s, 6); + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 7); + /* table[14] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 14); + /* table[28] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 28); + /* table[29] */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 29); + + /* table[9] */ + rsaz_1024_gather5_avx2(result, table_s, 8); + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 9); + /* table[18] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 18); + /* table[19] */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 19); + + /* table[11] */ + rsaz_1024_gather5_avx2(result, table_s, 10); + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 11); + /* table[22] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 22); + /* table[23] */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 23); + + /* table[13] */ + rsaz_1024_gather5_avx2(result, table_s, 12); + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 13); + /* table[26] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 26); + /* table[27] */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 27); + + /* table[15] */ + rsaz_1024_gather5_avx2(result, table_s, 14); + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 15); + /* table[30] */ + rsaz_1024_sqr_avx2(result, result, m, k0, 1); + rsaz_1024_scatter5_avx2(table_s, result, 30); + /* table[31] */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + rsaz_1024_scatter5_avx2(table_s, result, 31); +#endif + + /* load first window */ + p_str = (unsigned char *)exponent; + wvalue = p_str[127] >> 3; + rsaz_1024_gather5_avx2(result, table_s, wvalue); + + index = 1014; + + while (index > -1) { /* loop for the remaining 127 windows */ + + rsaz_1024_sqr_avx2(result, result, m, k0, 5); + + wvalue = *((unsigned short *)&p_str[index / 8]); + wvalue = (wvalue >> (index % 8)) & 31; + index -= 5; + + rsaz_1024_gather5_avx2(a_inv, table_s, wvalue); /* borrow a_inv */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + } + + /* square four times */ + rsaz_1024_sqr_avx2(result, result, m, k0, 4); + + wvalue = p_str[0] & 15; + + rsaz_1024_gather5_avx2(a_inv, table_s, wvalue); /* borrow a_inv */ + rsaz_1024_mul_avx2(result, result, a_inv, m, k0); + + /* from Montgomery */ + rsaz_1024_mul_avx2(result, result, one, m, k0); + + rsaz_1024_red2norm_avx2(result_norm, result); + + OPENSSL_cleanse(storage, sizeof(storage)); +} + +/* + * See crypto/bn/rsaz-x86_64.pl for further details. + */ +void rsaz_512_mul(void *ret, const void *a, const void *b, const void *n, + BN_ULONG k); +void rsaz_512_mul_scatter4(void *ret, const void *a, const void *n, + BN_ULONG k, const void *tbl, unsigned int power); +void rsaz_512_mul_gather4(void *ret, const void *a, const void *tbl, + const void *n, BN_ULONG k, unsigned int power); +void rsaz_512_mul_by_one(void *ret, const void *a, const void *n, BN_ULONG k); +void rsaz_512_sqr(void *ret, const void *a, const void *n, BN_ULONG k, + int cnt); +void rsaz_512_scatter4(void *tbl, const BN_ULONG *val, int power); +void rsaz_512_gather4(BN_ULONG *val, const void *tbl, int power); + +void RSAZ_512_mod_exp(BN_ULONG result[8], + const BN_ULONG base[8], const BN_ULONG exponent[8], + const BN_ULONG m[8], BN_ULONG k0, const BN_ULONG RR[8]) +{ + unsigned char storage[16 * 8 * 8 + 64 * 2 + 64]; /* 1.2KB */ + unsigned char *table = storage + (64 - ((size_t)storage % 64)); + BN_ULONG *a_inv = (BN_ULONG *)(table + 16 * 8 * 8); + BN_ULONG *temp = (BN_ULONG *)(table + 16 * 8 * 8 + 8 * 8); + unsigned char *p_str = (unsigned char *)exponent; + int index; + unsigned int wvalue; + + /* table[0] = 1_inv */ + temp[0] = 0 - m[0]; + temp[1] = ~m[1]; + temp[2] = ~m[2]; + temp[3] = ~m[3]; + temp[4] = ~m[4]; + temp[5] = ~m[5]; + temp[6] = ~m[6]; + temp[7] = ~m[7]; + rsaz_512_scatter4(table, temp, 0); + + /* table [1] = a_inv^1 */ + rsaz_512_mul(a_inv, base, RR, m, k0); + rsaz_512_scatter4(table, a_inv, 1); + + /* table [2] = a_inv^2 */ + rsaz_512_sqr(temp, a_inv, m, k0, 1); + rsaz_512_scatter4(table, temp, 2); + + for (index = 3; index < 16; index++) + rsaz_512_mul_scatter4(temp, a_inv, m, k0, table, index); + + /* load first window */ + wvalue = p_str[63]; + + rsaz_512_gather4(temp, table, wvalue >> 4); + rsaz_512_sqr(temp, temp, m, k0, 4); + rsaz_512_mul_gather4(temp, temp, table, m, k0, wvalue & 0xf); + + for (index = 62; index >= 0; index--) { + wvalue = p_str[index]; + + rsaz_512_sqr(temp, temp, m, k0, 4); + rsaz_512_mul_gather4(temp, temp, table, m, k0, wvalue >> 4); + + rsaz_512_sqr(temp, temp, m, k0, 4); + rsaz_512_mul_gather4(temp, temp, table, m, k0, wvalue & 0x0f); + } + + /* from Montgomery */ + rsaz_512_mul_by_one(result, temp, m, k0); + + OPENSSL_cleanse(storage, sizeof(storage)); +} + +#else + +# if defined(PEDANTIC) || defined(__DECC) || defined(__clang__) +static void *dummy = &dummy; +# endif + +#endif diff --git a/crypto/bn/rsaz_exp.h b/crypto/bn/rsaz_exp.h new file mode 100644 index 000000000000..33361de99572 --- /dev/null +++ b/crypto/bn/rsaz_exp.h @@ -0,0 +1,56 @@ +/****************************************************************************** +* Copyright(c) 2012, Intel Corp. +* Developers and authors: +* Shay Gueron (1, 2), and Vlad Krasnov (1) +* (1) Intel Corporation, Israel Development Center, Haifa, Israel +* (2) University of Haifa, Israel +****************************************************************************** +* LICENSE: +* This submission to OpenSSL is to be made available under the OpenSSL +* license, and only to the OpenSSL project, in order to allow integration +* into the publicly distributed code. +* The use of this code, or portions of this code, or concepts embedded in +* this code, or modification of this code and/or algorithm(s) in it, or the +* use of this code for any other purpose than stated above, requires special +* licensing. +****************************************************************************** +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE CONTRIBUTORS AND THE COPYRIGHT OWNERS +* ``AS IS''. ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS OR THE COPYRIGHT +* OWNERS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +* POSSIBILITY OF SUCH DAMAGE. +******************************************************************************/ + +#ifndef RSAZ_EXP_H +# define RSAZ_EXP_H + +# undef RSAZ_ENABLED +# if defined(OPENSSL_BN_ASM_MONT) && \ + (defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_AMD64) || defined(_M_X64)) +# define RSAZ_ENABLED + +# include + +void RSAZ_1024_mod_exp_avx2(BN_ULONG result[16], + const BN_ULONG base_norm[16], + const BN_ULONG exponent[16], + const BN_ULONG m_norm[16], const BN_ULONG RR[16], + BN_ULONG k0); +int rsaz_avx2_eligible(); + +void RSAZ_512_mod_exp(BN_ULONG result[8], + const BN_ULONG base_norm[8], const BN_ULONG exponent[8], + const BN_ULONG m_norm[8], BN_ULONG k0, + const BN_ULONG RR[8]); + +# endif + +#endif diff --git a/crypto/buffer/buf_str.c b/crypto/buffer/buf_str.c index fdde3d7db4ba..ebc5ab4646ce 100644 --- a/crypto/buffer/buf_str.c +++ b/crypto/buffer/buf_str.c @@ -60,6 +60,15 @@ #include "cryptlib.h" #include +size_t BUF_strnlen(const char *str, size_t maxlen) +{ + const char *p; + + for (p = str; maxlen-- != 0 && *p != '\0'; ++p) ; + + return p - str; +} + char *BUF_strdup(const char *str) { if (str == NULL) @@ -74,6 +83,8 @@ char *BUF_strndup(const char *str, size_t siz) if (str == NULL) return (NULL); + siz = BUF_strnlen(str, siz); + ret = OPENSSL_malloc(siz + 1); if (ret == NULL) { BUFerr(BUF_F_BUF_STRNDUP, ERR_R_MALLOC_FAILURE); diff --git a/crypto/buffer/buffer.h b/crypto/buffer/buffer.h index 632df93c657c..c343dd772f1e 100644 --- a/crypto/buffer/buffer.h +++ b/crypto/buffer/buffer.h @@ -84,6 +84,7 @@ BUF_MEM *BUF_MEM_new(void); void BUF_MEM_free(BUF_MEM *a); int BUF_MEM_grow(BUF_MEM *str, size_t len); int BUF_MEM_grow_clean(BUF_MEM *str, size_t len); +size_t BUF_strnlen(const char *str, size_t maxlen); char *BUF_strdup(const char *str); char *BUF_strndup(const char *str, size_t siz); void *BUF_memdup(const void *data, size_t siz); diff --git a/crypto/camellia/Makefile b/crypto/camellia/Makefile index 228f1dc713d5..ab1225e7d902 100644 --- a/crypto/camellia/Makefile +++ b/crypto/camellia/Makefile @@ -48,6 +48,8 @@ cmll-x86.s: asm/cmll-x86.pl ../perlasm/x86asm.pl $(PERL) asm/cmll-x86.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@ cmll-x86_64.s: asm/cmll-x86_64.pl $(PERL) asm/cmll-x86_64.pl $(PERLASM_SCHEME) > $@ +cmllt4-sparcv9.s: asm/cmllt4-sparcv9.pl ../perlasm/sparcv9_modes.pl + $(PERL) asm/cmllt4-sparcv9.pl $(CFLAGS) > $@ files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/camellia/asm/cmll-x86_64.pl b/crypto/camellia/asm/cmll-x86_64.pl index 9f4b82fa4821..d94f46b887e5 100755 --- a/crypto/camellia/asm/cmll-x86_64.pl +++ b/crypto/camellia/asm/cmll-x86_64.pl @@ -72,7 +72,7 @@ my $i=@_[0]; my $seed=defined(@_[1])?@_[1]:0; my $scale=$seed<0?-8:8; my $j=($i&1)*2; -my $s0=@S[($j)%4],$s1=@S[($j+1)%4],$s2=@S[($j+2)%4],$s3=@S[($j+3)%4]; +my ($s0,$s1,$s2,$s3)=(@S[($j)%4],@S[($j+1)%4],@S[($j+2)%4],@S[($j+3)%4]); $code.=<<___; xor $s0,$t0 # t0^=key[0] @@ -409,7 +409,7 @@ Camellia_Ekeygen: push %r15 .Lkey_prologue: - mov %rdi,$keyend # put away arguments, keyBitLength + mov %edi,${keyend}d # put away arguments, keyBitLength mov %rdx,$out # keyTable mov 0(%rsi),@S[0] # load 0-127 bits diff --git a/crypto/camellia/asm/cmllt4-sparcv9.pl b/crypto/camellia/asm/cmllt4-sparcv9.pl new file mode 100755 index 000000000000..a813168b42ec --- /dev/null +++ b/crypto/camellia/asm/cmllt4-sparcv9.pl @@ -0,0 +1,929 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by David S. Miller and Andy Polyakov +# . The module is licensed under 2-clause BSD +# license. October 2012. All rights reserved. +# ==================================================================== + +###################################################################### +# Camellia for SPARC T4. +# +# As with AES below results [for aligned data] are virtually identical +# to critical path lenths for 3-cycle instruction latency: +# +# 128-bit key 192/256- +# CBC encrypt 4.14/4.21(*) 5.46/5.52 +# (*) numbers after slash are for +# misaligned data. +# +# As with Intel AES-NI, question is if it's possible to improve +# performance of parallelizeable modes by interleaving round +# instructions. In Camellia every instruction is dependent on +# previous, which means that there is place for 2 additional ones +# in between two dependent. Can we expect 3x performance improvement? +# At least one can argue that it should be possible to break 2x +# barrier... For some reason not even 2x appears to be possible: +# +# 128-bit key 192/256- +# CBC decrypt 2.21/2.74 2.99/3.40 +# CTR 2.15/2.68(*) 2.93/3.34 +# (*) numbers after slash are for +# misaligned data. +# +# This is for 2x interleave. But compared to 1x interleave CBC decrypt +# improved by ... 0% for 128-bit key, and 11% for 192/256-bit one. +# So that out-of-order execution logic can take non-interleaved code +# to 1.87x, but can't take 2x interleaved one any further. There +# surely is some explanation... As result 3x interleave was not even +# attempted. Instead an effort was made to share specific modes +# implementations with AES module (therefore sparct4_modes.pl). +# +# To anchor to something else, software C implementation processes +# one byte in 38 cycles with 128-bit key on same processor. + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +push(@INC,"${dir}","${dir}../../perlasm"); +require "sparcv9_modes.pl"; + +&asm_init(@ARGV); + +$::evp=1; # if $evp is set to 0, script generates module with +# Camellia_[en|de]crypt, Camellia_set_key and Camellia_cbc_encrypt +# entry points. These are fully compatible with openssl/camellia.h. + +###################################################################### +# single-round subroutines +# +{ +my ($inp,$out,$key,$rounds,$tmp,$mask)=map("%o$_",(0..5)); + +$code=<<___; +.text + +.globl cmll_t4_encrypt +.align 32 +cmll_t4_encrypt: + andcc $inp, 7, %g1 ! is input aligned? + andn $inp, 7, $inp + + ldx [$key + 0], %g4 + ldx [$key + 8], %g5 + + ldx [$inp + 0], %o4 + bz,pt %icc, 1f + ldx [$inp + 8], %o5 + ldx [$inp + 16], $inp + sll %g1, 3, %g1 + sub %g0, %g1, %o3 + sllx %o4, %g1, %o4 + sllx %o5, %g1, %g1 + srlx %o5, %o3, %o5 + srlx $inp, %o3, %o3 + or %o5, %o4, %o4 + or %o3, %g1, %o5 +1: + ld [$key + 272], $rounds ! grandRounds, 3 or 4 + ldd [$key + 16], %f12 + ldd [$key + 24], %f14 + xor %g4, %o4, %o4 + xor %g5, %o5, %o5 + ldd [$key + 32], %f16 + ldd [$key + 40], %f18 + movxtod %o4, %f0 + movxtod %o5, %f2 + ldd [$key + 48], %f20 + ldd [$key + 56], %f22 + sub $rounds, 1, $rounds + ldd [$key + 64], %f24 + ldd [$key + 72], %f26 + add $key, 80, $key + +.Lenc: + camellia_f %f12, %f2, %f0, %f2 + ldd [$key + 0], %f12 + sub $rounds,1,$rounds + camellia_f %f14, %f0, %f2, %f0 + ldd [$key + 8], %f14 + camellia_f %f16, %f2, %f0, %f2 + ldd [$key + 16], %f16 + camellia_f %f18, %f0, %f2, %f0 + ldd [$key + 24], %f18 + camellia_f %f20, %f2, %f0, %f2 + ldd [$key + 32], %f20 + camellia_f %f22, %f0, %f2, %f0 + ldd [$key + 40], %f22 + camellia_fl %f24, %f0, %f0 + ldd [$key + 48], %f24 + camellia_fli %f26, %f2, %f2 + ldd [$key + 56], %f26 + brnz,pt $rounds, .Lenc + add $key, 64, $key + + andcc $out, 7, $tmp ! is output aligned? + camellia_f %f12, %f2, %f0, %f2 + camellia_f %f14, %f0, %f2, %f0 + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f18, %f0, %f2, %f0 + camellia_f %f20, %f2, %f0, %f4 + camellia_f %f22, %f0, %f4, %f2 + fxor %f24, %f4, %f0 + fxor %f26, %f2, %f2 + + bnz,pn %icc, 2f + nop + + std %f0, [$out + 0] + retl + std %f2, [$out + 8] + +2: alignaddrl $out, %g0, $out + mov 0xff, $mask + srl $mask, $tmp, $mask + + faligndata %f0, %f0, %f4 + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + + stda %f4, [$out + $mask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $mask, $mask + retl + stda %f8, [$out + $mask]0xc0 ! partial store +.type cmll_t4_encrypt,#function +.size cmll_t4_encrypt,.-cmll_t4_encrypt + +.globl cmll_t4_decrypt +.align 32 +cmll_t4_decrypt: + ld [$key + 272], $rounds ! grandRounds, 3 or 4 + andcc $inp, 7, %g1 ! is input aligned? + andn $inp, 7, $inp + + sll $rounds, 6, $rounds + add $rounds, $key, $key + + ldx [$inp + 0], %o4 + bz,pt %icc, 1f + ldx [$inp + 8], %o5 + ldx [$inp + 16], $inp + sll %g1, 3, %g1 + sub %g0, %g1, %g4 + sllx %o4, %g1, %o4 + sllx %o5, %g1, %g1 + srlx %o5, %g4, %o5 + srlx $inp, %g4, %g4 + or %o5, %o4, %o4 + or %g4, %g1, %o5 +1: + ldx [$key + 0], %g4 + ldx [$key + 8], %g5 + ldd [$key - 8], %f12 + ldd [$key - 16], %f14 + xor %g4, %o4, %o4 + xor %g5, %o5, %o5 + ldd [$key - 24], %f16 + ldd [$key - 32], %f18 + movxtod %o4, %f0 + movxtod %o5, %f2 + ldd [$key - 40], %f20 + ldd [$key - 48], %f22 + sub $rounds, 64, $rounds + ldd [$key - 56], %f24 + ldd [$key - 64], %f26 + sub $key, 64, $key + +.Ldec: + camellia_f %f12, %f2, %f0, %f2 + ldd [$key - 8], %f12 + sub $rounds, 64, $rounds + camellia_f %f14, %f0, %f2, %f0 + ldd [$key - 16], %f14 + camellia_f %f16, %f2, %f0, %f2 + ldd [$key - 24], %f16 + camellia_f %f18, %f0, %f2, %f0 + ldd [$key - 32], %f18 + camellia_f %f20, %f2, %f0, %f2 + ldd [$key - 40], %f20 + camellia_f %f22, %f0, %f2, %f0 + ldd [$key - 48], %f22 + camellia_fl %f24, %f0, %f0 + ldd [$key - 56], %f24 + camellia_fli %f26, %f2, %f2 + ldd [$key - 64], %f26 + brnz,pt $rounds, .Ldec + sub $key, 64, $key + + andcc $out, 7, $tmp ! is output aligned? + camellia_f %f12, %f2, %f0, %f2 + camellia_f %f14, %f0, %f2, %f0 + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f18, %f0, %f2, %f0 + camellia_f %f20, %f2, %f0, %f4 + camellia_f %f22, %f0, %f4, %f2 + fxor %f26, %f4, %f0 + fxor %f24, %f2, %f2 + + bnz,pn %icc, 2f + nop + + std %f0, [$out + 0] + retl + std %f2, [$out + 8] + +2: alignaddrl $out, %g0, $out + mov 0xff, $mask + srl $mask, $tmp, $mask + + faligndata %f0, %f0, %f4 + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + + stda %f4, [$out + $mask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $mask, $mask + retl + stda %f8, [$out + $mask]0xc0 ! partial store +.type cmll_t4_decrypt,#function +.size cmll_t4_decrypt,.-cmll_t4_decrypt +___ +} + +###################################################################### +# key setup subroutines +# +{ +sub ROTL128 { + my $rot = shift; + + "srlx %o4, 64-$rot, %g4\n\t". + "sllx %o4, $rot, %o4\n\t". + "srlx %o5, 64-$rot, %g5\n\t". + "sllx %o5, $rot, %o5\n\t". + "or %o4, %g5, %o4\n\t". + "or %o5, %g4, %o5"; +} + +my ($inp,$bits,$out,$tmp)=map("%o$_",(0..5)); +$code.=<<___; +.globl cmll_t4_set_key +.align 32 +cmll_t4_set_key: + and $inp, 7, $tmp + alignaddr $inp, %g0, $inp + cmp $bits, 192 + ldd [$inp + 0], %f0 + bl,pt %icc,.L128 + ldd [$inp + 8], %f2 + + be,pt %icc,.L192 + ldd [$inp + 16], %f4 + + brz,pt $tmp, .L256aligned + ldd [$inp + 24], %f6 + + ldd [$inp + 32], %f8 + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 + faligndata %f4, %f6, %f4 + b .L256aligned + faligndata %f6, %f8, %f6 + +.align 16 +.L192: + brz,a,pt $tmp, .L256aligned + fnot2 %f4, %f6 + + ldd [$inp + 24], %f6 + nop + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 + faligndata %f4, %f6, %f4 + fnot2 %f4, %f6 + +.L256aligned: + std %f0, [$out + 0] ! k[0, 1] + fsrc2 %f0, %f28 + std %f2, [$out + 8] ! k[2, 3] + fsrc2 %f2, %f30 + fxor %f4, %f0, %f0 + b .L128key + fxor %f6, %f2, %f2 + +.align 16 +.L128: + brz,pt $tmp, .L128aligned + nop + + ldd [$inp + 16], %f4 + nop + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 + +.L128aligned: + std %f0, [$out + 0] ! k[0, 1] + fsrc2 %f0, %f28 + std %f2, [$out + 8] ! k[2, 3] + fsrc2 %f2, %f30 + +.L128key: + mov %o7, %o5 +1: call .+8 + add %o7, SIGMA-1b, %o4 + mov %o5, %o7 + + ldd [%o4 + 0], %f16 + ldd [%o4 + 8], %f18 + ldd [%o4 + 16], %f20 + ldd [%o4 + 24], %f22 + + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f18, %f0, %f2, %f0 + fxor %f28, %f0, %f0 + fxor %f30, %f2, %f2 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f22, %f0, %f2, %f0 + + bge,pn %icc, .L256key + nop + std %f0, [$out + 0x10] ! k[ 4, 5] + std %f2, [$out + 0x18] ! k[ 6, 7] + + movdtox %f0, %o4 + movdtox %f2, %o5 + `&ROTL128(15)` + stx %o4, [$out + 0x30] ! k[12, 13] + stx %o5, [$out + 0x38] ! k[14, 15] + `&ROTL128(15)` + stx %o4, [$out + 0x40] ! k[16, 17] + stx %o5, [$out + 0x48] ! k[18, 19] + `&ROTL128(15)` + stx %o4, [$out + 0x60] ! k[24, 25] + `&ROTL128(15)` + stx %o4, [$out + 0x70] ! k[28, 29] + stx %o5, [$out + 0x78] ! k[30, 31] + `&ROTL128(34)` + stx %o4, [$out + 0xa0] ! k[40, 41] + stx %o5, [$out + 0xa8] ! k[42, 43] + `&ROTL128(17)` + stx %o4, [$out + 0xc0] ! k[48, 49] + stx %o5, [$out + 0xc8] ! k[50, 51] + + movdtox %f28, %o4 ! k[ 0, 1] + movdtox %f30, %o5 ! k[ 2, 3] + `&ROTL128(15)` + stx %o4, [$out + 0x20] ! k[ 8, 9] + stx %o5, [$out + 0x28] ! k[10, 11] + `&ROTL128(30)` + stx %o4, [$out + 0x50] ! k[20, 21] + stx %o5, [$out + 0x58] ! k[22, 23] + `&ROTL128(15)` + stx %o5, [$out + 0x68] ! k[26, 27] + `&ROTL128(17)` + stx %o4, [$out + 0x80] ! k[32, 33] + stx %o5, [$out + 0x88] ! k[34, 35] + `&ROTL128(17)` + stx %o4, [$out + 0x90] ! k[36, 37] + stx %o5, [$out + 0x98] ! k[38, 39] + `&ROTL128(17)` + stx %o4, [$out + 0xb0] ! k[44, 45] + stx %o5, [$out + 0xb8] ! k[46, 47] + + mov 3, $tmp + st $tmp, [$out + 0x110] + retl + xor %o0, %o0, %o0 + +.align 16 +.L256key: + ldd [%o4 + 32], %f24 + ldd [%o4 + 40], %f26 + + std %f0, [$out + 0x30] ! k[12, 13] + std %f2, [$out + 0x38] ! k[14, 15] + + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + camellia_f %f24, %f2, %f0, %f2 + camellia_f %f26, %f0, %f2, %f0 + + std %f0, [$out + 0x10] ! k[ 4, 5] + std %f2, [$out + 0x18] ! k[ 6, 7] + + movdtox %f0, %o4 + movdtox %f2, %o5 + `&ROTL128(30)` + stx %o4, [$out + 0x50] ! k[20, 21] + stx %o5, [$out + 0x58] ! k[22, 23] + `&ROTL128(30)` + stx %o4, [$out + 0xa0] ! k[40, 41] + stx %o5, [$out + 0xa8] ! k[42, 43] + `&ROTL128(51)` + stx %o4, [$out + 0x100] ! k[64, 65] + stx %o5, [$out + 0x108] ! k[66, 67] + + movdtox %f4, %o4 ! k[ 8, 9] + movdtox %f6, %o5 ! k[10, 11] + `&ROTL128(15)` + stx %o4, [$out + 0x20] ! k[ 8, 9] + stx %o5, [$out + 0x28] ! k[10, 11] + `&ROTL128(15)` + stx %o4, [$out + 0x40] ! k[16, 17] + stx %o5, [$out + 0x48] ! k[18, 19] + `&ROTL128(30)` + stx %o4, [$out + 0x90] ! k[36, 37] + stx %o5, [$out + 0x98] ! k[38, 39] + `&ROTL128(34)` + stx %o4, [$out + 0xd0] ! k[52, 53] + stx %o5, [$out + 0xd8] ! k[54, 55] + ldx [$out + 0x30], %o4 ! k[12, 13] + ldx [$out + 0x38], %o5 ! k[14, 15] + `&ROTL128(15)` + stx %o4, [$out + 0x30] ! k[12, 13] + stx %o5, [$out + 0x38] ! k[14, 15] + `&ROTL128(30)` + stx %o4, [$out + 0x70] ! k[28, 29] + stx %o5, [$out + 0x78] ! k[30, 31] + srlx %o4, 32, %g4 + srlx %o5, 32, %g5 + st %o4, [$out + 0xc0] ! k[48] + st %g5, [$out + 0xc4] ! k[49] + st %o5, [$out + 0xc8] ! k[50] + st %g4, [$out + 0xcc] ! k[51] + `&ROTL128(49)` + stx %o4, [$out + 0xe0] ! k[56, 57] + stx %o5, [$out + 0xe8] ! k[58, 59] + + movdtox %f28, %o4 ! k[ 0, 1] + movdtox %f30, %o5 ! k[ 2, 3] + `&ROTL128(45)` + stx %o4, [$out + 0x60] ! k[24, 25] + stx %o5, [$out + 0x68] ! k[26, 27] + `&ROTL128(15)` + stx %o4, [$out + 0x80] ! k[32, 33] + stx %o5, [$out + 0x88] ! k[34, 35] + `&ROTL128(17)` + stx %o4, [$out + 0xb0] ! k[44, 45] + stx %o5, [$out + 0xb8] ! k[46, 47] + `&ROTL128(34)` + stx %o4, [$out + 0xf0] ! k[60, 61] + stx %o5, [$out + 0xf8] ! k[62, 63] + + mov 4, $tmp + st $tmp, [$out + 0x110] + retl + xor %o0, %o0, %o0 +.type cmll_t4_set_key,#function +.size cmll_t4_set_key,.-cmll_t4_set_key +.align 32 +SIGMA: + .long 0xa09e667f, 0x3bcc908b, 0xb67ae858, 0x4caa73b2 + .long 0xc6ef372f, 0xe94f82be, 0x54ff53a5, 0xf1d36f1c + .long 0x10e527fa, 0xde682d1d, 0xb05688c2, 0xb3e6c1fd +.type SIGMA,#object +.size SIGMA,.-SIGMA +.asciz "Camellia for SPARC T4, David S. Miller, Andy Polyakov" +___ +} + +{{{ +my ($inp,$out,$len,$key,$ivec,$enc)=map("%i$_",(0..5)); +my ($ileft,$iright,$ooff,$omask,$ivoff)=map("%l$_",(1..7)); + +$code.=<<___; +.align 32 +_cmll128_load_enckey: + ldx [$key + 0], %g4 + ldx [$key + 8], %g5 +___ +for ($i=2; $i<26;$i++) { # load key schedule + $code.=<<___; + ldd [$key + `8*$i`], %f`12+2*$i` +___ +} +$code.=<<___; + retl + nop +.type _cmll128_load_enckey,#function +.size _cmll128_load_enckey,.-_cmll128_load_enckey +_cmll256_load_enckey=_cmll128_load_enckey + +.align 32 +_cmll256_load_deckey: + ldd [$key + 64], %f62 + ldd [$key + 72], %f60 + b .Load_deckey + add $key, 64, $key +_cmll128_load_deckey: + ldd [$key + 0], %f60 + ldd [$key + 8], %f62 +.Load_deckey: +___ +for ($i=2; $i<24;$i++) { # load key schedule + $code.=<<___; + ldd [$key + `8*$i`], %f`62-2*$i` +___ +} +$code.=<<___; + ldx [$key + 192], %g4 + retl + ldx [$key + 200], %g5 +.type _cmll256_load_deckey,#function +.size _cmll256_load_deckey,.-_cmll256_load_deckey + +.align 32 +_cmll128_encrypt_1x: +___ +for ($i=0; $i<3; $i++) { + $code.=<<___; + camellia_f %f`16+16*$i+0`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+2`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+4`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+6`, %f0, %f2, %f0 +___ +$code.=<<___ if ($i<2); + camellia_f %f`16+16*$i+8`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+10`, %f0, %f2, %f0 + camellia_fl %f`16+16*$i+12`, %f0, %f0 + camellia_fli %f`16+16*$i+14`, %f2, %f2 +___ +} +$code.=<<___; + camellia_f %f56, %f2, %f0, %f4 + camellia_f %f58, %f0, %f4, %f2 + fxor %f60, %f4, %f0 + retl + fxor %f62, %f2, %f2 +.type _cmll128_encrypt_1x,#function +.size _cmll128_encrypt_1x,.-_cmll128_encrypt_1x +_cmll128_decrypt_1x=_cmll128_encrypt_1x + +.align 32 +_cmll128_encrypt_2x: +___ +for ($i=0; $i<3; $i++) { + $code.=<<___; + camellia_f %f`16+16*$i+0`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+0`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+2`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+2`, %f4, %f6, %f4 + camellia_f %f`16+16*$i+4`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+4`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+6`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+6`, %f4, %f6, %f4 +___ +$code.=<<___ if ($i<2); + camellia_f %f`16+16*$i+8`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+8`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+10`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+10`, %f4, %f6, %f4 + camellia_fl %f`16+16*$i+12`, %f0, %f0 + camellia_fl %f`16+16*$i+12`, %f4, %f4 + camellia_fli %f`16+16*$i+14`, %f2, %f2 + camellia_fli %f`16+16*$i+14`, %f6, %f6 +___ +} +$code.=<<___; + camellia_f %f56, %f2, %f0, %f8 + camellia_f %f56, %f6, %f4, %f10 + camellia_f %f58, %f0, %f8, %f2 + camellia_f %f58, %f4, %f10, %f6 + fxor %f60, %f8, %f0 + fxor %f60, %f10, %f4 + fxor %f62, %f2, %f2 + retl + fxor %f62, %f6, %f6 +.type _cmll128_encrypt_2x,#function +.size _cmll128_encrypt_2x,.-_cmll128_encrypt_2x +_cmll128_decrypt_2x=_cmll128_encrypt_2x + +.align 32 +_cmll256_encrypt_1x: + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f18, %f0, %f2, %f0 + ldd [$key + 208], %f16 + ldd [$key + 216], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f22, %f0, %f2, %f0 + ldd [$key + 224], %f20 + ldd [$key + 232], %f22 + camellia_f %f24, %f2, %f0, %f2 + camellia_f %f26, %f0, %f2, %f0 + ldd [$key + 240], %f24 + ldd [$key + 248], %f26 + camellia_fl %f28, %f0, %f0 + camellia_fli %f30, %f2, %f2 + ldd [$key + 256], %f28 + ldd [$key + 264], %f30 +___ +for ($i=1; $i<3; $i++) { + $code.=<<___; + camellia_f %f`16+16*$i+0`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+2`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+4`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+6`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+8`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+10`, %f0, %f2, %f0 + camellia_fl %f`16+16*$i+12`, %f0, %f0 + camellia_fli %f`16+16*$i+14`, %f2, %f2 +___ +} +$code.=<<___; + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f18, %f0, %f2, %f0 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f22, %f0, %f2, %f0 + ldd [$key + 32], %f20 + ldd [$key + 40], %f22 + camellia_f %f24, %f2, %f0, %f4 + camellia_f %f26, %f0, %f4, %f2 + ldd [$key + 48], %f24 + ldd [$key + 56], %f26 + fxor %f28, %f4, %f0 + fxor %f30, %f2, %f2 + ldd [$key + 64], %f28 + retl + ldd [$key + 72], %f30 +.type _cmll256_encrypt_1x,#function +.size _cmll256_encrypt_1x,.-_cmll256_encrypt_1x + +.align 32 +_cmll256_encrypt_2x: + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f16, %f6, %f4, %f6 + camellia_f %f18, %f0, %f2, %f0 + camellia_f %f18, %f4, %f6, %f4 + ldd [$key + 208], %f16 + ldd [$key + 216], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f20, %f6, %f4, %f6 + camellia_f %f22, %f0, %f2, %f0 + camellia_f %f22, %f4, %f6, %f4 + ldd [$key + 224], %f20 + ldd [$key + 232], %f22 + camellia_f %f24, %f2, %f0, %f2 + camellia_f %f24, %f6, %f4, %f6 + camellia_f %f26, %f0, %f2, %f0 + camellia_f %f26, %f4, %f6, %f4 + ldd [$key + 240], %f24 + ldd [$key + 248], %f26 + camellia_fl %f28, %f0, %f0 + camellia_fl %f28, %f4, %f4 + camellia_fli %f30, %f2, %f2 + camellia_fli %f30, %f6, %f6 + ldd [$key + 256], %f28 + ldd [$key + 264], %f30 +___ +for ($i=1; $i<3; $i++) { + $code.=<<___; + camellia_f %f`16+16*$i+0`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+0`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+2`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+2`, %f4, %f6, %f4 + camellia_f %f`16+16*$i+4`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+4`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+6`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+6`, %f4, %f6, %f4 + camellia_f %f`16+16*$i+8`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+8`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+10`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+10`, %f4, %f6, %f4 + camellia_fl %f`16+16*$i+12`, %f0, %f0 + camellia_fl %f`16+16*$i+12`, %f4, %f4 + camellia_fli %f`16+16*$i+14`, %f2, %f2 + camellia_fli %f`16+16*$i+14`, %f6, %f6 +___ +} +$code.=<<___; + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f16, %f6, %f4, %f6 + camellia_f %f18, %f0, %f2, %f0 + camellia_f %f18, %f4, %f6, %f4 + ldd [$key + 16], %f16 + ldd [$key + 24], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f20, %f6, %f4, %f6 + camellia_f %f22, %f0, %f2, %f0 + camellia_f %f22, %f4, %f6, %f4 + ldd [$key + 32], %f20 + ldd [$key + 40], %f22 + camellia_f %f24, %f2, %f0, %f8 + camellia_f %f24, %f6, %f4, %f10 + camellia_f %f26, %f0, %f8, %f2 + camellia_f %f26, %f4, %f10, %f6 + ldd [$key + 48], %f24 + ldd [$key + 56], %f26 + fxor %f28, %f8, %f0 + fxor %f28, %f10, %f4 + fxor %f30, %f2, %f2 + fxor %f30, %f6, %f6 + ldd [$key + 64], %f28 + retl + ldd [$key + 72], %f30 +.type _cmll256_encrypt_2x,#function +.size _cmll256_encrypt_2x,.-_cmll256_encrypt_2x + +.align 32 +_cmll256_decrypt_1x: + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f18, %f0, %f2, %f0 + ldd [$key - 8], %f16 + ldd [$key - 16], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f22, %f0, %f2, %f0 + ldd [$key - 24], %f20 + ldd [$key - 32], %f22 + camellia_f %f24, %f2, %f0, %f2 + camellia_f %f26, %f0, %f2, %f0 + ldd [$key - 40], %f24 + ldd [$key - 48], %f26 + camellia_fl %f28, %f0, %f0 + camellia_fli %f30, %f2, %f2 + ldd [$key - 56], %f28 + ldd [$key - 64], %f30 +___ +for ($i=1; $i<3; $i++) { + $code.=<<___; + camellia_f %f`16+16*$i+0`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+2`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+4`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+6`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+8`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+10`, %f0, %f2, %f0 + camellia_fl %f`16+16*$i+12`, %f0, %f0 + camellia_fli %f`16+16*$i+14`, %f2, %f2 +___ +} +$code.=<<___; + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f18, %f0, %f2, %f0 + ldd [$key + 184], %f16 + ldd [$key + 176], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f22, %f0, %f2, %f0 + ldd [$key + 168], %f20 + ldd [$key + 160], %f22 + camellia_f %f24, %f2, %f0, %f4 + camellia_f %f26, %f0, %f4, %f2 + ldd [$key + 152], %f24 + ldd [$key + 144], %f26 + fxor %f30, %f4, %f0 + fxor %f28, %f2, %f2 + ldd [$key + 136], %f28 + retl + ldd [$key + 128], %f30 +.type _cmll256_decrypt_1x,#function +.size _cmll256_decrypt_1x,.-_cmll256_decrypt_1x + +.align 32 +_cmll256_decrypt_2x: + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f16, %f6, %f4, %f6 + camellia_f %f18, %f0, %f2, %f0 + camellia_f %f18, %f4, %f6, %f4 + ldd [$key - 8], %f16 + ldd [$key - 16], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f20, %f6, %f4, %f6 + camellia_f %f22, %f0, %f2, %f0 + camellia_f %f22, %f4, %f6, %f4 + ldd [$key - 24], %f20 + ldd [$key - 32], %f22 + camellia_f %f24, %f2, %f0, %f2 + camellia_f %f24, %f6, %f4, %f6 + camellia_f %f26, %f0, %f2, %f0 + camellia_f %f26, %f4, %f6, %f4 + ldd [$key - 40], %f24 + ldd [$key - 48], %f26 + camellia_fl %f28, %f0, %f0 + camellia_fl %f28, %f4, %f4 + camellia_fli %f30, %f2, %f2 + camellia_fli %f30, %f6, %f6 + ldd [$key - 56], %f28 + ldd [$key - 64], %f30 +___ +for ($i=1; $i<3; $i++) { + $code.=<<___; + camellia_f %f`16+16*$i+0`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+0`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+2`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+2`, %f4, %f6, %f4 + camellia_f %f`16+16*$i+4`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+4`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+6`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+6`, %f4, %f6, %f4 + camellia_f %f`16+16*$i+8`, %f2, %f0, %f2 + camellia_f %f`16+16*$i+8`, %f6, %f4, %f6 + camellia_f %f`16+16*$i+10`, %f0, %f2, %f0 + camellia_f %f`16+16*$i+10`, %f4, %f6, %f4 + camellia_fl %f`16+16*$i+12`, %f0, %f0 + camellia_fl %f`16+16*$i+12`, %f4, %f4 + camellia_fli %f`16+16*$i+14`, %f2, %f2 + camellia_fli %f`16+16*$i+14`, %f6, %f6 +___ +} +$code.=<<___; + camellia_f %f16, %f2, %f0, %f2 + camellia_f %f16, %f6, %f4, %f6 + camellia_f %f18, %f0, %f2, %f0 + camellia_f %f18, %f4, %f6, %f4 + ldd [$key + 184], %f16 + ldd [$key + 176], %f18 + camellia_f %f20, %f2, %f0, %f2 + camellia_f %f20, %f6, %f4, %f6 + camellia_f %f22, %f0, %f2, %f0 + camellia_f %f22, %f4, %f6, %f4 + ldd [$key + 168], %f20 + ldd [$key + 160], %f22 + camellia_f %f24, %f2, %f0, %f8 + camellia_f %f24, %f6, %f4, %f10 + camellia_f %f26, %f0, %f8, %f2 + camellia_f %f26, %f4, %f10, %f6 + ldd [$key + 152], %f24 + ldd [$key + 144], %f26 + fxor %f30, %f8, %f0 + fxor %f30, %f10, %f4 + fxor %f28, %f2, %f2 + fxor %f28, %f6, %f6 + ldd [$key + 136], %f28 + retl + ldd [$key + 128], %f30 +.type _cmll256_decrypt_2x,#function +.size _cmll256_decrypt_2x,.-_cmll256_decrypt_2x +___ + +&alg_cbc_encrypt_implement("cmll",128); +&alg_cbc_encrypt_implement("cmll",256); + +&alg_cbc_decrypt_implement("cmll",128); +&alg_cbc_decrypt_implement("cmll",256); + +if ($::evp) { + &alg_ctr32_implement("cmll",128); + &alg_ctr32_implement("cmll",256); +} +}}} + +if (!$::evp) { +$code.=<<___; +.global Camellia_encrypt +Camellia_encrypt=cmll_t4_encrypt +.global Camellia_decrypt +Camellia_decrypt=cmll_t4_decrypt +.global Camellia_set_key +.align 32 +Camellia_set_key: + andcc %o2, 7, %g0 ! double-check alignment + bnz,a,pn %icc, 1f + mov -1, %o0 + brz,a,pn %o0, 1f + mov -1, %o0 + brz,a,pn %o2, 1f + mov -1, %o0 + andncc %o1, 0x1c0, %g0 + bnz,a,pn %icc, 1f + mov -2, %o0 + cmp %o1, 128 + bl,a,pn %icc, 1f + mov -2, %o0 + b cmll_t4_set_key + nop +1: retl + nop +.type Camellia_set_key,#function +.size Camellia_set_key,.-Camellia_set_key +___ + +my ($inp,$out,$len,$key,$ivec,$enc)=map("%o$_",(0..5)); + +$code.=<<___; +.globl Camellia_cbc_encrypt +.align 32 +Camellia_cbc_encrypt: + ld [$key + 272], %g1 + nop + brz $enc, .Lcbc_decrypt + cmp %g1, 3 + + be,pt %icc, cmll128_t4_cbc_encrypt + nop + ba cmll256_t4_cbc_encrypt + nop + +.Lcbc_decrypt: + be,pt %icc, cmll128_t4_cbc_decrypt + nop + ba cmll256_t4_cbc_decrypt + nop +.type Camellia_cbc_encrypt,#function +.size Camellia_cbc_encrypt,.-Camellia_cbc_encrypt +___ +} + +&emit_assembler(); + +close STDOUT; diff --git a/crypto/cast/cast_lcl.h b/crypto/cast/cast_lcl.h index 7c4ad41b2157..b0f08294e373 100644 --- a/crypto/cast/cast_lcl.h +++ b/crypto/cast/cast_lcl.h @@ -152,6 +152,8 @@ #if defined(OPENSSL_SYS_WIN32) && defined(_MSC_VER) # define ROTL(a,n) (_lrotl(a,n)) +#elif defined(PEDANTIC) +# define ROTL(a,n) ((((a)<<(n))&0xffffffffL)|((a)>>((32-(n))&31))) #else # define ROTL(a,n) ((((a)<<(n))&0xffffffffL)|((a)>>(32-(n)))) #endif diff --git a/crypto/cms/Makefile b/crypto/cms/Makefile index b124b5dbf212..6f3a83202638 100644 --- a/crypto/cms/Makefile +++ b/crypto/cms/Makefile @@ -19,10 +19,10 @@ APPS= LIB=$(TOP)/libcrypto.a LIBSRC= cms_lib.c cms_asn1.c cms_att.c cms_io.c cms_smime.c cms_err.c \ cms_sd.c cms_dd.c cms_cd.c cms_env.c cms_enc.c cms_ess.c \ - cms_pwri.c + cms_pwri.c cms_kari.c LIBOBJ= cms_lib.o cms_asn1.o cms_att.o cms_io.o cms_smime.o cms_err.o \ cms_sd.o cms_dd.o cms_cd.o cms_env.o cms_enc.o cms_ess.o \ - cms_pwri.o + cms_pwri.o cms_kari.o SRC= $(LIBSRC) @@ -220,20 +220,39 @@ cms_io.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h cms_io.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h cms_io.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h cms.h cms_io.o: cms_io.c cms_lcl.h +cms_kari.o: ../../e_os.h ../../include/openssl/aes.h +cms_kari.o: ../../include/openssl/asn1.h ../../include/openssl/asn1t.h +cms_kari.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h +cms_kari.o: ../../include/openssl/cms.h ../../include/openssl/conf.h +cms_kari.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h +cms_kari.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h +cms_kari.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h +cms_kari.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h +cms_kari.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h +cms_kari.o: ../../include/openssl/opensslconf.h +cms_kari.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h +cms_kari.o: ../../include/openssl/pem.h ../../include/openssl/pem2.h +cms_kari.o: ../../include/openssl/pkcs7.h ../../include/openssl/rand.h +cms_kari.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h +cms_kari.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h +cms_kari.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h +cms_kari.o: ../../include/openssl/x509v3.h ../asn1/asn1_locl.h ../cryptlib.h +cms_kari.o: cms_kari.c cms_lcl.h cms_lib.o: ../../include/openssl/asn1.h ../../include/openssl/asn1t.h cms_lib.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h -cms_lib.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h -cms_lib.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h -cms_lib.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h -cms_lib.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h -cms_lib.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h -cms_lib.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h -cms_lib.o: ../../include/openssl/ossl_typ.h ../../include/openssl/pem.h -cms_lib.o: ../../include/openssl/pem2.h ../../include/openssl/pkcs7.h -cms_lib.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h -cms_lib.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h -cms_lib.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h cms.h -cms_lib.o: cms_lcl.h cms_lib.c +cms_lib.o: ../../include/openssl/conf.h ../../include/openssl/crypto.h +cms_lib.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h +cms_lib.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h +cms_lib.o: ../../include/openssl/err.h ../../include/openssl/evp.h +cms_lib.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h +cms_lib.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h +cms_lib.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h +cms_lib.o: ../../include/openssl/pem.h ../../include/openssl/pem2.h +cms_lib.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h +cms_lib.o: ../../include/openssl/sha.h ../../include/openssl/stack.h +cms_lib.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h +cms_lib.o: ../../include/openssl/x509_vfy.h ../../include/openssl/x509v3.h +cms_lib.o: cms.h cms_lcl.h cms_lib.c cms_pwri.o: ../../e_os.h ../../include/openssl/aes.h cms_pwri.o: ../../include/openssl/asn1.h ../../include/openssl/asn1t.h cms_pwri.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h @@ -283,4 +302,4 @@ cms_smime.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h cms_smime.o: ../../include/openssl/sha.h ../../include/openssl/stack.h cms_smime.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h cms_smime.o: ../../include/openssl/x509_vfy.h ../../include/openssl/x509v3.h -cms_smime.o: ../cryptlib.h cms_lcl.h cms_smime.c +cms_smime.o: ../asn1/asn1_locl.h ../cryptlib.h cms_lcl.h cms_smime.c diff --git a/crypto/cms/cms.h b/crypto/cms/cms.h index a2281eda5b0a..e6c7f964bf48 100644 --- a/crypto/cms/cms.h +++ b/crypto/cms/cms.h @@ -72,9 +72,12 @@ typedef struct CMS_RevocationInfoChoice_st CMS_RevocationInfoChoice; typedef struct CMS_RecipientInfo_st CMS_RecipientInfo; typedef struct CMS_ReceiptRequest_st CMS_ReceiptRequest; typedef struct CMS_Receipt_st CMS_Receipt; +typedef struct CMS_RecipientEncryptedKey_st CMS_RecipientEncryptedKey; +typedef struct CMS_OtherKeyAttribute_st CMS_OtherKeyAttribute; DECLARE_STACK_OF(CMS_SignerInfo) DECLARE_STACK_OF(GENERAL_NAMES) +DECLARE_STACK_OF(CMS_RecipientEncryptedKey) DECLARE_ASN1_FUNCTIONS(CMS_ContentInfo) DECLARE_ASN1_FUNCTIONS(CMS_ReceiptRequest) DECLARE_ASN1_PRINT_FUNCTION(CMS_ContentInfo) @@ -82,6 +85,7 @@ DECLARE_ASN1_PRINT_FUNCTION(CMS_ContentInfo) # define CMS_SIGNERINFO_ISSUER_SERIAL 0 # define CMS_SIGNERINFO_KEYIDENTIFIER 1 +# define CMS_RECIPINFO_NONE -1 # define CMS_RECIPINFO_TRANS 0 # define CMS_RECIPINFO_AGREE 1 # define CMS_RECIPINFO_KEK 2 @@ -111,6 +115,7 @@ DECLARE_ASN1_PRINT_FUNCTION(CMS_ContentInfo) # define CMS_REUSE_DIGEST 0x8000 # define CMS_USE_KEYID 0x10000 # define CMS_DEBUG_DECRYPT 0x20000 +# define CMS_KEY_PARAM 0x40000 const ASN1_OBJECT *CMS_get0_type(CMS_ContentInfo *cms); @@ -189,6 +194,7 @@ int CMS_decrypt_set1_password(CMS_ContentInfo *cms, STACK_OF(CMS_RecipientInfo) *CMS_get0_RecipientInfos(CMS_ContentInfo *cms); int CMS_RecipientInfo_type(CMS_RecipientInfo *ri); +EVP_PKEY_CTX *CMS_RecipientInfo_get0_pkey_ctx(CMS_RecipientInfo *ri); CMS_ContentInfo *CMS_EnvelopedData_create(const EVP_CIPHER *cipher); CMS_RecipientInfo *CMS_add1_recipient_cert(CMS_ContentInfo *cms, X509 *recip, unsigned int flags); @@ -234,6 +240,7 @@ CMS_RecipientInfo *CMS_add0_recipient_password(CMS_ContentInfo *cms, const EVP_CIPHER *kekciph); int CMS_RecipientInfo_decrypt(CMS_ContentInfo *cms, CMS_RecipientInfo *ri); +int CMS_RecipientInfo_encrypt(CMS_ContentInfo *cms, CMS_RecipientInfo *ri); int CMS_uncompress(CMS_ContentInfo *cms, BIO *dcont, BIO *out, unsigned int flags); @@ -256,6 +263,8 @@ int CMS_SignedData_init(CMS_ContentInfo *cms); CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, X509 *signer, EVP_PKEY *pk, const EVP_MD *md, unsigned int flags); +EVP_PKEY_CTX *CMS_SignerInfo_get0_pkey_ctx(CMS_SignerInfo *si); +EVP_MD_CTX *CMS_SignerInfo_get0_md_ctx(CMS_SignerInfo *si); STACK_OF(CMS_SignerInfo) *CMS_get0_SignerInfos(CMS_ContentInfo *cms); void CMS_SignerInfo_set1_signer_cert(CMS_SignerInfo *si, X509 *signer); @@ -268,6 +277,7 @@ int CMS_set1_signers_certs(CMS_ContentInfo *cms, STACK_OF(X509) *certs, void CMS_SignerInfo_get0_algs(CMS_SignerInfo *si, EVP_PKEY **pk, X509 **signer, X509_ALGOR **pdig, X509_ALGOR **psig); +ASN1_OCTET_STRING *CMS_SignerInfo_get0_signature(CMS_SignerInfo *si); int CMS_SignerInfo_sign(CMS_SignerInfo *si); int CMS_SignerInfo_verify(CMS_SignerInfo *si); int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain); @@ -331,8 +341,37 @@ void CMS_ReceiptRequest_get0_values(CMS_ReceiptRequest *rr, int *pallorfirst, STACK_OF(GENERAL_NAMES) **plist, STACK_OF(GENERAL_NAMES) **prto); - # endif +int CMS_RecipientInfo_kari_get0_alg(CMS_RecipientInfo *ri, + X509_ALGOR **palg, + ASN1_OCTET_STRING **pukm); +STACK_OF(CMS_RecipientEncryptedKey) +*CMS_RecipientInfo_kari_get0_reks(CMS_RecipientInfo *ri); + +int CMS_RecipientInfo_kari_get0_orig_id(CMS_RecipientInfo *ri, + X509_ALGOR **pubalg, + ASN1_BIT_STRING **pubkey, + ASN1_OCTET_STRING **keyid, + X509_NAME **issuer, + ASN1_INTEGER **sno); + +int CMS_RecipientInfo_kari_orig_id_cmp(CMS_RecipientInfo *ri, X509 *cert); + +int CMS_RecipientEncryptedKey_get0_id(CMS_RecipientEncryptedKey *rek, + ASN1_OCTET_STRING **keyid, + ASN1_GENERALIZEDTIME **tm, + CMS_OtherKeyAttribute **other, + X509_NAME **issuer, ASN1_INTEGER **sno); +int CMS_RecipientEncryptedKey_cert_cmp(CMS_RecipientEncryptedKey *rek, + X509 *cert); +int CMS_RecipientInfo_kari_set0_pkey(CMS_RecipientInfo *ri, EVP_PKEY *pk); +EVP_CIPHER_CTX *CMS_RecipientInfo_kari_get0_ctx(CMS_RecipientInfo *ri); +int CMS_RecipientInfo_kari_decrypt(CMS_ContentInfo *cms, + CMS_RecipientInfo *ri, + CMS_RecipientEncryptedKey *rek); + +int CMS_SharedInfo_encode(unsigned char **pder, X509_ALGOR *kekalg, + ASN1_OCTET_STRING *ukm, int keylen); /* BEGIN ERROR CODES */ /* @@ -377,6 +416,7 @@ void ERR_load_CMS_strings(void); # define CMS_F_CMS_ENVELOPEDDATA_CREATE 124 # define CMS_F_CMS_ENVELOPEDDATA_INIT_BIO 125 # define CMS_F_CMS_ENVELOPED_DATA_INIT 126 +# define CMS_F_CMS_ENV_ASN1_CTRL 171 # define CMS_F_CMS_FINAL 127 # define CMS_F_CMS_GET0_CERTIFICATE_CHOICES 128 # define CMS_F_CMS_GET0_CONTENT 129 @@ -388,6 +428,12 @@ void ERR_load_CMS_strings(void); # define CMS_F_CMS_RECEIPTREQUEST_CREATE0 159 # define CMS_F_CMS_RECEIPT_VERIFY 160 # define CMS_F_CMS_RECIPIENTINFO_DECRYPT 134 +# define CMS_F_CMS_RECIPIENTINFO_ENCRYPT 169 +# define CMS_F_CMS_RECIPIENTINFO_KARI_ENCRYPT 178 +# define CMS_F_CMS_RECIPIENTINFO_KARI_GET0_ALG 175 +# define CMS_F_CMS_RECIPIENTINFO_KARI_GET0_ORIG_ID 173 +# define CMS_F_CMS_RECIPIENTINFO_KARI_GET0_REKS 172 +# define CMS_F_CMS_RECIPIENTINFO_KARI_ORIG_ID_CMP 174 # define CMS_F_CMS_RECIPIENTINFO_KEKRI_DECRYPT 135 # define CMS_F_CMS_RECIPIENTINFO_KEKRI_ENCRYPT 136 # define CMS_F_CMS_RECIPIENTINFO_KEKRI_GET0_ID 137 @@ -401,6 +447,9 @@ void ERR_load_CMS_strings(void); # define CMS_F_CMS_RECIPIENTINFO_SET0_KEY 144 # define CMS_F_CMS_RECIPIENTINFO_SET0_PASSWORD 168 # define CMS_F_CMS_RECIPIENTINFO_SET0_PKEY 145 +# define CMS_F_CMS_SD_ASN1_CTRL 170 +# define CMS_F_CMS_SET1_IAS 176 +# define CMS_F_CMS_SET1_KEYID 177 # define CMS_F_CMS_SET1_SIGNERIDENTIFIER 146 # define CMS_F_CMS_SET_DETACHED 147 # define CMS_F_CMS_SIGN 148 @@ -452,6 +501,7 @@ void ERR_load_CMS_strings(void); # define CMS_R_NOT_A_SIGNED_RECEIPT 165 # define CMS_R_NOT_ENCRYPTED_DATA 122 # define CMS_R_NOT_KEK 123 +# define CMS_R_NOT_KEY_AGREEMENT 181 # define CMS_R_NOT_KEY_TRANSPORT 124 # define CMS_R_NOT_PWRI 177 # define CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE 125 diff --git a/crypto/cms/cms_asn1.c b/crypto/cms/cms_asn1.c index f9f267afdcd7..81a3407f1249 100644 --- a/crypto/cms/cms_asn1.c +++ b/crypto/cms/cms_asn1.c @@ -97,6 +97,8 @@ static int cms_si_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, EVP_PKEY_free(si->pkey); if (si->signer) X509_free(si->signer); + if (si->pctx) + EVP_MD_CTX_cleanup(&si->mctx); } return 1; } @@ -164,10 +166,21 @@ ASN1_CHOICE(CMS_KeyAgreeRecipientIdentifier) = { ASN1_IMP(CMS_KeyAgreeRecipientIdentifier, d.rKeyId, CMS_RecipientKeyIdentifier, 0) } ASN1_CHOICE_END(CMS_KeyAgreeRecipientIdentifier) -ASN1_SEQUENCE(CMS_RecipientEncryptedKey) = { +static int cms_rek_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + CMS_RecipientEncryptedKey *rek = (CMS_RecipientEncryptedKey *)*pval; + if (operation == ASN1_OP_FREE_POST) { + if (rek->pkey) + EVP_PKEY_free(rek->pkey); + } + return 1; +} + +ASN1_SEQUENCE_cb(CMS_RecipientEncryptedKey, cms_rek_cb) = { ASN1_SIMPLE(CMS_RecipientEncryptedKey, rid, CMS_KeyAgreeRecipientIdentifier), ASN1_SIMPLE(CMS_RecipientEncryptedKey, encryptedKey, ASN1_OCTET_STRING) -} ASN1_SEQUENCE_END(CMS_RecipientEncryptedKey) +} ASN1_SEQUENCE_END_cb(CMS_RecipientEncryptedKey, CMS_RecipientEncryptedKey) ASN1_SEQUENCE(CMS_OriginatorPublicKey) = { ASN1_SIMPLE(CMS_OriginatorPublicKey, algorithm, X509_ALGOR), @@ -180,13 +193,29 @@ ASN1_CHOICE(CMS_OriginatorIdentifierOrKey) = { ASN1_IMP(CMS_OriginatorIdentifierOrKey, d.originatorKey, CMS_OriginatorPublicKey, 1) } ASN1_CHOICE_END(CMS_OriginatorIdentifierOrKey) -ASN1_SEQUENCE(CMS_KeyAgreeRecipientInfo) = { +static int cms_kari_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + CMS_KeyAgreeRecipientInfo *kari = (CMS_KeyAgreeRecipientInfo *)*pval; + if (operation == ASN1_OP_NEW_POST) { + EVP_CIPHER_CTX_init(&kari->ctx); + EVP_CIPHER_CTX_set_flags(&kari->ctx, EVP_CIPHER_CTX_FLAG_WRAP_ALLOW); + kari->pctx = NULL; + } else if (operation == ASN1_OP_FREE_POST) { + if (kari->pctx) + EVP_PKEY_CTX_free(kari->pctx); + EVP_CIPHER_CTX_cleanup(&kari->ctx); + } + return 1; +} + +ASN1_SEQUENCE_cb(CMS_KeyAgreeRecipientInfo, cms_kari_cb) = { ASN1_SIMPLE(CMS_KeyAgreeRecipientInfo, version, LONG), ASN1_EXP(CMS_KeyAgreeRecipientInfo, originator, CMS_OriginatorIdentifierOrKey, 0), ASN1_EXP_OPT(CMS_KeyAgreeRecipientInfo, ukm, ASN1_OCTET_STRING, 1), ASN1_SIMPLE(CMS_KeyAgreeRecipientInfo, keyEncryptionAlgorithm, X509_ALGOR), ASN1_SEQUENCE_OF(CMS_KeyAgreeRecipientInfo, recipientEncryptedKeys, CMS_RecipientEncryptedKey) -} ASN1_SEQUENCE_END(CMS_KeyAgreeRecipientInfo) +} ASN1_SEQUENCE_END_cb(CMS_KeyAgreeRecipientInfo, CMS_KeyAgreeRecipientInfo) ASN1_SEQUENCE(CMS_KEKIdentifier) = { ASN1_SIMPLE(CMS_KEKIdentifier, keyIdentifier, ASN1_OCTET_STRING), @@ -225,6 +254,8 @@ static int cms_ri_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, EVP_PKEY_free(ktri->pkey); if (ktri->recip) X509_free(ktri->recip); + if (ktri->pctx) + EVP_PKEY_CTX_free(ktri->pctx); } else if (ri->type == CMS_RECIPINFO_KEK) { CMS_KEKRecipientInfo *kekri = ri->d.kekri; if (kekri->key) { @@ -379,3 +410,50 @@ ASN1_SEQUENCE(CMS_Receipt) = { ASN1_SIMPLE(CMS_Receipt, signedContentIdentifier, ASN1_OCTET_STRING), ASN1_SIMPLE(CMS_Receipt, originatorSignatureValue, ASN1_OCTET_STRING) } ASN1_SEQUENCE_END(CMS_Receipt) + +/* + * Utilities to encode the CMS_SharedInfo structure used during key + * derivation. + */ + +typedef struct { + X509_ALGOR *keyInfo; + ASN1_OCTET_STRING *entityUInfo; + ASN1_OCTET_STRING *suppPubInfo; +} CMS_SharedInfo; + +ASN1_SEQUENCE(CMS_SharedInfo) = { + ASN1_SIMPLE(CMS_SharedInfo, keyInfo, X509_ALGOR), + ASN1_EXP_OPT(CMS_SharedInfo, entityUInfo, ASN1_OCTET_STRING, 0), + ASN1_EXP_OPT(CMS_SharedInfo, suppPubInfo, ASN1_OCTET_STRING, 2), +} ASN1_SEQUENCE_END(CMS_SharedInfo) + +int CMS_SharedInfo_encode(unsigned char **pder, X509_ALGOR *kekalg, + ASN1_OCTET_STRING *ukm, int keylen) +{ + union { + CMS_SharedInfo *pecsi; + ASN1_VALUE *a; + } intsi = { + NULL + }; + + ASN1_OCTET_STRING oklen; + unsigned char kl[4]; + CMS_SharedInfo ecsi; + + keylen <<= 3; + kl[0] = (keylen >> 24) & 0xff; + kl[1] = (keylen >> 16) & 0xff; + kl[2] = (keylen >> 8) & 0xff; + kl[3] = keylen & 0xff; + oklen.length = 4; + oklen.data = kl; + oklen.type = V_ASN1_OCTET_STRING; + oklen.flags = 0; + ecsi.keyInfo = kekalg; + ecsi.entityUInfo = ukm; + ecsi.suppPubInfo = &oklen; + intsi.pecsi = &ecsi; + return ASN1_item_i2d(intsi.a, pder, ASN1_ITEM_rptr(CMS_SharedInfo)); +} diff --git a/crypto/cms/cms_env.c b/crypto/cms/cms_env.c index 1c3046ce9238..93c06cb00a8f 100644 --- a/crypto/cms/cms_env.c +++ b/crypto/cms/cms_env.c @@ -100,6 +100,36 @@ static CMS_EnvelopedData *cms_enveloped_data_init(CMS_ContentInfo *cms) return cms_get0_enveloped(cms); } +int cms_env_asn1_ctrl(CMS_RecipientInfo *ri, int cmd) +{ + EVP_PKEY *pkey; + int i; + if (ri->type == CMS_RECIPINFO_TRANS) + pkey = ri->d.ktri->pkey; + else if (ri->type == CMS_RECIPINFO_AGREE) { + EVP_PKEY_CTX *pctx = ri->d.kari->pctx; + if (!pctx) + return 0; + pkey = EVP_PKEY_CTX_get0_pkey(pctx); + if (!pkey) + return 0; + } else + return 0; + if (!pkey->ameth || !pkey->ameth->pkey_ctrl) + return 1; + i = pkey->ameth->pkey_ctrl(pkey, ASN1_PKEY_CTRL_CMS_ENVELOPE, cmd, ri); + if (i == -2) { + CMSerr(CMS_F_CMS_ENV_ASN1_CTRL, + CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); + return 0; + } + if (i <= 0) { + CMSerr(CMS_F_CMS_ENV_ASN1_CTRL, CMS_R_CTRL_FAILURE); + return 0; + } + return 1; +} + STACK_OF(CMS_RecipientInfo) *CMS_get0_RecipientInfos(CMS_ContentInfo *cms) { CMS_EnvelopedData *env; @@ -114,6 +144,15 @@ int CMS_RecipientInfo_type(CMS_RecipientInfo *ri) return ri->type; } +EVP_PKEY_CTX *CMS_RecipientInfo_get0_pkey_ctx(CMS_RecipientInfo *ri) +{ + if (ri->type == CMS_RECIPINFO_TRANS) + return ri->d.ktri->pctx; + else if (ri->type == CMS_RECIPINFO_AGREE) + return ri->d.kari->pctx; + return NULL; +} + CMS_ContentInfo *CMS_EnvelopedData_create(const EVP_CIPHER *cipher) { CMS_ContentInfo *cms; @@ -137,19 +176,63 @@ CMS_ContentInfo *CMS_EnvelopedData_create(const EVP_CIPHER *cipher) /* Key Transport Recipient Info (KTRI) routines */ +/* Initialise a ktri based on passed certificate and key */ + +static int cms_RecipientInfo_ktri_init(CMS_RecipientInfo *ri, X509 *recip, + EVP_PKEY *pk, unsigned int flags) +{ + CMS_KeyTransRecipientInfo *ktri; + int idtype; + + ri->d.ktri = M_ASN1_new_of(CMS_KeyTransRecipientInfo); + if (!ri->d.ktri) + return 0; + ri->type = CMS_RECIPINFO_TRANS; + + ktri = ri->d.ktri; + + if (flags & CMS_USE_KEYID) { + ktri->version = 2; + idtype = CMS_RECIPINFO_KEYIDENTIFIER; + } else { + ktri->version = 0; + idtype = CMS_RECIPINFO_ISSUER_SERIAL; + } + + /* + * Not a typo: RecipientIdentifier and SignerIdentifier are the same + * structure. + */ + + if (!cms_set1_SignerIdentifier(ktri->rid, recip, idtype)) + return 0; + + CRYPTO_add(&recip->references, 1, CRYPTO_LOCK_X509); + CRYPTO_add(&pk->references, 1, CRYPTO_LOCK_EVP_PKEY); + ktri->pkey = pk; + ktri->recip = recip; + + if (flags & CMS_KEY_PARAM) { + ktri->pctx = EVP_PKEY_CTX_new(ktri->pkey, NULL); + if (!ktri->pctx) + return 0; + if (EVP_PKEY_encrypt_init(ktri->pctx) <= 0) + return 0; + } else if (!cms_env_asn1_ctrl(ri, 0)) + return 0; + return 1; +} + /* - * Add a recipient certificate. For now only handle key transport. If we ever - * handle key agreement will need updating. + * Add a recipient certificate using appropriate type of RecipientInfo */ CMS_RecipientInfo *CMS_add1_recipient_cert(CMS_ContentInfo *cms, X509 *recip, unsigned int flags) { CMS_RecipientInfo *ri = NULL; - CMS_KeyTransRecipientInfo *ktri; CMS_EnvelopedData *env; EVP_PKEY *pk = NULL; - int i, type; env = cms_get0_enveloped(cms); if (!env) goto err; @@ -159,59 +242,36 @@ CMS_RecipientInfo *CMS_add1_recipient_cert(CMS_ContentInfo *cms, if (!ri) goto merr; - /* Initialize and add key transport recipient info */ - - ri->d.ktri = M_ASN1_new_of(CMS_KeyTransRecipientInfo); - if (!ri->d.ktri) - goto merr; - ri->type = CMS_RECIPINFO_TRANS; - - ktri = ri->d.ktri; - - X509_check_purpose(recip, -1, -1); pk = X509_get_pubkey(recip); if (!pk) { CMSerr(CMS_F_CMS_ADD1_RECIPIENT_CERT, CMS_R_ERROR_GETTING_PUBLIC_KEY); goto err; } - CRYPTO_add(&recip->references, 1, CRYPTO_LOCK_X509); - ktri->pkey = pk; - ktri->recip = recip; - if (flags & CMS_USE_KEYID) { - ktri->version = 2; - if (env->version < 2) - env->version = 2; - type = CMS_RECIPINFO_KEYIDENTIFIER; - } else { - ktri->version = 0; - type = CMS_RECIPINFO_ISSUER_SERIAL; - } + switch (cms_pkey_get_ri_type(pk)) { - /* - * Not a typo: RecipientIdentifier and SignerIdentifier are the same - * structure. - */ + case CMS_RECIPINFO_TRANS: + if (!cms_RecipientInfo_ktri_init(ri, recip, pk, flags)) + goto err; + break; - if (!cms_set1_SignerIdentifier(ktri->rid, recip, type)) + case CMS_RECIPINFO_AGREE: + if (!cms_RecipientInfo_kari_init(ri, recip, pk, flags)) + goto err; + break; + + default: + CMSerr(CMS_F_CMS_ADD1_RECIPIENT_CERT, + CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); goto err; - if (pk->ameth && pk->ameth->pkey_ctrl) { - i = pk->ameth->pkey_ctrl(pk, ASN1_PKEY_CTRL_CMS_ENVELOPE, 0, ri); - if (i == -2) { - CMSerr(CMS_F_CMS_ADD1_RECIPIENT_CERT, - CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); - goto err; - } - if (i <= 0) { - CMSerr(CMS_F_CMS_ADD1_RECIPIENT_CERT, CMS_R_CTRL_FAILURE); - goto err; - } } if (!sk_CMS_RecipientInfo_push(env->recipientInfos, ri)) goto merr; + EVP_PKEY_free(pk); + return ri; merr: @@ -219,6 +279,8 @@ CMS_RecipientInfo *CMS_add1_recipient_cert(CMS_ContentInfo *cms, err: if (ri) M_ASN1_free_of(ri, CMS_RecipientInfo); + if (pk) + EVP_PKEY_free(pk); return NULL; } @@ -288,7 +350,7 @@ static int cms_RecipientInfo_ktri_encrypt(CMS_ContentInfo *cms, { CMS_KeyTransRecipientInfo *ktri; CMS_EncryptedContentInfo *ec; - EVP_PKEY_CTX *pctx = NULL; + EVP_PKEY_CTX *pctx; unsigned char *ek = NULL; size_t eklen; @@ -301,12 +363,19 @@ static int cms_RecipientInfo_ktri_encrypt(CMS_ContentInfo *cms, ktri = ri->d.ktri; ec = cms->d.envelopedData->encryptedContentInfo; - pctx = EVP_PKEY_CTX_new(ktri->pkey, NULL); - if (!pctx) - return 0; + pctx = ktri->pctx; - if (EVP_PKEY_encrypt_init(pctx) <= 0) - goto err; + if (pctx) { + if (!cms_env_asn1_ctrl(ri, 0)) + goto err; + } else { + pctx = EVP_PKEY_CTX_new(ktri->pkey, NULL); + if (!pctx) + return 0; + + if (EVP_PKEY_encrypt_init(pctx) <= 0) + goto err; + } if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_ENCRYPT, EVP_PKEY_CTRL_CMS_ENCRYPT, 0, ri) <= 0) { @@ -333,8 +402,10 @@ static int cms_RecipientInfo_ktri_encrypt(CMS_ContentInfo *cms, ret = 1; err: - if (pctx) + if (pctx) { EVP_PKEY_CTX_free(pctx); + ktri->pctx = NULL; + } if (ek) OPENSSL_free(ek); return ret; @@ -347,7 +418,7 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms, CMS_RecipientInfo *ri) { CMS_KeyTransRecipientInfo *ktri = ri->d.ktri; - EVP_PKEY_CTX *pctx = NULL; + EVP_PKEY *pkey = ktri->pkey; unsigned char *ek = NULL; size_t eklen; int ret = 0; @@ -359,20 +430,23 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms, return 0; } - pctx = EVP_PKEY_CTX_new(ktri->pkey, NULL); - if (!pctx) + ktri->pctx = EVP_PKEY_CTX_new(pkey, NULL); + if (!ktri->pctx) return 0; - if (EVP_PKEY_decrypt_init(pctx) <= 0) + if (EVP_PKEY_decrypt_init(ktri->pctx) <= 0) goto err; - if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_DECRYPT, + if (!cms_env_asn1_ctrl(ri, 1)) + goto err; + + if (EVP_PKEY_CTX_ctrl(ktri->pctx, -1, EVP_PKEY_OP_DECRYPT, EVP_PKEY_CTRL_CMS_DECRYPT, 0, ri) <= 0) { CMSerr(CMS_F_CMS_RECIPIENTINFO_KTRI_DECRYPT, CMS_R_CTRL_ERROR); goto err; } - if (EVP_PKEY_decrypt(pctx, NULL, &eklen, + if (EVP_PKEY_decrypt(ktri->pctx, NULL, &eklen, ktri->encryptedKey->data, ktri->encryptedKey->length) <= 0) goto err; @@ -384,7 +458,7 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms, goto err; } - if (EVP_PKEY_decrypt(pctx, ek, &eklen, + if (EVP_PKEY_decrypt(ktri->pctx, ek, &eklen, ktri->encryptedKey->data, ktri->encryptedKey->length) <= 0) { CMSerr(CMS_F_CMS_RECIPIENTINFO_KTRI_DECRYPT, CMS_R_CMS_LIB); @@ -402,8 +476,10 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms, ec->keylen = eklen; err: - if (pctx) - EVP_PKEY_CTX_free(pctx); + if (ktri->pctx) { + EVP_PKEY_CTX_free(ktri->pctx); + ktri->pctx = NULL; + } if (!ret && ek) OPENSSL_free(ek); @@ -745,12 +821,99 @@ int CMS_RecipientInfo_decrypt(CMS_ContentInfo *cms, CMS_RecipientInfo *ri) } } +int CMS_RecipientInfo_encrypt(CMS_ContentInfo *cms, CMS_RecipientInfo *ri) +{ + switch (ri->type) { + case CMS_RECIPINFO_TRANS: + return cms_RecipientInfo_ktri_encrypt(cms, ri); + + case CMS_RECIPINFO_AGREE: + return cms_RecipientInfo_kari_encrypt(cms, ri); + + case CMS_RECIPINFO_KEK: + return cms_RecipientInfo_kekri_encrypt(cms, ri); + break; + + case CMS_RECIPINFO_PASS: + return cms_RecipientInfo_pwri_crypt(cms, ri, 1); + break; + + default: + CMSerr(CMS_F_CMS_RECIPIENTINFO_ENCRYPT, + CMS_R_UNSUPPORTED_RECIPIENT_TYPE); + return 0; + } +} + +/* Check structures and fixup version numbers (if necessary) */ + +static void cms_env_set_originfo_version(CMS_EnvelopedData *env) +{ + CMS_OriginatorInfo *org = env->originatorInfo; + int i; + if (org == NULL) + return; + for (i = 0; i < sk_CMS_CertificateChoices_num(org->certificates); i++) { + CMS_CertificateChoices *cch; + cch = sk_CMS_CertificateChoices_value(org->certificates, i); + if (cch->type == CMS_CERTCHOICE_OTHER) { + env->version = 4; + return; + } else if (cch->type == CMS_CERTCHOICE_V2ACERT) { + if (env->version < 3) + env->version = 3; + } + } + + for (i = 0; i < sk_CMS_RevocationInfoChoice_num(org->crls); i++) { + CMS_RevocationInfoChoice *rch; + rch = sk_CMS_RevocationInfoChoice_value(org->crls, i); + if (rch->type == CMS_REVCHOICE_OTHER) { + env->version = 4; + return; + } + } +} + +static void cms_env_set_version(CMS_EnvelopedData *env) +{ + int i; + CMS_RecipientInfo *ri; + + /* + * Can't set version higher than 4 so if 4 or more already nothing to do. + */ + if (env->version >= 4) + return; + + cms_env_set_originfo_version(env); + + if (env->version >= 3) + return; + + for (i = 0; i < sk_CMS_RecipientInfo_num(env->recipientInfos); i++) { + ri = sk_CMS_RecipientInfo_value(env->recipientInfos, i); + if (ri->type == CMS_RECIPINFO_PASS || ri->type == CMS_RECIPINFO_OTHER) { + env->version = 3; + return; + } else if (ri->type != CMS_RECIPINFO_TRANS + || ri->d.ktri->version != 0) { + env->version = 2; + } + } + if (env->version == 2) + return; + if (env->originatorInfo || env->unprotectedAttrs) + env->version = 2; + env->version = 0; +} + BIO *cms_EnvelopedData_init_bio(CMS_ContentInfo *cms) { CMS_EncryptedContentInfo *ec; STACK_OF(CMS_RecipientInfo) *rinfos; CMS_RecipientInfo *ri; - int i, r, ok = 0; + int i, ok = 0; BIO *ret; /* Get BIO first to set up key */ @@ -769,32 +932,13 @@ BIO *cms_EnvelopedData_init_bio(CMS_ContentInfo *cms) for (i = 0; i < sk_CMS_RecipientInfo_num(rinfos); i++) { ri = sk_CMS_RecipientInfo_value(rinfos, i); - - switch (ri->type) { - case CMS_RECIPINFO_TRANS: - r = cms_RecipientInfo_ktri_encrypt(cms, ri); - break; - - case CMS_RECIPINFO_KEK: - r = cms_RecipientInfo_kekri_encrypt(cms, ri); - break; - - case CMS_RECIPINFO_PASS: - r = cms_RecipientInfo_pwri_crypt(cms, ri, 1); - break; - - default: - CMSerr(CMS_F_CMS_ENVELOPEDDATA_INIT_BIO, - CMS_R_UNSUPPORTED_RECIPIENT_TYPE); - goto err; - } - - if (r <= 0) { + if (CMS_RecipientInfo_encrypt(cms, ri) <= 0) { CMSerr(CMS_F_CMS_ENVELOPEDDATA_INIT_BIO, CMS_R_ERROR_SETTING_RECIPIENTINFO); goto err; } } + cms_env_set_version(cms->d.envelopedData); ok = 1; @@ -812,3 +956,19 @@ BIO *cms_EnvelopedData_init_bio(CMS_ContentInfo *cms) return NULL; } + +/* + * Get RecipientInfo type (if any) supported by a key (public or private). To + * retain compatibility with previous behaviour if the ctrl value isn't + * supported we assume key transport. + */ +int cms_pkey_get_ri_type(EVP_PKEY *pk) +{ + if (pk->ameth && pk->ameth->pkey_ctrl) { + int i, r; + i = pk->ameth->pkey_ctrl(pk, ASN1_PKEY_CTRL_CMS_RI_TYPE, 0, &r); + if (i > 0) + return r; + } + return CMS_RECIPINFO_TRANS; +} diff --git a/crypto/cms/cms_err.c b/crypto/cms/cms_err.c index faf2fccd3b8c..15572ea348c8 100644 --- a/crypto/cms/cms_err.c +++ b/crypto/cms/cms_err.c @@ -1,6 +1,6 @@ /* crypto/cms/cms_err.c */ /* ==================================================================== - * Copyright (c) 1999-2009 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2013 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -110,6 +110,7 @@ static ERR_STRING_DATA CMS_str_functs[] = { {ERR_FUNC(CMS_F_CMS_ENVELOPEDDATA_INIT_BIO), "cms_EnvelopedData_init_bio"}, {ERR_FUNC(CMS_F_CMS_ENVELOPED_DATA_INIT), "CMS_ENVELOPED_DATA_INIT"}, + {ERR_FUNC(CMS_F_CMS_ENV_ASN1_CTRL), "cms_env_asn1_ctrl"}, {ERR_FUNC(CMS_F_CMS_FINAL), "CMS_final"}, {ERR_FUNC(CMS_F_CMS_GET0_CERTIFICATE_CHOICES), "CMS_GET0_CERTIFICATE_CHOICES"}, @@ -124,6 +125,17 @@ static ERR_STRING_DATA CMS_str_functs[] = { "CMS_ReceiptRequest_create0"}, {ERR_FUNC(CMS_F_CMS_RECEIPT_VERIFY), "cms_Receipt_verify"}, {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_DECRYPT), "CMS_RecipientInfo_decrypt"}, + {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_ENCRYPT), "CMS_RecipientInfo_encrypt"}, + {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_KARI_ENCRYPT), + "cms_RecipientInfo_kari_encrypt"}, + {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_KARI_GET0_ALG), + "CMS_RecipientInfo_kari_get0_alg"}, + {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_KARI_GET0_ORIG_ID), + "CMS_RecipientInfo_kari_get0_orig_id"}, + {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_KARI_GET0_REKS), + "CMS_RecipientInfo_kari_get0_reks"}, + {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_KARI_ORIG_ID_CMP), + "CMS_RecipientInfo_kari_orig_id_cmp"}, {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_KEKRI_DECRYPT), "CMS_RECIPIENTINFO_KEKRI_DECRYPT"}, {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_KEKRI_ENCRYPT), @@ -150,6 +162,9 @@ static ERR_STRING_DATA CMS_str_functs[] = { "CMS_RecipientInfo_set0_password"}, {ERR_FUNC(CMS_F_CMS_RECIPIENTINFO_SET0_PKEY), "CMS_RecipientInfo_set0_pkey"}, + {ERR_FUNC(CMS_F_CMS_SD_ASN1_CTRL), "CMS_SD_ASN1_CTRL"}, + {ERR_FUNC(CMS_F_CMS_SET1_IAS), "cms_set1_ias"}, + {ERR_FUNC(CMS_F_CMS_SET1_KEYID), "cms_set1_keyid"}, {ERR_FUNC(CMS_F_CMS_SET1_SIGNERIDENTIFIER), "cms_set1_SignerIdentifier"}, {ERR_FUNC(CMS_F_CMS_SET_DETACHED), "CMS_set_detached"}, {ERR_FUNC(CMS_F_CMS_SIGN), "CMS_sign"}, @@ -221,6 +236,7 @@ static ERR_STRING_DATA CMS_str_reasons[] = { {ERR_REASON(CMS_R_NOT_A_SIGNED_RECEIPT), "not a signed receipt"}, {ERR_REASON(CMS_R_NOT_ENCRYPTED_DATA), "not encrypted data"}, {ERR_REASON(CMS_R_NOT_KEK), "not kek"}, + {ERR_REASON(CMS_R_NOT_KEY_AGREEMENT), "not key agreement"}, {ERR_REASON(CMS_R_NOT_KEY_TRANSPORT), "not key transport"}, {ERR_REASON(CMS_R_NOT_PWRI), "not pwri"}, {ERR_REASON(CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE), diff --git a/crypto/cms/cms_kari.c b/crypto/cms/cms_kari.c new file mode 100644 index 000000000000..2cfcdb29cd50 --- /dev/null +++ b/crypto/cms/cms_kari.c @@ -0,0 +1,465 @@ +/* crypto/cms/cms_kari.c */ +/* + * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL + * project. + */ +/* ==================================================================== + * Copyright (c) 2013 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + */ + +#include "cryptlib.h" +#include +#include +#include +#include +#include +#include +#include +#include "cms_lcl.h" +#include "asn1_locl.h" + +DECLARE_ASN1_ITEM(CMS_KeyAgreeRecipientInfo) +DECLARE_ASN1_ITEM(CMS_RecipientEncryptedKey) +DECLARE_ASN1_ITEM(CMS_OriginatorPublicKey) +DECLARE_ASN1_ITEM(CMS_RecipientKeyIdentifier) + +/* Key Agreement Recipient Info (KARI) routines */ + +int CMS_RecipientInfo_kari_get0_alg(CMS_RecipientInfo *ri, + X509_ALGOR **palg, + ASN1_OCTET_STRING **pukm) +{ + if (ri->type != CMS_RECIPINFO_AGREE) { + CMSerr(CMS_F_CMS_RECIPIENTINFO_KARI_GET0_ALG, + CMS_R_NOT_KEY_AGREEMENT); + return 0; + } + if (palg) + *palg = ri->d.kari->keyEncryptionAlgorithm; + if (pukm) + *pukm = ri->d.kari->ukm; + return 1; +} + +/* Retrieve recipient encrypted keys from a kari */ + +STACK_OF(CMS_RecipientEncryptedKey) +*CMS_RecipientInfo_kari_get0_reks(CMS_RecipientInfo *ri) +{ + if (ri->type != CMS_RECIPINFO_AGREE) { + CMSerr(CMS_F_CMS_RECIPIENTINFO_KARI_GET0_REKS, + CMS_R_NOT_KEY_AGREEMENT); + return NULL; + } + return ri->d.kari->recipientEncryptedKeys; +} + +int CMS_RecipientInfo_kari_get0_orig_id(CMS_RecipientInfo *ri, + X509_ALGOR **pubalg, + ASN1_BIT_STRING **pubkey, + ASN1_OCTET_STRING **keyid, + X509_NAME **issuer, + ASN1_INTEGER **sno) +{ + CMS_OriginatorIdentifierOrKey *oik; + if (ri->type != CMS_RECIPINFO_AGREE) { + CMSerr(CMS_F_CMS_RECIPIENTINFO_KARI_GET0_ORIG_ID, + CMS_R_NOT_KEY_AGREEMENT); + return 0; + } + oik = ri->d.kari->originator; + if (issuer) + *issuer = NULL; + if (sno) + *sno = NULL; + if (keyid) + *keyid = NULL; + if (pubalg) + *pubalg = NULL; + if (pubkey) + *pubkey = NULL; + if (oik->type == CMS_OIK_ISSUER_SERIAL) { + if (issuer) + *issuer = oik->d.issuerAndSerialNumber->issuer; + if (sno) + *sno = oik->d.issuerAndSerialNumber->serialNumber; + } else if (oik->type == CMS_OIK_KEYIDENTIFIER) { + if (keyid) + *keyid = oik->d.subjectKeyIdentifier; + } else if (oik->type == CMS_OIK_PUBKEY) { + if (pubalg) + *pubalg = oik->d.originatorKey->algorithm; + if (pubkey) + *pubkey = oik->d.originatorKey->publicKey; + } else + return 0; + return 1; +} + +int CMS_RecipientInfo_kari_orig_id_cmp(CMS_RecipientInfo *ri, X509 *cert) +{ + CMS_OriginatorIdentifierOrKey *oik; + if (ri->type != CMS_RECIPINFO_AGREE) { + CMSerr(CMS_F_CMS_RECIPIENTINFO_KARI_ORIG_ID_CMP, + CMS_R_NOT_KEY_AGREEMENT); + return -2; + } + oik = ri->d.kari->originator; + if (oik->type == CMS_OIK_ISSUER_SERIAL) + return cms_ias_cert_cmp(oik->d.issuerAndSerialNumber, cert); + else if (oik->type == CMS_OIK_KEYIDENTIFIER) + return cms_keyid_cert_cmp(oik->d.subjectKeyIdentifier, cert); + return -1; +} + +int CMS_RecipientEncryptedKey_get0_id(CMS_RecipientEncryptedKey *rek, + ASN1_OCTET_STRING **keyid, + ASN1_GENERALIZEDTIME **tm, + CMS_OtherKeyAttribute **other, + X509_NAME **issuer, ASN1_INTEGER **sno) +{ + CMS_KeyAgreeRecipientIdentifier *rid = rek->rid; + if (rid->type == CMS_REK_ISSUER_SERIAL) { + if (issuer) + *issuer = rid->d.issuerAndSerialNumber->issuer; + if (sno) + *sno = rid->d.issuerAndSerialNumber->serialNumber; + if (keyid) + *keyid = NULL; + if (tm) + *tm = NULL; + if (other) + *other = NULL; + } else if (rid->type == CMS_REK_KEYIDENTIFIER) { + if (keyid) + *keyid = rid->d.rKeyId->subjectKeyIdentifier; + if (tm) + *tm = rid->d.rKeyId->date; + if (other) + *other = rid->d.rKeyId->other; + if (issuer) + *issuer = NULL; + if (sno) + *sno = NULL; + } else + return 0; + return 1; +} + +int CMS_RecipientEncryptedKey_cert_cmp(CMS_RecipientEncryptedKey *rek, + X509 *cert) +{ + CMS_KeyAgreeRecipientIdentifier *rid = rek->rid; + if (rid->type == CMS_REK_ISSUER_SERIAL) + return cms_ias_cert_cmp(rid->d.issuerAndSerialNumber, cert); + else if (rid->type == CMS_REK_KEYIDENTIFIER) + return cms_keyid_cert_cmp(rid->d.rKeyId->subjectKeyIdentifier, cert); + else + return -1; +} + +int CMS_RecipientInfo_kari_set0_pkey(CMS_RecipientInfo *ri, EVP_PKEY *pk) +{ + EVP_PKEY_CTX *pctx; + CMS_KeyAgreeRecipientInfo *kari = ri->d.kari; + if (kari->pctx) { + EVP_PKEY_CTX_free(kari->pctx); + kari->pctx = NULL; + } + if (!pk) + return 1; + pctx = EVP_PKEY_CTX_new(pk, NULL); + if (!pctx || !EVP_PKEY_derive_init(pctx)) + goto err; + kari->pctx = pctx; + return 1; + err: + if (pctx) + EVP_PKEY_CTX_free(pctx); + return 0; +} + +EVP_CIPHER_CTX *CMS_RecipientInfo_kari_get0_ctx(CMS_RecipientInfo *ri) +{ + if (ri->type == CMS_RECIPINFO_AGREE) + return &ri->d.kari->ctx; + return NULL; +} + +/* + * Derive KEK and decrypt/encrypt with it to produce either the original CEK + * or the encrypted CEK. + */ + +static int cms_kek_cipher(unsigned char **pout, size_t *poutlen, + const unsigned char *in, size_t inlen, + CMS_KeyAgreeRecipientInfo *kari, int enc) +{ + /* Key encryption key */ + unsigned char kek[EVP_MAX_KEY_LENGTH]; + size_t keklen; + int rv = 0; + unsigned char *out = NULL; + int outlen; + keklen = EVP_CIPHER_CTX_key_length(&kari->ctx); + if (keklen > EVP_MAX_KEY_LENGTH) + return 0; + /* Derive KEK */ + if (EVP_PKEY_derive(kari->pctx, kek, &keklen) <= 0) + goto err; + /* Set KEK in context */ + if (!EVP_CipherInit_ex(&kari->ctx, NULL, NULL, kek, NULL, enc)) + goto err; + /* obtain output length of ciphered key */ + if (!EVP_CipherUpdate(&kari->ctx, NULL, &outlen, in, inlen)) + goto err; + out = OPENSSL_malloc(outlen); + if (!out) + goto err; + if (!EVP_CipherUpdate(&kari->ctx, out, &outlen, in, inlen)) + goto err; + *pout = out; + *poutlen = (size_t)outlen; + rv = 1; + + err: + OPENSSL_cleanse(kek, keklen); + if (!rv && out) + OPENSSL_free(out); + EVP_CIPHER_CTX_cleanup(&kari->ctx); + EVP_PKEY_CTX_free(kari->pctx); + kari->pctx = NULL; + return rv; +} + +int CMS_RecipientInfo_kari_decrypt(CMS_ContentInfo *cms, + CMS_RecipientInfo *ri, + CMS_RecipientEncryptedKey *rek) +{ + int rv = 0; + unsigned char *enckey = NULL, *cek = NULL; + size_t enckeylen; + size_t ceklen; + CMS_EncryptedContentInfo *ec; + enckeylen = rek->encryptedKey->length; + enckey = rek->encryptedKey->data; + /* Setup all parameters to derive KEK */ + if (!cms_env_asn1_ctrl(ri, 1)) + goto err; + /* Attempt to decrypt CEK */ + if (!cms_kek_cipher(&cek, &ceklen, enckey, enckeylen, ri->d.kari, 0)) + goto err; + ec = cms->d.envelopedData->encryptedContentInfo; + if (ec->key) { + OPENSSL_cleanse(ec->key, ec->keylen); + OPENSSL_free(ec->key); + } + ec->key = cek; + ec->keylen = ceklen; + cek = NULL; + rv = 1; + err: + if (cek) + OPENSSL_free(cek); + return rv; +} + +/* Create ephemeral key and initialise context based on it */ +static int cms_kari_create_ephemeral_key(CMS_KeyAgreeRecipientInfo *kari, + EVP_PKEY *pk) +{ + EVP_PKEY_CTX *pctx = NULL; + EVP_PKEY *ekey = NULL; + int rv = 0; + pctx = EVP_PKEY_CTX_new(pk, NULL); + if (!pctx) + goto err; + if (EVP_PKEY_keygen_init(pctx) <= 0) + goto err; + if (EVP_PKEY_keygen(pctx, &ekey) <= 0) + goto err; + EVP_PKEY_CTX_free(pctx); + pctx = EVP_PKEY_CTX_new(ekey, NULL); + if (!pctx) + goto err; + if (EVP_PKEY_derive_init(pctx) <= 0) + goto err; + kari->pctx = pctx; + rv = 1; + err: + if (!rv && pctx) + EVP_PKEY_CTX_free(pctx); + if (ekey) + EVP_PKEY_free(ekey); + return rv; +} + +/* Initialise a ktri based on passed certificate and key */ + +int cms_RecipientInfo_kari_init(CMS_RecipientInfo *ri, X509 *recip, + EVP_PKEY *pk, unsigned int flags) +{ + CMS_KeyAgreeRecipientInfo *kari; + CMS_RecipientEncryptedKey *rek = NULL; + + ri->d.kari = M_ASN1_new_of(CMS_KeyAgreeRecipientInfo); + if (!ri->d.kari) + return 0; + ri->type = CMS_RECIPINFO_AGREE; + + kari = ri->d.kari; + kari->version = 3; + + rek = M_ASN1_new_of(CMS_RecipientEncryptedKey); + if (!sk_CMS_RecipientEncryptedKey_push(kari->recipientEncryptedKeys, rek)) { + M_ASN1_free_of(rek, CMS_RecipientEncryptedKey); + return 0; + } + + if (flags & CMS_USE_KEYID) { + rek->rid->type = CMS_REK_KEYIDENTIFIER; + rek->rid->d.rKeyId = M_ASN1_new_of(CMS_RecipientKeyIdentifier); + if (rek->rid->d.rKeyId == NULL) + return 0; + if (!cms_set1_keyid(&rek->rid->d.rKeyId->subjectKeyIdentifier, recip)) + return 0; + } else { + rek->rid->type = CMS_REK_ISSUER_SERIAL; + if (!cms_set1_ias(&rek->rid->d.issuerAndSerialNumber, recip)) + return 0; + } + + /* Create ephemeral key */ + if (!cms_kari_create_ephemeral_key(kari, pk)) + return 0; + + CRYPTO_add(&pk->references, 1, CRYPTO_LOCK_EVP_PKEY); + rek->pkey = pk; + return 1; +} + +static int cms_wrap_init(CMS_KeyAgreeRecipientInfo *kari, + const EVP_CIPHER *cipher) +{ + EVP_CIPHER_CTX *ctx = &kari->ctx; + const EVP_CIPHER *kekcipher; + int keylen = EVP_CIPHER_key_length(cipher); + /* If a suitable wrap algorithm is already set nothing to do */ + kekcipher = EVP_CIPHER_CTX_cipher(ctx); + + if (kekcipher) { + if (EVP_CIPHER_CTX_mode(ctx) != EVP_CIPH_WRAP_MODE) + return 0; + return 1; + } + /* + * Pick a cipher based on content encryption cipher. If it is DES3 use + * DES3 wrap otherwise use AES wrap similar to key size. + */ + if (EVP_CIPHER_type(cipher) == NID_des_ede3_cbc) + kekcipher = EVP_des_ede3_wrap(); + else if (keylen <= 16) + kekcipher = EVP_aes_128_wrap(); + else if (keylen <= 24) + kekcipher = EVP_aes_192_wrap(); + else + kekcipher = EVP_aes_256_wrap(); + return EVP_EncryptInit_ex(ctx, kekcipher, NULL, NULL, NULL); +} + +/* Encrypt content key in key agreement recipient info */ + +int cms_RecipientInfo_kari_encrypt(CMS_ContentInfo *cms, + CMS_RecipientInfo *ri) +{ + CMS_KeyAgreeRecipientInfo *kari; + CMS_EncryptedContentInfo *ec; + CMS_RecipientEncryptedKey *rek; + STACK_OF(CMS_RecipientEncryptedKey) *reks; + int i; + + if (ri->type != CMS_RECIPINFO_AGREE) { + CMSerr(CMS_F_CMS_RECIPIENTINFO_KARI_ENCRYPT, CMS_R_NOT_KEY_AGREEMENT); + return 0; + } + kari = ri->d.kari; + reks = kari->recipientEncryptedKeys; + ec = cms->d.envelopedData->encryptedContentInfo; + /* Initialise wrap algorithm parameters */ + if (!cms_wrap_init(kari, ec->cipher)) + return 0; + /* + * If no orignator key set up initialise for ephemeral key the public key + * ASN1 structure will set the actual public key value. + */ + if (kari->originator->type == -1) { + CMS_OriginatorIdentifierOrKey *oik = kari->originator; + oik->type = CMS_OIK_PUBKEY; + oik->d.originatorKey = M_ASN1_new_of(CMS_OriginatorPublicKey); + if (!oik->d.originatorKey) + return 0; + } + /* Initialise KDF algorithm */ + if (!cms_env_asn1_ctrl(ri, 0)) + return 0; + /* For each rek, derive KEK, encrypt CEK */ + for (i = 0; i < sk_CMS_RecipientEncryptedKey_num(reks); i++) { + unsigned char *enckey; + size_t enckeylen; + rek = sk_CMS_RecipientEncryptedKey_value(reks, i); + if (EVP_PKEY_derive_set_peer(kari->pctx, rek->pkey) <= 0) + return 0; + if (!cms_kek_cipher(&enckey, &enckeylen, ec->key, ec->keylen, + kari, 1)) + return 0; + ASN1_STRING_set0(rek->encryptedKey, enckey, enckeylen); + } + + return 1; + +} diff --git a/crypto/cms/cms_lcl.h b/crypto/cms/cms_lcl.h index 4f4c4c7434f3..20f2c25f5ae9 100644 --- a/crypto/cms/cms_lcl.h +++ b/crypto/cms/cms_lcl.h @@ -84,11 +84,9 @@ typedef struct CMS_KeyTransRecipientInfo_st CMS_KeyTransRecipientInfo; typedef struct CMS_OriginatorPublicKey_st CMS_OriginatorPublicKey; typedef struct CMS_OriginatorIdentifierOrKey_st CMS_OriginatorIdentifierOrKey; typedef struct CMS_KeyAgreeRecipientInfo_st CMS_KeyAgreeRecipientInfo; -typedef struct CMS_OtherKeyAttribute_st CMS_OtherKeyAttribute; typedef struct CMS_RecipientKeyIdentifier_st CMS_RecipientKeyIdentifier; typedef struct CMS_KeyAgreeRecipientIdentifier_st CMS_KeyAgreeRecipientIdentifier; -typedef struct CMS_RecipientEncryptedKey_st CMS_RecipientEncryptedKey; typedef struct CMS_KEKIdentifier_st CMS_KEKIdentifier; typedef struct CMS_KEKRecipientInfo_st CMS_KEKRecipientInfo; typedef struct CMS_PasswordRecipientInfo_st CMS_PasswordRecipientInfo; @@ -138,6 +136,9 @@ struct CMS_SignerInfo_st { /* Signing certificate and key */ X509 *signer; EVP_PKEY *pkey; + /* Digest and public key context for alternative parameters */ + EVP_MD_CTX mctx; + EVP_PKEY_CTX *pctx; }; struct CMS_SignerIdentifier_st { @@ -194,6 +195,8 @@ struct CMS_KeyTransRecipientInfo_st { /* Recipient Key and cert */ X509 *recip; EVP_PKEY *pkey; + /* Public key context for this operation */ + EVP_PKEY_CTX *pctx; }; struct CMS_KeyAgreeRecipientInfo_st { @@ -202,6 +205,10 @@ struct CMS_KeyAgreeRecipientInfo_st { ASN1_OCTET_STRING *ukm; X509_ALGOR *keyEncryptionAlgorithm; STACK_OF(CMS_RecipientEncryptedKey) *recipientEncryptedKeys; + /* Public key context associated with current operation */ + EVP_PKEY_CTX *pctx; + /* Cipher context for CEK wrapping */ + EVP_CIPHER_CTX ctx; }; struct CMS_OriginatorIdentifierOrKey_st { @@ -221,6 +228,8 @@ struct CMS_OriginatorPublicKey_st { struct CMS_RecipientEncryptedKey_st { CMS_KeyAgreeRecipientIdentifier *rid; ASN1_OCTET_STRING *encryptedKey; + /* Public key associated with this recipient */ + EVP_PKEY *pkey; }; struct CMS_KeyAgreeRecipientIdentifier_st { @@ -394,6 +403,13 @@ DECLARE_ASN1_ALLOC_FUNCTIONS(CMS_IssuerAndSerialNumber) # define CMS_RECIPINFO_ISSUER_SERIAL 0 # define CMS_RECIPINFO_KEYIDENTIFIER 1 +# define CMS_REK_ISSUER_SERIAL 0 +# define CMS_REK_KEYIDENTIFIER 1 + +# define CMS_OIK_ISSUER_SERIAL 0 +# define CMS_OIK_KEYIDENTIFIER 1 +# define CMS_OIK_PUBKEY 2 + BIO *cms_content_bio(CMS_ContentInfo *cms); CMS_ContentInfo *cms_Data_create(void); @@ -420,6 +436,11 @@ BIO *cms_DigestAlgorithm_init_bio(X509_ALGOR *digestAlgorithm); int cms_DigestAlgorithm_find_ctx(EVP_MD_CTX *mctx, BIO *chain, X509_ALGOR *mdalg); +int cms_ias_cert_cmp(CMS_IssuerAndSerialNumber *ias, X509 *cert); +int cms_keyid_cert_cmp(ASN1_OCTET_STRING *keyid, X509 *cert); +int cms_set1_ias(CMS_IssuerAndSerialNumber **pias, X509 *cert); +int cms_set1_keyid(ASN1_OCTET_STRING **pkeyid, X509 *cert); + BIO *cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec); BIO *cms_EncryptedData_init_bio(CMS_ContentInfo *cms); int cms_EncryptedContent_init(CMS_EncryptedContentInfo *ec, @@ -432,6 +453,13 @@ ASN1_OCTET_STRING *cms_encode_Receipt(CMS_SignerInfo *si); BIO *cms_EnvelopedData_init_bio(CMS_ContentInfo *cms); CMS_EnvelopedData *cms_get0_enveloped(CMS_ContentInfo *cms); +int cms_env_asn1_ctrl(CMS_RecipientInfo *ri, int cmd); +int cms_pkey_get_ri_type(EVP_PKEY *pk); +/* KARI routines */ +int cms_RecipientInfo_kari_init(CMS_RecipientInfo *ri, X509 *recip, + EVP_PKEY *pk, unsigned int flags); +int cms_RecipientInfo_kari_encrypt(CMS_ContentInfo *cms, + CMS_RecipientInfo *ri); /* PWRI routines */ int cms_RecipientInfo_pwri_crypt(CMS_ContentInfo *cms, CMS_RecipientInfo *ri, diff --git a/crypto/cms/cms_lib.c b/crypto/cms/cms_lib.c index e93846168038..d6cb60d02d15 100644 --- a/crypto/cms/cms_lib.c +++ b/crypto/cms/cms_lib.c @@ -53,7 +53,7 @@ */ #include -#include +#include #include #include #include @@ -593,3 +593,60 @@ STACK_OF(X509_CRL) *CMS_get1_crls(CMS_ContentInfo *cms) } return crls; } + +int cms_ias_cert_cmp(CMS_IssuerAndSerialNumber *ias, X509 *cert) +{ + int ret; + ret = X509_NAME_cmp(ias->issuer, X509_get_issuer_name(cert)); + if (ret) + return ret; + return ASN1_INTEGER_cmp(ias->serialNumber, X509_get_serialNumber(cert)); +} + +int cms_keyid_cert_cmp(ASN1_OCTET_STRING *keyid, X509 *cert) +{ + X509_check_purpose(cert, -1, -1); + if (!cert->skid) + return -1; + return ASN1_OCTET_STRING_cmp(keyid, cert->skid); +} + +int cms_set1_ias(CMS_IssuerAndSerialNumber **pias, X509 *cert) +{ + CMS_IssuerAndSerialNumber *ias; + ias = M_ASN1_new_of(CMS_IssuerAndSerialNumber); + if (!ias) + goto err; + if (!X509_NAME_set(&ias->issuer, X509_get_issuer_name(cert))) + goto err; + if (!ASN1_STRING_copy(ias->serialNumber, X509_get_serialNumber(cert))) + goto err; + if (*pias) + M_ASN1_free_of(*pias, CMS_IssuerAndSerialNumber); + *pias = ias; + return 1; + err: + if (ias) + M_ASN1_free_of(ias, CMS_IssuerAndSerialNumber); + CMSerr(CMS_F_CMS_SET1_IAS, ERR_R_MALLOC_FAILURE); + return 0; +} + +int cms_set1_keyid(ASN1_OCTET_STRING **pkeyid, X509 *cert) +{ + ASN1_OCTET_STRING *keyid = NULL; + X509_check_purpose(cert, -1, -1); + if (!cert->skid) { + CMSerr(CMS_F_CMS_SET1_KEYID, CMS_R_CERTIFICATE_HAS_NO_KEYID); + return 0; + } + keyid = ASN1_STRING_dup(cert->skid); + if (!keyid) { + CMSerr(CMS_F_CMS_SET1_KEYID, ERR_R_MALLOC_FAILURE); + return 0; + } + if (*pkeyid) + ASN1_OCTET_STRING_free(*pkeyid); + *pkeyid = keyid; + return 1; +} diff --git a/crypto/cms/cms_sd.c b/crypto/cms/cms_sd.c index 6daa2627dbab..721ffd5afb85 100644 --- a/crypto/cms/cms_sd.c +++ b/crypto/cms/cms_sd.c @@ -55,6 +55,7 @@ #include "cryptlib.h" #include #include +#include #include #include #include @@ -197,27 +198,13 @@ int cms_set1_SignerIdentifier(CMS_SignerIdentifier *sid, X509 *cert, int type) { switch (type) { case CMS_SIGNERINFO_ISSUER_SERIAL: - sid->d.issuerAndSerialNumber = - M_ASN1_new_of(CMS_IssuerAndSerialNumber); - if (!sid->d.issuerAndSerialNumber) - goto merr; - if (!X509_NAME_set(&sid->d.issuerAndSerialNumber->issuer, - X509_get_issuer_name(cert))) - goto merr; - if (!ASN1_STRING_copy(sid->d.issuerAndSerialNumber->serialNumber, - X509_get_serialNumber(cert))) - goto merr; + if (!cms_set1_ias(&sid->d.issuerAndSerialNumber, cert)) + return 0; break; case CMS_SIGNERINFO_KEYIDENTIFIER: - if (!cert->skid) { - CMSerr(CMS_F_CMS_SET1_SIGNERIDENTIFIER, - CMS_R_CERTIFICATE_HAS_NO_KEYID); + if (!cms_set1_keyid(&sid->d.subjectKeyIdentifier, cert)) return 0; - } - sid->d.subjectKeyIdentifier = ASN1_STRING_dup(cert->skid); - if (!sid->d.subjectKeyIdentifier) - goto merr; break; default: @@ -228,11 +215,6 @@ int cms_set1_SignerIdentifier(CMS_SignerIdentifier *sid, X509 *cert, int type) sid->type = type; return 1; - - merr: - CMSerr(CMS_F_CMS_SET1_SIGNERIDENTIFIER, ERR_R_MALLOC_FAILURE); - return 0; - } int cms_SignerIdentifier_get0_signer_id(CMS_SignerIdentifier *sid, @@ -255,23 +237,32 @@ int cms_SignerIdentifier_get0_signer_id(CMS_SignerIdentifier *sid, int cms_SignerIdentifier_cert_cmp(CMS_SignerIdentifier *sid, X509 *cert) { - int ret; - if (sid->type == CMS_SIGNERINFO_ISSUER_SERIAL) { - ret = X509_NAME_cmp(sid->d.issuerAndSerialNumber->issuer, - X509_get_issuer_name(cert)); - if (ret) - return ret; - return ASN1_INTEGER_cmp(sid->d.issuerAndSerialNumber->serialNumber, - X509_get_serialNumber(cert)); - } else if (sid->type == CMS_SIGNERINFO_KEYIDENTIFIER) { - X509_check_purpose(cert, -1, -1); - if (!cert->skid) - return -1; - return ASN1_OCTET_STRING_cmp(sid->d.subjectKeyIdentifier, cert->skid); - } else + if (sid->type == CMS_SIGNERINFO_ISSUER_SERIAL) + return cms_ias_cert_cmp(sid->d.issuerAndSerialNumber, cert); + else if (sid->type == CMS_SIGNERINFO_KEYIDENTIFIER) + return cms_keyid_cert_cmp(sid->d.subjectKeyIdentifier, cert); + else return -1; } +static int cms_sd_asn1_ctrl(CMS_SignerInfo *si, int cmd) +{ + EVP_PKEY *pkey = si->pkey; + int i; + if (!pkey->ameth || !pkey->ameth->pkey_ctrl) + return 1; + i = pkey->ameth->pkey_ctrl(pkey, ASN1_PKEY_CTRL_CMS_SIGN, cmd, si); + if (i == -2) { + CMSerr(CMS_F_CMS_SD_ASN1_CTRL, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); + return 0; + } + if (i <= 0) { + CMSerr(CMS_F_CMS_SD_ASN1_CTRL, CMS_R_CTRL_FAILURE); + return 0; + } + return 1; +} + CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, X509 *signer, EVP_PKEY *pk, const EVP_MD *md, unsigned int flags) @@ -298,6 +289,8 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, si->pkey = pk; si->signer = signer; + EVP_MD_CTX_init(&si->mctx); + si->pctx = NULL; if (flags & CMS_USE_KEYID) { si->version = 3; @@ -350,19 +343,8 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, } } - if (pk->ameth && pk->ameth->pkey_ctrl) { - i = pk->ameth->pkey_ctrl(pk, ASN1_PKEY_CTRL_CMS_SIGN, 0, si); - if (i == -2) { - CMSerr(CMS_F_CMS_ADD1_SIGNER, - CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); - goto err; - } - if (i <= 0) { - CMSerr(CMS_F_CMS_ADD1_SIGNER, CMS_R_CTRL_FAILURE); - goto err; - } - } - + if (!(flags & CMS_KEY_PARAM) && !cms_sd_asn1_ctrl(si, 0)) + goto err; if (!(flags & CMS_NOATTR)) { /* * Initialialize signed attributes strutucture so other attributes @@ -386,7 +368,8 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, if (flags & CMS_REUSE_DIGEST) { if (!cms_copy_messageDigest(cms, si)) goto err; - if (!(flags & CMS_PARTIAL) && !CMS_SignerInfo_sign(si)) + if (!(flags & (CMS_PARTIAL | CMS_KEY_PARAM)) && + !CMS_SignerInfo_sign(si)) goto err; } } @@ -397,6 +380,20 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, goto merr; } + if (flags & CMS_KEY_PARAM) { + if (flags & CMS_NOATTR) { + si->pctx = EVP_PKEY_CTX_new(si->pkey, NULL); + if (!si->pctx) + goto err; + if (EVP_PKEY_sign_init(si->pctx) <= 0) + goto err; + if (EVP_PKEY_CTX_set_signature_md(si->pctx, md) <= 0) + goto err; + } else if (EVP_DigestSignInit(&si->mctx, &si->pctx, md, NULL, pk) <= + 0) + goto err; + } + if (!sd->signerInfos) sd->signerInfos = sk_CMS_SignerInfo_new_null(); if (!sd->signerInfos || !sk_CMS_SignerInfo_push(sd->signerInfos, si)) @@ -443,6 +440,16 @@ static int cms_add1_signingTime(CMS_SignerInfo *si, ASN1_TIME *t) } +EVP_PKEY_CTX *CMS_SignerInfo_get0_pkey_ctx(CMS_SignerInfo *si) +{ + return si->pctx; +} + +EVP_MD_CTX *CMS_SignerInfo_get0_md_ctx(CMS_SignerInfo *si) +{ + return &si->mctx; +} + STACK_OF(CMS_SignerInfo) *CMS_get0_SignerInfos(CMS_ContentInfo *cms) { CMS_SignedData *sd; @@ -561,11 +568,17 @@ void CMS_SignerInfo_get0_algs(CMS_SignerInfo *si, EVP_PKEY **pk, *psig = si->signatureAlgorithm; } +ASN1_OCTET_STRING *CMS_SignerInfo_get0_signature(CMS_SignerInfo *si) +{ + return si->signature; +} + static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, CMS_SignerInfo *si, BIO *chain) { EVP_MD_CTX mctx; int r = 0; + EVP_PKEY_CTX *pctx = NULL; EVP_MD_CTX_init(&mctx); if (!si->pkey) { @@ -575,6 +588,9 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, if (!cms_DigestAlgorithm_find_ctx(&mctx, chain, si->digestAlgorithm)) goto err; + /* Set SignerInfo algortihm details if we used custom parametsr */ + if (si->pctx && !cms_sd_asn1_ctrl(si, 0)) + goto err; /* * If any signed attributes calculate and add messageDigest attribute @@ -596,6 +612,23 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, goto err; if (!CMS_SignerInfo_sign(si)) goto err; + } else if (si->pctx) { + unsigned char *sig; + size_t siglen; + unsigned char md[EVP_MAX_MD_SIZE]; + unsigned int mdlen; + pctx = si->pctx; + if (!EVP_DigestFinal_ex(&mctx, md, &mdlen)) + goto err; + siglen = EVP_PKEY_size(si->pkey); + sig = OPENSSL_malloc(siglen); + if (!sig) { + CMSerr(CMS_F_CMS_SIGNERINFO_CONTENT_SIGN, ERR_R_MALLOC_FAILURE); + goto err; + } + if (EVP_PKEY_sign(pctx, sig, &siglen, md, mdlen) <= 0) + goto err; + ASN1_STRING_set0(si->signature, sig, siglen); } else { unsigned char *sig; unsigned int siglen; @@ -616,6 +649,8 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, err: EVP_MD_CTX_cleanup(&mctx); + if (pctx) + EVP_PKEY_CTX_free(pctx); return r; } @@ -637,7 +672,7 @@ int cms_SignedData_final(CMS_ContentInfo *cms, BIO *chain) int CMS_SignerInfo_sign(CMS_SignerInfo *si) { - EVP_MD_CTX mctx; + EVP_MD_CTX *mctx = &si->mctx; EVP_PKEY_CTX *pctx; unsigned char *abuf = NULL; int alen; @@ -648,15 +683,18 @@ int CMS_SignerInfo_sign(CMS_SignerInfo *si) if (md == NULL) return 0; - EVP_MD_CTX_init(&mctx); - if (CMS_signed_get_attr_by_NID(si, NID_pkcs9_signingTime, -1) < 0) { if (!cms_add1_signingTime(si, NULL)) goto err; } - if (EVP_DigestSignInit(&mctx, &pctx, md, NULL, si->pkey) <= 0) - goto err; + if (si->pctx) + pctx = si->pctx; + else { + EVP_MD_CTX_init(mctx); + if (EVP_DigestSignInit(mctx, &pctx, md, NULL, si->pkey) <= 0) + goto err; + } if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_SIGN, EVP_PKEY_CTRL_CMS_SIGN, 0, si) <= 0) { @@ -668,15 +706,15 @@ int CMS_SignerInfo_sign(CMS_SignerInfo *si) ASN1_ITEM_rptr(CMS_Attributes_Sign)); if (!abuf) goto err; - if (EVP_DigestSignUpdate(&mctx, abuf, alen) <= 0) + if (EVP_DigestSignUpdate(mctx, abuf, alen) <= 0) goto err; - if (EVP_DigestSignFinal(&mctx, NULL, &siglen) <= 0) + if (EVP_DigestSignFinal(mctx, NULL, &siglen) <= 0) goto err; OPENSSL_free(abuf); abuf = OPENSSL_malloc(siglen); if (!abuf) goto err; - if (EVP_DigestSignFinal(&mctx, abuf, &siglen) <= 0) + if (EVP_DigestSignFinal(mctx, abuf, &siglen) <= 0) goto err; if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_SIGN, @@ -685,7 +723,7 @@ int CMS_SignerInfo_sign(CMS_SignerInfo *si) goto err; } - EVP_MD_CTX_cleanup(&mctx); + EVP_MD_CTX_cleanup(mctx); ASN1_STRING_set0(si->signature, abuf, siglen); @@ -694,15 +732,14 @@ int CMS_SignerInfo_sign(CMS_SignerInfo *si) err: if (abuf) OPENSSL_free(abuf); - EVP_MD_CTX_cleanup(&mctx); + EVP_MD_CTX_cleanup(mctx); return 0; } int CMS_SignerInfo_verify(CMS_SignerInfo *si) { - EVP_MD_CTX mctx; - EVP_PKEY_CTX *pctx; + EVP_MD_CTX *mctx = &si->mctx; unsigned char *abuf = NULL; int alen, r = -1; const EVP_MD *md = NULL; @@ -715,26 +752,29 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si) md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm); if (md == NULL) return -1; - EVP_MD_CTX_init(&mctx); - if (EVP_DigestVerifyInit(&mctx, &pctx, md, NULL, si->pkey) <= 0) + EVP_MD_CTX_init(mctx); + if (EVP_DigestVerifyInit(mctx, &si->pctx, md, NULL, si->pkey) <= 0) + goto err; + + if (!cms_sd_asn1_ctrl(si, 1)) goto err; alen = ASN1_item_i2d((ASN1_VALUE *)si->signedAttrs, &abuf, ASN1_ITEM_rptr(CMS_Attributes_Verify)); if (!abuf) goto err; - r = EVP_DigestVerifyUpdate(&mctx, abuf, alen); + r = EVP_DigestVerifyUpdate(mctx, abuf, alen); OPENSSL_free(abuf); if (r <= 0) { r = -1; goto err; } - r = EVP_DigestVerifyFinal(&mctx, + r = EVP_DigestVerifyFinal(mctx, si->signature->data, si->signature->length); if (r <= 0) CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY, CMS_R_VERIFICATION_FAILURE); err: - EVP_MD_CTX_cleanup(&mctx); + EVP_MD_CTX_cleanup(mctx); return r; } @@ -773,7 +813,10 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) { ASN1_OCTET_STRING *os = NULL; EVP_MD_CTX mctx; + EVP_PKEY_CTX *pkctx = NULL; int r = -1; + unsigned char mval[EVP_MAX_MD_SIZE]; + unsigned int mlen; EVP_MD_CTX_init(&mctx); /* If we have any signed attributes look for messageDigest value */ if (CMS_signed_get_attr_count(si) >= 0) { @@ -790,16 +833,15 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) if (!cms_DigestAlgorithm_find_ctx(&mctx, chain, si->digestAlgorithm)) goto err; + if (EVP_DigestFinal_ex(&mctx, mval, &mlen) <= 0) { + CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY_CONTENT, + CMS_R_UNABLE_TO_FINALIZE_CONTEXT); + goto err; + } + /* If messageDigest found compare it */ if (os) { - unsigned char mval[EVP_MAX_MD_SIZE]; - unsigned int mlen; - if (EVP_DigestFinal_ex(&mctx, mval, &mlen) <= 0) { - CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY_CONTENT, - CMS_R_UNABLE_TO_FINALIZE_CONTEXT); - goto err; - } if (mlen != (unsigned int)os->length) { CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY_CONTENT, CMS_R_MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH); @@ -813,8 +855,17 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) } else r = 1; } else { - r = EVP_VerifyFinal(&mctx, si->signature->data, - si->signature->length, si->pkey); + const EVP_MD *md = EVP_MD_CTX_md(&mctx); + pkctx = EVP_PKEY_CTX_new(si->pkey, NULL); + if (EVP_PKEY_verify_init(pkctx) <= 0) + goto err; + if (EVP_PKEY_CTX_set_signature_md(pkctx, md) <= 0) + goto err; + si->pctx = pkctx; + if (!cms_sd_asn1_ctrl(si, 1)) + goto err; + r = EVP_PKEY_verify(pkctx, si->signature->data, + si->signature->length, mval, mlen); if (r <= 0) { CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY_CONTENT, CMS_R_VERIFICATION_FAILURE); @@ -823,6 +874,8 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) } err: + if (pkctx) + EVP_PKEY_CTX_free(pkctx); EVP_MD_CTX_cleanup(&mctx); return r; diff --git a/crypto/cms/cms_smime.c b/crypto/cms/cms_smime.c index 8b37560821ad..5522a376acb6 100644 --- a/crypto/cms/cms_smime.c +++ b/crypto/cms/cms_smime.c @@ -59,6 +59,7 @@ #include #include #include "cms_lcl.h" +#include "asn1_locl.h" static int cms_copy_content(BIO *out, BIO *in, unsigned int flags) { @@ -373,7 +374,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, tmpin = BIO_new_mem_buf(ptr, len); if (tmpin == NULL) { CMSerr(CMS_F_CMS_VERIFY, ERR_R_MALLOC_FAILURE); - return 0; + goto err2; } } else tmpin = dcont; @@ -404,6 +405,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, else BIO_free_all(cmsbio); + err2: if (cms_certs) sk_X509_pop_free(cms_certs, X509_free); if (crls) @@ -567,25 +569,63 @@ CMS_ContentInfo *CMS_encrypt(STACK_OF(X509) *certs, BIO *data, return NULL; } +static int cms_kari_set1_pkey(CMS_ContentInfo *cms, CMS_RecipientInfo *ri, + EVP_PKEY *pk, X509 *cert) +{ + int i; + STACK_OF(CMS_RecipientEncryptedKey) *reks; + CMS_RecipientEncryptedKey *rek; + reks = CMS_RecipientInfo_kari_get0_reks(ri); + if (!cert) + return 0; + for (i = 0; i < sk_CMS_RecipientEncryptedKey_num(reks); i++) { + int rv; + rek = sk_CMS_RecipientEncryptedKey_value(reks, i); + if (CMS_RecipientEncryptedKey_cert_cmp(rek, cert)) + continue; + CMS_RecipientInfo_kari_set0_pkey(ri, pk); + rv = CMS_RecipientInfo_kari_decrypt(cms, ri, rek); + CMS_RecipientInfo_kari_set0_pkey(ri, NULL); + if (rv > 0) + return 1; + return -1; + } + return 0; +} + int CMS_decrypt_set1_pkey(CMS_ContentInfo *cms, EVP_PKEY *pk, X509 *cert) { STACK_OF(CMS_RecipientInfo) *ris; CMS_RecipientInfo *ri; - int i, r; - int debug = 0, ri_match = 0; + int i, r, ri_type; + int debug = 0, match_ri = 0; ris = CMS_get0_RecipientInfos(cms); if (ris) debug = cms->d.envelopedData->encryptedContentInfo->debug; + ri_type = cms_pkey_get_ri_type(pk); + if (ri_type == CMS_RECIPINFO_NONE) { + CMSerr(CMS_F_CMS_DECRYPT_SET1_PKEY, + CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); + return 0; + } + for (i = 0; i < sk_CMS_RecipientInfo_num(ris); i++) { ri = sk_CMS_RecipientInfo_value(ris, i); - if (CMS_RecipientInfo_type(ri) != CMS_RECIPINFO_TRANS) + if (CMS_RecipientInfo_type(ri) != ri_type) continue; - ri_match = 1; + match_ri = 1; + if (ri_type == CMS_RECIPINFO_AGREE) { + r = cms_kari_set1_pkey(cms, ri, pk, cert); + if (r > 0) + return 1; + if (r < 0) + return 0; + } /* * If we have a cert try matching RecipientInfo otherwise try them * all. */ - if (!cert || (CMS_RecipientInfo_ktri_cert_cmp(ri, cert) == 0)) { + else if (!cert || !CMS_RecipientInfo_ktri_cert_cmp(ri, cert)) { CMS_RecipientInfo_set0_pkey(ri, pk); r = CMS_RecipientInfo_decrypt(cms, ri); CMS_RecipientInfo_set0_pkey(ri, NULL); @@ -613,7 +653,7 @@ int CMS_decrypt_set1_pkey(CMS_ContentInfo *cms, EVP_PKEY *pk, X509 *cert) } } /* If no cert and not debugging always return success */ - if (ri_match && !cert && !debug) { + if (match_ri && !cert && !debug) { ERR_clear_error(); return 1; } diff --git a/crypto/cryptlib.c b/crypto/cryptlib.c index c654a5c84f2c..ca0e3ccc0c7a 100644 --- a/crypto/cryptlib.c +++ b/crypto/cryptlib.c @@ -653,7 +653,7 @@ const char *CRYPTO_get_lock_name(int type) defined(__x86_64) || defined(__x86_64__) || \ defined(_M_AMD64) || defined(_M_X64) -unsigned int OPENSSL_ia32cap_P[2]; +extern unsigned int OPENSSL_ia32cap_P[4]; unsigned long *OPENSSL_ia32cap_loc(void) { if (sizeof(long) == 4) @@ -663,6 +663,9 @@ unsigned long *OPENSSL_ia32cap_loc(void) * is 32-bit. */ OPENSSL_ia32cap_P[1] = 0; + + OPENSSL_ia32cap_P[2] = 0; + return (unsigned long *)OPENSSL_ia32cap_P; } @@ -676,7 +679,7 @@ typedef unsigned long long IA32CAP; void OPENSSL_cpuid_setup(void) { static int trigger = 0; - IA32CAP OPENSSL_ia32_cpuid(void); + IA32CAP OPENSSL_ia32_cpuid(unsigned int *); IA32CAP vec; char *env; @@ -694,9 +697,23 @@ void OPENSSL_cpuid_setup(void) vec = strtoul(env + off, NULL, 0); # endif if (off) - vec = OPENSSL_ia32_cpuid() & ~vec; + vec = OPENSSL_ia32_cpuid(OPENSSL_ia32cap_P) & ~vec; + else if (env[0] == ':') + vec = OPENSSL_ia32_cpuid(OPENSSL_ia32cap_P); + + OPENSSL_ia32cap_P[2] = 0; + if ((env = strchr(env, ':'))) { + unsigned int vecx; + env++; + off = (env[0] == '~') ? 1 : 0; + vecx = strtoul(env + off, NULL, 0); + if (off) + OPENSSL_ia32cap_P[2] &= ~vecx; + else + OPENSSL_ia32cap_P[2] = vecx; + } } else - vec = OPENSSL_ia32_cpuid(); + vec = OPENSSL_ia32_cpuid(OPENSSL_ia32cap_P); /* * |(1<<10) sets a reserved bit to signal that variable @@ -706,6 +723,8 @@ void OPENSSL_cpuid_setup(void) OPENSSL_ia32cap_P[0] = (unsigned int)vec | (1 << 10); OPENSSL_ia32cap_P[1] = (unsigned int)(vec >> 32); } +# else +unsigned int OPENSSL_ia32cap_P[4]; # endif #else @@ -857,8 +876,12 @@ void OPENSSL_showfatal(const char *fmta, ...) if ((h = GetStdHandle(STD_ERROR_HANDLE)) != NULL && GetFileType(h) != FILE_TYPE_UNKNOWN) { /* must be console application */ + int len; + DWORD out; + va_start(ap, fmta); - vfprintf(stderr, fmta, ap); + len = _vsnprintf((char *)buf, sizeof(buf), fmta, ap); + WriteFile(h, buf, len < 0 ? sizeof(buf) : (DWORD) len, &out, NULL); va_end(ap); return; } @@ -965,7 +988,9 @@ void OpenSSLDie(const char *file, int line, const char *assertion) /* * Win32 abort() customarily shows a dialog, but we just did that... */ +# if !defined(_WIN32_WCE) raise(SIGABRT); +# endif _exit(3); #endif } diff --git a/crypto/cversion.c b/crypto/cversion.c index 9e6f50d78182..bfff6995c12e 100644 --- a/crypto/cversion.c +++ b/crypto/cversion.c @@ -68,7 +68,11 @@ const char *SSLeay_version(int t) return OPENSSL_VERSION_TEXT; if (t == SSLEAY_BUILT_ON) { #ifdef DATE +# ifdef OPENSSL_USE_BUILD_DATE return (DATE); +# else + return ("built on: reproducible build, date unspecified"); +# endif #else return ("built on: date not available"); #endif diff --git a/crypto/des/Makefile b/crypto/des/Makefile index fbc77c163cd3..8b5166ca9ff5 100644 --- a/crypto/des/Makefile +++ b/crypto/des/Makefile @@ -61,6 +61,8 @@ des: des.o cbc3_enc.o lib des_enc-sparc.S: asm/des_enc.m4 m4 -B 8192 asm/des_enc.m4 > des_enc-sparc.S +dest4-sparcv9.s: asm/dest4-sparcv9.pl + $(PERL) asm/dest4-sparcv9.pl $(CFLAGS) > $@ des-586.s: asm/des-586.pl ../perlasm/x86asm.pl ../perlasm/cbc.pl $(PERL) asm/des-586.pl $(PERLASM_SCHEME) $(CFLAGS) > $@ diff --git a/crypto/des/asm/des-586.pl b/crypto/des/asm/des-586.pl index 5b5f39cebd13..bd6a7dd6b76b 100644 --- a/crypto/des/asm/des-586.pl +++ b/crypto/des/asm/des-586.pl @@ -25,6 +25,7 @@ $small_footprint=1 if (grep(/\-DOPENSSL_SMALL_FOOTPRINT/,@ARGV)); # the folded loop is only 3% slower than unrolled, but >7 times smaller &public_label("DES_SPtrans"); +&static_label("des_sptrans"); &DES_encrypt_internal(); &DES_decrypt_internal(); @@ -158,7 +159,7 @@ sub DES_encrypt &call (&label("pic_point")); &set_label("pic_point"); &blindpop($trans); - &lea ($trans,&DWP(&label("DES_SPtrans")."-".&label("pic_point"),$trans)); + &lea ($trans,&DWP(&label("des_sptrans")."-".&label("pic_point"),$trans)); &mov( "ecx", &wparam(1) ); @@ -315,6 +316,7 @@ sub FP_new sub DES_SPtrans { &set_label("DES_SPtrans",64); + &set_label("des_sptrans"); &data_word(0x02080800, 0x00080000, 0x02000002, 0x02080802); &data_word(0x02000000, 0x00080802, 0x00080002, 0x02000002); &data_word(0x00080802, 0x02080800, 0x02080000, 0x00000802); diff --git a/crypto/des/asm/des_enc.m4 b/crypto/des/asm/des_enc.m4 index 328059547820..dda08e126dc6 100644 --- a/crypto/des/asm/des_enc.m4 +++ b/crypto/des/asm/des_enc.m4 @@ -46,6 +46,8 @@ .ident "des_enc.m4 2.1" .file "des_enc-sparc.S" +#include + #if defined(__SUNPRO_C) && defined(__sparcv9) # define ABI64 /* They've said -xarch=v9 at command line */ #elif defined(__GNUC__) && defined(__arch64__) diff --git a/crypto/des/asm/dest4-sparcv9.pl b/crypto/des/asm/dest4-sparcv9.pl new file mode 100755 index 000000000000..1dc60243d4fb --- /dev/null +++ b/crypto/des/asm/dest4-sparcv9.pl @@ -0,0 +1,617 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by David S. Miller and Andy Polyakov +# . The module is licensed under 2-clause BSD +# license. March 2013. All rights reserved. +# ==================================================================== + +###################################################################### +# DES for SPARC T4. +# +# As with other hardware-assisted ciphers CBC encrypt results [for +# aligned data] are virtually identical to critical path lengths: +# +# DES Triple-DES +# CBC encrypt 4.14/4.15(*) 11.7/11.7 +# CBC decrypt 1.77/4.11(**) 6.42/7.47 +# +# (*) numbers after slash are for +# misaligned data; +# (**) this is result for largest +# block size, unlike all other +# cases smaller blocks results +# are better[?]; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +push(@INC,"${dir}","${dir}../../perlasm"); +require "sparcv9_modes.pl"; + +&asm_init(@ARGV); + +$code.=<<___ if ($::abibits==64); +.register %g2,#scratch +.register %g3,#scratch +___ + +$code.=<<___; +.text +___ + +{ my ($inp,$out)=("%o0","%o1"); + +$code.=<<___; +.align 32 +.globl des_t4_key_expand +.type des_t4_key_expand,#function +des_t4_key_expand: + andcc $inp, 0x7, %g0 + alignaddr $inp, %g0, $inp + bz,pt %icc, 1f + ldd [$inp + 0x00], %f0 + ldd [$inp + 0x08], %f2 + faligndata %f0, %f2, %f0 +1: des_kexpand %f0, 0, %f0 + des_kexpand %f0, 1, %f2 + std %f0, [$out + 0x00] + des_kexpand %f2, 3, %f6 + std %f2, [$out + 0x08] + des_kexpand %f2, 2, %f4 + des_kexpand %f6, 3, %f10 + std %f6, [$out + 0x18] + des_kexpand %f6, 2, %f8 + std %f4, [$out + 0x10] + des_kexpand %f10, 3, %f14 + std %f10, [$out + 0x28] + des_kexpand %f10, 2, %f12 + std %f8, [$out + 0x20] + des_kexpand %f14, 1, %f16 + std %f14, [$out + 0x38] + des_kexpand %f16, 3, %f20 + std %f12, [$out + 0x30] + des_kexpand %f16, 2, %f18 + std %f16, [$out + 0x40] + des_kexpand %f20, 3, %f24 + std %f20, [$out + 0x50] + des_kexpand %f20, 2, %f22 + std %f18, [$out + 0x48] + des_kexpand %f24, 3, %f28 + std %f24, [$out + 0x60] + des_kexpand %f24, 2, %f26 + std %f22, [$out + 0x58] + des_kexpand %f28, 1, %f30 + std %f28, [$out + 0x70] + std %f26, [$out + 0x68] + retl + std %f30, [$out + 0x78] +.size des_t4_key_expand,.-des_t4_key_expand +___ +} +{ my ($inp,$out,$len,$key,$ivec) = map("%o$_",(0..4)); + my ($ileft,$iright,$omask) = map("%g$_",(1..3)); + +$code.=<<___; +.globl des_t4_cbc_encrypt +.align 32 +des_t4_cbc_encrypt: + cmp $len, 0 + be,pn $::size_t_cc, .Lcbc_abort + nop + ld [$ivec + 0], %f0 ! load ivec + ld [$ivec + 4], %f1 + + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 0xff, $omask + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + sub %g0, $ileft, $iright + and $out, 7, %g4 + alignaddrl $out, %g0, $out + srl $omask, %g4, $omask + srlx $len, 3, $len + movrz %g4, 0, $omask + prefetch [$out], 22 + + ldd [$key + 0x00], %f4 ! load key schedule + ldd [$key + 0x08], %f6 + ldd [$key + 0x10], %f8 + ldd [$key + 0x18], %f10 + ldd [$key + 0x20], %f12 + ldd [$key + 0x28], %f14 + ldd [$key + 0x30], %f16 + ldd [$key + 0x38], %f18 + ldd [$key + 0x40], %f20 + ldd [$key + 0x48], %f22 + ldd [$key + 0x50], %f24 + ldd [$key + 0x58], %f26 + ldd [$key + 0x60], %f28 + ldd [$key + 0x68], %f30 + ldd [$key + 0x70], %f32 + ldd [$key + 0x78], %f34 + +.Ldes_cbc_enc_loop: + ldx [$inp + 0], %g4 + brz,pt $ileft, 4f + nop + + ldx [$inp + 8], %g5 + sllx %g4, $ileft, %g4 + srlx %g5, $iright, %g5 + or %g5, %g4, %g4 +4: + movxtod %g4, %f2 + prefetch [$inp + 8+63], 20 + add $inp, 8, $inp + fxor %f2, %f0, %f0 ! ^= ivec + prefetch [$out + 63], 22 + + des_ip %f0, %f0 + des_round %f4, %f6, %f0, %f0 + des_round %f8, %f10, %f0, %f0 + des_round %f12, %f14, %f0, %f0 + des_round %f16, %f18, %f0, %f0 + des_round %f20, %f22, %f0, %f0 + des_round %f24, %f26, %f0, %f0 + des_round %f28, %f30, %f0, %f0 + des_round %f32, %f34, %f0, %f0 + des_iip %f0, %f0 + + brnz,pn $omask, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + brnz,pt $len, .Ldes_cbc_enc_loop + add $out, 8, $out + + st %f0, [$ivec + 0] ! write out ivec + retl + st %f1, [$ivec + 4] +.Lcbc_abort: + retl + nop + +.align 16 +2: ldxa [$inp]0x82, %g4 ! avoid read-after-write hazard + ! and ~4x deterioration + ! in inp==out case + faligndata %f0, %f0, %f2 ! handle unaligned output + + stda %f2, [$out + $omask]0xc0 ! partial store + add $out, 8, $out + orn %g0, $omask, $omask + stda %f2, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .Ldes_cbc_enc_loop+4 + orn %g0, $omask, $omask + + st %f0, [$ivec + 0] ! write out ivec + retl + st %f1, [$ivec + 4] +.type des_t4_cbc_encrypt,#function +.size des_t4_cbc_encrypt,.-des_t4_cbc_encrypt + +.globl des_t4_cbc_decrypt +.align 32 +des_t4_cbc_decrypt: + cmp $len, 0 + be,pn $::size_t_cc, .Lcbc_abort + nop + ld [$ivec + 0], %f2 ! load ivec + ld [$ivec + 4], %f3 + + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 0xff, $omask + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + sub %g0, $ileft, $iright + and $out, 7, %g4 + alignaddrl $out, %g0, $out + srl $omask, %g4, $omask + srlx $len, 3, $len + movrz %g4, 0, $omask + prefetch [$out], 22 + + ldd [$key + 0x78], %f4 ! load key schedule + ldd [$key + 0x70], %f6 + ldd [$key + 0x68], %f8 + ldd [$key + 0x60], %f10 + ldd [$key + 0x58], %f12 + ldd [$key + 0x50], %f14 + ldd [$key + 0x48], %f16 + ldd [$key + 0x40], %f18 + ldd [$key + 0x38], %f20 + ldd [$key + 0x30], %f22 + ldd [$key + 0x28], %f24 + ldd [$key + 0x20], %f26 + ldd [$key + 0x18], %f28 + ldd [$key + 0x10], %f30 + ldd [$key + 0x08], %f32 + ldd [$key + 0x00], %f34 + +.Ldes_cbc_dec_loop: + ldx [$inp + 0], %g4 + brz,pt $ileft, 4f + nop + + ldx [$inp + 8], %g5 + sllx %g4, $ileft, %g4 + srlx %g5, $iright, %g5 + or %g5, %g4, %g4 +4: + movxtod %g4, %f0 + prefetch [$inp + 8+63], 20 + add $inp, 8, $inp + prefetch [$out + 63], 22 + + des_ip %f0, %f0 + des_round %f4, %f6, %f0, %f0 + des_round %f8, %f10, %f0, %f0 + des_round %f12, %f14, %f0, %f0 + des_round %f16, %f18, %f0, %f0 + des_round %f20, %f22, %f0, %f0 + des_round %f24, %f26, %f0, %f0 + des_round %f28, %f30, %f0, %f0 + des_round %f32, %f34, %f0, %f0 + des_iip %f0, %f0 + + fxor %f2, %f0, %f0 ! ^= ivec + movxtod %g4, %f2 + + brnz,pn $omask, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + brnz,pt $len, .Ldes_cbc_dec_loop + add $out, 8, $out + + st %f2, [$ivec + 0] ! write out ivec + retl + st %f3, [$ivec + 4] + +.align 16 +2: ldxa [$inp]0x82, %g4 ! avoid read-after-write hazard + ! and ~4x deterioration + ! in inp==out case + faligndata %f0, %f0, %f0 ! handle unaligned output + + stda %f0, [$out + $omask]0xc0 ! partial store + add $out, 8, $out + orn %g0, $omask, $omask + stda %f0, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .Ldes_cbc_dec_loop+4 + orn %g0, $omask, $omask + + st %f2, [$ivec + 0] ! write out ivec + retl + st %f3, [$ivec + 4] +.type des_t4_cbc_decrypt,#function +.size des_t4_cbc_decrypt,.-des_t4_cbc_decrypt +___ + +# One might wonder why does one have back-to-back des_iip/des_ip +# pairs between EDE passes. Indeed, aren't they inverse of each other? +# They almost are. Outcome of the pair is 32-bit words being swapped +# in target register. Consider pair of des_iip/des_ip as a way to +# perform the due swap, it's actually fastest way in this case. + +$code.=<<___; +.globl des_t4_ede3_cbc_encrypt +.align 32 +des_t4_ede3_cbc_encrypt: + cmp $len, 0 + be,pn $::size_t_cc, .Lcbc_abort + nop + ld [$ivec + 0], %f0 ! load ivec + ld [$ivec + 4], %f1 + + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 0xff, $omask + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + sub %g0, $ileft, $iright + and $out, 7, %g4 + alignaddrl $out, %g0, $out + srl $omask, %g4, $omask + srlx $len, 3, $len + movrz %g4, 0, $omask + prefetch [$out], 22 + + ldd [$key + 0x00], %f4 ! load key schedule + ldd [$key + 0x08], %f6 + ldd [$key + 0x10], %f8 + ldd [$key + 0x18], %f10 + ldd [$key + 0x20], %f12 + ldd [$key + 0x28], %f14 + ldd [$key + 0x30], %f16 + ldd [$key + 0x38], %f18 + ldd [$key + 0x40], %f20 + ldd [$key + 0x48], %f22 + ldd [$key + 0x50], %f24 + ldd [$key + 0x58], %f26 + ldd [$key + 0x60], %f28 + ldd [$key + 0x68], %f30 + ldd [$key + 0x70], %f32 + ldd [$key + 0x78], %f34 + +.Ldes_ede3_cbc_enc_loop: + ldx [$inp + 0], %g4 + brz,pt $ileft, 4f + nop + + ldx [$inp + 8], %g5 + sllx %g4, $ileft, %g4 + srlx %g5, $iright, %g5 + or %g5, %g4, %g4 +4: + movxtod %g4, %f2 + prefetch [$inp + 8+63], 20 + add $inp, 8, $inp + fxor %f2, %f0, %f0 ! ^= ivec + prefetch [$out + 63], 22 + + des_ip %f0, %f0 + des_round %f4, %f6, %f0, %f0 + des_round %f8, %f10, %f0, %f0 + des_round %f12, %f14, %f0, %f0 + des_round %f16, %f18, %f0, %f0 + ldd [$key + 0x100-0x08], %f36 + ldd [$key + 0x100-0x10], %f38 + des_round %f20, %f22, %f0, %f0 + ldd [$key + 0x100-0x18], %f40 + ldd [$key + 0x100-0x20], %f42 + des_round %f24, %f26, %f0, %f0 + ldd [$key + 0x100-0x28], %f44 + ldd [$key + 0x100-0x30], %f46 + des_round %f28, %f30, %f0, %f0 + ldd [$key + 0x100-0x38], %f48 + ldd [$key + 0x100-0x40], %f50 + des_round %f32, %f34, %f0, %f0 + ldd [$key + 0x100-0x48], %f52 + ldd [$key + 0x100-0x50], %f54 + des_iip %f0, %f0 + + ldd [$key + 0x100-0x58], %f56 + ldd [$key + 0x100-0x60], %f58 + des_ip %f0, %f0 + ldd [$key + 0x100-0x68], %f60 + ldd [$key + 0x100-0x70], %f62 + des_round %f36, %f38, %f0, %f0 + ldd [$key + 0x100-0x78], %f36 + ldd [$key + 0x100-0x80], %f38 + des_round %f40, %f42, %f0, %f0 + des_round %f44, %f46, %f0, %f0 + des_round %f48, %f50, %f0, %f0 + ldd [$key + 0x100+0x00], %f40 + ldd [$key + 0x100+0x08], %f42 + des_round %f52, %f54, %f0, %f0 + ldd [$key + 0x100+0x10], %f44 + ldd [$key + 0x100+0x18], %f46 + des_round %f56, %f58, %f0, %f0 + ldd [$key + 0x100+0x20], %f48 + ldd [$key + 0x100+0x28], %f50 + des_round %f60, %f62, %f0, %f0 + ldd [$key + 0x100+0x30], %f52 + ldd [$key + 0x100+0x38], %f54 + des_round %f36, %f38, %f0, %f0 + ldd [$key + 0x100+0x40], %f56 + ldd [$key + 0x100+0x48], %f58 + des_iip %f0, %f0 + + ldd [$key + 0x100+0x50], %f60 + ldd [$key + 0x100+0x58], %f62 + des_ip %f0, %f0 + ldd [$key + 0x100+0x60], %f36 + ldd [$key + 0x100+0x68], %f38 + des_round %f40, %f42, %f0, %f0 + ldd [$key + 0x100+0x70], %f40 + ldd [$key + 0x100+0x78], %f42 + des_round %f44, %f46, %f0, %f0 + des_round %f48, %f50, %f0, %f0 + des_round %f52, %f54, %f0, %f0 + des_round %f56, %f58, %f0, %f0 + des_round %f60, %f62, %f0, %f0 + des_round %f36, %f38, %f0, %f0 + des_round %f40, %f42, %f0, %f0 + des_iip %f0, %f0 + + brnz,pn $omask, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + brnz,pt $len, .Ldes_ede3_cbc_enc_loop + add $out, 8, $out + + st %f0, [$ivec + 0] ! write out ivec + retl + st %f1, [$ivec + 4] + +.align 16 +2: ldxa [$inp]0x82, %g4 ! avoid read-after-write hazard + ! and ~2x deterioration + ! in inp==out case + faligndata %f0, %f0, %f2 ! handle unaligned output + + stda %f2, [$out + $omask]0xc0 ! partial store + add $out, 8, $out + orn %g0, $omask, $omask + stda %f2, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .Ldes_ede3_cbc_enc_loop+4 + orn %g0, $omask, $omask + + st %f0, [$ivec + 0] ! write out ivec + retl + st %f1, [$ivec + 4] +.type des_t4_ede3_cbc_encrypt,#function +.size des_t4_ede3_cbc_encrypt,.-des_t4_ede3_cbc_encrypt + +.globl des_t4_ede3_cbc_decrypt +.align 32 +des_t4_ede3_cbc_decrypt: + cmp $len, 0 + be,pn $::size_t_cc, .Lcbc_abort + nop + ld [$ivec + 0], %f2 ! load ivec + ld [$ivec + 4], %f3 + + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 0xff, $omask + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + sub %g0, $ileft, $iright + and $out, 7, %g4 + alignaddrl $out, %g0, $out + srl $omask, %g4, $omask + srlx $len, 3, $len + movrz %g4, 0, $omask + prefetch [$out], 22 + + ldd [$key + 0x100+0x78], %f4 ! load key schedule + ldd [$key + 0x100+0x70], %f6 + ldd [$key + 0x100+0x68], %f8 + ldd [$key + 0x100+0x60], %f10 + ldd [$key + 0x100+0x58], %f12 + ldd [$key + 0x100+0x50], %f14 + ldd [$key + 0x100+0x48], %f16 + ldd [$key + 0x100+0x40], %f18 + ldd [$key + 0x100+0x38], %f20 + ldd [$key + 0x100+0x30], %f22 + ldd [$key + 0x100+0x28], %f24 + ldd [$key + 0x100+0x20], %f26 + ldd [$key + 0x100+0x18], %f28 + ldd [$key + 0x100+0x10], %f30 + ldd [$key + 0x100+0x08], %f32 + ldd [$key + 0x100+0x00], %f34 + +.Ldes_ede3_cbc_dec_loop: + ldx [$inp + 0], %g4 + brz,pt $ileft, 4f + nop + + ldx [$inp + 8], %g5 + sllx %g4, $ileft, %g4 + srlx %g5, $iright, %g5 + or %g5, %g4, %g4 +4: + movxtod %g4, %f0 + prefetch [$inp + 8+63], 20 + add $inp, 8, $inp + prefetch [$out + 63], 22 + + des_ip %f0, %f0 + des_round %f4, %f6, %f0, %f0 + des_round %f8, %f10, %f0, %f0 + des_round %f12, %f14, %f0, %f0 + des_round %f16, %f18, %f0, %f0 + ldd [$key + 0x80+0x00], %f36 + ldd [$key + 0x80+0x08], %f38 + des_round %f20, %f22, %f0, %f0 + ldd [$key + 0x80+0x10], %f40 + ldd [$key + 0x80+0x18], %f42 + des_round %f24, %f26, %f0, %f0 + ldd [$key + 0x80+0x20], %f44 + ldd [$key + 0x80+0x28], %f46 + des_round %f28, %f30, %f0, %f0 + ldd [$key + 0x80+0x30], %f48 + ldd [$key + 0x80+0x38], %f50 + des_round %f32, %f34, %f0, %f0 + ldd [$key + 0x80+0x40], %f52 + ldd [$key + 0x80+0x48], %f54 + des_iip %f0, %f0 + + ldd [$key + 0x80+0x50], %f56 + ldd [$key + 0x80+0x58], %f58 + des_ip %f0, %f0 + ldd [$key + 0x80+0x60], %f60 + ldd [$key + 0x80+0x68], %f62 + des_round %f36, %f38, %f0, %f0 + ldd [$key + 0x80+0x70], %f36 + ldd [$key + 0x80+0x78], %f38 + des_round %f40, %f42, %f0, %f0 + des_round %f44, %f46, %f0, %f0 + des_round %f48, %f50, %f0, %f0 + ldd [$key + 0x80-0x08], %f40 + ldd [$key + 0x80-0x10], %f42 + des_round %f52, %f54, %f0, %f0 + ldd [$key + 0x80-0x18], %f44 + ldd [$key + 0x80-0x20], %f46 + des_round %f56, %f58, %f0, %f0 + ldd [$key + 0x80-0x28], %f48 + ldd [$key + 0x80-0x30], %f50 + des_round %f60, %f62, %f0, %f0 + ldd [$key + 0x80-0x38], %f52 + ldd [$key + 0x80-0x40], %f54 + des_round %f36, %f38, %f0, %f0 + ldd [$key + 0x80-0x48], %f56 + ldd [$key + 0x80-0x50], %f58 + des_iip %f0, %f0 + + ldd [$key + 0x80-0x58], %f60 + ldd [$key + 0x80-0x60], %f62 + des_ip %f0, %f0 + ldd [$key + 0x80-0x68], %f36 + ldd [$key + 0x80-0x70], %f38 + des_round %f40, %f42, %f0, %f0 + ldd [$key + 0x80-0x78], %f40 + ldd [$key + 0x80-0x80], %f42 + des_round %f44, %f46, %f0, %f0 + des_round %f48, %f50, %f0, %f0 + des_round %f52, %f54, %f0, %f0 + des_round %f56, %f58, %f0, %f0 + des_round %f60, %f62, %f0, %f0 + des_round %f36, %f38, %f0, %f0 + des_round %f40, %f42, %f0, %f0 + des_iip %f0, %f0 + + fxor %f2, %f0, %f0 ! ^= ivec + movxtod %g4, %f2 + + brnz,pn $omask, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + brnz,pt $len, .Ldes_ede3_cbc_dec_loop + add $out, 8, $out + + st %f2, [$ivec + 0] ! write out ivec + retl + st %f3, [$ivec + 4] + +.align 16 +2: ldxa [$inp]0x82, %g4 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f0 ! handle unaligned output + + stda %f0, [$out + $omask]0xc0 ! partial store + add $out, 8, $out + orn %g0, $omask, $omask + stda %f0, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .Ldes_ede3_cbc_dec_loop+4 + orn %g0, $omask, $omask + + st %f2, [$ivec + 0] ! write out ivec + retl + st %f3, [$ivec + 4] +.type des_t4_ede3_cbc_decrypt,#function +.size des_t4_ede3_cbc_decrypt,.-des_t4_ede3_cbc_decrypt +___ +} +$code.=<<___; +.asciz "DES for SPARC T4, David S. Miller, Andy Polyakov" +.align 4 +___ + +&emit_assembler(); + +close STDOUT; diff --git a/crypto/des/des_locl.h b/crypto/des/des_locl.h index 1a8e41dd29c7..23ea9d32a7be 100644 --- a/crypto/des/des_locl.h +++ b/crypto/des/des_locl.h @@ -162,8 +162,10 @@ } \ } -# if (defined(OPENSSL_SYS_WIN32) && defined(_MSC_VER)) || defined(__ICC) +# if (defined(OPENSSL_SYS_WIN32) && defined(_MSC_VER)) # define ROTATE(a,n) (_lrotr(a,n)) +# elif defined(__ICC) +# define ROTATE(a,n) (_rotr(a,n)) # elif defined(__GNUC__) && __GNUC__>=2 && !defined(__STRICT_ANSI__) && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) && !defined(PEDANTIC) # if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) # define ROTATE(a,n) ({ register unsigned int ret; \ diff --git a/crypto/des/read_pwd.c b/crypto/des/read_pwd.c index 16ba0a9eb2a1..514a7063b4bf 100644 --- a/crypto/des/read_pwd.c +++ b/crypto/des/read_pwd.c @@ -172,7 +172,7 @@ # include #endif -#if defined(OPENSSL_SYS_MSDOS) && !defined(__CYGWIN32__) && !defined(OPENSSL_SYS_WINCE) +#if defined(OPENSSL_SYS_MSDOS) && !defined(OPENSSL_SYS_WINCE) # include # define fgets(a,b,c) noecho_fgets(a,b,c) #endif diff --git a/crypto/dh/Makefile b/crypto/dh/Makefile index 6d574f407d39..46fa5ac57b47 100644 --- a/crypto/dh/Makefile +++ b/crypto/dh/Makefile @@ -18,9 +18,9 @@ APPS= LIB=$(TOP)/libcrypto.a LIBSRC= dh_asn1.c dh_gen.c dh_key.c dh_lib.c dh_check.c dh_err.c dh_depr.c \ - dh_ameth.c dh_pmeth.c dh_prn.c + dh_ameth.c dh_pmeth.c dh_prn.c dh_rfc5114.c dh_kdf.c LIBOBJ= dh_asn1.o dh_gen.o dh_key.o dh_lib.o dh_check.o dh_err.o dh_depr.o \ - dh_ameth.o dh_pmeth.o dh_prn.o + dh_ameth.o dh_pmeth.o dh_prn.o dh_rfc5114.o dh_kdf.o SRC= $(LIBSRC) @@ -80,13 +80,13 @@ clean: dh_ameth.o: ../../e_os.h ../../include/openssl/asn1.h dh_ameth.o: ../../include/openssl/bio.h ../../include/openssl/bn.h -dh_ameth.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h -dh_ameth.o: ../../include/openssl/dh.h ../../include/openssl/e_os2.h -dh_ameth.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h -dh_ameth.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h -dh_ameth.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h -dh_ameth.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h -dh_ameth.o: ../../include/openssl/opensslconf.h +dh_ameth.o: ../../include/openssl/buffer.h ../../include/openssl/cms.h +dh_ameth.o: ../../include/openssl/crypto.h ../../include/openssl/dh.h +dh_ameth.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h +dh_ameth.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h +dh_ameth.o: ../../include/openssl/err.h ../../include/openssl/evp.h +dh_ameth.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h +dh_ameth.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h dh_ameth.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h dh_ameth.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h dh_ameth.o: ../../include/openssl/sha.h ../../include/openssl/stack.h @@ -134,6 +134,19 @@ dh_gen.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h dh_gen.o: ../../include/openssl/ossl_typ.h ../../include/openssl/safestack.h dh_gen.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h dh_gen.o: ../cryptlib.h dh_gen.c +dh_kdf.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h +dh_kdf.o: ../../include/openssl/buffer.h ../../include/openssl/cms.h +dh_kdf.o: ../../include/openssl/crypto.h ../../include/openssl/dh.h +dh_kdf.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h +dh_kdf.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h +dh_kdf.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h +dh_kdf.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h +dh_kdf.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h +dh_kdf.o: ../../include/openssl/ossl_typ.h ../../include/openssl/pkcs7.h +dh_kdf.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h +dh_kdf.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h +dh_kdf.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h +dh_kdf.o: dh_kdf.c dh_key.o: ../../e_os.h ../../include/openssl/bio.h ../../include/openssl/bn.h dh_key.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h dh_key.o: ../../include/openssl/dh.h ../../include/openssl/e_os2.h @@ -160,11 +173,12 @@ dh_pmeth.o: ../../e_os.h ../../include/openssl/asn1.h dh_pmeth.o: ../../include/openssl/asn1t.h ../../include/openssl/bio.h dh_pmeth.o: ../../include/openssl/bn.h ../../include/openssl/buffer.h dh_pmeth.o: ../../include/openssl/crypto.h ../../include/openssl/dh.h -dh_pmeth.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h -dh_pmeth.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h -dh_pmeth.o: ../../include/openssl/err.h ../../include/openssl/evp.h -dh_pmeth.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h -dh_pmeth.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h +dh_pmeth.o: ../../include/openssl/dsa.h ../../include/openssl/e_os2.h +dh_pmeth.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h +dh_pmeth.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h +dh_pmeth.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h +dh_pmeth.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h +dh_pmeth.o: ../../include/openssl/opensslconf.h dh_pmeth.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h dh_pmeth.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h dh_pmeth.o: ../../include/openssl/sha.h ../../include/openssl/stack.h @@ -180,3 +194,11 @@ dh_prn.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h dh_prn.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h dh_prn.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h dh_prn.o: ../../include/openssl/symhacks.h ../cryptlib.h dh_prn.c +dh_rfc5114.o: ../../e_os.h ../../include/openssl/bio.h +dh_rfc5114.o: ../../include/openssl/bn.h ../../include/openssl/buffer.h +dh_rfc5114.o: ../../include/openssl/crypto.h ../../include/openssl/dh.h +dh_rfc5114.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h +dh_rfc5114.o: ../../include/openssl/lhash.h ../../include/openssl/opensslconf.h +dh_rfc5114.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h +dh_rfc5114.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h +dh_rfc5114.o: ../../include/openssl/symhacks.h ../cryptlib.h dh_rfc5114.c diff --git a/crypto/dh/dh.h b/crypto/dh/dh.h index 4cbaa9784dba..0502f1a9cc14 100644 --- a/crypto/dh/dh.h +++ b/crypto/dh/dh.h @@ -167,6 +167,9 @@ struct dh_st { # define DH_CHECK_P_NOT_SAFE_PRIME 0x02 # define DH_UNABLE_TO_CHECK_GENERATOR 0x04 # define DH_NOT_SUITABLE_GENERATOR 0x08 +# define DH_CHECK_Q_NOT_PRIME 0x10 +# define DH_CHECK_INVALID_Q_VALUE 0x20 +# define DH_CHECK_INVALID_J_VALUE 0x40 /* DH_check_pub_key error codes */ # define DH_CHECK_PUBKEY_TOO_SMALL 0x01 @@ -217,8 +220,11 @@ int DH_check(const DH *dh, int *codes); int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *codes); int DH_generate_key(DH *dh); int DH_compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh); +int DH_compute_key_padded(unsigned char *key, const BIGNUM *pub_key, DH *dh); DH *d2i_DHparams(DH **a, const unsigned char **pp, long length); int i2d_DHparams(const DH *a, unsigned char **pp); +DH *d2i_DHxparams(DH **a, const unsigned char **pp, long length); +int i2d_DHxparams(const DH *a, unsigned char **pp); # ifndef OPENSSL_NO_FP_API int DHparams_print_fp(FILE *fp, const DH *x); # endif @@ -228,16 +234,109 @@ int DHparams_print(BIO *bp, const DH *x); int DHparams_print(char *bp, const DH *x); # endif +/* RFC 5114 parameters */ +DH *DH_get_1024_160(void); +DH *DH_get_2048_224(void); +DH *DH_get_2048_256(void); + +/* RFC2631 KDF */ +int DH_KDF_X9_42(unsigned char *out, size_t outlen, + const unsigned char *Z, size_t Zlen, + ASN1_OBJECT *key_oid, + const unsigned char *ukm, size_t ukmlen, const EVP_MD *md); + # define EVP_PKEY_CTX_set_dh_paramgen_prime_len(ctx, len) \ EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DH, EVP_PKEY_OP_PARAMGEN, \ EVP_PKEY_CTRL_DH_PARAMGEN_PRIME_LEN, len, NULL) +# define EVP_PKEY_CTX_set_dh_paramgen_subprime_len(ctx, len) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DH, EVP_PKEY_OP_PARAMGEN, \ + EVP_PKEY_CTRL_DH_PARAMGEN_SUBPRIME_LEN, len, NULL) + +# define EVP_PKEY_CTX_set_dh_paramgen_type(ctx, typ) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DH, EVP_PKEY_OP_PARAMGEN, \ + EVP_PKEY_CTRL_DH_PARAMGEN_TYPE, typ, NULL) + # define EVP_PKEY_CTX_set_dh_paramgen_generator(ctx, gen) \ EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DH, EVP_PKEY_OP_PARAMGEN, \ EVP_PKEY_CTRL_DH_PARAMGEN_GENERATOR, gen, NULL) +# define EVP_PKEY_CTX_set_dh_rfc5114(ctx, gen) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, EVP_PKEY_OP_PARAMGEN, \ + EVP_PKEY_CTRL_DH_RFC5114, gen, NULL) + +# define EVP_PKEY_CTX_set_dhx_rfc5114(ctx, gen) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, EVP_PKEY_OP_PARAMGEN, \ + EVP_PKEY_CTRL_DH_RFC5114, gen, NULL) + +# define EVP_PKEY_CTX_set_dh_kdf_type(ctx, kdf) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_DH_KDF_TYPE, kdf, NULL) + +# define EVP_PKEY_CTX_get_dh_kdf_type(ctx) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_DH_KDF_TYPE, -2, NULL) + +# define EVP_PKEY_CTX_set0_dh_kdf_oid(ctx, oid) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_DH_KDF_OID, 0, (void *)oid) + +# define EVP_PKEY_CTX_get0_dh_kdf_oid(ctx, poid) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_GET_DH_KDF_OID, 0, (void *)poid) + +# define EVP_PKEY_CTX_set_dh_kdf_md(ctx, md) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_DH_KDF_MD, 0, (void *)md) + +# define EVP_PKEY_CTX_get_dh_kdf_md(ctx, pmd) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_GET_DH_KDF_MD, 0, (void *)pmd) + +# define EVP_PKEY_CTX_set_dh_kdf_outlen(ctx, len) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_DH_KDF_OUTLEN, len, NULL) + +# define EVP_PKEY_CTX_get_dh_kdf_outlen(ctx, plen) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_GET_DH_KDF_OUTLEN, 0, (void *)plen) + +# define EVP_PKEY_CTX_set0_dh_kdf_ukm(ctx, p, plen) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_DH_KDF_UKM, plen, (void *)p) + +# define EVP_PKEY_CTX_get0_dh_kdf_ukm(ctx, p) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DHX, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_GET_DH_KDF_UKM, 0, (void *)p) + # define EVP_PKEY_CTRL_DH_PARAMGEN_PRIME_LEN (EVP_PKEY_ALG_CTRL + 1) # define EVP_PKEY_CTRL_DH_PARAMGEN_GENERATOR (EVP_PKEY_ALG_CTRL + 2) +# define EVP_PKEY_CTRL_DH_RFC5114 (EVP_PKEY_ALG_CTRL + 3) +# define EVP_PKEY_CTRL_DH_PARAMGEN_SUBPRIME_LEN (EVP_PKEY_ALG_CTRL + 4) +# define EVP_PKEY_CTRL_DH_PARAMGEN_TYPE (EVP_PKEY_ALG_CTRL + 5) +# define EVP_PKEY_CTRL_DH_KDF_TYPE (EVP_PKEY_ALG_CTRL + 6) +# define EVP_PKEY_CTRL_DH_KDF_MD (EVP_PKEY_ALG_CTRL + 7) +# define EVP_PKEY_CTRL_GET_DH_KDF_MD (EVP_PKEY_ALG_CTRL + 8) +# define EVP_PKEY_CTRL_DH_KDF_OUTLEN (EVP_PKEY_ALG_CTRL + 9) +# define EVP_PKEY_CTRL_GET_DH_KDF_OUTLEN (EVP_PKEY_ALG_CTRL + 10) +# define EVP_PKEY_CTRL_DH_KDF_UKM (EVP_PKEY_ALG_CTRL + 11) +# define EVP_PKEY_CTRL_GET_DH_KDF_UKM (EVP_PKEY_ALG_CTRL + 12) +# define EVP_PKEY_CTRL_DH_KDF_OID (EVP_PKEY_ALG_CTRL + 13) +# define EVP_PKEY_CTRL_GET_DH_KDF_OID (EVP_PKEY_ALG_CTRL + 14) + +/* KDF types */ +# define EVP_PKEY_DH_KDF_NONE 1 +# define EVP_PKEY_DH_KDF_X9_42 2 /* BEGIN ERROR CODES */ /* @@ -252,6 +351,9 @@ void ERR_load_DH_strings(void); # define DH_F_COMPUTE_KEY 102 # define DH_F_DHPARAMS_PRINT_FP 101 # define DH_F_DH_BUILTIN_GENPARAMS 106 +# define DH_F_DH_CMS_DECRYPT 117 +# define DH_F_DH_CMS_SET_PEERKEY 118 +# define DH_F_DH_CMS_SET_SHARED_INFO 119 # define DH_F_DH_COMPUTE_KEY 114 # define DH_F_DH_GENERATE_KEY 115 # define DH_F_DH_GENERATE_PARAMETERS_EX 116 @@ -273,6 +375,7 @@ void ERR_load_DH_strings(void); # define DH_R_BN_ERROR 106 # define DH_R_DECODE_ERROR 104 # define DH_R_INVALID_PUBKEY 102 +# define DH_R_KDF_PARAMETER_ERROR 112 # define DH_R_KEYS_NOT_SET 108 # define DH_R_KEY_SIZE_TOO_SMALL 110 # define DH_R_MODULUS_TOO_LARGE 103 @@ -280,6 +383,8 @@ void ERR_load_DH_strings(void); # define DH_R_NO_PARAMETERS_SET 107 # define DH_R_NO_PRIVATE_VALUE 100 # define DH_R_PARAMETER_ENCODING_ERROR 105 +# define DH_R_PEER_KEY_ERROR 113 +# define DH_R_SHARED_INFO_ERROR 114 #ifdef __cplusplus } diff --git a/crypto/dh/dh_ameth.c b/crypto/dh/dh_ameth.c index 873eb2e22de8..ac72468bd14b 100644 --- a/crypto/dh/dh_ameth.c +++ b/crypto/dh/dh_ameth.c @@ -63,6 +63,31 @@ #include #include #include "asn1_locl.h" +#ifndef OPENSSL_NO_CMS +# include +#endif + +extern const EVP_PKEY_ASN1_METHOD dhx_asn1_meth; + +/* + * i2d/d2i like DH parameter functions which use the appropriate routine for + * PKCS#3 DH or X9.42 DH. + */ + +static DH *d2i_dhp(const EVP_PKEY *pkey, const unsigned char **pp, + long length) +{ + if (pkey->ameth == &dhx_asn1_meth) + return d2i_DHxparams(NULL, pp, length); + return d2i_DHparams(NULL, pp, length); +} + +static int i2d_dhp(const EVP_PKEY *pkey, const DH *a, unsigned char **pp) +{ + if (pkey->ameth == &dhx_asn1_meth) + return i2d_DHxparams(a, pp); + return i2d_DHparams(a, pp); +} static void int_dh_free(EVP_PKEY *pkey) { @@ -94,7 +119,7 @@ static int dh_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey) pm = pstr->data; pmlen = pstr->length; - if (!(dh = d2i_DHparams(NULL, &pm, pmlen))) { + if (!(dh = d2i_dhp(pkey, &pm, pmlen))) { DHerr(DH_F_DH_PUB_DECODE, DH_R_DECODE_ERROR); goto err; } @@ -111,7 +136,7 @@ static int dh_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey) } ASN1_INTEGER_free(public_key); - EVP_PKEY_assign_DH(pkey, dh); + EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, dh); return 1; err: @@ -139,7 +164,7 @@ static int dh_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) DHerr(DH_F_DH_PUB_ENCODE, ERR_R_MALLOC_FAILURE); goto err; } - str->length = i2d_DHparams(dh, &str->data); + str->length = i2d_dhp(pkey, dh, &str->data); if (str->length <= 0) { DHerr(DH_F_DH_PUB_ENCODE, ERR_R_MALLOC_FAILURE); goto err; @@ -159,7 +184,7 @@ static int dh_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) goto err; } - if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(EVP_PKEY_DH), + if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id), ptype, str, penc, penclen)) return 1; @@ -204,7 +229,7 @@ static int dh_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8) pstr = pval; pm = pstr->data; pmlen = pstr->length; - if (!(dh = d2i_DHparams(NULL, &pm, pmlen))) + if (!(dh = d2i_dhp(pkey, &pm, pmlen))) goto decerr; /* We have parameters now set private key */ if (!(dh->priv_key = ASN1_INTEGER_to_BN(privkey, NULL))) { @@ -215,7 +240,7 @@ static int dh_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8) if (!DH_generate_key(dh)) goto dherr; - EVP_PKEY_assign_DH(pkey, dh); + EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, dh); ASN1_STRING_clear_free(privkey); @@ -243,7 +268,7 @@ static int dh_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) goto err; } - params->length = i2d_DHparams(pkey->pkey.dh, ¶ms->data); + params->length = i2d_dhp(pkey, pkey->pkey.dh, ¶ms->data); if (params->length <= 0) { DHerr(DH_F_DH_PRIV_ENCODE, ERR_R_MALLOC_FAILURE); goto err; @@ -263,7 +288,7 @@ static int dh_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) ASN1_STRING_clear_free(prkey); prkey = NULL; - if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_dhKeyAgreement), 0, + if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0, V_ASN1_SEQUENCE, params, dp, dplen)) goto err; @@ -292,17 +317,17 @@ static int dh_param_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen) { DH *dh; - if (!(dh = d2i_DHparams(NULL, pder, derlen))) { + if (!(dh = d2i_dhp(pkey, pder, derlen))) { DHerr(DH_F_DH_PARAM_DECODE, ERR_R_DH_LIB); return 0; } - EVP_PKEY_assign_DH(pkey, dh); + EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, dh); return 1; } static int dh_param_encode(const EVP_PKEY *pkey, unsigned char **pder) { - return i2d_DHparams(pkey->pkey.dh, pder); + return i2d_dhp(pkey, pkey->pkey.dh, pder); } static int do_dh_print(BIO *bp, const DH *x, int indent, @@ -334,15 +359,18 @@ static int do_dh_print(BIO *bp, const DH *x, int indent, } update_buflen(x->g, &buf_len); + update_buflen(x->q, &buf_len); + update_buflen(x->j, &buf_len); + update_buflen(x->counter, &buf_len); update_buflen(pub_key, &buf_len); update_buflen(priv_key, &buf_len); if (ptype == 2) - ktype = "PKCS#3 DH Private-Key"; + ktype = "DH Private-Key"; else if (ptype == 1) - ktype = "PKCS#3 DH Public-Key"; + ktype = "DH Public-Key"; else - ktype = "PKCS#3 DH Parameters"; + ktype = "DH Parameters"; m = OPENSSL_malloc(buf_len + 10); if (m == NULL) { @@ -364,6 +392,29 @@ static int do_dh_print(BIO *bp, const DH *x, int indent, goto err; if (!ASN1_bn_print(bp, "generator:", x->g, m, indent)) goto err; + if (x->q && !ASN1_bn_print(bp, "subgroup order:", x->q, m, indent)) + goto err; + if (x->j && !ASN1_bn_print(bp, "subgroup factor:", x->j, m, indent)) + goto err; + if (x->seed) { + int i; + BIO_indent(bp, indent, 128); + BIO_puts(bp, "seed:"); + for (i = 0; i < x->seedlen; i++) { + if ((i % 15) == 0) { + if (BIO_puts(bp, "\n") <= 0 + || !BIO_indent(bp, indent + 4, 128)) + goto err; + } + if (BIO_printf(bp, "%02x%s", x->seed[i], + ((i + 1) == x->seedlen) ? "" : ":") <= 0) + goto err; + } + if (BIO_write(bp, "\n", 1) <= 0) + return (0); + } + if (x->counter && !ASN1_bn_print(bp, "counter:", x->counter, m, indent)) + goto err; if (x->length != 0) { BIO_indent(bp, indent, 128); if (BIO_printf(bp, "recommended-private-length: %d bits\n", @@ -396,29 +447,76 @@ static int dh_cmp_parameters(const EVP_PKEY *a, const EVP_PKEY *b) if (BN_cmp(a->pkey.dh->p, b->pkey.dh->p) || BN_cmp(a->pkey.dh->g, b->pkey.dh->g)) return 0; - else - return 1; + else if (a->ameth == &dhx_asn1_meth) { + if (BN_cmp(a->pkey.dh->q, b->pkey.dh->q)) + return 0; + } + return 1; } -static int dh_copy_parameters(EVP_PKEY *to, const EVP_PKEY *from) +static int int_dh_bn_cpy(BIGNUM **dst, const BIGNUM *src) { BIGNUM *a; + if (src) { + a = BN_dup(src); + if (!a) + return 0; + } else + a = NULL; + if (*dst) + BN_free(*dst); + *dst = a; + return 1; +} - if ((a = BN_dup(from->pkey.dh->p)) == NULL) +static int int_dh_param_copy(DH *to, const DH *from, int is_x942) +{ + if (is_x942 == -1) + is_x942 = ! !from->q; + if (!int_dh_bn_cpy(&to->p, from->p)) return 0; - if (to->pkey.dh->p != NULL) - BN_free(to->pkey.dh->p); - to->pkey.dh->p = a; - - if ((a = BN_dup(from->pkey.dh->g)) == NULL) + if (!int_dh_bn_cpy(&to->g, from->g)) return 0; - if (to->pkey.dh->g != NULL) - BN_free(to->pkey.dh->g); - to->pkey.dh->g = a; - + if (is_x942) { + if (!int_dh_bn_cpy(&to->q, from->q)) + return 0; + if (!int_dh_bn_cpy(&to->j, from->j)) + return 0; + if (to->seed) { + OPENSSL_free(to->seed); + to->seed = NULL; + to->seedlen = 0; + } + if (from->seed) { + to->seed = BUF_memdup(from->seed, from->seedlen); + if (!to->seed) + return 0; + to->seedlen = from->seedlen; + } + } else + to->length = from->length; return 1; } +DH *DHparams_dup(DH *dh) +{ + DH *ret; + ret = DH_new(); + if (!ret) + return NULL; + if (!int_dh_param_copy(ret, dh, -1)) { + DH_free(ret); + return NULL; + } + return ret; +} + +static int dh_copy_parameters(EVP_PKEY *to, const EVP_PKEY *from) +{ + return int_dh_param_copy(to->pkey.dh, from->pkey.dh, + from->ameth == &dhx_asn1_meth); +} + static int dh_missing_parameters(const EVP_PKEY *a) { if (!a->pkey.dh->p || !a->pkey.dh->g) @@ -459,6 +557,33 @@ int DHparams_print(BIO *bp, const DH *x) return do_dh_print(bp, x, 4, NULL, 0); } +#ifndef OPENSSL_NO_CMS +static int dh_cms_decrypt(CMS_RecipientInfo *ri); +static int dh_cms_encrypt(CMS_RecipientInfo *ri); +#endif + +static int dh_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2) +{ + switch (op) { +#ifndef OPENSSL_NO_CMS + + case ASN1_PKEY_CTRL_CMS_ENVELOPE: + if (arg1 == 1) + return dh_cms_decrypt(arg2); + else if (arg1 == 0) + return dh_cms_encrypt(arg2); + return -2; + + case ASN1_PKEY_CTRL_CMS_RI_TYPE: + *(int *)arg2 = CMS_RECIPINFO_AGREE; + return 1; +#endif + default: + return -2; + } + +} + const EVP_PKEY_ASN1_METHOD dh_asn1_meth = { EVP_PKEY_DH, EVP_PKEY_DH, @@ -490,3 +615,343 @@ const EVP_PKEY_ASN1_METHOD dh_asn1_meth = { int_dh_free, 0 }; + +const EVP_PKEY_ASN1_METHOD dhx_asn1_meth = { + EVP_PKEY_DHX, + EVP_PKEY_DHX, + 0, + + "X9.42 DH", + "OpenSSL X9.42 DH method", + + dh_pub_decode, + dh_pub_encode, + dh_pub_cmp, + dh_public_print, + + dh_priv_decode, + dh_priv_encode, + dh_private_print, + + int_dh_size, + dh_bits, + + dh_param_decode, + dh_param_encode, + dh_missing_parameters, + dh_copy_parameters, + dh_cmp_parameters, + dh_param_print, + 0, + + int_dh_free, + dh_pkey_ctrl +}; + +#ifndef OPENSSL_NO_CMS + +static int dh_cms_set_peerkey(EVP_PKEY_CTX *pctx, + X509_ALGOR *alg, ASN1_BIT_STRING *pubkey) +{ + ASN1_OBJECT *aoid; + int atype; + void *aval; + ASN1_INTEGER *public_key = NULL; + int rv = 0; + EVP_PKEY *pkpeer = NULL, *pk = NULL; + DH *dhpeer = NULL; + const unsigned char *p; + int plen; + + X509_ALGOR_get0(&aoid, &atype, &aval, alg); + if (OBJ_obj2nid(aoid) != NID_dhpublicnumber) + goto err; + /* Only absent parameters allowed in RFC XXXX */ + if (atype != V_ASN1_UNDEF && atype == V_ASN1_NULL) + goto err; + + pk = EVP_PKEY_CTX_get0_pkey(pctx); + if (!pk) + goto err; + if (pk->type != EVP_PKEY_DHX) + goto err; + /* Get parameters from parent key */ + dhpeer = DHparams_dup(pk->pkey.dh); + /* We have parameters now set public key */ + plen = ASN1_STRING_length(pubkey); + p = ASN1_STRING_data(pubkey); + if (!p || !plen) + goto err; + + if (!(public_key = d2i_ASN1_INTEGER(NULL, &p, plen))) { + DHerr(DH_F_DH_CMS_SET_PEERKEY, DH_R_DECODE_ERROR); + goto err; + } + + /* We have parameters now set public key */ + if (!(dhpeer->pub_key = ASN1_INTEGER_to_BN(public_key, NULL))) { + DHerr(DH_F_DH_CMS_SET_PEERKEY, DH_R_BN_DECODE_ERROR); + goto err; + } + + pkpeer = EVP_PKEY_new(); + if (!pkpeer) + goto err; + EVP_PKEY_assign(pkpeer, pk->ameth->pkey_id, dhpeer); + dhpeer = NULL; + if (EVP_PKEY_derive_set_peer(pctx, pkpeer) > 0) + rv = 1; + err: + if (public_key) + ASN1_INTEGER_free(public_key); + if (pkpeer) + EVP_PKEY_free(pkpeer); + if (dhpeer) + DH_free(dhpeer); + return rv; +} + +static int dh_cms_set_shared_info(EVP_PKEY_CTX *pctx, CMS_RecipientInfo *ri) +{ + int rv = 0; + + X509_ALGOR *alg, *kekalg = NULL; + ASN1_OCTET_STRING *ukm; + const unsigned char *p; + unsigned char *dukm = NULL; + size_t dukmlen = 0; + int keylen, plen; + const EVP_CIPHER *kekcipher; + EVP_CIPHER_CTX *kekctx; + + if (!CMS_RecipientInfo_kari_get0_alg(ri, &alg, &ukm)) + goto err; + + /* + * For DH we only have one OID permissible. If ever any more get defined + * we will need something cleverer. + */ + if (OBJ_obj2nid(alg->algorithm) != NID_id_smime_alg_ESDH) { + DHerr(DH_F_DH_CMS_SET_SHARED_INFO, DH_R_KDF_PARAMETER_ERROR); + goto err; + } + + if (EVP_PKEY_CTX_set_dh_kdf_type(pctx, EVP_PKEY_DH_KDF_X9_42) <= 0) + goto err; + + if (EVP_PKEY_CTX_set_dh_kdf_md(pctx, EVP_sha1()) <= 0) + goto err; + + if (alg->parameter->type != V_ASN1_SEQUENCE) + goto err; + + p = alg->parameter->value.sequence->data; + plen = alg->parameter->value.sequence->length; + kekalg = d2i_X509_ALGOR(NULL, &p, plen); + if (!kekalg) + goto err; + kekctx = CMS_RecipientInfo_kari_get0_ctx(ri); + if (!kekctx) + goto err; + kekcipher = EVP_get_cipherbyobj(kekalg->algorithm); + if (!kekcipher || EVP_CIPHER_mode(kekcipher) != EVP_CIPH_WRAP_MODE) + goto err; + if (!EVP_EncryptInit_ex(kekctx, kekcipher, NULL, NULL, NULL)) + goto err; + if (EVP_CIPHER_asn1_to_param(kekctx, kekalg->parameter) <= 0) + goto err; + + keylen = EVP_CIPHER_CTX_key_length(kekctx); + if (EVP_PKEY_CTX_set_dh_kdf_outlen(pctx, keylen) <= 0) + goto err; + /* Use OBJ_nid2obj to ensure we use built in OID that isn't freed */ + if (EVP_PKEY_CTX_set0_dh_kdf_oid(pctx, + OBJ_nid2obj(EVP_CIPHER_type(kekcipher))) + <= 0) + goto err; + + if (ukm) { + dukmlen = ASN1_STRING_length(ukm); + dukm = BUF_memdup(ASN1_STRING_data(ukm), dukmlen); + if (!dukm) + goto err; + } + + if (EVP_PKEY_CTX_set0_dh_kdf_ukm(pctx, dukm, dukmlen) <= 0) + goto err; + dukm = NULL; + + rv = 1; + err: + if (kekalg) + X509_ALGOR_free(kekalg); + if (dukm) + OPENSSL_free(dukm); + return rv; +} + +static int dh_cms_decrypt(CMS_RecipientInfo *ri) +{ + EVP_PKEY_CTX *pctx; + pctx = CMS_RecipientInfo_get0_pkey_ctx(ri); + if (!pctx) + return 0; + /* See if we need to set peer key */ + if (!EVP_PKEY_CTX_get0_peerkey(pctx)) { + X509_ALGOR *alg; + ASN1_BIT_STRING *pubkey; + if (!CMS_RecipientInfo_kari_get0_orig_id(ri, &alg, &pubkey, + NULL, NULL, NULL)) + return 0; + if (!alg || !pubkey) + return 0; + if (!dh_cms_set_peerkey(pctx, alg, pubkey)) { + DHerr(DH_F_DH_CMS_DECRYPT, DH_R_PEER_KEY_ERROR); + return 0; + } + } + /* Set DH derivation parameters and initialise unwrap context */ + if (!dh_cms_set_shared_info(pctx, ri)) { + DHerr(DH_F_DH_CMS_DECRYPT, DH_R_SHARED_INFO_ERROR); + return 0; + } + return 1; +} + +static int dh_cms_encrypt(CMS_RecipientInfo *ri) +{ + EVP_PKEY_CTX *pctx; + EVP_PKEY *pkey; + EVP_CIPHER_CTX *ctx; + int keylen; + X509_ALGOR *talg, *wrap_alg = NULL; + ASN1_OBJECT *aoid; + ASN1_BIT_STRING *pubkey; + ASN1_STRING *wrap_str; + ASN1_OCTET_STRING *ukm; + unsigned char *penc = NULL, *dukm = NULL; + int penclen; + size_t dukmlen = 0; + int rv = 0; + int kdf_type, wrap_nid; + const EVP_MD *kdf_md; + pctx = CMS_RecipientInfo_get0_pkey_ctx(ri); + if (!pctx) + return 0; + /* Get ephemeral key */ + pkey = EVP_PKEY_CTX_get0_pkey(pctx); + if (!CMS_RecipientInfo_kari_get0_orig_id(ri, &talg, &pubkey, + NULL, NULL, NULL)) + goto err; + X509_ALGOR_get0(&aoid, NULL, NULL, talg); + /* Is everything uninitialised? */ + if (aoid == OBJ_nid2obj(NID_undef)) { + ASN1_INTEGER *pubk; + pubk = BN_to_ASN1_INTEGER(pkey->pkey.dh->pub_key, NULL); + if (!pubk) + goto err; + /* Set the key */ + + penclen = i2d_ASN1_INTEGER(pubk, &penc); + ASN1_INTEGER_free(pubk); + if (penclen <= 0) + goto err; + ASN1_STRING_set0(pubkey, penc, penclen); + pubkey->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + pubkey->flags |= ASN1_STRING_FLAG_BITS_LEFT; + + penc = NULL; + X509_ALGOR_set0(talg, OBJ_nid2obj(NID_dhpublicnumber), + V_ASN1_UNDEF, NULL); + } + + /* See if custom paraneters set */ + kdf_type = EVP_PKEY_CTX_get_dh_kdf_type(pctx); + if (kdf_type <= 0) + goto err; + if (!EVP_PKEY_CTX_get_dh_kdf_md(pctx, &kdf_md)) + goto err; + + if (kdf_type == EVP_PKEY_DH_KDF_NONE) { + kdf_type = EVP_PKEY_DH_KDF_X9_42; + if (EVP_PKEY_CTX_set_dh_kdf_type(pctx, kdf_type) <= 0) + goto err; + } else if (kdf_type != EVP_PKEY_DH_KDF_X9_42) + /* Unknown KDF */ + goto err; + if (kdf_md == NULL) { + /* Only SHA1 supported */ + kdf_md = EVP_sha1(); + if (EVP_PKEY_CTX_set_dh_kdf_md(pctx, kdf_md) <= 0) + goto err; + } else if (EVP_MD_type(kdf_md) != NID_sha1) + /* Unsupported digest */ + goto err; + + if (!CMS_RecipientInfo_kari_get0_alg(ri, &talg, &ukm)) + goto err; + + /* Get wrap NID */ + ctx = CMS_RecipientInfo_kari_get0_ctx(ri); + wrap_nid = EVP_CIPHER_CTX_type(ctx); + if (EVP_PKEY_CTX_set0_dh_kdf_oid(pctx, OBJ_nid2obj(wrap_nid)) <= 0) + goto err; + keylen = EVP_CIPHER_CTX_key_length(ctx); + + /* Package wrap algorithm in an AlgorithmIdentifier */ + + wrap_alg = X509_ALGOR_new(); + if (!wrap_alg) + goto err; + wrap_alg->algorithm = OBJ_nid2obj(wrap_nid); + wrap_alg->parameter = ASN1_TYPE_new(); + if (!wrap_alg->parameter) + goto err; + if (EVP_CIPHER_param_to_asn1(ctx, wrap_alg->parameter) <= 0) + goto err; + if (ASN1_TYPE_get(wrap_alg->parameter) == NID_undef) { + ASN1_TYPE_free(wrap_alg->parameter); + wrap_alg->parameter = NULL; + } + + if (EVP_PKEY_CTX_set_dh_kdf_outlen(pctx, keylen) <= 0) + goto err; + + if (ukm) { + dukmlen = ASN1_STRING_length(ukm); + dukm = BUF_memdup(ASN1_STRING_data(ukm), dukmlen); + if (!dukm) + goto err; + } + + if (EVP_PKEY_CTX_set0_dh_kdf_ukm(pctx, dukm, dukmlen) <= 0) + goto err; + dukm = NULL; + + /* + * Now need to wrap encoding of wrap AlgorithmIdentifier into parameter + * of another AlgorithmIdentifier. + */ + penc = NULL; + penclen = i2d_X509_ALGOR(wrap_alg, &penc); + if (!penc || !penclen) + goto err; + wrap_str = ASN1_STRING_new(); + if (!wrap_str) + goto err; + ASN1_STRING_set0(wrap_str, penc, penclen); + penc = NULL; + X509_ALGOR_set0(talg, OBJ_nid2obj(NID_id_smime_alg_ESDH), + V_ASN1_SEQUENCE, wrap_str); + + rv = 1; + + err: + if (penc) + OPENSSL_free(penc); + if (wrap_alg) + X509_ALGOR_free(wrap_alg); + return rv; +} + +#endif diff --git a/crypto/dh/dh_asn1.c b/crypto/dh/dh_asn1.c index e6ee3cfc66cc..f470214399b6 100644 --- a/crypto/dh/dh_asn1.c +++ b/crypto/dh/dh_asn1.c @@ -89,7 +89,101 @@ ASN1_SEQUENCE_cb(DHparams, dh_cb) = { IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(DH, DHparams, DHparams) -DH *DHparams_dup(DH *dh) +/* + * Internal only structures for handling X9.42 DH: this gets translated to or + * from a DH structure straight away. + */ + +typedef struct { + ASN1_BIT_STRING *seed; + BIGNUM *counter; +} int_dhvparams; + +typedef struct { + BIGNUM *p; + BIGNUM *q; + BIGNUM *g; + BIGNUM *j; + int_dhvparams *vparams; +} int_dhx942_dh; + +ASN1_SEQUENCE(DHvparams) = { + ASN1_SIMPLE(int_dhvparams, seed, ASN1_BIT_STRING), + ASN1_SIMPLE(int_dhvparams, counter, BIGNUM) +} ASN1_SEQUENCE_END_name(int_dhvparams, DHvparams) + +ASN1_SEQUENCE(DHxparams) = { + ASN1_SIMPLE(int_dhx942_dh, p, BIGNUM), + ASN1_SIMPLE(int_dhx942_dh, g, BIGNUM), + ASN1_SIMPLE(int_dhx942_dh, q, BIGNUM), + ASN1_OPT(int_dhx942_dh, j, BIGNUM), + ASN1_OPT(int_dhx942_dh, vparams, DHvparams), +} ASN1_SEQUENCE_END_name(int_dhx942_dh, DHxparams) + +int_dhx942_dh *d2i_int_dhx(int_dhx942_dh **a, + const unsigned char **pp, long length); +int i2d_int_dhx(const int_dhx942_dh *a, unsigned char **pp); + +IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(int_dhx942_dh, DHxparams, int_dhx) + +/* Application leve function: read in X9.42 DH parameters into DH structure */ + +DH *d2i_DHxparams(DH **a, const unsigned char **pp, long length) { - return ASN1_item_dup(ASN1_ITEM_rptr(DHparams), dh); + int_dhx942_dh *dhx = NULL; + DH *dh = NULL; + dh = DH_new(); + if (!dh) + return NULL; + dhx = d2i_int_dhx(NULL, pp, length); + if (!dhx) { + DH_free(dh); + return NULL; + } + + if (a) { + if (*a) + DH_free(*a); + *a = dh; + } + + dh->p = dhx->p; + dh->q = dhx->q; + dh->g = dhx->g; + dh->j = dhx->j; + + if (dhx->vparams) { + dh->seed = dhx->vparams->seed->data; + dh->seedlen = dhx->vparams->seed->length; + dh->counter = dhx->vparams->counter; + dhx->vparams->seed->data = NULL; + ASN1_BIT_STRING_free(dhx->vparams->seed); + OPENSSL_free(dhx->vparams); + dhx->vparams = NULL; + } + + OPENSSL_free(dhx); + return dh; +} + +int i2d_DHxparams(const DH *dh, unsigned char **pp) +{ + int_dhx942_dh dhx; + int_dhvparams dhv; + ASN1_BIT_STRING bs; + dhx.p = dh->p; + dhx.g = dh->g; + dhx.q = dh->q; + dhx.j = dh->j; + if (dh->counter && dh->seed && dh->seedlen > 0) { + bs.flags = ASN1_STRING_FLAG_BITS_LEFT; + bs.data = dh->seed; + bs.length = dh->seedlen; + dhv.seed = &bs; + dhv.counter = dh->counter; + dhx.vparams = &dhv; + } else + dhx.vparams = NULL; + + return i2d_int_dhx(&dhx, pp); } diff --git a/crypto/dh/dh_check.c b/crypto/dh/dh_check.c index c39ed97ba627..347467c6a433 100644 --- a/crypto/dh/dh_check.c +++ b/crypto/dh/dh_check.c @@ -76,17 +76,43 @@ int DH_check(const DH *dh, int *ret) int ok = 0; BN_CTX *ctx = NULL; BN_ULONG l; - BIGNUM *q = NULL; + BIGNUM *t1 = NULL, *t2 = NULL; *ret = 0; ctx = BN_CTX_new(); if (ctx == NULL) goto err; - q = BN_new(); - if (q == NULL) + BN_CTX_start(ctx); + t1 = BN_CTX_get(ctx); + if (t1 == NULL) + goto err; + t2 = BN_CTX_get(ctx); + if (t2 == NULL) goto err; - if (BN_is_word(dh->g, DH_GENERATOR_2)) { + if (dh->q) { + if (BN_cmp(dh->g, BN_value_one()) <= 0) + *ret |= DH_NOT_SUITABLE_GENERATOR; + else if (BN_cmp(dh->g, dh->p) >= 0) + *ret |= DH_NOT_SUITABLE_GENERATOR; + else { + /* Check g^q == 1 mod p */ + if (!BN_mod_exp(t1, dh->g, dh->q, dh->p, ctx)) + goto err; + if (!BN_is_one(t1)) + *ret |= DH_NOT_SUITABLE_GENERATOR; + } + if (!BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL)) + *ret |= DH_CHECK_Q_NOT_PRIME; + /* Check p == 1 mod q i.e. q divides p - 1 */ + if (!BN_div(t1, t2, dh->p, dh->q, ctx)) + goto err; + if (!BN_is_one(t2)) + *ret |= DH_CHECK_INVALID_Q_VALUE; + if (dh->j && BN_cmp(dh->j, t1)) + *ret |= DH_CHECK_INVALID_J_VALUE; + + } else if (BN_is_word(dh->g, DH_GENERATOR_2)) { l = BN_mod_word(dh->p, 24); if (l != 11) *ret |= DH_NOT_SUITABLE_GENERATOR; @@ -107,18 +133,18 @@ int DH_check(const DH *dh, int *ret) if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL)) *ret |= DH_CHECK_P_NOT_PRIME; - else { - if (!BN_rshift1(q, dh->p)) + else if (!dh->q) { + if (!BN_rshift1(t1, dh->p)) goto err; - if (!BN_is_prime_ex(q, BN_prime_checks, ctx, NULL)) + if (!BN_is_prime_ex(t1, BN_prime_checks, ctx, NULL)) *ret |= DH_CHECK_P_NOT_SAFE_PRIME; } ok = 1; err: - if (ctx != NULL) + if (ctx != NULL) { + BN_CTX_end(ctx); BN_CTX_free(ctx); - if (q != NULL) - BN_free(q); + } return (ok); } diff --git a/crypto/dh/dh_err.c b/crypto/dh/dh_err.c index 6ed5eb789016..b890cca81748 100644 --- a/crypto/dh/dh_err.c +++ b/crypto/dh/dh_err.c @@ -1,6 +1,6 @@ /* crypto/dh/dh_err.c */ /* ==================================================================== - * Copyright (c) 1999-2011 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2013 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -73,6 +73,9 @@ static ERR_STRING_DATA DH_str_functs[] = { {ERR_FUNC(DH_F_COMPUTE_KEY), "COMPUTE_KEY"}, {ERR_FUNC(DH_F_DHPARAMS_PRINT_FP), "DHparams_print_fp"}, {ERR_FUNC(DH_F_DH_BUILTIN_GENPARAMS), "DH_BUILTIN_GENPARAMS"}, + {ERR_FUNC(DH_F_DH_CMS_DECRYPT), "DH_CMS_DECRYPT"}, + {ERR_FUNC(DH_F_DH_CMS_SET_PEERKEY), "DH_CMS_SET_PEERKEY"}, + {ERR_FUNC(DH_F_DH_CMS_SET_SHARED_INFO), "DH_CMS_SET_SHARED_INFO"}, {ERR_FUNC(DH_F_DH_COMPUTE_KEY), "DH_compute_key"}, {ERR_FUNC(DH_F_DH_GENERATE_KEY), "DH_generate_key"}, {ERR_FUNC(DH_F_DH_GENERATE_PARAMETERS_EX), "DH_generate_parameters_ex"}, @@ -96,6 +99,7 @@ static ERR_STRING_DATA DH_str_reasons[] = { {ERR_REASON(DH_R_BN_ERROR), "bn error"}, {ERR_REASON(DH_R_DECODE_ERROR), "decode error"}, {ERR_REASON(DH_R_INVALID_PUBKEY), "invalid public key"}, + {ERR_REASON(DH_R_KDF_PARAMETER_ERROR), "kdf parameter error"}, {ERR_REASON(DH_R_KEYS_NOT_SET), "keys not set"}, {ERR_REASON(DH_R_KEY_SIZE_TOO_SMALL), "key size too small"}, {ERR_REASON(DH_R_MODULUS_TOO_LARGE), "modulus too large"}, @@ -103,6 +107,8 @@ static ERR_STRING_DATA DH_str_reasons[] = { {ERR_REASON(DH_R_NO_PARAMETERS_SET), "no parameters set"}, {ERR_REASON(DH_R_NO_PRIVATE_VALUE), "no private value"}, {ERR_REASON(DH_R_PARAMETER_ENCODING_ERROR), "parameter encoding error"}, + {ERR_REASON(DH_R_PEER_KEY_ERROR), "peer key error"}, + {ERR_REASON(DH_R_SHARED_INFO_ERROR), "shared info error"}, {0, NULL} }; diff --git a/crypto/dh/dh_kdf.c b/crypto/dh/dh_kdf.c new file mode 100644 index 000000000000..a882cb286e0e --- /dev/null +++ b/crypto/dh/dh_kdf.c @@ -0,0 +1,187 @@ +/* crypto/dh/dh_kdf.c */ +/* + * Written by Stephen Henson for the OpenSSL project. + */ +/* ==================================================================== + * Copyright (c) 2013 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + */ + +#include +#include +#include +#include +#include + +/* Key derivation from X9.42/RFC2631 */ + +#define DH_KDF_MAX (1L << 30) + +/* Skip past an ASN1 structure: for OBJECT skip content octets too */ + +static int skip_asn1(unsigned char **pp, long *plen, int exptag) +{ + const unsigned char *q = *pp; + int i, tag, xclass; + long tmplen; + i = ASN1_get_object(&q, &tmplen, &tag, &xclass, *plen); + if (i & 0x80) + return 0; + if (tag != exptag || xclass != V_ASN1_UNIVERSAL) + return 0; + if (tag == V_ASN1_OBJECT) + q += tmplen; + *plen -= q - *pp; + *pp = (unsigned char *)q; + return 1; +} + +/* + * Encode the DH shared info structure, return an offset to the counter value + * so we can update the structure without reencoding it. + */ + +static int dh_sharedinfo_encode(unsigned char **pder, unsigned char **pctr, + ASN1_OBJECT *key_oid, size_t outlen, + const unsigned char *ukm, size_t ukmlen) +{ + unsigned char *p; + int derlen; + long tlen; + /* "magic" value to check offset is sane */ + static unsigned char ctr[4] = { 0xF3, 0x17, 0x22, 0x53 }; + X509_ALGOR atmp; + ASN1_OCTET_STRING ctr_oct, ukm_oct, *pukm_oct; + ASN1_TYPE ctr_atype; + if (ukmlen > DH_KDF_MAX || outlen > DH_KDF_MAX) + return 0; + ctr_oct.data = ctr; + ctr_oct.length = 4; + ctr_oct.flags = 0; + ctr_oct.type = V_ASN1_OCTET_STRING; + ctr_atype.type = V_ASN1_OCTET_STRING; + ctr_atype.value.octet_string = &ctr_oct; + atmp.algorithm = key_oid; + atmp.parameter = &ctr_atype; + if (ukm) { + ukm_oct.type = V_ASN1_OCTET_STRING; + ukm_oct.flags = 0; + ukm_oct.data = (unsigned char *)ukm; + ukm_oct.length = ukmlen; + pukm_oct = &ukm_oct; + } else + pukm_oct = NULL; + derlen = CMS_SharedInfo_encode(pder, &atmp, pukm_oct, outlen); + if (derlen <= 0) + return 0; + p = *pder; + tlen = derlen; + if (!skip_asn1(&p, &tlen, V_ASN1_SEQUENCE)) + return 0; + if (!skip_asn1(&p, &tlen, V_ASN1_SEQUENCE)) + return 0; + if (!skip_asn1(&p, &tlen, V_ASN1_OBJECT)) + return 0; + if (!skip_asn1(&p, &tlen, V_ASN1_OCTET_STRING)) + return 0; + if (CRYPTO_memcmp(p, ctr, 4)) + return 0; + *pctr = p; + return derlen; +} + +int DH_KDF_X9_42(unsigned char *out, size_t outlen, + const unsigned char *Z, size_t Zlen, + ASN1_OBJECT *key_oid, + const unsigned char *ukm, size_t ukmlen, const EVP_MD *md) +{ + EVP_MD_CTX mctx; + int rv = 0; + unsigned int i; + size_t mdlen; + unsigned char *der = NULL, *ctr; + int derlen; + if (Zlen > DH_KDF_MAX) + return 0; + mdlen = EVP_MD_size(md); + EVP_MD_CTX_init(&mctx); + derlen = dh_sharedinfo_encode(&der, &ctr, key_oid, outlen, ukm, ukmlen); + if (derlen == 0) + goto err; + for (i = 1;; i++) { + unsigned char mtmp[EVP_MAX_MD_SIZE]; + EVP_DigestInit_ex(&mctx, md, NULL); + if (!EVP_DigestUpdate(&mctx, Z, Zlen)) + goto err; + ctr[3] = i & 0xFF; + ctr[2] = (i >> 8) & 0xFF; + ctr[1] = (i >> 16) & 0xFF; + ctr[0] = (i >> 24) & 0xFF; + if (!EVP_DigestUpdate(&mctx, der, derlen)) + goto err; + if (outlen >= mdlen) { + if (!EVP_DigestFinal(&mctx, out, NULL)) + goto err; + outlen -= mdlen; + if (outlen == 0) + break; + out += mdlen; + } else { + if (!EVP_DigestFinal(&mctx, mtmp, NULL)) + goto err; + memcpy(out, mtmp, outlen); + OPENSSL_cleanse(mtmp, mdlen); + break; + } + } + rv = 1; + err: + if (der) + OPENSSL_free(der); + EVP_MD_CTX_cleanup(&mctx); + return rv; +} diff --git a/crypto/dh/dh_key.c b/crypto/dh/dh_key.c index 9e1d8e58225b..1d80fb2c5f60 100644 --- a/crypto/dh/dh_key.c +++ b/crypto/dh/dh_key.c @@ -94,6 +94,20 @@ int DH_compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh) return dh->meth->compute_key(key, pub_key, dh); } +int DH_compute_key_padded(unsigned char *key, const BIGNUM *pub_key, DH *dh) +{ + int rv, pad; + rv = dh->meth->compute_key(key, pub_key, dh); + if (rv <= 0) + return rv; + pad = BN_num_bytes(dh->p) - rv; + if (pad > 0) { + memmove(key + pad, key, rv); + memset(key, 0, pad); + } + return rv + pad; +} + static DH_METHOD dh_ossl = { "OpenSSL DH Method", generate_key, diff --git a/crypto/dh/dh_pmeth.c b/crypto/dh/dh_pmeth.c index 65bc3882c7f2..b58e3fa86fad 100644 --- a/crypto/dh/dh_pmeth.c +++ b/crypto/dh/dh_pmeth.c @@ -63,6 +63,10 @@ #include #include #include +#ifndef OPENSSL_NO_DSA +# include +#endif +#include #include "evp_locl.h" /* DH pkey context structure */ @@ -72,9 +76,23 @@ typedef struct { int prime_len; int generator; int use_dsa; + int subprime_len; + /* message digest used for parameter generation */ + const EVP_MD *md; + int rfc5114_param; /* Keygen callback info */ int gentmp[2]; - /* message digest */ + /* KDF (if any) to use for DH */ + char kdf_type; + /* OID to use for KDF */ + ASN1_OBJECT *kdf_oid; + /* Message digest to use for key derivation */ + const EVP_MD *kdf_md; + /* User key material */ + unsigned char *kdf_ukm; + size_t kdf_ukmlen; + /* KDF output length */ + size_t kdf_outlen; } DH_PKEY_CTX; static int pkey_dh_init(EVP_PKEY_CTX *ctx) @@ -84,8 +102,18 @@ static int pkey_dh_init(EVP_PKEY_CTX *ctx) if (!dctx) return 0; dctx->prime_len = 1024; + dctx->subprime_len = -1; dctx->generator = 2; dctx->use_dsa = 0; + dctx->md = NULL; + dctx->rfc5114_param = 0; + + dctx->kdf_type = EVP_PKEY_DH_KDF_NONE; + dctx->kdf_oid = NULL; + dctx->kdf_md = NULL; + dctx->kdf_ukm = NULL; + dctx->kdf_ukmlen = 0; + dctx->kdf_outlen = 0; ctx->data = dctx; ctx->keygen_info = dctx->gentmp; @@ -102,16 +130,35 @@ static int pkey_dh_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src) sctx = src->data; dctx = dst->data; dctx->prime_len = sctx->prime_len; + dctx->subprime_len = sctx->subprime_len; dctx->generator = sctx->generator; dctx->use_dsa = sctx->use_dsa; + dctx->md = sctx->md; + dctx->rfc5114_param = sctx->rfc5114_param; + + dctx->kdf_type = sctx->kdf_type; + dctx->kdf_oid = OBJ_dup(sctx->kdf_oid); + if (!dctx->kdf_oid) + return 0; + dctx->kdf_md = sctx->kdf_md; + if (dctx->kdf_ukm) { + dctx->kdf_ukm = BUF_memdup(sctx->kdf_ukm, sctx->kdf_ukmlen); + dctx->kdf_ukmlen = sctx->kdf_ukmlen; + } + dctx->kdf_outlen = sctx->kdf_outlen; return 1; } static void pkey_dh_cleanup(EVP_PKEY_CTX *ctx) { DH_PKEY_CTX *dctx = ctx->data; - if (dctx) + if (dctx) { + if (dctx->kdf_ukm) + OPENSSL_free(dctx->kdf_ukm); + if (dctx->kdf_oid) + ASN1_OBJECT_free(dctx->kdf_oid); OPENSSL_free(dctx); + } } static int pkey_dh_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) @@ -124,14 +171,89 @@ static int pkey_dh_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) dctx->prime_len = p1; return 1; + case EVP_PKEY_CTRL_DH_PARAMGEN_SUBPRIME_LEN: + if (dctx->use_dsa == 0) + return -2; + dctx->subprime_len = p1; + return 1; + case EVP_PKEY_CTRL_DH_PARAMGEN_GENERATOR: + if (dctx->use_dsa) + return -2; dctx->generator = p1; return 1; + case EVP_PKEY_CTRL_DH_PARAMGEN_TYPE: +#ifdef OPENSSL_NO_DSA + if (p1 != 0) + return -2; +#else + if (p1 < 0 || p1 > 2) + return -2; +#endif + dctx->use_dsa = p1; + return 1; + + case EVP_PKEY_CTRL_DH_RFC5114: + if (p1 < 1 || p1 > 3) + return -2; + dctx->rfc5114_param = p1; + return 1; + case EVP_PKEY_CTRL_PEER_KEY: /* Default behaviour is OK */ return 1; + case EVP_PKEY_CTRL_DH_KDF_TYPE: + if (p1 == -2) + return dctx->kdf_type; + if (p1 != EVP_PKEY_DH_KDF_NONE && p1 != EVP_PKEY_DH_KDF_X9_42) + return -2; + dctx->kdf_type = p1; + return 1; + + case EVP_PKEY_CTRL_DH_KDF_MD: + dctx->kdf_md = p2; + return 1; + + case EVP_PKEY_CTRL_GET_DH_KDF_MD: + *(const EVP_MD **)p2 = dctx->kdf_md; + return 1; + + case EVP_PKEY_CTRL_DH_KDF_OUTLEN: + if (p1 <= 0) + return -2; + dctx->kdf_outlen = (size_t)p1; + return 1; + + case EVP_PKEY_CTRL_GET_DH_KDF_OUTLEN: + *(int *)p2 = dctx->kdf_outlen; + return 1; + + case EVP_PKEY_CTRL_DH_KDF_UKM: + if (dctx->kdf_ukm) + OPENSSL_free(dctx->kdf_ukm); + dctx->kdf_ukm = p2; + if (p2) + dctx->kdf_ukmlen = p1; + else + dctx->kdf_ukmlen = 0; + return 1; + + case EVP_PKEY_CTRL_GET_DH_KDF_UKM: + *(unsigned char **)p2 = dctx->kdf_ukm; + return dctx->kdf_ukmlen; + + case EVP_PKEY_CTRL_DH_KDF_OID: + if (dctx->kdf_oid) + ASN1_OBJECT_free(dctx->kdf_oid); + dctx->kdf_oid = p2; + return 1; + + case EVP_PKEY_CTRL_GET_DH_KDF_OID: + *(ASN1_OBJECT **)p2 = dctx->kdf_oid; + return 1; + default: return -2; @@ -146,30 +268,139 @@ static int pkey_dh_ctrl_str(EVP_PKEY_CTX *ctx, len = atoi(value); return EVP_PKEY_CTX_set_dh_paramgen_prime_len(ctx, len); } + if (!strcmp(type, "dh_rfc5114")) { + DH_PKEY_CTX *dctx = ctx->data; + int len; + len = atoi(value); + if (len < 0 || len > 3) + return -2; + dctx->rfc5114_param = len; + return 1; + } if (!strcmp(type, "dh_paramgen_generator")) { int len; len = atoi(value); return EVP_PKEY_CTX_set_dh_paramgen_generator(ctx, len); } + if (!strcmp(type, "dh_paramgen_subprime_len")) { + int len; + len = atoi(value); + return EVP_PKEY_CTX_set_dh_paramgen_subprime_len(ctx, len); + } + if (!strcmp(type, "dh_paramgen_type")) { + int typ; + typ = atoi(value); + return EVP_PKEY_CTX_set_dh_paramgen_type(ctx, typ); + } return -2; } +#ifndef OPENSSL_NO_DSA + +extern int dsa_builtin_paramgen(DSA *ret, size_t bits, size_t qbits, + const EVP_MD *evpmd, + const unsigned char *seed_in, size_t seed_len, + unsigned char *seed_out, int *counter_ret, + unsigned long *h_ret, BN_GENCB *cb); + +extern int dsa_builtin_paramgen2(DSA *ret, size_t L, size_t N, + const EVP_MD *evpmd, + const unsigned char *seed_in, + size_t seed_len, int idx, + unsigned char *seed_out, int *counter_ret, + unsigned long *h_ret, BN_GENCB *cb); + +static DSA *dsa_dh_generate(DH_PKEY_CTX *dctx, BN_GENCB *pcb) +{ + DSA *ret; + int rv = 0; + int prime_len = dctx->prime_len; + int subprime_len = dctx->subprime_len; + const EVP_MD *md = dctx->md; + if (dctx->use_dsa > 2) + return NULL; + ret = DSA_new(); + if (!ret) + return NULL; + if (subprime_len == -1) { + if (prime_len >= 2048) + subprime_len = 256; + else + subprime_len = 160; + } + if (md == NULL) { + if (prime_len >= 2048) + md = EVP_sha256(); + else + md = EVP_sha1(); + } + if (dctx->use_dsa == 1) + rv = dsa_builtin_paramgen(ret, prime_len, subprime_len, md, + NULL, 0, NULL, NULL, NULL, pcb); + else if (dctx->use_dsa == 2) + rv = dsa_builtin_paramgen2(ret, prime_len, subprime_len, md, + NULL, 0, -1, NULL, NULL, NULL, pcb); + if (rv <= 0) { + DSA_free(ret); + return NULL; + } + return ret; +} + +#endif + static int pkey_dh_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) { DH *dh = NULL; DH_PKEY_CTX *dctx = ctx->data; BN_GENCB *pcb, cb; int ret; + if (dctx->rfc5114_param) { + switch (dctx->rfc5114_param) { + case 1: + dh = DH_get_1024_160(); + break; + + case 2: + dh = DH_get_2048_224(); + break; + + case 3: + dh = DH_get_2048_256(); + break; + + default: + return -2; + } + EVP_PKEY_assign(pkey, EVP_PKEY_DHX, dh); + return 1; + } + if (ctx->pkey_gencb) { pcb = &cb; evp_pkey_set_cb_translate(pcb, ctx); } else pcb = NULL; +#ifndef OPENSSL_NO_DSA + if (dctx->use_dsa) { + DSA *dsa_dh; + dsa_dh = dsa_dh_generate(dctx, pcb); + if (!dsa_dh) + return 0; + dh = DSA_dup_DH(dsa_dh); + DSA_free(dsa_dh); + if (!dh) + return 0; + EVP_PKEY_assign(pkey, EVP_PKEY_DHX, dh); + return 1; + } +#endif dh = DH_new(); if (!dh) return 0; ret = DH_generate_parameters_ex(dh, dctx->prime_len, dctx->generator, pcb); + if (ret) EVP_PKEY_assign_DH(pkey, dh); else @@ -187,7 +418,7 @@ static int pkey_dh_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) dh = DH_new(); if (!dh) return 0; - EVP_PKEY_assign_DH(pkey, dh); + EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, dh); /* Note: if error return, pkey is freed by parent routine */ if (!EVP_PKEY_copy_parameters(pkey, ctx->pkey)) return 0; @@ -198,21 +429,96 @@ static int pkey_dh_derive(EVP_PKEY_CTX *ctx, unsigned char *key, size_t *keylen) { int ret; + DH *dh; + DH_PKEY_CTX *dctx = ctx->data; + BIGNUM *dhpub; if (!ctx->pkey || !ctx->peerkey) { DHerr(DH_F_PKEY_DH_DERIVE, DH_R_KEYS_NOT_SET); return 0; } - ret = DH_compute_key(key, ctx->peerkey->pkey.dh->pub_key, - ctx->pkey->pkey.dh); - if (ret < 0) + dh = ctx->pkey->pkey.dh; + dhpub = ctx->peerkey->pkey.dh->pub_key; + if (dctx->kdf_type == EVP_PKEY_DH_KDF_NONE) { + if (key == NULL) { + *keylen = DH_size(dh); + return 1; + } + ret = DH_compute_key(key, dhpub, dh); + if (ret < 0) + return ret; + *keylen = ret; + return 1; + } else if (dctx->kdf_type == EVP_PKEY_DH_KDF_X9_42) { + unsigned char *Z = NULL; + size_t Zlen = 0; + if (!dctx->kdf_outlen || !dctx->kdf_oid) + return 0; + if (key == NULL) { + *keylen = dctx->kdf_outlen; + return 1; + } + if (*keylen != dctx->kdf_outlen) + return 0; + ret = 0; + Zlen = DH_size(dh); + Z = OPENSSL_malloc(Zlen); + if (!Z) { + goto err; + } + if (DH_compute_key_padded(Z, dhpub, dh) <= 0) + goto err; + if (!DH_KDF_X9_42(key, *keylen, Z, Zlen, dctx->kdf_oid, + dctx->kdf_ukm, dctx->kdf_ukmlen, dctx->kdf_md)) + goto err; + *keylen = dctx->kdf_outlen; + ret = 1; + err: + if (Z) { + OPENSSL_cleanse(Z, Zlen); + OPENSSL_free(Z); + } return ret; - *keylen = ret; + } return 1; } const EVP_PKEY_METHOD dh_pkey_meth = { EVP_PKEY_DH, - EVP_PKEY_FLAG_AUTOARGLEN, + 0, + pkey_dh_init, + pkey_dh_copy, + pkey_dh_cleanup, + + 0, + pkey_dh_paramgen, + + 0, + pkey_dh_keygen, + + 0, + 0, + + 0, + 0, + + 0, 0, + + 0, 0, 0, 0, + + 0, 0, + + 0, 0, + + 0, + pkey_dh_derive, + + pkey_dh_ctrl, + pkey_dh_ctrl_str +}; + +const EVP_PKEY_METHOD dhx_pkey_meth = { + EVP_PKEY_DHX, + 0, pkey_dh_init, pkey_dh_copy, pkey_dh_cleanup, diff --git a/crypto/dh/dh_rfc5114.c b/crypto/dh/dh_rfc5114.c new file mode 100644 index 000000000000..e96e2aa3fc52 --- /dev/null +++ b/crypto/dh/dh_rfc5114.c @@ -0,0 +1,285 @@ +/* + * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project + * 2011. + */ +/* ==================================================================== + * Copyright (c) 2011 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + +#include +#include "cryptlib.h" +#include +#include + +/* DH parameters from RFC5114 */ + +#if BN_BITS2 == 64 +static const BN_ULONG dh1024_160_p[] = { + 0xDF1FB2BC2E4A4371ULL, 0xE68CFDA76D4DA708ULL, 0x45BF37DF365C1A65ULL, + 0xA151AF5F0DC8B4BDULL, 0xFAA31A4FF55BCCC0ULL, 0x4EFFD6FAE5644738ULL, + 0x98488E9C219A7372ULL, 0xACCBDD7D90C4BD70ULL, 0x24975C3CD49B83BFULL, + 0x13ECB4AEA9061123ULL, 0x9838EF1E2EE652C0ULL, 0x6073E28675A23D18ULL, + 0x9A6A9DCA52D23B61ULL, 0x52C99FBCFB06A3C6ULL, 0xDE92DE5EAE5D54ECULL, + 0xB10B8F96A080E01DULL +}; + +static const BN_ULONG dh1024_160_g[] = { + 0x855E6EEB22B3B2E5ULL, 0x858F4DCEF97C2A24ULL, 0x2D779D5918D08BC8ULL, + 0xD662A4D18E73AFA3ULL, 0x1DBF0A0169B6A28AULL, 0xA6A24C087A091F53ULL, + 0x909D0D2263F80A76ULL, 0xD7FBD7D3B9A92EE1ULL, 0x5E91547F9E2749F4ULL, + 0x160217B4B01B886AULL, 0x777E690F5504F213ULL, 0x266FEA1E5C41564BULL, + 0xD6406CFF14266D31ULL, 0xF8104DD258AC507FULL, 0x6765A442EFB99905ULL, + 0xA4D1CBD5C3FD3412ULL +}; + +static const BN_ULONG dh1024_160_q[] = { + 0x64B7CB9D49462353ULL, 0x81A8DF278ABA4E7DULL, 0x00000000F518AA87ULL +}; + +static const BN_ULONG dh2048_224_p[] = { + 0x0AC4DFFE0C10E64FULL, 0xCF9DE5384E71B81CULL, 0x7EF363E2FFA31F71ULL, + 0xE3FB73C16B8E75B9ULL, 0xC9B53DCF4BA80A29ULL, 0x23F10B0E16E79763ULL, + 0xC52172E413042E9BULL, 0xBE60E69CC928B2B9ULL, 0x80CD86A1B9E587E8ULL, + 0x315D75E198C641A4ULL, 0xCDF93ACC44328387ULL, 0x15987D9ADC0A486DULL, + 0x7310F7121FD5A074ULL, 0x278273C7DE31EFDCULL, 0x1602E714415D9330ULL, + 0x81286130BC8985DBULL, 0xB3BF8A3170918836ULL, 0x6A00E0A0B9C49708ULL, + 0xC6BA0B2C8BBC27BEULL, 0xC9F98D11ED34DBF6ULL, 0x7AD5B7D0B6C12207ULL, + 0xD91E8FEF55B7394BULL, 0x9037C9EDEFDA4DF8ULL, 0x6D3F8152AD6AC212ULL, + 0x1DE6B85A1274A0A6ULL, 0xEB3D688A309C180EULL, 0xAF9A3C407BA1DF15ULL, + 0xE6FA141DF95A56DBULL, 0xB54B1597B61D0A75ULL, 0xA20D64E5683B9FD1ULL, + 0xD660FAA79559C51FULL, 0xAD107E1E9123A9D0ULL +}; + +static const BN_ULONG dh2048_224_g[] = { + 0x84B890D3191F2BFAULL, 0x81BC087F2A7065B3ULL, 0x19C418E1F6EC0179ULL, + 0x7B5A0F1C71CFFF4CULL, 0xEDFE72FE9B6AA4BDULL, 0x81E1BCFE94B30269ULL, + 0x566AFBB48D6C0191ULL, 0xB539CCE3409D13CDULL, 0x6AA21E7F5F2FF381ULL, + 0xD9E263E4770589EFULL, 0x10E183EDD19963DDULL, 0xB70A8137150B8EEBULL, + 0x051AE3D428C8F8ACULL, 0xBB77A86F0C1AB15BULL, 0x6E3025E316A330EFULL, + 0x19529A45D6F83456ULL, 0xF180EB34118E98D1ULL, 0xB5F6C6B250717CBEULL, + 0x09939D54DA7460CDULL, 0xE247150422EA1ED4ULL, 0xB8A762D0521BC98AULL, + 0xF4D027275AC1348BULL, 0xC17669101999024AULL, 0xBE5E9001A8D66AD7ULL, + 0xC57DB17C620A8652ULL, 0xAB739D7700C29F52ULL, 0xDD921F01A70C4AFAULL, + 0xA6824A4E10B9A6F0ULL, 0x74866A08CFE4FFE3ULL, 0x6CDEBE7B89998CAFULL, + 0x9DF30B5C8FFDAC50ULL, 0xAC4032EF4F2D9AE3ULL +}; + +static const BN_ULONG dh2048_224_q[] = { + 0xBF389A99B36371EBULL, 0x1F80535A4738CEBCULL, 0xC58D93FE99717710ULL, + 0x00000000801C0D34ULL +}; + +static const BN_ULONG dh2048_256_p[] = { + 0xDB094AE91E1A1597ULL, 0x693877FAD7EF09CAULL, 0x6116D2276E11715FULL, + 0xA4B54330C198AF12ULL, 0x75F26375D7014103ULL, 0xC3A3960A54E710C3ULL, + 0xDED4010ABD0BE621ULL, 0xC0B857F689962856ULL, 0xB3CA3F7971506026ULL, + 0x1CCACB83E6B486F6ULL, 0x67E144E514056425ULL, 0xF6A167B5A41825D9ULL, + 0x3AD8347796524D8EULL, 0xF13C6D9A51BFA4ABULL, 0x2D52526735488A0EULL, + 0xB63ACAE1CAA6B790ULL, 0x4FDB70C581B23F76ULL, 0xBC39A0BF12307F5CULL, + 0xB941F54EB1E59BB8ULL, 0x6C5BFC11D45F9088ULL, 0x22E0B1EF4275BF7BULL, + 0x91F9E6725B4758C0ULL, 0x5A8A9D306BCF67EDULL, 0x209E0C6497517ABDULL, + 0x3BF4296D830E9A7CULL, 0x16C3D91134096FAAULL, 0xFAF7DF4561B2AA30ULL, + 0xE00DF8F1D61957D4ULL, 0x5D2CEED4435E3B00ULL, 0x8CEEF608660DD0F2ULL, + 0xFFBBD19C65195999ULL, 0x87A8E61DB4B6663CULL +}; + +static const BN_ULONG dh2048_256_g[] = { + 0x664B4C0F6CC41659ULL, 0x5E2327CFEF98C582ULL, 0xD647D148D4795451ULL, + 0x2F63078490F00EF8ULL, 0x184B523D1DB246C3ULL, 0xC7891428CDC67EB6ULL, + 0x7FD028370DF92B52ULL, 0xB3353BBB64E0EC37ULL, 0xECD06E1557CD0915ULL, + 0xB7D2BBD2DF016199ULL, 0xC8484B1E052588B9ULL, 0xDB2A3B7313D3FE14ULL, + 0xD052B985D182EA0AULL, 0xA4BD1BFFE83B9C80ULL, 0xDFC967C1FB3F2E55ULL, + 0xB5045AF2767164E1ULL, 0x1D14348F6F2F9193ULL, 0x64E67982428EBC83ULL, + 0x8AC376D282D6ED38ULL, 0x777DE62AAAB8A862ULL, 0xDDF463E5E9EC144BULL, + 0x0196F931C77A57F2ULL, 0xA55AE31341000A65ULL, 0x901228F8C28CBB18ULL, + 0xBC3773BF7E8C6F62ULL, 0xBE3A6C1B0C6B47B1ULL, 0xFF4FED4AAC0BB555ULL, + 0x10DBC15077BE463FULL, 0x07F4793A1A0BA125ULL, 0x4CA7B18F21EF2054ULL, + 0x2E77506660EDBD48ULL, 0x3FB32C9B73134D0BULL +}; + +static const BN_ULONG dh2048_256_q[] = { + 0xA308B0FE64F5FBD3ULL, 0x99B1A47D1EB3750BULL, 0xB447997640129DA2ULL, + 0x8CF83642A709A097ULL +}; + +#elif BN_BITS2 == 32 + +static const BN_ULONG dh1024_160_p[] = { + 0x2E4A4371, 0xDF1FB2BC, 0x6D4DA708, 0xE68CFDA7, 0x365C1A65, 0x45BF37DF, + 0x0DC8B4BD, 0xA151AF5F, 0xF55BCCC0, 0xFAA31A4F, 0xE5644738, 0x4EFFD6FA, + 0x219A7372, 0x98488E9C, 0x90C4BD70, 0xACCBDD7D, 0xD49B83BF, 0x24975C3C, + 0xA9061123, 0x13ECB4AE, 0x2EE652C0, 0x9838EF1E, 0x75A23D18, 0x6073E286, + 0x52D23B61, 0x9A6A9DCA, 0xFB06A3C6, 0x52C99FBC, 0xAE5D54EC, 0xDE92DE5E, + 0xA080E01D, 0xB10B8F96 +}; + +static const BN_ULONG dh1024_160_g[] = { + 0x22B3B2E5, 0x855E6EEB, 0xF97C2A24, 0x858F4DCE, 0x18D08BC8, 0x2D779D59, + 0x8E73AFA3, 0xD662A4D1, 0x69B6A28A, 0x1DBF0A01, 0x7A091F53, 0xA6A24C08, + 0x63F80A76, 0x909D0D22, 0xB9A92EE1, 0xD7FBD7D3, 0x9E2749F4, 0x5E91547F, + 0xB01B886A, 0x160217B4, 0x5504F213, 0x777E690F, 0x5C41564B, 0x266FEA1E, + 0x14266D31, 0xD6406CFF, 0x58AC507F, 0xF8104DD2, 0xEFB99905, 0x6765A442, + 0xC3FD3412, 0xA4D1CBD5 +}; + +static const BN_ULONG dh1024_160_q[] = { + 0x49462353, 0x64B7CB9D, 0x8ABA4E7D, 0x81A8DF27, 0xF518AA87 +}; + +static const BN_ULONG dh2048_224_p[] = { + 0x0C10E64F, 0x0AC4DFFE, 0x4E71B81C, 0xCF9DE538, 0xFFA31F71, 0x7EF363E2, + 0x6B8E75B9, 0xE3FB73C1, 0x4BA80A29, 0xC9B53DCF, 0x16E79763, 0x23F10B0E, + 0x13042E9B, 0xC52172E4, 0xC928B2B9, 0xBE60E69C, 0xB9E587E8, 0x80CD86A1, + 0x98C641A4, 0x315D75E1, 0x44328387, 0xCDF93ACC, 0xDC0A486D, 0x15987D9A, + 0x1FD5A074, 0x7310F712, 0xDE31EFDC, 0x278273C7, 0x415D9330, 0x1602E714, + 0xBC8985DB, 0x81286130, 0x70918836, 0xB3BF8A31, 0xB9C49708, 0x6A00E0A0, + 0x8BBC27BE, 0xC6BA0B2C, 0xED34DBF6, 0xC9F98D11, 0xB6C12207, 0x7AD5B7D0, + 0x55B7394B, 0xD91E8FEF, 0xEFDA4DF8, 0x9037C9ED, 0xAD6AC212, 0x6D3F8152, + 0x1274A0A6, 0x1DE6B85A, 0x309C180E, 0xEB3D688A, 0x7BA1DF15, 0xAF9A3C40, + 0xF95A56DB, 0xE6FA141D, 0xB61D0A75, 0xB54B1597, 0x683B9FD1, 0xA20D64E5, + 0x9559C51F, 0xD660FAA7, 0x9123A9D0, 0xAD107E1E +}; + +static const BN_ULONG dh2048_224_g[] = { + 0x191F2BFA, 0x84B890D3, 0x2A7065B3, 0x81BC087F, 0xF6EC0179, 0x19C418E1, + 0x71CFFF4C, 0x7B5A0F1C, 0x9B6AA4BD, 0xEDFE72FE, 0x94B30269, 0x81E1BCFE, + 0x8D6C0191, 0x566AFBB4, 0x409D13CD, 0xB539CCE3, 0x5F2FF381, 0x6AA21E7F, + 0x770589EF, 0xD9E263E4, 0xD19963DD, 0x10E183ED, 0x150B8EEB, 0xB70A8137, + 0x28C8F8AC, 0x051AE3D4, 0x0C1AB15B, 0xBB77A86F, 0x16A330EF, 0x6E3025E3, + 0xD6F83456, 0x19529A45, 0x118E98D1, 0xF180EB34, 0x50717CBE, 0xB5F6C6B2, + 0xDA7460CD, 0x09939D54, 0x22EA1ED4, 0xE2471504, 0x521BC98A, 0xB8A762D0, + 0x5AC1348B, 0xF4D02727, 0x1999024A, 0xC1766910, 0xA8D66AD7, 0xBE5E9001, + 0x620A8652, 0xC57DB17C, 0x00C29F52, 0xAB739D77, 0xA70C4AFA, 0xDD921F01, + 0x10B9A6F0, 0xA6824A4E, 0xCFE4FFE3, 0x74866A08, 0x89998CAF, 0x6CDEBE7B, + 0x8FFDAC50, 0x9DF30B5C, 0x4F2D9AE3, 0xAC4032EF +}; + +static const BN_ULONG dh2048_224_q[] = { + 0xB36371EB, 0xBF389A99, 0x4738CEBC, 0x1F80535A, 0x99717710, 0xC58D93FE, + 0x801C0D34 +}; + +static const BN_ULONG dh2048_256_p[] = { + 0x1E1A1597, 0xDB094AE9, 0xD7EF09CA, 0x693877FA, 0x6E11715F, 0x6116D227, + 0xC198AF12, 0xA4B54330, 0xD7014103, 0x75F26375, 0x54E710C3, 0xC3A3960A, + 0xBD0BE621, 0xDED4010A, 0x89962856, 0xC0B857F6, 0x71506026, 0xB3CA3F79, + 0xE6B486F6, 0x1CCACB83, 0x14056425, 0x67E144E5, 0xA41825D9, 0xF6A167B5, + 0x96524D8E, 0x3AD83477, 0x51BFA4AB, 0xF13C6D9A, 0x35488A0E, 0x2D525267, + 0xCAA6B790, 0xB63ACAE1, 0x81B23F76, 0x4FDB70C5, 0x12307F5C, 0xBC39A0BF, + 0xB1E59BB8, 0xB941F54E, 0xD45F9088, 0x6C5BFC11, 0x4275BF7B, 0x22E0B1EF, + 0x5B4758C0, 0x91F9E672, 0x6BCF67ED, 0x5A8A9D30, 0x97517ABD, 0x209E0C64, + 0x830E9A7C, 0x3BF4296D, 0x34096FAA, 0x16C3D911, 0x61B2AA30, 0xFAF7DF45, + 0xD61957D4, 0xE00DF8F1, 0x435E3B00, 0x5D2CEED4, 0x660DD0F2, 0x8CEEF608, + 0x65195999, 0xFFBBD19C, 0xB4B6663C, 0x87A8E61D +}; + +static const BN_ULONG dh2048_256_g[] = { + 0x6CC41659, 0x664B4C0F, 0xEF98C582, 0x5E2327CF, 0xD4795451, 0xD647D148, + 0x90F00EF8, 0x2F630784, 0x1DB246C3, 0x184B523D, 0xCDC67EB6, 0xC7891428, + 0x0DF92B52, 0x7FD02837, 0x64E0EC37, 0xB3353BBB, 0x57CD0915, 0xECD06E15, + 0xDF016199, 0xB7D2BBD2, 0x052588B9, 0xC8484B1E, 0x13D3FE14, 0xDB2A3B73, + 0xD182EA0A, 0xD052B985, 0xE83B9C80, 0xA4BD1BFF, 0xFB3F2E55, 0xDFC967C1, + 0x767164E1, 0xB5045AF2, 0x6F2F9193, 0x1D14348F, 0x428EBC83, 0x64E67982, + 0x82D6ED38, 0x8AC376D2, 0xAAB8A862, 0x777DE62A, 0xE9EC144B, 0xDDF463E5, + 0xC77A57F2, 0x0196F931, 0x41000A65, 0xA55AE313, 0xC28CBB18, 0x901228F8, + 0x7E8C6F62, 0xBC3773BF, 0x0C6B47B1, 0xBE3A6C1B, 0xAC0BB555, 0xFF4FED4A, + 0x77BE463F, 0x10DBC150, 0x1A0BA125, 0x07F4793A, 0x21EF2054, 0x4CA7B18F, + 0x60EDBD48, 0x2E775066, 0x73134D0B, 0x3FB32C9B +}; + +static const BN_ULONG dh2048_256_q[] = { + 0x64F5FBD3, 0xA308B0FE, 0x1EB3750B, 0x99B1A47D, 0x40129DA2, 0xB4479976, + 0xA709A097, 0x8CF83642 +}; + +#else +# error "unsupported BN_BITS2" +#endif + +/* Macro to make a BIGNUM from static data */ + +#define make_dh_bn(x) static const BIGNUM _bignum_##x = { (BN_ULONG *) x, \ + sizeof(x)/sizeof(BN_ULONG),\ + sizeof(x)/sizeof(BN_ULONG),\ + 0, BN_FLG_STATIC_DATA } + +/* + * Macro to make a DH structure from BIGNUM data. NB: although just copying + * the BIGNUM static pointers would be more efficient we can't as they get + * wiped using BN_clear_free() when DH_free() is called. + */ + +#define make_dh(x) \ +DH * DH_get_##x(void) \ + { \ + DH *dh; \ + make_dh_bn(dh##x##_p); \ + make_dh_bn(dh##x##_q); \ + make_dh_bn(dh##x##_g); \ + dh = DH_new(); \ + if (!dh) \ + return NULL; \ + dh->p = BN_dup(&_bignum_dh##x##_p); \ + dh->g = BN_dup(&_bignum_dh##x##_g); \ + dh->q = BN_dup(&_bignum_dh##x##_q); \ + if (!dh->p || !dh->q || !dh->g) \ + { \ + DH_free(dh); \ + return NULL; \ + } \ + return dh; \ + } + +make_dh(1024_160) +make_dh(2048_224) +make_dh(2048_256) diff --git a/crypto/dh/dhtest.c b/crypto/dh/dhtest.c index 5a4ee9a35d83..c9dd76bc75e1 100644 --- a/crypto/dh/dhtest.c +++ b/crypto/dh/dhtest.c @@ -96,6 +96,8 @@ static int MS_CALLBACK cb(int p, int n, BN_GENCB *arg); static const char rnd_seed[] = "string to make the random number generator think it has entropy"; +static int run_rfc5114_tests(void); + int main(int argc, char *argv[]) { BN_GENCB _cb; @@ -199,6 +201,8 @@ int main(int argc, char *argv[]) ret = 1; } else ret = 0; + if (!run_rfc5114_tests()) + ret = 1; err: ERR_print_errors_fp(stderr); @@ -238,4 +242,323 @@ static int MS_CALLBACK cb(int p, int n, BN_GENCB *arg) # endif return 1; } + +/* Test data from RFC 5114 */ + +static const unsigned char dhtest_1024_160_xA[] = { + 0xB9, 0xA3, 0xB3, 0xAE, 0x8F, 0xEF, 0xC1, 0xA2, 0x93, 0x04, 0x96, 0x50, + 0x70, 0x86, 0xF8, 0x45, 0x5D, 0x48, 0x94, 0x3E +}; + +static const unsigned char dhtest_1024_160_yA[] = { + 0x2A, 0x85, 0x3B, 0x3D, 0x92, 0x19, 0x75, 0x01, 0xB9, 0x01, 0x5B, 0x2D, + 0xEB, 0x3E, 0xD8, 0x4F, 0x5E, 0x02, 0x1D, 0xCC, 0x3E, 0x52, 0xF1, 0x09, + 0xD3, 0x27, 0x3D, 0x2B, 0x75, 0x21, 0x28, 0x1C, 0xBA, 0xBE, 0x0E, 0x76, + 0xFF, 0x57, 0x27, 0xFA, 0x8A, 0xCC, 0xE2, 0x69, 0x56, 0xBA, 0x9A, 0x1F, + 0xCA, 0x26, 0xF2, 0x02, 0x28, 0xD8, 0x69, 0x3F, 0xEB, 0x10, 0x84, 0x1D, + 0x84, 0xA7, 0x36, 0x00, 0x54, 0xEC, 0xE5, 0xA7, 0xF5, 0xB7, 0xA6, 0x1A, + 0xD3, 0xDF, 0xB3, 0xC6, 0x0D, 0x2E, 0x43, 0x10, 0x6D, 0x87, 0x27, 0xDA, + 0x37, 0xDF, 0x9C, 0xCE, 0x95, 0xB4, 0x78, 0x75, 0x5D, 0x06, 0xBC, 0xEA, + 0x8F, 0x9D, 0x45, 0x96, 0x5F, 0x75, 0xA5, 0xF3, 0xD1, 0xDF, 0x37, 0x01, + 0x16, 0x5F, 0xC9, 0xE5, 0x0C, 0x42, 0x79, 0xCE, 0xB0, 0x7F, 0x98, 0x95, + 0x40, 0xAE, 0x96, 0xD5, 0xD8, 0x8E, 0xD7, 0x76 +}; + +static const unsigned char dhtest_1024_160_xB[] = { + 0x93, 0x92, 0xC9, 0xF9, 0xEB, 0x6A, 0x7A, 0x6A, 0x90, 0x22, 0xF7, 0xD8, + 0x3E, 0x72, 0x23, 0xC6, 0x83, 0x5B, 0xBD, 0xDA +}; + +static const unsigned char dhtest_1024_160_yB[] = { + 0x71, 0x7A, 0x6C, 0xB0, 0x53, 0x37, 0x1F, 0xF4, 0xA3, 0xB9, 0x32, 0x94, + 0x1C, 0x1E, 0x56, 0x63, 0xF8, 0x61, 0xA1, 0xD6, 0xAD, 0x34, 0xAE, 0x66, + 0x57, 0x6D, 0xFB, 0x98, 0xF6, 0xC6, 0xCB, 0xF9, 0xDD, 0xD5, 0xA5, 0x6C, + 0x78, 0x33, 0xF6, 0xBC, 0xFD, 0xFF, 0x09, 0x55, 0x82, 0xAD, 0x86, 0x8E, + 0x44, 0x0E, 0x8D, 0x09, 0xFD, 0x76, 0x9E, 0x3C, 0xEC, 0xCD, 0xC3, 0xD3, + 0xB1, 0xE4, 0xCF, 0xA0, 0x57, 0x77, 0x6C, 0xAA, 0xF9, 0x73, 0x9B, 0x6A, + 0x9F, 0xEE, 0x8E, 0x74, 0x11, 0xF8, 0xD6, 0xDA, 0xC0, 0x9D, 0x6A, 0x4E, + 0xDB, 0x46, 0xCC, 0x2B, 0x5D, 0x52, 0x03, 0x09, 0x0E, 0xAE, 0x61, 0x26, + 0x31, 0x1E, 0x53, 0xFD, 0x2C, 0x14, 0xB5, 0x74, 0xE6, 0xA3, 0x10, 0x9A, + 0x3D, 0xA1, 0xBE, 0x41, 0xBD, 0xCE, 0xAA, 0x18, 0x6F, 0x5C, 0xE0, 0x67, + 0x16, 0xA2, 0xB6, 0xA0, 0x7B, 0x3C, 0x33, 0xFE +}; + +static const unsigned char dhtest_1024_160_Z[] = { + 0x5C, 0x80, 0x4F, 0x45, 0x4D, 0x30, 0xD9, 0xC4, 0xDF, 0x85, 0x27, 0x1F, + 0x93, 0x52, 0x8C, 0x91, 0xDF, 0x6B, 0x48, 0xAB, 0x5F, 0x80, 0xB3, 0xB5, + 0x9C, 0xAA, 0xC1, 0xB2, 0x8F, 0x8A, 0xCB, 0xA9, 0xCD, 0x3E, 0x39, 0xF3, + 0xCB, 0x61, 0x45, 0x25, 0xD9, 0x52, 0x1D, 0x2E, 0x64, 0x4C, 0x53, 0xB8, + 0x07, 0xB8, 0x10, 0xF3, 0x40, 0x06, 0x2F, 0x25, 0x7D, 0x7D, 0x6F, 0xBF, + 0xE8, 0xD5, 0xE8, 0xF0, 0x72, 0xE9, 0xB6, 0xE9, 0xAF, 0xDA, 0x94, 0x13, + 0xEA, 0xFB, 0x2E, 0x8B, 0x06, 0x99, 0xB1, 0xFB, 0x5A, 0x0C, 0xAC, 0xED, + 0xDE, 0xAE, 0xAD, 0x7E, 0x9C, 0xFB, 0xB3, 0x6A, 0xE2, 0xB4, 0x20, 0x83, + 0x5B, 0xD8, 0x3A, 0x19, 0xFB, 0x0B, 0x5E, 0x96, 0xBF, 0x8F, 0xA4, 0xD0, + 0x9E, 0x34, 0x55, 0x25, 0x16, 0x7E, 0xCD, 0x91, 0x55, 0x41, 0x6F, 0x46, + 0xF4, 0x08, 0xED, 0x31, 0xB6, 0x3C, 0x6E, 0x6D +}; + +static const unsigned char dhtest_2048_224_xA[] = { + 0x22, 0xE6, 0x26, 0x01, 0xDB, 0xFF, 0xD0, 0x67, 0x08, 0xA6, 0x80, 0xF7, + 0x47, 0xF3, 0x61, 0xF7, 0x6D, 0x8F, 0x4F, 0x72, 0x1A, 0x05, 0x48, 0xE4, + 0x83, 0x29, 0x4B, 0x0C +}; + +static const unsigned char dhtest_2048_224_yA[] = { + 0x1B, 0x3A, 0x63, 0x45, 0x1B, 0xD8, 0x86, 0xE6, 0x99, 0xE6, 0x7B, 0x49, + 0x4E, 0x28, 0x8B, 0xD7, 0xF8, 0xE0, 0xD3, 0x70, 0xBA, 0xDD, 0xA7, 0xA0, + 0xEF, 0xD2, 0xFD, 0xE7, 0xD8, 0xF6, 0x61, 0x45, 0xCC, 0x9F, 0x28, 0x04, + 0x19, 0x97, 0x5E, 0xB8, 0x08, 0x87, 0x7C, 0x8A, 0x4C, 0x0C, 0x8E, 0x0B, + 0xD4, 0x8D, 0x4A, 0x54, 0x01, 0xEB, 0x1E, 0x87, 0x76, 0xBF, 0xEE, 0xE1, + 0x34, 0xC0, 0x38, 0x31, 0xAC, 0x27, 0x3C, 0xD9, 0xD6, 0x35, 0xAB, 0x0C, + 0xE0, 0x06, 0xA4, 0x2A, 0x88, 0x7E, 0x3F, 0x52, 0xFB, 0x87, 0x66, 0xB6, + 0x50, 0xF3, 0x80, 0x78, 0xBC, 0x8E, 0xE8, 0x58, 0x0C, 0xEF, 0xE2, 0x43, + 0x96, 0x8C, 0xFC, 0x4F, 0x8D, 0xC3, 0xDB, 0x08, 0x45, 0x54, 0x17, 0x1D, + 0x41, 0xBF, 0x2E, 0x86, 0x1B, 0x7B, 0xB4, 0xD6, 0x9D, 0xD0, 0xE0, 0x1E, + 0xA3, 0x87, 0xCB, 0xAA, 0x5C, 0xA6, 0x72, 0xAF, 0xCB, 0xE8, 0xBD, 0xB9, + 0xD6, 0x2D, 0x4C, 0xE1, 0x5F, 0x17, 0xDD, 0x36, 0xF9, 0x1E, 0xD1, 0xEE, + 0xDD, 0x65, 0xCA, 0x4A, 0x06, 0x45, 0x5C, 0xB9, 0x4C, 0xD4, 0x0A, 0x52, + 0xEC, 0x36, 0x0E, 0x84, 0xB3, 0xC9, 0x26, 0xE2, 0x2C, 0x43, 0x80, 0xA3, + 0xBF, 0x30, 0x9D, 0x56, 0x84, 0x97, 0x68, 0xB7, 0xF5, 0x2C, 0xFD, 0xF6, + 0x55, 0xFD, 0x05, 0x3A, 0x7E, 0xF7, 0x06, 0x97, 0x9E, 0x7E, 0x58, 0x06, + 0xB1, 0x7D, 0xFA, 0xE5, 0x3A, 0xD2, 0xA5, 0xBC, 0x56, 0x8E, 0xBB, 0x52, + 0x9A, 0x7A, 0x61, 0xD6, 0x8D, 0x25, 0x6F, 0x8F, 0xC9, 0x7C, 0x07, 0x4A, + 0x86, 0x1D, 0x82, 0x7E, 0x2E, 0xBC, 0x8C, 0x61, 0x34, 0x55, 0x31, 0x15, + 0xB7, 0x0E, 0x71, 0x03, 0x92, 0x0A, 0xA1, 0x6D, 0x85, 0xE5, 0x2B, 0xCB, + 0xAB, 0x8D, 0x78, 0x6A, 0x68, 0x17, 0x8F, 0xA8, 0xFF, 0x7C, 0x2F, 0x5C, + 0x71, 0x64, 0x8D, 0x6F +}; + +static const unsigned char dhtest_2048_224_xB[] = { + 0x4F, 0xF3, 0xBC, 0x96, 0xC7, 0xFC, 0x6A, 0x6D, 0x71, 0xD3, 0xB3, 0x63, + 0x80, 0x0A, 0x7C, 0xDF, 0xEF, 0x6F, 0xC4, 0x1B, 0x44, 0x17, 0xEA, 0x15, + 0x35, 0x3B, 0x75, 0x90 +}; + +static const unsigned char dhtest_2048_224_yB[] = { + 0x4D, 0xCE, 0xE9, 0x92, 0xA9, 0x76, 0x2A, 0x13, 0xF2, 0xF8, 0x38, 0x44, + 0xAD, 0x3D, 0x77, 0xEE, 0x0E, 0x31, 0xC9, 0x71, 0x8B, 0x3D, 0xB6, 0xC2, + 0x03, 0x5D, 0x39, 0x61, 0x18, 0x2C, 0x3E, 0x0B, 0xA2, 0x47, 0xEC, 0x41, + 0x82, 0xD7, 0x60, 0xCD, 0x48, 0xD9, 0x95, 0x99, 0x97, 0x06, 0x22, 0xA1, + 0x88, 0x1B, 0xBA, 0x2D, 0xC8, 0x22, 0x93, 0x9C, 0x78, 0xC3, 0x91, 0x2C, + 0x66, 0x61, 0xFA, 0x54, 0x38, 0xB2, 0x07, 0x66, 0x22, 0x2B, 0x75, 0xE2, + 0x4C, 0x2E, 0x3A, 0xD0, 0xC7, 0x28, 0x72, 0x36, 0x12, 0x95, 0x25, 0xEE, + 0x15, 0xB5, 0xDD, 0x79, 0x98, 0xAA, 0x04, 0xC4, 0xA9, 0x69, 0x6C, 0xAC, + 0xD7, 0x17, 0x20, 0x83, 0xA9, 0x7A, 0x81, 0x66, 0x4E, 0xAD, 0x2C, 0x47, + 0x9E, 0x44, 0x4E, 0x4C, 0x06, 0x54, 0xCC, 0x19, 0xE2, 0x8D, 0x77, 0x03, + 0xCE, 0xE8, 0xDA, 0xCD, 0x61, 0x26, 0xF5, 0xD6, 0x65, 0xEC, 0x52, 0xC6, + 0x72, 0x55, 0xDB, 0x92, 0x01, 0x4B, 0x03, 0x7E, 0xB6, 0x21, 0xA2, 0xAC, + 0x8E, 0x36, 0x5D, 0xE0, 0x71, 0xFF, 0xC1, 0x40, 0x0A, 0xCF, 0x07, 0x7A, + 0x12, 0x91, 0x3D, 0xD8, 0xDE, 0x89, 0x47, 0x34, 0x37, 0xAB, 0x7B, 0xA3, + 0x46, 0x74, 0x3C, 0x1B, 0x21, 0x5D, 0xD9, 0xC1, 0x21, 0x64, 0xA7, 0xE4, + 0x05, 0x31, 0x18, 0xD1, 0x99, 0xBE, 0xC8, 0xEF, 0x6F, 0xC5, 0x61, 0x17, + 0x0C, 0x84, 0xC8, 0x7D, 0x10, 0xEE, 0x9A, 0x67, 0x4A, 0x1F, 0xA8, 0xFF, + 0xE1, 0x3B, 0xDF, 0xBA, 0x1D, 0x44, 0xDE, 0x48, 0x94, 0x6D, 0x68, 0xDC, + 0x0C, 0xDD, 0x77, 0x76, 0x35, 0xA7, 0xAB, 0x5B, 0xFB, 0x1E, 0x4B, 0xB7, + 0xB8, 0x56, 0xF9, 0x68, 0x27, 0x73, 0x4C, 0x18, 0x41, 0x38, 0xE9, 0x15, + 0xD9, 0xC3, 0x00, 0x2E, 0xBC, 0xE5, 0x31, 0x20, 0x54, 0x6A, 0x7E, 0x20, + 0x02, 0x14, 0x2B, 0x6C +}; + +static const unsigned char dhtest_2048_224_Z[] = { + 0x34, 0xD9, 0xBD, 0xDC, 0x1B, 0x42, 0x17, 0x6C, 0x31, 0x3F, 0xEA, 0x03, + 0x4C, 0x21, 0x03, 0x4D, 0x07, 0x4A, 0x63, 0x13, 0xBB, 0x4E, 0xCD, 0xB3, + 0x70, 0x3F, 0xFF, 0x42, 0x45, 0x67, 0xA4, 0x6B, 0xDF, 0x75, 0x53, 0x0E, + 0xDE, 0x0A, 0x9D, 0xA5, 0x22, 0x9D, 0xE7, 0xD7, 0x67, 0x32, 0x28, 0x6C, + 0xBC, 0x0F, 0x91, 0xDA, 0x4C, 0x3C, 0x85, 0x2F, 0xC0, 0x99, 0xC6, 0x79, + 0x53, 0x1D, 0x94, 0xC7, 0x8A, 0xB0, 0x3D, 0x9D, 0xEC, 0xB0, 0xA4, 0xE4, + 0xCA, 0x8B, 0x2B, 0xB4, 0x59, 0x1C, 0x40, 0x21, 0xCF, 0x8C, 0xE3, 0xA2, + 0x0A, 0x54, 0x1D, 0x33, 0x99, 0x40, 0x17, 0xD0, 0x20, 0x0A, 0xE2, 0xC9, + 0x51, 0x6E, 0x2F, 0xF5, 0x14, 0x57, 0x79, 0x26, 0x9E, 0x86, 0x2B, 0x0F, + 0xB4, 0x74, 0xA2, 0xD5, 0x6D, 0xC3, 0x1E, 0xD5, 0x69, 0xA7, 0x70, 0x0B, + 0x4C, 0x4A, 0xB1, 0x6B, 0x22, 0xA4, 0x55, 0x13, 0x53, 0x1E, 0xF5, 0x23, + 0xD7, 0x12, 0x12, 0x07, 0x7B, 0x5A, 0x16, 0x9B, 0xDE, 0xFF, 0xAD, 0x7A, + 0xD9, 0x60, 0x82, 0x84, 0xC7, 0x79, 0x5B, 0x6D, 0x5A, 0x51, 0x83, 0xB8, + 0x70, 0x66, 0xDE, 0x17, 0xD8, 0xD6, 0x71, 0xC9, 0xEB, 0xD8, 0xEC, 0x89, + 0x54, 0x4D, 0x45, 0xEC, 0x06, 0x15, 0x93, 0xD4, 0x42, 0xC6, 0x2A, 0xB9, + 0xCE, 0x3B, 0x1C, 0xB9, 0x94, 0x3A, 0x1D, 0x23, 0xA5, 0xEA, 0x3B, 0xCF, + 0x21, 0xA0, 0x14, 0x71, 0xE6, 0x7E, 0x00, 0x3E, 0x7F, 0x8A, 0x69, 0xC7, + 0x28, 0xBE, 0x49, 0x0B, 0x2F, 0xC8, 0x8C, 0xFE, 0xB9, 0x2D, 0xB6, 0xA2, + 0x15, 0xE5, 0xD0, 0x3C, 0x17, 0xC4, 0x64, 0xC9, 0xAC, 0x1A, 0x46, 0xE2, + 0x03, 0xE1, 0x3F, 0x95, 0x29, 0x95, 0xFB, 0x03, 0xC6, 0x9D, 0x3C, 0xC4, + 0x7F, 0xCB, 0x51, 0x0B, 0x69, 0x98, 0xFF, 0xD3, 0xAA, 0x6D, 0xE7, 0x3C, + 0xF9, 0xF6, 0x38, 0x69 +}; + +static const unsigned char dhtest_2048_256_xA[] = { + 0x08, 0x81, 0x38, 0x2C, 0xDB, 0x87, 0x66, 0x0C, 0x6D, 0xC1, 0x3E, 0x61, + 0x49, 0x38, 0xD5, 0xB9, 0xC8, 0xB2, 0xF2, 0x48, 0x58, 0x1C, 0xC5, 0xE3, + 0x1B, 0x35, 0x45, 0x43, 0x97, 0xFC, 0xE5, 0x0E +}; + +static const unsigned char dhtest_2048_256_yA[] = { + 0x2E, 0x93, 0x80, 0xC8, 0x32, 0x3A, 0xF9, 0x75, 0x45, 0xBC, 0x49, 0x41, + 0xDE, 0xB0, 0xEC, 0x37, 0x42, 0xC6, 0x2F, 0xE0, 0xEC, 0xE8, 0x24, 0xA6, + 0xAB, 0xDB, 0xE6, 0x6C, 0x59, 0xBE, 0xE0, 0x24, 0x29, 0x11, 0xBF, 0xB9, + 0x67, 0x23, 0x5C, 0xEB, 0xA3, 0x5A, 0xE1, 0x3E, 0x4E, 0xC7, 0x52, 0xBE, + 0x63, 0x0B, 0x92, 0xDC, 0x4B, 0xDE, 0x28, 0x47, 0xA9, 0xC6, 0x2C, 0xB8, + 0x15, 0x27, 0x45, 0x42, 0x1F, 0xB7, 0xEB, 0x60, 0xA6, 0x3C, 0x0F, 0xE9, + 0x15, 0x9F, 0xCC, 0xE7, 0x26, 0xCE, 0x7C, 0xD8, 0x52, 0x3D, 0x74, 0x50, + 0x66, 0x7E, 0xF8, 0x40, 0xE4, 0x91, 0x91, 0x21, 0xEB, 0x5F, 0x01, 0xC8, + 0xC9, 0xB0, 0xD3, 0xD6, 0x48, 0xA9, 0x3B, 0xFB, 0x75, 0x68, 0x9E, 0x82, + 0x44, 0xAC, 0x13, 0x4A, 0xF5, 0x44, 0x71, 0x1C, 0xE7, 0x9A, 0x02, 0xDC, + 0xC3, 0x42, 0x26, 0x68, 0x47, 0x80, 0xDD, 0xDC, 0xB4, 0x98, 0x59, 0x41, + 0x06, 0xC3, 0x7F, 0x5B, 0xC7, 0x98, 0x56, 0x48, 0x7A, 0xF5, 0xAB, 0x02, + 0x2A, 0x2E, 0x5E, 0x42, 0xF0, 0x98, 0x97, 0xC1, 0xA8, 0x5A, 0x11, 0xEA, + 0x02, 0x12, 0xAF, 0x04, 0xD9, 0xB4, 0xCE, 0xBC, 0x93, 0x7C, 0x3C, 0x1A, + 0x3E, 0x15, 0xA8, 0xA0, 0x34, 0x2E, 0x33, 0x76, 0x15, 0xC8, 0x4E, 0x7F, + 0xE3, 0xB8, 0xB9, 0xB8, 0x7F, 0xB1, 0xE7, 0x3A, 0x15, 0xAF, 0x12, 0xA3, + 0x0D, 0x74, 0x6E, 0x06, 0xDF, 0xC3, 0x4F, 0x29, 0x0D, 0x79, 0x7C, 0xE5, + 0x1A, 0xA1, 0x3A, 0xA7, 0x85, 0xBF, 0x66, 0x58, 0xAF, 0xF5, 0xE4, 0xB0, + 0x93, 0x00, 0x3C, 0xBE, 0xAF, 0x66, 0x5B, 0x3C, 0x2E, 0x11, 0x3A, 0x3A, + 0x4E, 0x90, 0x52, 0x69, 0x34, 0x1D, 0xC0, 0x71, 0x14, 0x26, 0x68, 0x5F, + 0x4E, 0xF3, 0x7E, 0x86, 0x8A, 0x81, 0x26, 0xFF, 0x3F, 0x22, 0x79, 0xB5, + 0x7C, 0xA6, 0x7E, 0x29 +}; + +static const unsigned char dhtest_2048_256_xB[] = { + 0x7D, 0x62, 0xA7, 0xE3, 0xEF, 0x36, 0xDE, 0x61, 0x7B, 0x13, 0xD1, 0xAF, + 0xB8, 0x2C, 0x78, 0x0D, 0x83, 0xA2, 0x3B, 0xD4, 0xEE, 0x67, 0x05, 0x64, + 0x51, 0x21, 0xF3, 0x71, 0xF5, 0x46, 0xA5, 0x3D +}; + +static const unsigned char dhtest_2048_256_yB[] = { + 0x57, 0x5F, 0x03, 0x51, 0xBD, 0x2B, 0x1B, 0x81, 0x74, 0x48, 0xBD, 0xF8, + 0x7A, 0x6C, 0x36, 0x2C, 0x1E, 0x28, 0x9D, 0x39, 0x03, 0xA3, 0x0B, 0x98, + 0x32, 0xC5, 0x74, 0x1F, 0xA2, 0x50, 0x36, 0x3E, 0x7A, 0xCB, 0xC7, 0xF7, + 0x7F, 0x3D, 0xAC, 0xBC, 0x1F, 0x13, 0x1A, 0xDD, 0x8E, 0x03, 0x36, 0x7E, + 0xFF, 0x8F, 0xBB, 0xB3, 0xE1, 0xC5, 0x78, 0x44, 0x24, 0x80, 0x9B, 0x25, + 0xAF, 0xE4, 0xD2, 0x26, 0x2A, 0x1A, 0x6F, 0xD2, 0xFA, 0xB6, 0x41, 0x05, + 0xCA, 0x30, 0xA6, 0x74, 0xE0, 0x7F, 0x78, 0x09, 0x85, 0x20, 0x88, 0x63, + 0x2F, 0xC0, 0x49, 0x23, 0x37, 0x91, 0xAD, 0x4E, 0xDD, 0x08, 0x3A, 0x97, + 0x8B, 0x88, 0x3E, 0xE6, 0x18, 0xBC, 0x5E, 0x0D, 0xD0, 0x47, 0x41, 0x5F, + 0x2D, 0x95, 0xE6, 0x83, 0xCF, 0x14, 0x82, 0x6B, 0x5F, 0xBE, 0x10, 0xD3, + 0xCE, 0x41, 0xC6, 0xC1, 0x20, 0xC7, 0x8A, 0xB2, 0x00, 0x08, 0xC6, 0x98, + 0xBF, 0x7F, 0x0B, 0xCA, 0xB9, 0xD7, 0xF4, 0x07, 0xBE, 0xD0, 0xF4, 0x3A, + 0xFB, 0x29, 0x70, 0xF5, 0x7F, 0x8D, 0x12, 0x04, 0x39, 0x63, 0xE6, 0x6D, + 0xDD, 0x32, 0x0D, 0x59, 0x9A, 0xD9, 0x93, 0x6C, 0x8F, 0x44, 0x13, 0x7C, + 0x08, 0xB1, 0x80, 0xEC, 0x5E, 0x98, 0x5C, 0xEB, 0xE1, 0x86, 0xF3, 0xD5, + 0x49, 0x67, 0x7E, 0x80, 0x60, 0x73, 0x31, 0xEE, 0x17, 0xAF, 0x33, 0x80, + 0xA7, 0x25, 0xB0, 0x78, 0x23, 0x17, 0xD7, 0xDD, 0x43, 0xF5, 0x9D, 0x7A, + 0xF9, 0x56, 0x8A, 0x9B, 0xB6, 0x3A, 0x84, 0xD3, 0x65, 0xF9, 0x22, 0x44, + 0xED, 0x12, 0x09, 0x88, 0x21, 0x93, 0x02, 0xF4, 0x29, 0x24, 0xC7, 0xCA, + 0x90, 0xB8, 0x9D, 0x24, 0xF7, 0x1B, 0x0A, 0xB6, 0x97, 0x82, 0x3D, 0x7D, + 0xEB, 0x1A, 0xFF, 0x5B, 0x0E, 0x8E, 0x4A, 0x45, 0xD4, 0x9F, 0x7F, 0x53, + 0x75, 0x7E, 0x19, 0x13 +}; + +static const unsigned char dhtest_2048_256_Z[] = { + 0x86, 0xC7, 0x0B, 0xF8, 0xD0, 0xBB, 0x81, 0xBB, 0x01, 0x07, 0x8A, 0x17, + 0x21, 0x9C, 0xB7, 0xD2, 0x72, 0x03, 0xDB, 0x2A, 0x19, 0xC8, 0x77, 0xF1, + 0xD1, 0xF1, 0x9F, 0xD7, 0xD7, 0x7E, 0xF2, 0x25, 0x46, 0xA6, 0x8F, 0x00, + 0x5A, 0xD5, 0x2D, 0xC8, 0x45, 0x53, 0xB7, 0x8F, 0xC6, 0x03, 0x30, 0xBE, + 0x51, 0xEA, 0x7C, 0x06, 0x72, 0xCA, 0xC1, 0x51, 0x5E, 0x4B, 0x35, 0xC0, + 0x47, 0xB9, 0xA5, 0x51, 0xB8, 0x8F, 0x39, 0xDC, 0x26, 0xDA, 0x14, 0xA0, + 0x9E, 0xF7, 0x47, 0x74, 0xD4, 0x7C, 0x76, 0x2D, 0xD1, 0x77, 0xF9, 0xED, + 0x5B, 0xC2, 0xF1, 0x1E, 0x52, 0xC8, 0x79, 0xBD, 0x95, 0x09, 0x85, 0x04, + 0xCD, 0x9E, 0xEC, 0xD8, 0xA8, 0xF9, 0xB3, 0xEF, 0xBD, 0x1F, 0x00, 0x8A, + 0xC5, 0x85, 0x30, 0x97, 0xD9, 0xD1, 0x83, 0x7F, 0x2B, 0x18, 0xF7, 0x7C, + 0xD7, 0xBE, 0x01, 0xAF, 0x80, 0xA7, 0xC7, 0xB5, 0xEA, 0x3C, 0xA5, 0x4C, + 0xC0, 0x2D, 0x0C, 0x11, 0x6F, 0xEE, 0x3F, 0x95, 0xBB, 0x87, 0x39, 0x93, + 0x85, 0x87, 0x5D, 0x7E, 0x86, 0x74, 0x7E, 0x67, 0x6E, 0x72, 0x89, 0x38, + 0xAC, 0xBF, 0xF7, 0x09, 0x8E, 0x05, 0xBE, 0x4D, 0xCF, 0xB2, 0x40, 0x52, + 0xB8, 0x3A, 0xEF, 0xFB, 0x14, 0x78, 0x3F, 0x02, 0x9A, 0xDB, 0xDE, 0x7F, + 0x53, 0xFA, 0xE9, 0x20, 0x84, 0x22, 0x40, 0x90, 0xE0, 0x07, 0xCE, 0xE9, + 0x4D, 0x4B, 0xF2, 0xBA, 0xCE, 0x9F, 0xFD, 0x4B, 0x57, 0xD2, 0xAF, 0x7C, + 0x72, 0x4D, 0x0C, 0xAA, 0x19, 0xBF, 0x05, 0x01, 0xF6, 0xF1, 0x7B, 0x4A, + 0xA1, 0x0F, 0x42, 0x5E, 0x3E, 0xA7, 0x60, 0x80, 0xB4, 0xB9, 0xD6, 0xB3, + 0xCE, 0xFE, 0xA1, 0x15, 0xB2, 0xCE, 0xB8, 0x78, 0x9B, 0xB8, 0xA3, 0xB0, + 0xEA, 0x87, 0xFE, 0xBE, 0x63, 0xB6, 0xC8, 0xF8, 0x46, 0xEC, 0x6D, 0xB0, + 0xC2, 0x6C, 0x5D, 0x7C +}; + +typedef struct { + DH *(*get_param) (void); + const unsigned char *xA; + size_t xA_len; + const unsigned char *yA; + size_t yA_len; + const unsigned char *xB; + size_t xB_len; + const unsigned char *yB; + size_t yB_len; + const unsigned char *Z; + size_t Z_len; +} rfc5114_td; + +# define make_rfc5114_td(pre) { \ + DH_get_##pre, \ + dhtest_##pre##_xA, sizeof(dhtest_##pre##_xA), \ + dhtest_##pre##_yA, sizeof(dhtest_##pre##_yA), \ + dhtest_##pre##_xB, sizeof(dhtest_##pre##_xB), \ + dhtest_##pre##_yB, sizeof(dhtest_##pre##_yB), \ + dhtest_##pre##_Z, sizeof(dhtest_##pre##_Z) \ + } + +static const rfc5114_td rfctd[] = { + make_rfc5114_td(1024_160), + make_rfc5114_td(2048_224), + make_rfc5114_td(2048_256) +}; + +static int run_rfc5114_tests(void) +{ + int i; + for (i = 0; i < (int)(sizeof(rfctd) / sizeof(rfc5114_td)); i++) { + DH *dhA, *dhB; + unsigned char *Z1 = NULL, *Z2 = NULL; + const rfc5114_td *td = rfctd + i; + /* Set up DH structures setting key components */ + dhA = td->get_param(); + dhB = td->get_param(); + if (!dhA || !dhB) + goto bad_err; + + dhA->priv_key = BN_bin2bn(td->xA, td->xA_len, NULL); + dhA->pub_key = BN_bin2bn(td->yA, td->yA_len, NULL); + + dhB->priv_key = BN_bin2bn(td->xB, td->xB_len, NULL); + dhB->pub_key = BN_bin2bn(td->yB, td->yB_len, NULL); + + if (!dhA->priv_key || !dhA->pub_key + || !dhB->priv_key || !dhB->pub_key) + goto bad_err; + + if ((td->Z_len != (size_t)DH_size(dhA)) + || (td->Z_len != (size_t)DH_size(dhB))) + goto err; + + Z1 = OPENSSL_malloc(DH_size(dhA)); + Z2 = OPENSSL_malloc(DH_size(dhB)); + /* + * Work out shared secrets using both sides and compare with expected + * values. + */ + if (!DH_compute_key(Z1, dhB->pub_key, dhA)) + goto bad_err; + if (!DH_compute_key(Z2, dhA->pub_key, dhB)) + goto bad_err; + + if (memcmp(Z1, td->Z, td->Z_len)) + goto err; + if (memcmp(Z2, td->Z, td->Z_len)) + goto err; + + printf("RFC5114 parameter test %d OK\n", i + 1); + + DH_free(dhA); + DH_free(dhB); + OPENSSL_free(Z1); + OPENSSL_free(Z2); + + } + return 1; + bad_err: + fprintf(stderr, "Initalisation error RFC5114 set %d\n", i + 1); + ERR_print_errors_fp(stderr); + return 0; + err: + fprintf(stderr, "Test failed RFC5114 set %d\n", i + 1); + return 0; +} + #endif diff --git a/crypto/dsa/dsa.h b/crypto/dsa/dsa.h index a2f0ee786370..545358fd02b2 100644 --- a/crypto/dsa/dsa.h +++ b/crypto/dsa/dsa.h @@ -287,6 +287,7 @@ void ERR_load_DSA_strings(void); # define DSA_F_DO_DSA_PRINT 104 # define DSA_F_DSAPARAMS_PRINT 100 # define DSA_F_DSAPARAMS_PRINT_FP 101 +# define DSA_F_DSA_BUILTIN_PARAMGEN2 126 # define DSA_F_DSA_DO_SIGN 112 # define DSA_F_DSA_DO_VERIFY 113 # define DSA_F_DSA_GENERATE_KEY 124 @@ -316,12 +317,14 @@ void ERR_load_DSA_strings(void); # define DSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE 100 # define DSA_R_DECODE_ERROR 104 # define DSA_R_INVALID_DIGEST_TYPE 106 +# define DSA_R_INVALID_PARAMETERS 112 # define DSA_R_MISSING_PARAMETERS 101 # define DSA_R_MODULUS_TOO_LARGE 103 # define DSA_R_NEED_NEW_SETUP_VALUES 110 # define DSA_R_NON_FIPS_DSA_METHOD 111 # define DSA_R_NO_PARAMETERS_SET 107 # define DSA_R_PARAMETER_ENCODING_ERROR 105 +# define DSA_R_Q_NOT_PRIME 113 #ifdef __cplusplus } diff --git a/crypto/dsa/dsa_ameth.c b/crypto/dsa/dsa_ameth.c index a2840eaed095..2a5cd71371a7 100644 --- a/crypto/dsa/dsa_ameth.c +++ b/crypto/dsa/dsa_ameth.c @@ -601,10 +601,14 @@ static int dsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2) X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0); } return 1; + + case ASN1_PKEY_CTRL_CMS_RI_TYPE: + *(int *)arg2 = CMS_RECIPINFO_NONE; + return 1; #endif case ASN1_PKEY_CTRL_DEFAULT_MD_NID: - *(int *)arg2 = NID_sha1; + *(int *)arg2 = NID_sha256; return 2; default: diff --git a/crypto/dsa/dsa_err.c b/crypto/dsa/dsa_err.c index 746f5dfe6dc7..f5ddc66b8a73 100644 --- a/crypto/dsa/dsa_err.c +++ b/crypto/dsa/dsa_err.c @@ -1,6 +1,6 @@ /* crypto/dsa/dsa_err.c */ /* ==================================================================== - * Copyright (c) 1999-2011 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2013 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -74,6 +74,7 @@ static ERR_STRING_DATA DSA_str_functs[] = { {ERR_FUNC(DSA_F_DO_DSA_PRINT), "DO_DSA_PRINT"}, {ERR_FUNC(DSA_F_DSAPARAMS_PRINT), "DSAparams_print"}, {ERR_FUNC(DSA_F_DSAPARAMS_PRINT_FP), "DSAparams_print_fp"}, + {ERR_FUNC(DSA_F_DSA_BUILTIN_PARAMGEN2), "DSA_BUILTIN_PARAMGEN2"}, {ERR_FUNC(DSA_F_DSA_DO_SIGN), "DSA_do_sign"}, {ERR_FUNC(DSA_F_DSA_DO_VERIFY), "DSA_do_verify"}, {ERR_FUNC(DSA_F_DSA_GENERATE_KEY), "DSA_generate_key"}, @@ -107,12 +108,14 @@ static ERR_STRING_DATA DSA_str_reasons[] = { "data too large for key size"}, {ERR_REASON(DSA_R_DECODE_ERROR), "decode error"}, {ERR_REASON(DSA_R_INVALID_DIGEST_TYPE), "invalid digest type"}, + {ERR_REASON(DSA_R_INVALID_PARAMETERS), "invalid parameters"}, {ERR_REASON(DSA_R_MISSING_PARAMETERS), "missing parameters"}, {ERR_REASON(DSA_R_MODULUS_TOO_LARGE), "modulus too large"}, {ERR_REASON(DSA_R_NEED_NEW_SETUP_VALUES), "need new setup values"}, {ERR_REASON(DSA_R_NON_FIPS_DSA_METHOD), "non fips dsa method"}, {ERR_REASON(DSA_R_NO_PARAMETERS_SET), "no parameters set"}, {ERR_REASON(DSA_R_PARAMETER_ENCODING_ERROR), "parameter encoding error"}, + {ERR_REASON(DSA_R_Q_NOT_PRIME), "q not prime"}, {0, NULL} }; diff --git a/crypto/dsa/dsa_gen.c b/crypto/dsa/dsa_gen.c index d686ab0af770..5a328aaab5b4 100644 --- a/crypto/dsa/dsa_gen.c +++ b/crypto/dsa/dsa_gen.c @@ -86,6 +86,8 @@ # include "dsa_locl.h" # ifdef OPENSSL_FIPS +/* Workaround bug in prototype */ +# define fips_dsa_builtin_paramgen2 fips_dsa_paramgen_bad # include # endif @@ -383,4 +385,371 @@ int dsa_builtin_paramgen(DSA *ret, size_t bits, size_t qbits, BN_MONT_CTX_free(mont); return ok; } + +# ifdef OPENSSL_FIPS +# undef fips_dsa_builtin_paramgen2 +extern int fips_dsa_builtin_paramgen2(DSA *ret, size_t L, size_t N, + const EVP_MD *evpmd, + const unsigned char *seed_in, + size_t seed_len, int idx, + unsigned char *seed_out, + int *counter_ret, unsigned long *h_ret, + BN_GENCB *cb); +# endif + +/* + * This is a parameter generation algorithm for the DSA2 algorithm as + * described in FIPS 186-3. + */ + +int dsa_builtin_paramgen2(DSA *ret, size_t L, size_t N, + const EVP_MD *evpmd, const unsigned char *seed_in, + size_t seed_len, int idx, unsigned char *seed_out, + int *counter_ret, unsigned long *h_ret, + BN_GENCB *cb) +{ + int ok = -1; + unsigned char *seed = NULL, *seed_tmp = NULL; + unsigned char md[EVP_MAX_MD_SIZE]; + int mdsize; + BIGNUM *r0, *W, *X, *c, *test; + BIGNUM *g = NULL, *q = NULL, *p = NULL; + BN_MONT_CTX *mont = NULL; + int i, k, n = 0, m = 0, qsize = N >> 3; + int counter = 0; + int r = 0; + BN_CTX *ctx = NULL; + EVP_MD_CTX mctx; + unsigned int h = 2; + +# ifdef OPENSSL_FIPS + + if (FIPS_mode()) + return fips_dsa_builtin_paramgen2(ret, L, N, evpmd, + seed_in, seed_len, idx, + seed_out, counter_ret, h_ret, cb); +# endif + + EVP_MD_CTX_init(&mctx); + + if (evpmd == NULL) { + if (N == 160) + evpmd = EVP_sha1(); + else if (N == 224) + evpmd = EVP_sha224(); + else + evpmd = EVP_sha256(); + } + + mdsize = EVP_MD_size(evpmd); + /* If unverificable g generation only don't need seed */ + if (!ret->p || !ret->q || idx >= 0) { + if (seed_len == 0) + seed_len = mdsize; + + seed = OPENSSL_malloc(seed_len); + + if (seed_out) + seed_tmp = seed_out; + else + seed_tmp = OPENSSL_malloc(seed_len); + + if (!seed || !seed_tmp) + goto err; + + if (seed_in) + memcpy(seed, seed_in, seed_len); + + } + + if ((ctx = BN_CTX_new()) == NULL) + goto err; + + if ((mont = BN_MONT_CTX_new()) == NULL) + goto err; + + BN_CTX_start(ctx); + r0 = BN_CTX_get(ctx); + g = BN_CTX_get(ctx); + W = BN_CTX_get(ctx); + X = BN_CTX_get(ctx); + c = BN_CTX_get(ctx); + test = BN_CTX_get(ctx); + + /* if p, q already supplied generate g only */ + if (ret->p && ret->q) { + p = ret->p; + q = ret->q; + if (idx >= 0) + memcpy(seed_tmp, seed, seed_len); + goto g_only; + } else { + p = BN_CTX_get(ctx); + q = BN_CTX_get(ctx); + } + + if (!BN_lshift(test, BN_value_one(), L - 1)) + goto err; + for (;;) { + for (;;) { /* find q */ + unsigned char *pmd; + /* step 1 */ + if (!BN_GENCB_call(cb, 0, m++)) + goto err; + + if (!seed_in) { + if (RAND_pseudo_bytes(seed, seed_len) < 0) + goto err; + } + /* step 2 */ + if (!EVP_Digest(seed, seed_len, md, NULL, evpmd, NULL)) + goto err; + /* Take least significant bits of md */ + if (mdsize > qsize) + pmd = md + mdsize - qsize; + else + pmd = md; + + if (mdsize < qsize) + memset(md + mdsize, 0, qsize - mdsize); + + /* step 3 */ + pmd[0] |= 0x80; + pmd[qsize - 1] |= 0x01; + if (!BN_bin2bn(pmd, qsize, q)) + goto err; + + /* step 4 */ + r = BN_is_prime_fasttest_ex(q, DSS_prime_checks, ctx, + seed_in ? 1 : 0, cb); + if (r > 0) + break; + if (r != 0) + goto err; + /* Provided seed didn't produce a prime: error */ + if (seed_in) { + ok = 0; + DSAerr(DSA_F_DSA_BUILTIN_PARAMGEN2, DSA_R_Q_NOT_PRIME); + goto err; + } + + /* do a callback call */ + /* step 5 */ + } + /* Copy seed to seed_out before we mess with it */ + if (seed_out) + memcpy(seed_out, seed, seed_len); + + if (!BN_GENCB_call(cb, 2, 0)) + goto err; + if (!BN_GENCB_call(cb, 3, 0)) + goto err; + + /* step 6 */ + counter = 0; + /* "offset = 1" */ + + n = (L - 1) / (mdsize << 3); + + for (;;) { + if ((counter != 0) && !BN_GENCB_call(cb, 0, counter)) + goto err; + + /* step 7 */ + BN_zero(W); + /* now 'buf' contains "SEED + offset - 1" */ + for (k = 0; k <= n; k++) { + /* + * obtain "SEED + offset + k" by incrementing: + */ + for (i = seed_len - 1; i >= 0; i--) { + seed[i]++; + if (seed[i] != 0) + break; + } + + if (!EVP_Digest(seed, seed_len, md, NULL, evpmd, NULL)) + goto err; + + /* step 8 */ + if (!BN_bin2bn(md, mdsize, r0)) + goto err; + if (!BN_lshift(r0, r0, (mdsize << 3) * k)) + goto err; + if (!BN_add(W, W, r0)) + goto err; + } + + /* more of step 8 */ + if (!BN_mask_bits(W, L - 1)) + goto err; + if (!BN_copy(X, W)) + goto err; + if (!BN_add(X, X, test)) + goto err; + + /* step 9 */ + if (!BN_lshift1(r0, q)) + goto err; + if (!BN_mod(c, X, r0, ctx)) + goto err; + if (!BN_sub(r0, c, BN_value_one())) + goto err; + if (!BN_sub(p, X, r0)) + goto err; + + /* step 10 */ + if (BN_cmp(p, test) >= 0) { + /* step 11 */ + r = BN_is_prime_fasttest_ex(p, DSS_prime_checks, ctx, 1, cb); + if (r > 0) + goto end; /* found it */ + if (r != 0) + goto err; + } + + /* step 13 */ + counter++; + /* "offset = offset + n + 1" */ + + /* step 14 */ + if (counter >= (int)(4 * L)) + break; + } + if (seed_in) { + ok = 0; + DSAerr(DSA_F_DSA_BUILTIN_PARAMGEN2, DSA_R_INVALID_PARAMETERS); + goto err; + } + } + end: + if (!BN_GENCB_call(cb, 2, 1)) + goto err; + + g_only: + + /* We now need to generate g */ + /* Set r0=(p-1)/q */ + if (!BN_sub(test, p, BN_value_one())) + goto err; + if (!BN_div(r0, NULL, test, q, ctx)) + goto err; + + if (idx < 0) { + if (!BN_set_word(test, h)) + goto err; + } else + h = 1; + if (!BN_MONT_CTX_set(mont, p, ctx)) + goto err; + + for (;;) { + static const unsigned char ggen[4] = { 0x67, 0x67, 0x65, 0x6e }; + if (idx >= 0) { + md[0] = idx & 0xff; + md[1] = (h >> 8) & 0xff; + md[2] = h & 0xff; + if (!EVP_DigestInit_ex(&mctx, evpmd, NULL)) + goto err; + if (!EVP_DigestUpdate(&mctx, seed_tmp, seed_len)) + goto err; + if (!EVP_DigestUpdate(&mctx, ggen, sizeof(ggen))) + goto err; + if (!EVP_DigestUpdate(&mctx, md, 3)) + goto err; + if (!EVP_DigestFinal_ex(&mctx, md, NULL)) + goto err; + if (!BN_bin2bn(md, mdsize, test)) + goto err; + } + /* g=test^r0%p */ + if (!BN_mod_exp_mont(g, test, r0, p, ctx, mont)) + goto err; + if (!BN_is_one(g)) + break; + if (idx < 0 && !BN_add(test, test, BN_value_one())) + goto err; + h++; + if (idx >= 0 && h > 0xffff) + goto err; + } + + if (!BN_GENCB_call(cb, 3, 1)) + goto err; + + ok = 1; + err: + if (ok == 1) { + if (p != ret->p) { + if (ret->p) + BN_free(ret->p); + ret->p = BN_dup(p); + } + if (q != ret->q) { + if (ret->q) + BN_free(ret->q); + ret->q = BN_dup(q); + } + if (ret->g) + BN_free(ret->g); + ret->g = BN_dup(g); + if (ret->p == NULL || ret->q == NULL || ret->g == NULL) { + ok = -1; + goto err; + } + if (counter_ret != NULL) + *counter_ret = counter; + if (h_ret != NULL) + *h_ret = h; + } + if (seed) + OPENSSL_free(seed); + if (seed_out != seed_tmp) + OPENSSL_free(seed_tmp); + if (ctx) { + BN_CTX_end(ctx); + BN_CTX_free(ctx); + } + if (mont != NULL) + BN_MONT_CTX_free(mont); + EVP_MD_CTX_cleanup(&mctx); + return ok; +} + +int dsa_paramgen_check_g(DSA *dsa) +{ + BN_CTX *ctx; + BIGNUM *tmp; + BN_MONT_CTX *mont = NULL; + int rv = -1; + ctx = BN_CTX_new(); + if (!ctx) + return -1; + BN_CTX_start(ctx); + if (BN_cmp(dsa->g, BN_value_one()) <= 0) + return 0; + if (BN_cmp(dsa->g, dsa->p) >= 0) + return 0; + tmp = BN_CTX_get(ctx); + if (!tmp) + goto err; + if ((mont = BN_MONT_CTX_new()) == NULL) + goto err; + if (!BN_MONT_CTX_set(mont, dsa->p, ctx)) + goto err; + /* Work out g^q mod p */ + if (!BN_mod_exp_mont(tmp, dsa->g, dsa->q, dsa->p, ctx, mont)) + goto err; + if (!BN_cmp(tmp, BN_value_one())) + rv = 1; + else + rv = 0; + err: + BN_CTX_end(ctx); + if (mont) + BN_MONT_CTX_free(mont); + BN_CTX_free(ctx); + return rv; + +} #endif diff --git a/crypto/dsa/dsa_locl.h b/crypto/dsa/dsa_locl.h index f32ee964d0e3..9c23c3ef90e2 100644 --- a/crypto/dsa/dsa_locl.h +++ b/crypto/dsa/dsa_locl.h @@ -59,3 +59,11 @@ int dsa_builtin_paramgen(DSA *ret, size_t bits, size_t qbits, size_t seed_len, unsigned char *seed_out, int *counter_ret, unsigned long *h_ret, BN_GENCB *cb); + +int dsa_builtin_paramgen2(DSA *ret, size_t L, size_t N, + const EVP_MD *evpmd, const unsigned char *seed_in, + size_t seed_len, int idx, unsigned char *seed_out, + int *counter_ret, unsigned long *h_ret, + BN_GENCB *cb); + +int dsa_paramgen_check_g(DSA *dsa); diff --git a/crypto/dsa/dsa_ossl.c b/crypto/dsa/dsa_ossl.c index 6edb26d97397..f0ec8faa84cc 100644 --- a/crypto/dsa/dsa_ossl.c +++ b/crypto/dsa/dsa_ossl.c @@ -398,11 +398,7 @@ static int dsa_do_verify(const unsigned char *dgst, int dgst_len, ret = (BN_ucmp(&u1, sig->r) == 0); err: - /* - * XXX: surely this is wrong - if ret is 0, it just didn't verify; there - * is no error in BN. Test should be ret == -1 (Ben) - */ - if (ret != 1) + if (ret < 0) DSAerr(DSA_F_DSA_DO_VERIFY, ERR_R_BN_LIB); if (ctx != NULL) BN_CTX_free(ctx); diff --git a/crypto/dsa/dsa_pmeth.c b/crypto/dsa/dsa_pmeth.c index 0d480f6a707e..42b8bb086251 100644 --- a/crypto/dsa/dsa_pmeth.c +++ b/crypto/dsa/dsa_pmeth.c @@ -197,6 +197,10 @@ static int pkey_dsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) dctx->md = p2; return 1; + case EVP_PKEY_CTRL_GET_MD: + *(const EVP_MD **)p2 = dctx->md; + return 1; + case EVP_PKEY_CTRL_DIGESTINIT: case EVP_PKEY_CTRL_PKCS7_SIGN: case EVP_PKEY_CTRL_CMS_SIGN: diff --git a/crypto/ebcdic.c b/crypto/ebcdic.c index 4b7652c0ecf7..fd6df92b468b 100644 --- a/crypto/ebcdic.c +++ b/crypto/ebcdic.c @@ -3,7 +3,7 @@ #ifndef CHARSET_EBCDIC # include -# if defined(PEDANTIC) || defined(__DECC) || defined(OPENSSL_SYS_MACOSX) +# if defined(PEDANTIC) || defined(__DECC) || defined(OPENSSL_SYS_MACOSX) || defined(__clang__) static void *dummy = &dummy; # endif diff --git a/crypto/ec/Makefile b/crypto/ec/Makefile index 2753b28bec0a..359ef4e40fd4 100644 --- a/crypto/ec/Makefile +++ b/crypto/ec/Makefile @@ -11,6 +11,8 @@ MAKEFILE= Makefile AR= ar r CFLAGS= $(INCLUDES) $(CFLAG) +ASFLAGS= $(INCLUDES) $(ASFLAG) +AFLAGS= $(ASFLAGS) GENERAL=Makefile TEST=ectest.c @@ -27,7 +29,7 @@ LIBOBJ= ec_lib.o ecp_smpl.o ecp_mont.o ecp_nist.o ec_cvt.o ec_mult.o\ ec_err.o ec_curve.o ec_check.o ec_print.o ec_asn1.o ec_key.o\ ec2_smpl.o ec2_mult.o ec_ameth.o ec_pmeth.o eck_prn.o \ ecp_nistp224.o ecp_nistp256.o ecp_nistp521.o ecp_nistputil.o \ - ecp_oct.o ec2_oct.o ec_oct.o + ecp_oct.o ec2_oct.o ec_oct.o $(EC_ASM) SRC= $(LIBSRC) @@ -46,6 +48,12 @@ lib: $(LIBOBJ) $(RANLIB) $(LIB) || echo Never mind. @touch lib +ecp_nistz256-x86_64.s: asm/ecp_nistz256-x86_64.pl + $(PERL) asm/ecp_nistz256-x86_64.pl $(PERLASM_SCHEME) > $@ + +ecp_nistz256-avx2.s: asm/ecp_nistz256-avx2.pl + $(PERL) asm/ecp_nistz256-avx2.pl $(PERLASM_SCHEME) > $@ + files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO @@ -110,14 +118,14 @@ ec2_smpl.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h ec2_smpl.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h ec2_smpl.o: ../../include/openssl/symhacks.h ec2_smpl.c ec_lcl.h ec_ameth.o: ../../e_os.h ../../include/openssl/asn1.h -ec_ameth.o: ../../include/openssl/bio.h ../../include/openssl/bn.h -ec_ameth.o: ../../include/openssl/buffer.h ../../include/openssl/cms.h -ec_ameth.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h -ec_ameth.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h -ec_ameth.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h -ec_ameth.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h -ec_ameth.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h -ec_ameth.o: ../../include/openssl/opensslconf.h +ec_ameth.o: ../../include/openssl/asn1t.h ../../include/openssl/bio.h +ec_ameth.o: ../../include/openssl/bn.h ../../include/openssl/buffer.h +ec_ameth.o: ../../include/openssl/cms.h ../../include/openssl/crypto.h +ec_ameth.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h +ec_ameth.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h +ec_ameth.o: ../../include/openssl/err.h ../../include/openssl/evp.h +ec_ameth.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h +ec_ameth.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h ec_ameth.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h ec_ameth.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h ec_ameth.o: ../../include/openssl/sha.h ../../include/openssl/stack.h @@ -198,18 +206,19 @@ ec_oct.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h ec_oct.o: ../../include/openssl/symhacks.h ec_lcl.h ec_oct.c ec_pmeth.o: ../../e_os.h ../../include/openssl/asn1.h ec_pmeth.o: ../../include/openssl/asn1t.h ../../include/openssl/bio.h -ec_pmeth.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h -ec_pmeth.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h -ec_pmeth.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h -ec_pmeth.o: ../../include/openssl/err.h ../../include/openssl/evp.h -ec_pmeth.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h -ec_pmeth.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h +ec_pmeth.o: ../../include/openssl/bn.h ../../include/openssl/buffer.h +ec_pmeth.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h +ec_pmeth.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h +ec_pmeth.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h +ec_pmeth.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h +ec_pmeth.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h +ec_pmeth.o: ../../include/openssl/opensslconf.h ec_pmeth.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h ec_pmeth.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h ec_pmeth.o: ../../include/openssl/sha.h ../../include/openssl/stack.h ec_pmeth.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h ec_pmeth.o: ../../include/openssl/x509_vfy.h ../cryptlib.h ../evp/evp_locl.h -ec_pmeth.o: ec_pmeth.c +ec_pmeth.o: ec_lcl.h ec_pmeth.c ec_print.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h ec_print.o: ../../include/openssl/bn.h ../../include/openssl/crypto.h ec_print.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h diff --git a/crypto/ec/asm/ecp_nistz256-avx2.pl b/crypto/ec/asm/ecp_nistz256-avx2.pl new file mode 100755 index 000000000000..4c220aa645f1 --- /dev/null +++ b/crypto/ec/asm/ecp_nistz256-avx2.pl @@ -0,0 +1,2093 @@ +#!/usr/bin/env perl + +############################################################################## +# # +# Copyright 2014 Intel Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# # +############################################################################## +# # +# Developers and authors: # +# Shay Gueron (1, 2), and Vlad Krasnov (1) # +# (1) Intel Corporation, Israel Development Center # +# (2) University of Haifa # +# Reference: # +# S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with# +# 256 Bit Primes" # +# # +############################################################################## + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); + $addx = ($1>=2.23); +} + +if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); + $addx = ($1>=2.10); +} + +if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + $addx = ($1>=12); +} + +if (!$addx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { + my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 + $avx = ($ver>=3.0) + ($ver>=3.01); + $addx = ($ver>=3.03); +} + +if ($avx>=2) {{ +$digit_size = "\$29"; +$n_digits = "\$9"; + +$code.=<<___; +.text + +.align 64 +.LAVX2_AND_MASK: +.LAVX2_POLY: +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x000001ff, 0x000001ff, 0x000001ff, 0x000001ff +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00040000, 0x00040000, 0x00040000, 0x00040000 +.quad 0x1fe00000, 0x1fe00000, 0x1fe00000, 0x1fe00000 +.quad 0x00ffffff, 0x00ffffff, 0x00ffffff, 0x00ffffff + +.LAVX2_POLY_x2: +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x400007FC, 0x400007FC, 0x400007FC, 0x400007FC +.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE +.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE +.quad 0x400FFFFE, 0x400FFFFE, 0x400FFFFE, 0x400FFFFE +.quad 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE +.quad 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC + +.LAVX2_POLY_x8: +.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8 +.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8 +.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8 +.quad 0x80000FF8, 0x80000FF8, 0x80000FF8, 0x80000FF8 +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC +.quad 0x801FFFFC, 0x801FFFFC, 0x801FFFFC, 0x801FFFFC +.quad 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC +.quad 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8 + +.LONE: +.quad 0x00000020, 0x00000020, 0x00000020, 0x00000020 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x1fffc000, 0x1fffc000, 0x1fffc000, 0x1fffc000 +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1f7fffff, 0x1f7fffff, 0x1f7fffff, 0x1f7fffff +.quad 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +# RR = 2^266 mod p in AVX2 format, to transform from the native OpenSSL +# Montgomery form (*2^256) to our format (*2^261) + +.LTO_MONT_AVX2: +.quad 0x00000400, 0x00000400, 0x00000400, 0x00000400 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x1ff80000, 0x1ff80000, 0x1ff80000, 0x1ff80000 +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x0fffffff, 0x0fffffff, 0x0fffffff, 0x0fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x00000003, 0x00000003, 0x00000003, 0x00000003 + +.LFROM_MONT_AVX2: +.quad 0x00000001, 0x00000001, 0x00000001, 0x00000001 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 +.quad 0x1ffffe00, 0x1ffffe00, 0x1ffffe00, 0x1ffffe00 +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff +.quad 0x1ffbffff, 0x1ffbffff, 0x1ffbffff, 0x1ffbffff +.quad 0x001fffff, 0x001fffff, 0x001fffff, 0x001fffff +.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +.LIntOne: +.long 1,1,1,1,1,1,1,1 +___ + +{ +# This function recieves a pointer to an array of four affine points +# (X, Y, <1>) and rearanges the data for AVX2 execution, while +# converting it to 2^29 radix redundant form + +my ($X0,$X1,$X2,$X3, $Y0,$Y1,$Y2,$Y3, + $T0,$T1,$T2,$T3, $T4,$T5,$T6,$T7)=map("%ymm$_",(0..15)); + +$code.=<<___; +.globl ecp_nistz256_avx2_transpose_convert +.type ecp_nistz256_avx2_transpose_convert,\@function,2 +.align 64 +ecp_nistz256_avx2_transpose_convert: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -8-16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + # Load the data + vmovdqa 32*0(%rsi), $X0 + lea 112(%rsi), %rax # size optimization + vmovdqa 32*1(%rsi), $Y0 + lea .LAVX2_AND_MASK(%rip), %rdx + vmovdqa 32*2(%rsi), $X1 + vmovdqa 32*3(%rsi), $Y1 + vmovdqa 32*4-112(%rax), $X2 + vmovdqa 32*5-112(%rax), $Y2 + vmovdqa 32*6-112(%rax), $X3 + vmovdqa 32*7-112(%rax), $Y3 + + # Transpose X and Y independently + vpunpcklqdq $X1, $X0, $T0 # T0 = [B2 A2 B0 A0] + vpunpcklqdq $X3, $X2, $T1 # T1 = [D2 C2 D0 C0] + vpunpckhqdq $X1, $X0, $T2 # T2 = [B3 A3 B1 A1] + vpunpckhqdq $X3, $X2, $T3 # T3 = [D3 C3 D1 C1] + + vpunpcklqdq $Y1, $Y0, $T4 + vpunpcklqdq $Y3, $Y2, $T5 + vpunpckhqdq $Y1, $Y0, $T6 + vpunpckhqdq $Y3, $Y2, $T7 + + vperm2i128 \$0x20, $T1, $T0, $X0 # X0 = [D0 C0 B0 A0] + vperm2i128 \$0x20, $T3, $T2, $X1 # X1 = [D1 C1 B1 A1] + vperm2i128 \$0x31, $T1, $T0, $X2 # X2 = [D2 C2 B2 A2] + vperm2i128 \$0x31, $T3, $T2, $X3 # X3 = [D3 C3 B3 A3] + + vperm2i128 \$0x20, $T5, $T4, $Y0 + vperm2i128 \$0x20, $T7, $T6, $Y1 + vperm2i128 \$0x31, $T5, $T4, $Y2 + vperm2i128 \$0x31, $T7, $T6, $Y3 + vmovdqa (%rdx), $T7 + + vpand (%rdx), $X0, $T0 # out[0] = in[0] & mask; + vpsrlq \$29, $X0, $X0 + vpand $T7, $X0, $T1 # out[1] = (in[0] >> shift) & mask; + vpsrlq \$29, $X0, $X0 + vpsllq \$6, $X1, $T2 + vpxor $X0, $T2, $T2 + vpand $T7, $T2, $T2 # out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask; + vpsrlq \$23, $X1, $X1 + vpand $T7, $X1, $T3 # out[3] = (in[1] >> ((shift*3)%64)) & mask; + vpsrlq \$29, $X1, $X1 + vpsllq \$12, $X2, $T4 + vpxor $X1, $T4, $T4 + vpand $T7, $T4, $T4 # out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask; + vpsrlq \$17, $X2, $X2 + vpand $T7, $X2, $T5 # out[5] = (in[2] >> ((shift*5)%64)) & mask; + vpsrlq \$29, $X2, $X2 + vpsllq \$18, $X3, $T6 + vpxor $X2, $T6, $T6 + vpand $T7, $T6, $T6 # out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask; + vpsrlq \$11, $X3, $X3 + vmovdqa $T0, 32*0(%rdi) + lea 112(%rdi), %rax # size optimization + vpand $T7, $X3, $T0 # out[7] = (in[3] >> ((shift*7)%64)) & mask; + vpsrlq \$29, $X3, $X3 # out[8] = (in[3] >> ((shift*8)%64)) & mask; + + vmovdqa $T1, 32*1(%rdi) + vmovdqa $T2, 32*2(%rdi) + vmovdqa $T3, 32*3(%rdi) + vmovdqa $T4, 32*4-112(%rax) + vmovdqa $T5, 32*5-112(%rax) + vmovdqa $T6, 32*6-112(%rax) + vmovdqa $T0, 32*7-112(%rax) + vmovdqa $X3, 32*8-112(%rax) + lea 448(%rdi), %rax # size optimization + + vpand $T7, $Y0, $T0 # out[0] = in[0] & mask; + vpsrlq \$29, $Y0, $Y0 + vpand $T7, $Y0, $T1 # out[1] = (in[0] >> shift) & mask; + vpsrlq \$29, $Y0, $Y0 + vpsllq \$6, $Y1, $T2 + vpxor $Y0, $T2, $T2 + vpand $T7, $T2, $T2 # out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask; + vpsrlq \$23, $Y1, $Y1 + vpand $T7, $Y1, $T3 # out[3] = (in[1] >> ((shift*3)%64)) & mask; + vpsrlq \$29, $Y1, $Y1 + vpsllq \$12, $Y2, $T4 + vpxor $Y1, $T4, $T4 + vpand $T7, $T4, $T4 # out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask; + vpsrlq \$17, $Y2, $Y2 + vpand $T7, $Y2, $T5 # out[5] = (in[2] >> ((shift*5)%64)) & mask; + vpsrlq \$29, $Y2, $Y2 + vpsllq \$18, $Y3, $T6 + vpxor $Y2, $T6, $T6 + vpand $T7, $T6, $T6 # out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask; + vpsrlq \$11, $Y3, $Y3 + vmovdqa $T0, 32*9-448(%rax) + vpand $T7, $Y3, $T0 # out[7] = (in[3] >> ((shift*7)%64)) & mask; + vpsrlq \$29, $Y3, $Y3 # out[8] = (in[3] >> ((shift*8)%64)) & mask; + + vmovdqa $T1, 32*10-448(%rax) + vmovdqa $T2, 32*11-448(%rax) + vmovdqa $T3, 32*12-448(%rax) + vmovdqa $T4, 32*13-448(%rax) + vmovdqa $T5, 32*14-448(%rax) + vmovdqa $T6, 32*15-448(%rax) + vmovdqa $T0, 32*16-448(%rax) + vmovdqa $Y3, 32*17-448(%rax) + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*0(%rsp), %xmm6 + movaps 16*1(%rsp), %xmm7 + movaps 16*2(%rsp), %xmm8 + movaps 16*3(%rsp), %xmm9 + movaps 16*4(%rsp), %xmm10 + movaps 16*5(%rsp), %xmm11 + movaps 16*6(%rsp), %xmm12 + movaps 16*7(%rsp), %xmm13 + movaps 16*8(%rsp), %xmm14 + movaps 16*9(%rsp), %xmm15 + lea 8+16*10(%rsp), %rsp +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_transpose_convert,.-ecp_nistz256_avx2_transpose_convert +___ +} +{ +################################################################################ +# This function recieves a pointer to an array of four AVX2 formatted points +# (X, Y, Z) convert the data to normal representation, and rearanges the data + +my ($D0,$D1,$D2,$D3, $D4,$D5,$D6,$D7, $D8)=map("%ymm$_",(0..8)); +my ($T0,$T1,$T2,$T3, $T4,$T5,$T6)=map("%ymm$_",(9..15)); + +$code.=<<___; + +.globl ecp_nistz256_avx2_convert_transpose_back +.type ecp_nistz256_avx2_convert_transpose_back,\@function,2 +.align 32 +ecp_nistz256_avx2_convert_transpose_back: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -8-16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + mov \$3, %ecx + +.Lconv_loop: + vmovdqa 32*0(%rsi), $D0 + lea 160(%rsi), %rax # size optimization + vmovdqa 32*1(%rsi), $D1 + vmovdqa 32*2(%rsi), $D2 + vmovdqa 32*3(%rsi), $D3 + vmovdqa 32*4-160(%rax), $D4 + vmovdqa 32*5-160(%rax), $D5 + vmovdqa 32*6-160(%rax), $D6 + vmovdqa 32*7-160(%rax), $D7 + vmovdqa 32*8-160(%rax), $D8 + + vpsllq \$29, $D1, $D1 + vpsllq \$58, $D2, $T0 + vpaddq $D1, $D0, $D0 + vpaddq $T0, $D0, $D0 # out[0] = (in[0]) ^ (in[1] << shift*1) ^ (in[2] << shift*2); + + vpsrlq \$6, $D2, $D2 + vpsllq \$23, $D3, $D3 + vpsllq \$52, $D4, $T1 + vpaddq $D2, $D3, $D3 + vpaddq $D3, $T1, $D1 # out[1] = (in[2] >> (64*1-shift*2)) ^ (in[3] << shift*3%64) ^ (in[4] << shift*4%64); + + vpsrlq \$12, $D4, $D4 + vpsllq \$17, $D5, $D5 + vpsllq \$46, $D6, $T2 + vpaddq $D4, $D5, $D5 + vpaddq $D5, $T2, $D2 # out[2] = (in[4] >> (64*2-shift*4)) ^ (in[5] << shift*5%64) ^ (in[6] << shift*6%64); + + vpsrlq \$18, $D6, $D6 + vpsllq \$11, $D7, $D7 + vpsllq \$40, $D8, $T3 + vpaddq $D6, $D7, $D7 + vpaddq $D7, $T3, $D3 # out[3] = (in[6] >> (64*3-shift*6)) ^ (in[7] << shift*7%64) ^ (in[8] << shift*8%64); + + vpunpcklqdq $D1, $D0, $T0 # T0 = [B2 A2 B0 A0] + vpunpcklqdq $D3, $D2, $T1 # T1 = [D2 C2 D0 C0] + vpunpckhqdq $D1, $D0, $T2 # T2 = [B3 A3 B1 A1] + vpunpckhqdq $D3, $D2, $T3 # T3 = [D3 C3 D1 C1] + + vperm2i128 \$0x20, $T1, $T0, $D0 # X0 = [D0 C0 B0 A0] + vperm2i128 \$0x20, $T3, $T2, $D1 # X1 = [D1 C1 B1 A1] + vperm2i128 \$0x31, $T1, $T0, $D2 # X2 = [D2 C2 B2 A2] + vperm2i128 \$0x31, $T3, $T2, $D3 # X3 = [D3 C3 B3 A3] + + vmovdqa $D0, 32*0(%rdi) + vmovdqa $D1, 32*3(%rdi) + vmovdqa $D2, 32*6(%rdi) + vmovdqa $D3, 32*9(%rdi) + + lea 32*9(%rsi), %rsi + lea 32*1(%rdi), %rdi + + dec %ecx + jnz .Lconv_loop + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*0(%rsp), %xmm6 + movaps 16*1(%rsp), %xmm7 + movaps 16*2(%rsp), %xmm8 + movaps 16*3(%rsp), %xmm9 + movaps 16*4(%rsp), %xmm10 + movaps 16*5(%rsp), %xmm11 + movaps 16*6(%rsp), %xmm12 + movaps 16*7(%rsp), %xmm13 + movaps 16*8(%rsp), %xmm14 + movaps 16*9(%rsp), %xmm15 + lea 8+16*10(%rsp), %rsp +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_convert_transpose_back,.-ecp_nistz256_avx2_convert_transpose_back +___ +} +{ +my ($r_ptr,$a_ptr,$b_ptr,$itr)=("%rdi","%rsi","%rdx","%ecx"); +my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4,$ACC5,$ACC6,$ACC7,$ACC8)=map("%ymm$_",(0..8)); +my ($B,$Y,$T0,$AND_MASK,$OVERFLOW)=map("%ymm$_",(9..13)); + +sub NORMALIZE { +my $ret=<<___; + vpsrlq $digit_size, $ACC0, $T0 + vpand $AND_MASK, $ACC0, $ACC0 + vpaddq $T0, $ACC1, $ACC1 + + vpsrlq $digit_size, $ACC1, $T0 + vpand $AND_MASK, $ACC1, $ACC1 + vpaddq $T0, $ACC2, $ACC2 + + vpsrlq $digit_size, $ACC2, $T0 + vpand $AND_MASK, $ACC2, $ACC2 + vpaddq $T0, $ACC3, $ACC3 + + vpsrlq $digit_size, $ACC3, $T0 + vpand $AND_MASK, $ACC3, $ACC3 + vpaddq $T0, $ACC4, $ACC4 + + vpsrlq $digit_size, $ACC4, $T0 + vpand $AND_MASK, $ACC4, $ACC4 + vpaddq $T0, $ACC5, $ACC5 + + vpsrlq $digit_size, $ACC5, $T0 + vpand $AND_MASK, $ACC5, $ACC5 + vpaddq $T0, $ACC6, $ACC6 + + vpsrlq $digit_size, $ACC6, $T0 + vpand $AND_MASK, $ACC6, $ACC6 + vpaddq $T0, $ACC7, $ACC7 + + vpsrlq $digit_size, $ACC7, $T0 + vpand $AND_MASK, $ACC7, $ACC7 + vpaddq $T0, $ACC8, $ACC8 + #vpand $AND_MASK, $ACC8, $ACC8 +___ + $ret; +} + +sub STORE { +my $ret=<<___; + vmovdqa $ACC0, 32*0(%rdi) + lea 160(%rdi), %rax # size optimization + vmovdqa $ACC1, 32*1(%rdi) + vmovdqa $ACC2, 32*2(%rdi) + vmovdqa $ACC3, 32*3(%rdi) + vmovdqa $ACC4, 32*4-160(%rax) + vmovdqa $ACC5, 32*5-160(%rax) + vmovdqa $ACC6, 32*6-160(%rax) + vmovdqa $ACC7, 32*7-160(%rax) + vmovdqa $ACC8, 32*8-160(%rax) +___ + $ret; +} + +$code.=<<___; +.type avx2_normalize,\@abi-omnipotent +.align 32 +avx2_normalize: + vpsrlq $digit_size, $ACC0, $T0 + vpand $AND_MASK, $ACC0, $ACC0 + vpaddq $T0, $ACC1, $ACC1 + + vpsrlq $digit_size, $ACC1, $T0 + vpand $AND_MASK, $ACC1, $ACC1 + vpaddq $T0, $ACC2, $ACC2 + + vpsrlq $digit_size, $ACC2, $T0 + vpand $AND_MASK, $ACC2, $ACC2 + vpaddq $T0, $ACC3, $ACC3 + + vpsrlq $digit_size, $ACC3, $T0 + vpand $AND_MASK, $ACC3, $ACC3 + vpaddq $T0, $ACC4, $ACC4 + + vpsrlq $digit_size, $ACC4, $T0 + vpand $AND_MASK, $ACC4, $ACC4 + vpaddq $T0, $ACC5, $ACC5 + + vpsrlq $digit_size, $ACC5, $T0 + vpand $AND_MASK, $ACC5, $ACC5 + vpaddq $T0, $ACC6, $ACC6 + + vpsrlq $digit_size, $ACC6, $T0 + vpand $AND_MASK, $ACC6, $ACC6 + vpaddq $T0, $ACC7, $ACC7 + + vpsrlq $digit_size, $ACC7, $T0 + vpand $AND_MASK, $ACC7, $ACC7 + vpaddq $T0, $ACC8, $ACC8 + #vpand $AND_MASK, $ACC8, $ACC8 + + ret +.size avx2_normalize,.-avx2_normalize + +.type avx2_normalize_n_store,\@abi-omnipotent +.align 32 +avx2_normalize_n_store: + vpsrlq $digit_size, $ACC0, $T0 + vpand $AND_MASK, $ACC0, $ACC0 + vpaddq $T0, $ACC1, $ACC1 + + vpsrlq $digit_size, $ACC1, $T0 + vpand $AND_MASK, $ACC1, $ACC1 + vmovdqa $ACC0, 32*0(%rdi) + lea 160(%rdi), %rax # size optimization + vpaddq $T0, $ACC2, $ACC2 + + vpsrlq $digit_size, $ACC2, $T0 + vpand $AND_MASK, $ACC2, $ACC2 + vmovdqa $ACC1, 32*1(%rdi) + vpaddq $T0, $ACC3, $ACC3 + + vpsrlq $digit_size, $ACC3, $T0 + vpand $AND_MASK, $ACC3, $ACC3 + vmovdqa $ACC2, 32*2(%rdi) + vpaddq $T0, $ACC4, $ACC4 + + vpsrlq $digit_size, $ACC4, $T0 + vpand $AND_MASK, $ACC4, $ACC4 + vmovdqa $ACC3, 32*3(%rdi) + vpaddq $T0, $ACC5, $ACC5 + + vpsrlq $digit_size, $ACC5, $T0 + vpand $AND_MASK, $ACC5, $ACC5 + vmovdqa $ACC4, 32*4-160(%rax) + vpaddq $T0, $ACC6, $ACC6 + + vpsrlq $digit_size, $ACC6, $T0 + vpand $AND_MASK, $ACC6, $ACC6 + vmovdqa $ACC5, 32*5-160(%rax) + vpaddq $T0, $ACC7, $ACC7 + + vpsrlq $digit_size, $ACC7, $T0 + vpand $AND_MASK, $ACC7, $ACC7 + vmovdqa $ACC6, 32*6-160(%rax) + vpaddq $T0, $ACC8, $ACC8 + #vpand $AND_MASK, $ACC8, $ACC8 + vmovdqa $ACC7, 32*7-160(%rax) + vmovdqa $ACC8, 32*8-160(%rax) + + ret +.size avx2_normalize_n_store,.-avx2_normalize_n_store + +################################################################################ +# void avx2_mul_x4(void* RESULTx4, void *Ax4, void *Bx4); +.type avx2_mul_x4,\@abi-omnipotent +.align 32 +avx2_mul_x4: + lea .LAVX2_POLY(%rip), %rax + + vpxor $ACC0, $ACC0, $ACC0 + vpxor $ACC1, $ACC1, $ACC1 + vpxor $ACC2, $ACC2, $ACC2 + vpxor $ACC3, $ACC3, $ACC3 + vpxor $ACC4, $ACC4, $ACC4 + vpxor $ACC5, $ACC5, $ACC5 + vpxor $ACC6, $ACC6, $ACC6 + vpxor $ACC7, $ACC7, $ACC7 + + vmovdqa 32*7(%rax), %ymm14 + vmovdqa 32*8(%rax), %ymm15 + + mov $n_digits, $itr + lea -512($a_ptr), $a_ptr # strategic bias to control u-op density + jmp .Lavx2_mul_x4_loop + +.align 32 +.Lavx2_mul_x4_loop: + vmovdqa 32*0($b_ptr), $B + lea 32*1($b_ptr), $b_ptr + + vpmuludq 32*0+512($a_ptr), $B, $T0 + vpmuludq 32*1+512($a_ptr), $B, $OVERFLOW # borrow $OVERFLOW + vpaddq $T0, $ACC0, $ACC0 + vpmuludq 32*2+512($a_ptr), $B, $T0 + vpaddq $OVERFLOW, $ACC1, $ACC1 + vpand $AND_MASK, $ACC0, $Y + vpmuludq 32*3+512($a_ptr), $B, $OVERFLOW + vpaddq $T0, $ACC2, $ACC2 + vpmuludq 32*4+512($a_ptr), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*5+512($a_ptr), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*6+512($a_ptr), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*7+512($a_ptr), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + # Skip some multiplications, optimizing for the constant poly + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*8+512($a_ptr), $B, $ACC8 + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + .byte 0x67 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $OVERFLOW + .byte 0x67 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $T0 + vpaddq $OVERFLOW, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $OVERFLOW + vpaddq $T0, $ACC7, $ACC6 + vpaddq $OVERFLOW, $ACC8, $ACC7 + + dec $itr + jnz .Lavx2_mul_x4_loop + + vpxor $ACC8, $ACC8, $ACC8 + + ret +.size avx2_mul_x4,.-avx2_mul_x4 + +# Function optimized for the constant 1 +################################################################################ +# void avx2_mul_by1_x4(void* RESULTx4, void *Ax4); +.type avx2_mul_by1_x4,\@abi-omnipotent +.align 32 +avx2_mul_by1_x4: + lea .LAVX2_POLY(%rip), %rax + + vpxor $ACC0, $ACC0, $ACC0 + vpxor $ACC1, $ACC1, $ACC1 + vpxor $ACC2, $ACC2, $ACC2 + vpxor $ACC3, $ACC3, $ACC3 + vpxor $ACC4, $ACC4, $ACC4 + vpxor $ACC5, $ACC5, $ACC5 + vpxor $ACC6, $ACC6, $ACC6 + vpxor $ACC7, $ACC7, $ACC7 + vpxor $ACC8, $ACC8, $ACC8 + + vmovdqa 32*3+.LONE(%rip), %ymm14 + vmovdqa 32*7+.LONE(%rip), %ymm15 + + mov $n_digits, $itr + jmp .Lavx2_mul_by1_x4_loop + +.align 32 +.Lavx2_mul_by1_x4_loop: + vmovdqa 32*0($a_ptr), $B + .byte 0x48,0x8d,0xb6,0x20,0,0,0 # lea 32*1($a_ptr), $a_ptr + + vpsllq \$5, $B, $OVERFLOW + vpmuludq %ymm14, $B, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC3 + .byte 0x67 + vpmuludq $AND_MASK, $B, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $T0, $ACC4, $ACC4 + vpaddq $T0, $ACC5, $ACC5 + vpaddq $T0, $ACC6, $ACC6 + vpsllq \$23, $B, $T0 + + .byte 0x67,0x67 + vpmuludq %ymm15, $B, $OVERFLOW + vpsubq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + .byte 0x67,0x67 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $OVERFLOW + vmovdqa $ACC5, $ACC4 + vpmuludq 32*7(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC6, $ACC5 + vpaddq $T0, $ACC7, $ACC6 + vpmuludq 32*8(%rax), $Y, $ACC7 + + dec $itr + jnz .Lavx2_mul_by1_x4_loop + + ret +.size avx2_mul_by1_x4,.-avx2_mul_by1_x4 + +################################################################################ +# void avx2_sqr_x4(void* RESULTx4, void *Ax4, void *Bx4); +.type avx2_sqr_x4,\@abi-omnipotent +.align 32 +avx2_sqr_x4: + lea .LAVX2_POLY(%rip), %rax + + vmovdqa 32*7(%rax), %ymm14 + vmovdqa 32*8(%rax), %ymm15 + + vmovdqa 32*0($a_ptr), $B + vmovdqa 32*1($a_ptr), $ACC1 + vmovdqa 32*2($a_ptr), $ACC2 + vmovdqa 32*3($a_ptr), $ACC3 + vmovdqa 32*4($a_ptr), $ACC4 + vmovdqa 32*5($a_ptr), $ACC5 + vmovdqa 32*6($a_ptr), $ACC6 + vmovdqa 32*7($a_ptr), $ACC7 + vpaddq $ACC1, $ACC1, $ACC1 # 2*$ACC0..7 + vmovdqa 32*8($a_ptr), $ACC8 + vpaddq $ACC2, $ACC2, $ACC2 + vmovdqa $ACC1, 32*0(%rcx) + vpaddq $ACC3, $ACC3, $ACC3 + vmovdqa $ACC2, 32*1(%rcx) + vpaddq $ACC4, $ACC4, $ACC4 + vmovdqa $ACC3, 32*2(%rcx) + vpaddq $ACC5, $ACC5, $ACC5 + vmovdqa $ACC4, 32*3(%rcx) + vpaddq $ACC6, $ACC6, $ACC6 + vmovdqa $ACC5, 32*4(%rcx) + vpaddq $ACC7, $ACC7, $ACC7 + vmovdqa $ACC6, 32*5(%rcx) + vpaddq $ACC8, $ACC8, $ACC8 + vmovdqa $ACC7, 32*6(%rcx) + vmovdqa $ACC8, 32*7(%rcx) + + #itr 1 + vpmuludq $B, $B, $ACC0 + vpmuludq $B, $ACC1, $ACC1 + vpand $AND_MASK, $ACC0, $Y + vpmuludq $B, $ACC2, $ACC2 + vpmuludq $B, $ACC3, $ACC3 + vpmuludq $B, $ACC4, $ACC4 + vpmuludq $B, $ACC5, $ACC5 + vpmuludq $B, $ACC6, $ACC6 + vpmuludq $AND_MASK, $Y, $T0 + vpmuludq $B, $ACC7, $ACC7 + vpmuludq $B, $ACC8, $ACC8 + vmovdqa 32*1($a_ptr), $B + + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 2 + vpmuludq $B, $B, $OVERFLOW + vpand $AND_MASK, $ACC0, $Y + vpmuludq 32*1(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC1, $ACC1 + vpmuludq 32*2(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC2, $ACC2 + vpmuludq 32*3(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*2($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 3 + vpmuludq $B, $B, $T0 + vpand $AND_MASK, $ACC0, $Y + vpmuludq 32*2(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC2, $ACC2 + vpmuludq 32*3(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*3($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 4 + vpmuludq $B, $B, $OVERFLOW + vpmuludq 32*3(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC3, $ACC3 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*4($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 5 + vpmuludq $B, $B, $T0 + vpmuludq 32*4(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC4, $ACC4 + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*5($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3+.LAVX2_POLY(%rip), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 6 + vpmuludq $B, $B, $OVERFLOW + vpmuludq 32*5(%rcx), $B, $T0 + vpaddq $OVERFLOW, $ACC5, $ACC5 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*6($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 7 + vpmuludq $B, $B, $T0 + vpmuludq 32*6(%rcx), $B, $OVERFLOW + vpaddq $T0, $ACC6, $ACC6 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*7($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 8 + vpmuludq $B, $B, $OVERFLOW + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC7 + vpmuludq 32*7(%rcx), $B, $ACC8 + vmovdqa 32*8($a_ptr), $B + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpand $AND_MASK, $ACC0, $Y + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + #itr 9 + vpmuludq $B, $B, $ACC8 + + vpmuludq $AND_MASK, $Y, $T0 + vpaddq $T0, $ACC0, $OVERFLOW + vpsrlq $digit_size, $OVERFLOW, $OVERFLOW + vpaddq $T0, $ACC1, $ACC0 + vpaddq $T0, $ACC2, $ACC1 + vpmuludq 32*3(%rax), $Y, $T0 + vpaddq $OVERFLOW, $ACC0, $ACC0 + vpaddq $T0, $ACC3, $ACC2 + vmovdqa $ACC4, $ACC3 + vpsllq \$18, $Y, $T0 + vmovdqa $ACC5, $ACC4 + vpmuludq %ymm14, $Y, $OVERFLOW + vpaddq $T0, $ACC6, $ACC5 + vpmuludq %ymm15, $Y, $T0 + vpaddq $OVERFLOW, $ACC7, $ACC6 + vpaddq $T0, $ACC8, $ACC7 + + vpxor $ACC8, $ACC8, $ACC8 + + ret +.size avx2_sqr_x4,.-avx2_sqr_x4 + +################################################################################ +# void avx2_sub_x4(void* RESULTx4, void *Ax4, void *Bx4); +.type avx2_sub_x4,\@abi-omnipotent +.align 32 +avx2_sub_x4: + vmovdqa 32*0($a_ptr), $ACC0 + lea 160($a_ptr), $a_ptr + lea .LAVX2_POLY_x8+128(%rip), %rax + lea 128($b_ptr), $b_ptr + vmovdqa 32*1-160($a_ptr), $ACC1 + vmovdqa 32*2-160($a_ptr), $ACC2 + vmovdqa 32*3-160($a_ptr), $ACC3 + vmovdqa 32*4-160($a_ptr), $ACC4 + vmovdqa 32*5-160($a_ptr), $ACC5 + vmovdqa 32*6-160($a_ptr), $ACC6 + vmovdqa 32*7-160($a_ptr), $ACC7 + vmovdqa 32*8-160($a_ptr), $ACC8 + + vpaddq 32*0-128(%rax), $ACC0, $ACC0 + vpaddq 32*1-128(%rax), $ACC1, $ACC1 + vpaddq 32*2-128(%rax), $ACC2, $ACC2 + vpaddq 32*3-128(%rax), $ACC3, $ACC3 + vpaddq 32*4-128(%rax), $ACC4, $ACC4 + vpaddq 32*5-128(%rax), $ACC5, $ACC5 + vpaddq 32*6-128(%rax), $ACC6, $ACC6 + vpaddq 32*7-128(%rax), $ACC7, $ACC7 + vpaddq 32*8-128(%rax), $ACC8, $ACC8 + + vpsubq 32*0-128($b_ptr), $ACC0, $ACC0 + vpsubq 32*1-128($b_ptr), $ACC1, $ACC1 + vpsubq 32*2-128($b_ptr), $ACC2, $ACC2 + vpsubq 32*3-128($b_ptr), $ACC3, $ACC3 + vpsubq 32*4-128($b_ptr), $ACC4, $ACC4 + vpsubq 32*5-128($b_ptr), $ACC5, $ACC5 + vpsubq 32*6-128($b_ptr), $ACC6, $ACC6 + vpsubq 32*7-128($b_ptr), $ACC7, $ACC7 + vpsubq 32*8-128($b_ptr), $ACC8, $ACC8 + + ret +.size avx2_sub_x4,.-avx2_sub_x4 + +.type avx2_select_n_store,\@abi-omnipotent +.align 32 +avx2_select_n_store: + vmovdqa `8+32*9*8`(%rsp), $Y + vpor `8+32*9*8+32`(%rsp), $Y, $Y + + vpandn $ACC0, $Y, $ACC0 + vpandn $ACC1, $Y, $ACC1 + vpandn $ACC2, $Y, $ACC2 + vpandn $ACC3, $Y, $ACC3 + vpandn $ACC4, $Y, $ACC4 + vpandn $ACC5, $Y, $ACC5 + vpandn $ACC6, $Y, $ACC6 + vmovdqa `8+32*9*8+32`(%rsp), $B + vpandn $ACC7, $Y, $ACC7 + vpandn `8+32*9*8`(%rsp), $B, $B + vpandn $ACC8, $Y, $ACC8 + + vpand 32*0(%rsi), $B, $T0 + lea 160(%rsi), %rax + vpand 32*1(%rsi), $B, $Y + vpxor $T0, $ACC0, $ACC0 + vpand 32*2(%rsi), $B, $T0 + vpxor $Y, $ACC1, $ACC1 + vpand 32*3(%rsi), $B, $Y + vpxor $T0, $ACC2, $ACC2 + vpand 32*4-160(%rax), $B, $T0 + vpxor $Y, $ACC3, $ACC3 + vpand 32*5-160(%rax), $B, $Y + vpxor $T0, $ACC4, $ACC4 + vpand 32*6-160(%rax), $B, $T0 + vpxor $Y, $ACC5, $ACC5 + vpand 32*7-160(%rax), $B, $Y + vpxor $T0, $ACC6, $ACC6 + vpand 32*8-160(%rax), $B, $T0 + vmovdqa `8+32*9*8+32`(%rsp), $B + vpxor $Y, $ACC7, $ACC7 + + vpand 32*0(%rdx), $B, $Y + lea 160(%rdx), %rax + vpxor $T0, $ACC8, $ACC8 + vpand 32*1(%rdx), $B, $T0 + vpxor $Y, $ACC0, $ACC0 + vpand 32*2(%rdx), $B, $Y + vpxor $T0, $ACC1, $ACC1 + vpand 32*3(%rdx), $B, $T0 + vpxor $Y, $ACC2, $ACC2 + vpand 32*4-160(%rax), $B, $Y + vpxor $T0, $ACC3, $ACC3 + vpand 32*5-160(%rax), $B, $T0 + vpxor $Y, $ACC4, $ACC4 + vpand 32*6-160(%rax), $B, $Y + vpxor $T0, $ACC5, $ACC5 + vpand 32*7-160(%rax), $B, $T0 + vpxor $Y, $ACC6, $ACC6 + vpand 32*8-160(%rax), $B, $Y + vpxor $T0, $ACC7, $ACC7 + vpxor $Y, $ACC8, $ACC8 + `&STORE` + + ret +.size avx2_select_n_store,.-avx2_select_n_store +___ +$code.=<<___ if (0); # inlined +################################################################################ +# void avx2_mul_by2_x4(void* RESULTx4, void *Ax4); +.type avx2_mul_by2_x4,\@abi-omnipotent +.align 32 +avx2_mul_by2_x4: + vmovdqa 32*0($a_ptr), $ACC0 + lea 160($a_ptr), %rax + vmovdqa 32*1($a_ptr), $ACC1 + vmovdqa 32*2($a_ptr), $ACC2 + vmovdqa 32*3($a_ptr), $ACC3 + vmovdqa 32*4-160(%rax), $ACC4 + vmovdqa 32*5-160(%rax), $ACC5 + vmovdqa 32*6-160(%rax), $ACC6 + vmovdqa 32*7-160(%rax), $ACC7 + vmovdqa 32*8-160(%rax), $ACC8 + + vpaddq $ACC0, $ACC0, $ACC0 + vpaddq $ACC1, $ACC1, $ACC1 + vpaddq $ACC2, $ACC2, $ACC2 + vpaddq $ACC3, $ACC3, $ACC3 + vpaddq $ACC4, $ACC4, $ACC4 + vpaddq $ACC5, $ACC5, $ACC5 + vpaddq $ACC6, $ACC6, $ACC6 + vpaddq $ACC7, $ACC7, $ACC7 + vpaddq $ACC8, $ACC8, $ACC8 + + ret +.size avx2_mul_by2_x4,.-avx2_mul_by2_x4 +___ +my ($r_ptr_in,$a_ptr_in,$b_ptr_in)=("%rdi","%rsi","%rdx"); +my ($r_ptr,$a_ptr,$b_ptr)=("%r8","%r9","%r10"); + +$code.=<<___; +################################################################################ +# void ecp_nistz256_avx2_point_add_affine_x4(void* RESULTx4, void *Ax4, void *Bx4); +.globl ecp_nistz256_avx2_point_add_affine_x4 +.type ecp_nistz256_avx2_point_add_affine_x4,\@function,3 +.align 32 +ecp_nistz256_avx2_point_add_affine_x4: + mov %rsp, %rax + push %rbp + vzeroupper +___ +$code.=<<___ if ($win64); + lea -16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + lea -8(%rax), %rbp + +# Result + 32*0 = Result.X +# Result + 32*9 = Result.Y +# Result + 32*18 = Result.Z + +# A + 32*0 = A.X +# A + 32*9 = A.Y +# A + 32*18 = A.Z + +# B + 32*0 = B.X +# B + 32*9 = B.Y + + sub \$`32*9*8+32*2+32*8`, %rsp + and \$-64, %rsp + + mov $r_ptr_in, $r_ptr + mov $a_ptr_in, $a_ptr + mov $b_ptr_in, $b_ptr + + vmovdqa 32*0($a_ptr_in), %ymm0 + vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK + vpxor %ymm1, %ymm1, %ymm1 + lea 256($a_ptr_in), %rax # size optimization + vpor 32*1($a_ptr_in), %ymm0, %ymm0 + vpor 32*2($a_ptr_in), %ymm0, %ymm0 + vpor 32*3($a_ptr_in), %ymm0, %ymm0 + vpor 32*4-256(%rax), %ymm0, %ymm0 + lea 256(%rax), %rcx # size optimization + vpor 32*5-256(%rax), %ymm0, %ymm0 + vpor 32*6-256(%rax), %ymm0, %ymm0 + vpor 32*7-256(%rax), %ymm0, %ymm0 + vpor 32*8-256(%rax), %ymm0, %ymm0 + vpor 32*9-256(%rax), %ymm0, %ymm0 + vpor 32*10-256(%rax), %ymm0, %ymm0 + vpor 32*11-256(%rax), %ymm0, %ymm0 + vpor 32*12-512(%rcx), %ymm0, %ymm0 + vpor 32*13-512(%rcx), %ymm0, %ymm0 + vpor 32*14-512(%rcx), %ymm0, %ymm0 + vpor 32*15-512(%rcx), %ymm0, %ymm0 + vpor 32*16-512(%rcx), %ymm0, %ymm0 + vpor 32*17-512(%rcx), %ymm0, %ymm0 + vpcmpeqq %ymm1, %ymm0, %ymm0 + vmovdqa %ymm0, `32*9*8`(%rsp) + + vpxor %ymm1, %ymm1, %ymm1 + vmovdqa 32*0($b_ptr), %ymm0 + lea 256($b_ptr), %rax # size optimization + vpor 32*1($b_ptr), %ymm0, %ymm0 + vpor 32*2($b_ptr), %ymm0, %ymm0 + vpor 32*3($b_ptr), %ymm0, %ymm0 + vpor 32*4-256(%rax), %ymm0, %ymm0 + lea 256(%rax), %rcx # size optimization + vpor 32*5-256(%rax), %ymm0, %ymm0 + vpor 32*6-256(%rax), %ymm0, %ymm0 + vpor 32*7-256(%rax), %ymm0, %ymm0 + vpor 32*8-256(%rax), %ymm0, %ymm0 + vpor 32*9-256(%rax), %ymm0, %ymm0 + vpor 32*10-256(%rax), %ymm0, %ymm0 + vpor 32*11-256(%rax), %ymm0, %ymm0 + vpor 32*12-512(%rcx), %ymm0, %ymm0 + vpor 32*13-512(%rcx), %ymm0, %ymm0 + vpor 32*14-512(%rcx), %ymm0, %ymm0 + vpor 32*15-512(%rcx), %ymm0, %ymm0 + vpor 32*16-512(%rcx), %ymm0, %ymm0 + vpor 32*17-512(%rcx), %ymm0, %ymm0 + vpcmpeqq %ymm1, %ymm0, %ymm0 + vmovdqa %ymm0, `32*9*8+32`(%rsp) + + # Z1^2 = Z1*Z1 + lea `32*9*2`($a_ptr), %rsi + lea `32*9*2`(%rsp), %rdi + lea `32*9*8+32*2`(%rsp), %rcx # temporary vector + call avx2_sqr_x4 + call avx2_normalize_n_store + + # U2 = X2*Z1^2 + lea `32*9*0`($b_ptr), %rsi + lea `32*9*2`(%rsp), %rdx + lea `32*9*0`(%rsp), %rdi + call avx2_mul_x4 + #call avx2_normalize + `&STORE` + + # S2 = Z1*Z1^2 = Z1^3 + lea `32*9*2`($a_ptr), %rsi + lea `32*9*2`(%rsp), %rdx + lea `32*9*1`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # S2 = S2*Y2 = Y2*Z1^3 + lea `32*9*1`($b_ptr), %rsi + lea `32*9*1`(%rsp), %rdx + lea `32*9*1`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # H = U2 - U1 = U2 - X1 + lea `32*9*0`(%rsp), %rsi + lea `32*9*0`($a_ptr), %rdx + lea `32*9*3`(%rsp), %rdi + call avx2_sub_x4 + call avx2_normalize_n_store + + # R = S2 - S1 = S2 - Y1 + lea `32*9*1`(%rsp), %rsi + lea `32*9*1`($a_ptr), %rdx + lea `32*9*4`(%rsp), %rdi + call avx2_sub_x4 + call avx2_normalize_n_store + + # Z3 = H*Z1*Z2 + lea `32*9*3`(%rsp), %rsi + lea `32*9*2`($a_ptr), %rdx + lea `32*9*2`($r_ptr), %rdi + call avx2_mul_x4 + call avx2_normalize + + lea .LONE(%rip), %rsi + lea `32*9*2`($a_ptr), %rdx + call avx2_select_n_store + + # R^2 = R^2 + lea `32*9*4`(%rsp), %rsi + lea `32*9*6`(%rsp), %rdi + lea `32*9*8+32*2`(%rsp), %rcx # temporary vector + call avx2_sqr_x4 + call avx2_normalize_n_store + + # H^2 = H^2 + lea `32*9*3`(%rsp), %rsi + lea `32*9*5`(%rsp), %rdi + call avx2_sqr_x4 + call avx2_normalize_n_store + + # H^3 = H^2*H + lea `32*9*3`(%rsp), %rsi + lea `32*9*5`(%rsp), %rdx + lea `32*9*7`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # U2 = U1*H^2 + lea `32*9*0`($a_ptr), %rsi + lea `32*9*5`(%rsp), %rdx + lea `32*9*0`(%rsp), %rdi + call avx2_mul_x4 + #call avx2_normalize + `&STORE` + + # Hsqr = U2*2 + #lea 32*9*0(%rsp), %rsi + #lea 32*9*5(%rsp), %rdi + #call avx2_mul_by2_x4 + + vpaddq $ACC0, $ACC0, $ACC0 # inlined avx2_mul_by2_x4 + lea `32*9*5`(%rsp), %rdi + vpaddq $ACC1, $ACC1, $ACC1 + vpaddq $ACC2, $ACC2, $ACC2 + vpaddq $ACC3, $ACC3, $ACC3 + vpaddq $ACC4, $ACC4, $ACC4 + vpaddq $ACC5, $ACC5, $ACC5 + vpaddq $ACC6, $ACC6, $ACC6 + vpaddq $ACC7, $ACC7, $ACC7 + vpaddq $ACC8, $ACC8, $ACC8 + call avx2_normalize_n_store + + # X3 = R^2 - H^3 + #lea 32*9*6(%rsp), %rsi + #lea 32*9*7(%rsp), %rdx + #lea 32*9*5(%rsp), %rcx + #lea 32*9*0($r_ptr), %rdi + #call avx2_sub_x4 + #NORMALIZE + #STORE + + # X3 = X3 - U2*2 + #lea 32*9*0($r_ptr), %rsi + #lea 32*9*0($r_ptr), %rdi + #call avx2_sub_x4 + #NORMALIZE + #STORE + + lea `32*9*6+128`(%rsp), %rsi + lea .LAVX2_POLY_x2+128(%rip), %rax + lea `32*9*7+128`(%rsp), %rdx + lea `32*9*5+128`(%rsp), %rcx + lea `32*9*0`($r_ptr), %rdi + + vmovdqa 32*0-128(%rsi), $ACC0 + vmovdqa 32*1-128(%rsi), $ACC1 + vmovdqa 32*2-128(%rsi), $ACC2 + vmovdqa 32*3-128(%rsi), $ACC3 + vmovdqa 32*4-128(%rsi), $ACC4 + vmovdqa 32*5-128(%rsi), $ACC5 + vmovdqa 32*6-128(%rsi), $ACC6 + vmovdqa 32*7-128(%rsi), $ACC7 + vmovdqa 32*8-128(%rsi), $ACC8 + + vpaddq 32*0-128(%rax), $ACC0, $ACC0 + vpaddq 32*1-128(%rax), $ACC1, $ACC1 + vpaddq 32*2-128(%rax), $ACC2, $ACC2 + vpaddq 32*3-128(%rax), $ACC3, $ACC3 + vpaddq 32*4-128(%rax), $ACC4, $ACC4 + vpaddq 32*5-128(%rax), $ACC5, $ACC5 + vpaddq 32*6-128(%rax), $ACC6, $ACC6 + vpaddq 32*7-128(%rax), $ACC7, $ACC7 + vpaddq 32*8-128(%rax), $ACC8, $ACC8 + + vpsubq 32*0-128(%rdx), $ACC0, $ACC0 + vpsubq 32*1-128(%rdx), $ACC1, $ACC1 + vpsubq 32*2-128(%rdx), $ACC2, $ACC2 + vpsubq 32*3-128(%rdx), $ACC3, $ACC3 + vpsubq 32*4-128(%rdx), $ACC4, $ACC4 + vpsubq 32*5-128(%rdx), $ACC5, $ACC5 + vpsubq 32*6-128(%rdx), $ACC6, $ACC6 + vpsubq 32*7-128(%rdx), $ACC7, $ACC7 + vpsubq 32*8-128(%rdx), $ACC8, $ACC8 + + vpsubq 32*0-128(%rcx), $ACC0, $ACC0 + vpsubq 32*1-128(%rcx), $ACC1, $ACC1 + vpsubq 32*2-128(%rcx), $ACC2, $ACC2 + vpsubq 32*3-128(%rcx), $ACC3, $ACC3 + vpsubq 32*4-128(%rcx), $ACC4, $ACC4 + vpsubq 32*5-128(%rcx), $ACC5, $ACC5 + vpsubq 32*6-128(%rcx), $ACC6, $ACC6 + vpsubq 32*7-128(%rcx), $ACC7, $ACC7 + vpsubq 32*8-128(%rcx), $ACC8, $ACC8 + call avx2_normalize + + lea 32*0($b_ptr), %rsi + lea 32*0($a_ptr), %rdx + call avx2_select_n_store + + # H = U2 - X3 + lea `32*9*0`(%rsp), %rsi + lea `32*9*0`($r_ptr), %rdx + lea `32*9*3`(%rsp), %rdi + call avx2_sub_x4 + call avx2_normalize_n_store + + # + lea `32*9*3`(%rsp), %rsi + lea `32*9*4`(%rsp), %rdx + lea `32*9*3`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # + lea `32*9*7`(%rsp), %rsi + lea `32*9*1`($a_ptr), %rdx + lea `32*9*1`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # + lea `32*9*3`(%rsp), %rsi + lea `32*9*1`(%rsp), %rdx + lea `32*9*1`($r_ptr), %rdi + call avx2_sub_x4 + call avx2_normalize + + lea 32*9($b_ptr), %rsi + lea 32*9($a_ptr), %rdx + call avx2_select_n_store + + #lea 32*9*0($r_ptr), %rsi + #lea 32*9*0($r_ptr), %rdi + #call avx2_mul_by1_x4 + #NORMALIZE + #STORE + + lea `32*9*1`($r_ptr), %rsi + lea `32*9*1`($r_ptr), %rdi + call avx2_mul_by1_x4 + call avx2_normalize_n_store + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps %xmm6, -16*10(%rbp) + movaps %xmm7, -16*9(%rbp) + movaps %xmm8, -16*8(%rbp) + movaps %xmm9, -16*7(%rbp) + movaps %xmm10, -16*6(%rbp) + movaps %xmm11, -16*5(%rbp) + movaps %xmm12, -16*4(%rbp) + movaps %xmm13, -16*3(%rbp) + movaps %xmm14, -16*2(%rbp) + movaps %xmm15, -16*1(%rbp) +___ +$code.=<<___; + mov %rbp, %rsp + pop %rbp + ret +.size ecp_nistz256_avx2_point_add_affine_x4,.-ecp_nistz256_avx2_point_add_affine_x4 + +################################################################################ +# void ecp_nistz256_avx2_point_add_affines_x4(void* RESULTx4, void *Ax4, void *Bx4); +.globl ecp_nistz256_avx2_point_add_affines_x4 +.type ecp_nistz256_avx2_point_add_affines_x4,\@function,3 +.align 32 +ecp_nistz256_avx2_point_add_affines_x4: + mov %rsp, %rax + push %rbp + vzeroupper +___ +$code.=<<___ if ($win64); + lea -16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + lea -8(%rax), %rbp + +# Result + 32*0 = Result.X +# Result + 32*9 = Result.Y +# Result + 32*18 = Result.Z + +# A + 32*0 = A.X +# A + 32*9 = A.Y + +# B + 32*0 = B.X +# B + 32*9 = B.Y + + sub \$`32*9*8+32*2+32*8`, %rsp + and \$-64, %rsp + + mov $r_ptr_in, $r_ptr + mov $a_ptr_in, $a_ptr + mov $b_ptr_in, $b_ptr + + vmovdqa 32*0($a_ptr_in), %ymm0 + vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK + vpxor %ymm1, %ymm1, %ymm1 + lea 256($a_ptr_in), %rax # size optimization + vpor 32*1($a_ptr_in), %ymm0, %ymm0 + vpor 32*2($a_ptr_in), %ymm0, %ymm0 + vpor 32*3($a_ptr_in), %ymm0, %ymm0 + vpor 32*4-256(%rax), %ymm0, %ymm0 + lea 256(%rax), %rcx # size optimization + vpor 32*5-256(%rax), %ymm0, %ymm0 + vpor 32*6-256(%rax), %ymm0, %ymm0 + vpor 32*7-256(%rax), %ymm0, %ymm0 + vpor 32*8-256(%rax), %ymm0, %ymm0 + vpor 32*9-256(%rax), %ymm0, %ymm0 + vpor 32*10-256(%rax), %ymm0, %ymm0 + vpor 32*11-256(%rax), %ymm0, %ymm0 + vpor 32*12-512(%rcx), %ymm0, %ymm0 + vpor 32*13-512(%rcx), %ymm0, %ymm0 + vpor 32*14-512(%rcx), %ymm0, %ymm0 + vpor 32*15-512(%rcx), %ymm0, %ymm0 + vpor 32*16-512(%rcx), %ymm0, %ymm0 + vpor 32*17-512(%rcx), %ymm0, %ymm0 + vpcmpeqq %ymm1, %ymm0, %ymm0 + vmovdqa %ymm0, `32*9*8`(%rsp) + + vpxor %ymm1, %ymm1, %ymm1 + vmovdqa 32*0($b_ptr), %ymm0 + lea 256($b_ptr), %rax # size optimization + vpor 32*1($b_ptr), %ymm0, %ymm0 + vpor 32*2($b_ptr), %ymm0, %ymm0 + vpor 32*3($b_ptr), %ymm0, %ymm0 + vpor 32*4-256(%rax), %ymm0, %ymm0 + lea 256(%rax), %rcx # size optimization + vpor 32*5-256(%rax), %ymm0, %ymm0 + vpor 32*6-256(%rax), %ymm0, %ymm0 + vpor 32*7-256(%rax), %ymm0, %ymm0 + vpor 32*8-256(%rax), %ymm0, %ymm0 + vpor 32*9-256(%rax), %ymm0, %ymm0 + vpor 32*10-256(%rax), %ymm0, %ymm0 + vpor 32*11-256(%rax), %ymm0, %ymm0 + vpor 32*12-512(%rcx), %ymm0, %ymm0 + vpor 32*13-512(%rcx), %ymm0, %ymm0 + vpor 32*14-512(%rcx), %ymm0, %ymm0 + vpor 32*15-512(%rcx), %ymm0, %ymm0 + vpor 32*16-512(%rcx), %ymm0, %ymm0 + vpor 32*17-512(%rcx), %ymm0, %ymm0 + vpcmpeqq %ymm1, %ymm0, %ymm0 + vmovdqa %ymm0, `32*9*8+32`(%rsp) + + # H = U2 - U1 = X2 - X1 + lea `32*9*0`($b_ptr), %rsi + lea `32*9*0`($a_ptr), %rdx + lea `32*9*3`(%rsp), %rdi + call avx2_sub_x4 + call avx2_normalize_n_store + + # R = S2 - S1 = Y2 - Y1 + lea `32*9*1`($b_ptr), %rsi + lea `32*9*1`($a_ptr), %rdx + lea `32*9*4`(%rsp), %rdi + call avx2_sub_x4 + call avx2_normalize_n_store + + # Z3 = H*Z1*Z2 = H + lea `32*9*3`(%rsp), %rsi + lea `32*9*2`($r_ptr), %rdi + call avx2_mul_by1_x4 + call avx2_normalize + + vmovdqa `32*9*8`(%rsp), $B + vpor `32*9*8+32`(%rsp), $B, $B + + vpandn $ACC0, $B, $ACC0 + lea .LONE+128(%rip), %rax + vpandn $ACC1, $B, $ACC1 + vpandn $ACC2, $B, $ACC2 + vpandn $ACC3, $B, $ACC3 + vpandn $ACC4, $B, $ACC4 + vpandn $ACC5, $B, $ACC5 + vpandn $ACC6, $B, $ACC6 + vpandn $ACC7, $B, $ACC7 + + vpand 32*0-128(%rax), $B, $T0 + vpandn $ACC8, $B, $ACC8 + vpand 32*1-128(%rax), $B, $Y + vpxor $T0, $ACC0, $ACC0 + vpand 32*2-128(%rax), $B, $T0 + vpxor $Y, $ACC1, $ACC1 + vpand 32*3-128(%rax), $B, $Y + vpxor $T0, $ACC2, $ACC2 + vpand 32*4-128(%rax), $B, $T0 + vpxor $Y, $ACC3, $ACC3 + vpand 32*5-128(%rax), $B, $Y + vpxor $T0, $ACC4, $ACC4 + vpand 32*6-128(%rax), $B, $T0 + vpxor $Y, $ACC5, $ACC5 + vpand 32*7-128(%rax), $B, $Y + vpxor $T0, $ACC6, $ACC6 + vpand 32*8-128(%rax), $B, $T0 + vpxor $Y, $ACC7, $ACC7 + vpxor $T0, $ACC8, $ACC8 + `&STORE` + + # R^2 = R^2 + lea `32*9*4`(%rsp), %rsi + lea `32*9*6`(%rsp), %rdi + lea `32*9*8+32*2`(%rsp), %rcx # temporary vector + call avx2_sqr_x4 + call avx2_normalize_n_store + + # H^2 = H^2 + lea `32*9*3`(%rsp), %rsi + lea `32*9*5`(%rsp), %rdi + call avx2_sqr_x4 + call avx2_normalize_n_store + + # H^3 = H^2*H + lea `32*9*3`(%rsp), %rsi + lea `32*9*5`(%rsp), %rdx + lea `32*9*7`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # U2 = U1*H^2 + lea `32*9*0`($a_ptr), %rsi + lea `32*9*5`(%rsp), %rdx + lea `32*9*0`(%rsp), %rdi + call avx2_mul_x4 + #call avx2_normalize + `&STORE` + + # Hsqr = U2*2 + #lea 32*9*0(%rsp), %rsi + #lea 32*9*5(%rsp), %rdi + #call avx2_mul_by2_x4 + + vpaddq $ACC0, $ACC0, $ACC0 # inlined avx2_mul_by2_x4 + lea `32*9*5`(%rsp), %rdi + vpaddq $ACC1, $ACC1, $ACC1 + vpaddq $ACC2, $ACC2, $ACC2 + vpaddq $ACC3, $ACC3, $ACC3 + vpaddq $ACC4, $ACC4, $ACC4 + vpaddq $ACC5, $ACC5, $ACC5 + vpaddq $ACC6, $ACC6, $ACC6 + vpaddq $ACC7, $ACC7, $ACC7 + vpaddq $ACC8, $ACC8, $ACC8 + call avx2_normalize_n_store + + # X3 = R^2 - H^3 + #lea 32*9*6(%rsp), %rsi + #lea 32*9*7(%rsp), %rdx + #lea 32*9*5(%rsp), %rcx + #lea 32*9*0($r_ptr), %rdi + #call avx2_sub_x4 + #NORMALIZE + #STORE + + # X3 = X3 - U2*2 + #lea 32*9*0($r_ptr), %rsi + #lea 32*9*0($r_ptr), %rdi + #call avx2_sub_x4 + #NORMALIZE + #STORE + + lea `32*9*6+128`(%rsp), %rsi + lea .LAVX2_POLY_x2+128(%rip), %rax + lea `32*9*7+128`(%rsp), %rdx + lea `32*9*5+128`(%rsp), %rcx + lea `32*9*0`($r_ptr), %rdi + + vmovdqa 32*0-128(%rsi), $ACC0 + vmovdqa 32*1-128(%rsi), $ACC1 + vmovdqa 32*2-128(%rsi), $ACC2 + vmovdqa 32*3-128(%rsi), $ACC3 + vmovdqa 32*4-128(%rsi), $ACC4 + vmovdqa 32*5-128(%rsi), $ACC5 + vmovdqa 32*6-128(%rsi), $ACC6 + vmovdqa 32*7-128(%rsi), $ACC7 + vmovdqa 32*8-128(%rsi), $ACC8 + + vpaddq 32*0-128(%rax), $ACC0, $ACC0 + vpaddq 32*1-128(%rax), $ACC1, $ACC1 + vpaddq 32*2-128(%rax), $ACC2, $ACC2 + vpaddq 32*3-128(%rax), $ACC3, $ACC3 + vpaddq 32*4-128(%rax), $ACC4, $ACC4 + vpaddq 32*5-128(%rax), $ACC5, $ACC5 + vpaddq 32*6-128(%rax), $ACC6, $ACC6 + vpaddq 32*7-128(%rax), $ACC7, $ACC7 + vpaddq 32*8-128(%rax), $ACC8, $ACC8 + + vpsubq 32*0-128(%rdx), $ACC0, $ACC0 + vpsubq 32*1-128(%rdx), $ACC1, $ACC1 + vpsubq 32*2-128(%rdx), $ACC2, $ACC2 + vpsubq 32*3-128(%rdx), $ACC3, $ACC3 + vpsubq 32*4-128(%rdx), $ACC4, $ACC4 + vpsubq 32*5-128(%rdx), $ACC5, $ACC5 + vpsubq 32*6-128(%rdx), $ACC6, $ACC6 + vpsubq 32*7-128(%rdx), $ACC7, $ACC7 + vpsubq 32*8-128(%rdx), $ACC8, $ACC8 + + vpsubq 32*0-128(%rcx), $ACC0, $ACC0 + vpsubq 32*1-128(%rcx), $ACC1, $ACC1 + vpsubq 32*2-128(%rcx), $ACC2, $ACC2 + vpsubq 32*3-128(%rcx), $ACC3, $ACC3 + vpsubq 32*4-128(%rcx), $ACC4, $ACC4 + vpsubq 32*5-128(%rcx), $ACC5, $ACC5 + vpsubq 32*6-128(%rcx), $ACC6, $ACC6 + vpsubq 32*7-128(%rcx), $ACC7, $ACC7 + vpsubq 32*8-128(%rcx), $ACC8, $ACC8 + call avx2_normalize + + lea 32*0($b_ptr), %rsi + lea 32*0($a_ptr), %rdx + call avx2_select_n_store + + # H = U2 - X3 + lea `32*9*0`(%rsp), %rsi + lea `32*9*0`($r_ptr), %rdx + lea `32*9*3`(%rsp), %rdi + call avx2_sub_x4 + call avx2_normalize_n_store + + # H = H*R + lea `32*9*3`(%rsp), %rsi + lea `32*9*4`(%rsp), %rdx + lea `32*9*3`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # S2 = S1 * H^3 + lea `32*9*7`(%rsp), %rsi + lea `32*9*1`($a_ptr), %rdx + lea `32*9*1`(%rsp), %rdi + call avx2_mul_x4 + call avx2_normalize_n_store + + # + lea `32*9*3`(%rsp), %rsi + lea `32*9*1`(%rsp), %rdx + lea `32*9*1`($r_ptr), %rdi + call avx2_sub_x4 + call avx2_normalize + + lea 32*9($b_ptr), %rsi + lea 32*9($a_ptr), %rdx + call avx2_select_n_store + + #lea 32*9*0($r_ptr), %rsi + #lea 32*9*0($r_ptr), %rdi + #call avx2_mul_by1_x4 + #NORMALIZE + #STORE + + lea `32*9*1`($r_ptr), %rsi + lea `32*9*1`($r_ptr), %rdi + call avx2_mul_by1_x4 + call avx2_normalize_n_store + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps %xmm6, -16*10(%rbp) + movaps %xmm7, -16*9(%rbp) + movaps %xmm8, -16*8(%rbp) + movaps %xmm9, -16*7(%rbp) + movaps %xmm10, -16*6(%rbp) + movaps %xmm11, -16*5(%rbp) + movaps %xmm12, -16*4(%rbp) + movaps %xmm13, -16*3(%rbp) + movaps %xmm14, -16*2(%rbp) + movaps %xmm15, -16*1(%rbp) +___ +$code.=<<___; + mov %rbp, %rsp + pop %rbp + ret +.size ecp_nistz256_avx2_point_add_affines_x4,.-ecp_nistz256_avx2_point_add_affines_x4 + +################################################################################ +# void ecp_nistz256_avx2_to_mont(void* RESULTx4, void *Ax4); +.globl ecp_nistz256_avx2_to_mont +.type ecp_nistz256_avx2_to_mont,\@function,2 +.align 32 +ecp_nistz256_avx2_to_mont: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -8-16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK + lea .LTO_MONT_AVX2(%rip), %rdx + call avx2_mul_x4 + call avx2_normalize_n_store + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*0(%rsp), %xmm6 + movaps 16*1(%rsp), %xmm7 + movaps 16*2(%rsp), %xmm8 + movaps 16*3(%rsp), %xmm9 + movaps 16*4(%rsp), %xmm10 + movaps 16*5(%rsp), %xmm11 + movaps 16*6(%rsp), %xmm12 + movaps 16*7(%rsp), %xmm13 + movaps 16*8(%rsp), %xmm14 + movaps 16*9(%rsp), %xmm15 + lea 8+16*10(%rsp), %rsp +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_to_mont,.-ecp_nistz256_avx2_to_mont + +################################################################################ +# void ecp_nistz256_avx2_from_mont(void* RESULTx4, void *Ax4); +.globl ecp_nistz256_avx2_from_mont +.type ecp_nistz256_avx2_from_mont,\@function,2 +.align 32 +ecp_nistz256_avx2_from_mont: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -8-16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK + lea .LFROM_MONT_AVX2(%rip), %rdx + call avx2_mul_x4 + call avx2_normalize_n_store + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*0(%rsp), %xmm6 + movaps 16*1(%rsp), %xmm7 + movaps 16*2(%rsp), %xmm8 + movaps 16*3(%rsp), %xmm9 + movaps 16*4(%rsp), %xmm10 + movaps 16*5(%rsp), %xmm11 + movaps 16*6(%rsp), %xmm12 + movaps 16*7(%rsp), %xmm13 + movaps 16*8(%rsp), %xmm14 + movaps 16*9(%rsp), %xmm15 + lea 8+16*10(%rsp), %rsp +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_from_mont,.-ecp_nistz256_avx2_from_mont + +################################################################################ +# void ecp_nistz256_avx2_set1(void* RESULTx4); +.globl ecp_nistz256_avx2_set1 +.type ecp_nistz256_avx2_set1,\@function,1 +.align 32 +ecp_nistz256_avx2_set1: + lea .LONE+128(%rip), %rax + lea 128(%rdi), %rdi + vzeroupper + vmovdqa 32*0-128(%rax), %ymm0 + vmovdqa 32*1-128(%rax), %ymm1 + vmovdqa 32*2-128(%rax), %ymm2 + vmovdqa 32*3-128(%rax), %ymm3 + vmovdqa 32*4-128(%rax), %ymm4 + vmovdqa 32*5-128(%rax), %ymm5 + vmovdqa %ymm0, 32*0-128(%rdi) + vmovdqa 32*6-128(%rax), %ymm0 + vmovdqa %ymm1, 32*1-128(%rdi) + vmovdqa 32*7-128(%rax), %ymm1 + vmovdqa %ymm2, 32*2-128(%rdi) + vmovdqa 32*8-128(%rax), %ymm2 + vmovdqa %ymm3, 32*3-128(%rdi) + vmovdqa %ymm4, 32*4-128(%rdi) + vmovdqa %ymm5, 32*5-128(%rdi) + vmovdqa %ymm0, 32*6-128(%rdi) + vmovdqa %ymm1, 32*7-128(%rdi) + vmovdqa %ymm2, 32*8-128(%rdi) + + vzeroupper + ret +.size ecp_nistz256_avx2_set1,.-ecp_nistz256_avx2_set1 +___ +} +{ +################################################################################ +# void ecp_nistz256_avx2_multi_select_w7(void* RESULT, void *in, +# int index0, int index1, int index2, int index3); +################################################################################ + +my ($val,$in_t,$index0,$index1,$index2,$index3)=("%rdi","%rsi","%edx","%ecx","%r8d","%r9d"); +my ($INDEX0,$INDEX1,$INDEX2,$INDEX3)=map("%ymm$_",(0..3)); +my ($R0a,$R0b,$R1a,$R1b,$R2a,$R2b,$R3a,$R3b)=map("%ymm$_",(4..11)); +my ($M0,$T0,$T1,$TMP0)=map("%ymm$_",(12..15)); + +$code.=<<___; +.globl ecp_nistz256_avx2_multi_select_w7 +.type ecp_nistz256_avx2_multi_select_w7,\@function,6 +.align 32 +ecp_nistz256_avx2_multi_select_w7: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -8-16*10(%rsp), %rsp + vmovaps %xmm6, -8-16*10(%rax) + vmovaps %xmm7, -8-16*9(%rax) + vmovaps %xmm8, -8-16*8(%rax) + vmovaps %xmm9, -8-16*7(%rax) + vmovaps %xmm10, -8-16*6(%rax) + vmovaps %xmm11, -8-16*5(%rax) + vmovaps %xmm12, -8-16*4(%rax) + vmovaps %xmm13, -8-16*3(%rax) + vmovaps %xmm14, -8-16*2(%rax) + vmovaps %xmm15, -8-16*1(%rax) +___ +$code.=<<___; + lea .LIntOne(%rip), %rax + + vmovd $index0, %xmm0 + vmovd $index1, %xmm1 + vmovd $index2, %xmm2 + vmovd $index3, %xmm3 + + vpxor $R0a, $R0a, $R0a + vpxor $R0b, $R0b, $R0b + vpxor $R1a, $R1a, $R1a + vpxor $R1b, $R1b, $R1b + vpxor $R2a, $R2a, $R2a + vpxor $R2b, $R2b, $R2b + vpxor $R3a, $R3a, $R3a + vpxor $R3b, $R3b, $R3b + vmovdqa (%rax), $M0 + + vpermd $INDEX0, $R0a, $INDEX0 + vpermd $INDEX1, $R0a, $INDEX1 + vpermd $INDEX2, $R0a, $INDEX2 + vpermd $INDEX3, $R0a, $INDEX3 + + mov \$64, %ecx + lea 112($val), $val # size optimization + jmp .Lmulti_select_loop_avx2 + +# INDEX=0, corresponds to the point at infty (0,0) +.align 32 +.Lmulti_select_loop_avx2: + vpcmpeqd $INDEX0, $M0, $TMP0 + + vmovdqa `32*0+32*64*2*0`($in_t), $T0 + vmovdqa `32*1+32*64*2*0`($in_t), $T1 + vpand $TMP0, $T0, $T0 + vpand $TMP0, $T1, $T1 + vpxor $T0, $R0a, $R0a + vpxor $T1, $R0b, $R0b + + vpcmpeqd $INDEX1, $M0, $TMP0 + + vmovdqa `32*0+32*64*2*1`($in_t), $T0 + vmovdqa `32*1+32*64*2*1`($in_t), $T1 + vpand $TMP0, $T0, $T0 + vpand $TMP0, $T1, $T1 + vpxor $T0, $R1a, $R1a + vpxor $T1, $R1b, $R1b + + vpcmpeqd $INDEX2, $M0, $TMP0 + + vmovdqa `32*0+32*64*2*2`($in_t), $T0 + vmovdqa `32*1+32*64*2*2`($in_t), $T1 + vpand $TMP0, $T0, $T0 + vpand $TMP0, $T1, $T1 + vpxor $T0, $R2a, $R2a + vpxor $T1, $R2b, $R2b + + vpcmpeqd $INDEX3, $M0, $TMP0 + + vmovdqa `32*0+32*64*2*3`($in_t), $T0 + vmovdqa `32*1+32*64*2*3`($in_t), $T1 + vpand $TMP0, $T0, $T0 + vpand $TMP0, $T1, $T1 + vpxor $T0, $R3a, $R3a + vpxor $T1, $R3b, $R3b + + vpaddd (%rax), $M0, $M0 # increment + lea 32*2($in_t), $in_t + + dec %ecx + jnz .Lmulti_select_loop_avx2 + + vmovdqu $R0a, 32*0-112($val) + vmovdqu $R0b, 32*1-112($val) + vmovdqu $R1a, 32*2-112($val) + vmovdqu $R1b, 32*3-112($val) + vmovdqu $R2a, 32*4-112($val) + vmovdqu $R2b, 32*5-112($val) + vmovdqu $R3a, 32*6-112($val) + vmovdqu $R3b, 32*7-112($val) + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*0(%rsp), %xmm6 + movaps 16*1(%rsp), %xmm7 + movaps 16*2(%rsp), %xmm8 + movaps 16*3(%rsp), %xmm9 + movaps 16*4(%rsp), %xmm10 + movaps 16*5(%rsp), %xmm11 + movaps 16*6(%rsp), %xmm12 + movaps 16*7(%rsp), %xmm13 + movaps 16*8(%rsp), %xmm14 + movaps 16*9(%rsp), %xmm15 + lea 8+16*10(%rsp), %rsp +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_multi_select_w7,.-ecp_nistz256_avx2_multi_select_w7 + +.extern OPENSSL_ia32cap_P +.globl ecp_nistz_avx2_eligible +.type ecp_nistz_avx2_eligible,\@abi-omnipotent +.align 32 +ecp_nistz_avx2_eligible: + mov OPENSSL_ia32cap_P+8(%rip),%eax + shr \$5,%eax + and \$1,%eax + ret +.size ecp_nistz_avx2_eligible,.-ecp_nistz_avx2_eligible +___ +} +}} else {{ # assembler is too old +$code.=<<___; +.text + +.globl ecp_nistz256_avx2_transpose_convert +.globl ecp_nistz256_avx2_convert_transpose_back +.globl ecp_nistz256_avx2_point_add_affine_x4 +.globl ecp_nistz256_avx2_point_add_affines_x4 +.globl ecp_nistz256_avx2_to_mont +.globl ecp_nistz256_avx2_from_mont +.globl ecp_nistz256_avx2_set1 +.globl ecp_nistz256_avx2_multi_select_w7 +.type ecp_nistz256_avx2_multi_select_w7,\@abi-omnipotent +ecp_nistz256_avx2_transpose_convert: +ecp_nistz256_avx2_convert_transpose_back: +ecp_nistz256_avx2_point_add_affine_x4: +ecp_nistz256_avx2_point_add_affines_x4: +ecp_nistz256_avx2_to_mont: +ecp_nistz256_avx2_from_mont: +ecp_nistz256_avx2_set1: +ecp_nistz256_avx2_multi_select_w7: + .byte 0x0f,0x0b # ud2 + ret +.size ecp_nistz256_avx2_multi_select_w7,.-ecp_nistz256_avx2_multi_select_w7 + +.globl ecp_nistz_avx2_eligible +.type ecp_nistz_avx2_eligible,\@abi-omnipotent +ecp_nistz_avx2_eligible: + xor %eax,%eax + ret +.size ecp_nistz_avx2_eligible,.-ecp_nistz_avx2_eligible +___ +}} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/geo; + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/ec/asm/ecp_nistz256-x86_64.pl b/crypto/ec/asm/ecp_nistz256-x86_64.pl new file mode 100755 index 000000000000..84379fce1cb9 --- /dev/null +++ b/crypto/ec/asm/ecp_nistz256-x86_64.pl @@ -0,0 +1,2997 @@ +#!/usr/bin/env perl + +############################################################################## +# # +# Copyright 2014 Intel Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# # +############################################################################## +# # +# Developers and authors: # +# Shay Gueron (1, 2), and Vlad Krasnov (1) # +# (1) Intel Corporation, Israel Development Center # +# (2) University of Haifa # +# Reference: # +# S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with# +# 256 Bit Primes" # +# # +############################################################################## + +# Further optimization by : +# +# this/original with/without -DECP_NISTZ256_ASM(*) +# Opteron +12-49% +110-150% +# Bulldozer +14-45% +175-210% +# P4 +18-46% n/a :-( +# Westmere +12-34% +80-87% +# Sandy Bridge +9-35% +110-120% +# Ivy Bridge +9-35% +110-125% +# Haswell +8-37% +140-160% +# Broadwell +18-58% +145-210% +# Atom +15-50% +130-180% +# VIA Nano +43-160% +300-480% +# +# (*) "without -DECP_NISTZ256_ASM" refers to build with +# "enable-ec_nistp_64_gcc_128"; +# +# Ranges denote minimum and maximum improvement coefficients depending +# on benchmark. Lower coefficients are for ECDSA sign, relatively fastest +# server-side operation. Keep in mind that +100% means 2x improvement. + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); + $addx = ($1>=2.23); +} + +if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); + $addx = ($1>=2.10); +} + +if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + $addx = ($1>=12); +} + +if (!$addx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { + my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 + $avx = ($ver>=3.0) + ($ver>=3.01); + $addx = ($ver>=3.03); +} + +$code.=<<___; +.text +.extern OPENSSL_ia32cap_P + +# The polynomial +.align 64 +.Lpoly: +.quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 + +# 2^512 mod P precomputed for NIST P256 polynomial +.LRR: +.quad 0x0000000000000003, 0xfffffffbffffffff, 0xfffffffffffffffe, 0x00000004fffffffd + +.LOne: +.long 1,1,1,1,1,1,1,1 +.LTwo: +.long 2,2,2,2,2,2,2,2 +.LThree: +.long 3,3,3,3,3,3,3,3 +.LONE_mont: +.quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe +___ + +{ +################################################################################ +# void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]); + +my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11)); +my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13"); +my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx"); + +$code.=<<___; + +.globl ecp_nistz256_mul_by_2 +.type ecp_nistz256_mul_by_2,\@function,2 +.align 64 +ecp_nistz256_mul_by_2: + push %r12 + push %r13 + + mov 8*0($a_ptr), $a0 + mov 8*1($a_ptr), $a1 + add $a0, $a0 # a0:a3+a0:a3 + mov 8*2($a_ptr), $a2 + adc $a1, $a1 + mov 8*3($a_ptr), $a3 + lea .Lpoly(%rip), $a_ptr + mov $a0, $t0 + adc $a2, $a2 + adc $a3, $a3 + mov $a1, $t1 + sbb $t4, $t4 + + sub 8*0($a_ptr), $a0 + mov $a2, $t2 + sbb 8*1($a_ptr), $a1 + sbb 8*2($a_ptr), $a2 + mov $a3, $t3 + sbb 8*3($a_ptr), $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + pop %r13 + pop %r12 + ret +.size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2 + +################################################################################ +# void ecp_nistz256_div_by_2(uint64_t res[4], uint64_t a[4]); +.globl ecp_nistz256_div_by_2 +.type ecp_nistz256_div_by_2,\@function,2 +.align 32 +ecp_nistz256_div_by_2: + push %r12 + push %r13 + + mov 8*0($a_ptr), $a0 + mov 8*1($a_ptr), $a1 + mov 8*2($a_ptr), $a2 + mov $a0, $t0 + mov 8*3($a_ptr), $a3 + lea .Lpoly(%rip), $a_ptr + + mov $a1, $t1 + xor $t4, $t4 + add 8*0($a_ptr), $a0 + mov $a2, $t2 + adc 8*1($a_ptr), $a1 + adc 8*2($a_ptr), $a2 + mov $a3, $t3 + adc 8*3($a_ptr), $a3 + adc \$0, $t4 + xor $a_ptr, $a_ptr # borrow $a_ptr + test \$1, $t0 + + cmovz $t0, $a0 + cmovz $t1, $a1 + cmovz $t2, $a2 + cmovz $t3, $a3 + cmovz $a_ptr, $t4 + + mov $a1, $t0 # a0:a3>>1 + shr \$1, $a0 + shl \$63, $t0 + mov $a2, $t1 + shr \$1, $a1 + or $t0, $a0 + shl \$63, $t1 + mov $a3, $t2 + shr \$1, $a2 + or $t1, $a1 + shl \$63, $t2 + shr \$1, $a3 + shl \$63, $t4 + or $t2, $a2 + or $t4, $a3 + + mov $a0, 8*0($r_ptr) + mov $a1, 8*1($r_ptr) + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + pop %r13 + pop %r12 + ret +.size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2 + +################################################################################ +# void ecp_nistz256_mul_by_3(uint64_t res[4], uint64_t a[4]); +.globl ecp_nistz256_mul_by_3 +.type ecp_nistz256_mul_by_3,\@function,2 +.align 32 +ecp_nistz256_mul_by_3: + push %r12 + push %r13 + + mov 8*0($a_ptr), $a0 + xor $t4, $t4 + mov 8*1($a_ptr), $a1 + add $a0, $a0 # a0:a3+a0:a3 + mov 8*2($a_ptr), $a2 + adc $a1, $a1 + mov 8*3($a_ptr), $a3 + mov $a0, $t0 + adc $a2, $a2 + adc $a3, $a3 + mov $a1, $t1 + adc \$0, $t4 + + sub \$-1, $a0 + mov $a2, $t2 + sbb .Lpoly+8*1(%rip), $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb .Lpoly+8*3(%rip), $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + cmovz $t2, $a2 + cmovz $t3, $a3 + + xor $t4, $t4 + add 8*0($a_ptr), $a0 # a0:a3+=a_ptr[0:3] + adc 8*1($a_ptr), $a1 + mov $a0, $t0 + adc 8*2($a_ptr), $a2 + adc 8*3($a_ptr), $a3 + mov $a1, $t1 + adc \$0, $t4 + + sub \$-1, $a0 + mov $a2, $t2 + sbb .Lpoly+8*1(%rip), $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb .Lpoly+8*3(%rip), $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + pop %r13 + pop %r12 + ret +.size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3 + +################################################################################ +# void ecp_nistz256_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +.globl ecp_nistz256_add +.type ecp_nistz256_add,\@function,3 +.align 32 +ecp_nistz256_add: + push %r12 + push %r13 + + mov 8*0($a_ptr), $a0 + xor $t4, $t4 + mov 8*1($a_ptr), $a1 + mov 8*2($a_ptr), $a2 + mov 8*3($a_ptr), $a3 + lea .Lpoly(%rip), $a_ptr + + add 8*0($b_ptr), $a0 + adc 8*1($b_ptr), $a1 + mov $a0, $t0 + adc 8*2($b_ptr), $a2 + adc 8*3($b_ptr), $a3 + mov $a1, $t1 + adc \$0, $t4 + + sub 8*0($a_ptr), $a0 + mov $a2, $t2 + sbb 8*1($a_ptr), $a1 + sbb 8*2($a_ptr), $a2 + mov $a3, $t3 + sbb 8*3($a_ptr), $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + pop %r13 + pop %r12 + ret +.size ecp_nistz256_add,.-ecp_nistz256_add + +################################################################################ +# void ecp_nistz256_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +.globl ecp_nistz256_sub +.type ecp_nistz256_sub,\@function,3 +.align 32 +ecp_nistz256_sub: + push %r12 + push %r13 + + mov 8*0($a_ptr), $a0 + xor $t4, $t4 + mov 8*1($a_ptr), $a1 + mov 8*2($a_ptr), $a2 + mov 8*3($a_ptr), $a3 + lea .Lpoly(%rip), $a_ptr + + sub 8*0($b_ptr), $a0 + sbb 8*1($b_ptr), $a1 + mov $a0, $t0 + sbb 8*2($b_ptr), $a2 + sbb 8*3($b_ptr), $a3 + mov $a1, $t1 + sbb \$0, $t4 + + add 8*0($a_ptr), $a0 + mov $a2, $t2 + adc 8*1($a_ptr), $a1 + adc 8*2($a_ptr), $a2 + mov $a3, $t3 + adc 8*3($a_ptr), $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + pop %r13 + pop %r12 + ret +.size ecp_nistz256_sub,.-ecp_nistz256_sub + +################################################################################ +# void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]); +.globl ecp_nistz256_neg +.type ecp_nistz256_neg,\@function,2 +.align 32 +ecp_nistz256_neg: + push %r12 + push %r13 + + xor $a0, $a0 + xor $a1, $a1 + xor $a2, $a2 + xor $a3, $a3 + xor $t4, $t4 + + sub 8*0($a_ptr), $a0 + sbb 8*1($a_ptr), $a1 + sbb 8*2($a_ptr), $a2 + mov $a0, $t0 + sbb 8*3($a_ptr), $a3 + lea .Lpoly(%rip), $a_ptr + mov $a1, $t1 + sbb \$0, $t4 + + add 8*0($a_ptr), $a0 + mov $a2, $t2 + adc 8*1($a_ptr), $a1 + adc 8*2($a_ptr), $a2 + mov $a3, $t3 + adc 8*3($a_ptr), $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + pop %r13 + pop %r12 + ret +.size ecp_nistz256_neg,.-ecp_nistz256_neg +___ +} +{ +my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx"); +my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15)); +my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax"); +my ($poly1,$poly3)=($acc6,$acc7); + +$code.=<<___; +################################################################################ +# void ecp_nistz256_to_mont( +# uint64_t res[4], +# uint64_t in[4]); +.globl ecp_nistz256_to_mont +.type ecp_nistz256_to_mont,\@function,2 +.align 32 +ecp_nistz256_to_mont: +___ +$code.=<<___ if ($addx); + mov \$0x80100, %ecx + and OPENSSL_ia32cap_P+8(%rip), %ecx +___ +$code.=<<___; + lea .LRR(%rip), $b_org + jmp .Lmul_mont +.size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont + +################################################################################ +# void ecp_nistz256_mul_mont( +# uint64_t res[4], +# uint64_t a[4], +# uint64_t b[4]); + +.globl ecp_nistz256_mul_mont +.type ecp_nistz256_mul_mont,\@function,3 +.align 32 +ecp_nistz256_mul_mont: +___ +$code.=<<___ if ($addx); + mov \$0x80100, %ecx + and OPENSSL_ia32cap_P+8(%rip), %ecx +___ +$code.=<<___; +.Lmul_mont: + push %rbp + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($addx); + cmp \$0x80100, %ecx + je .Lmul_montx +___ +$code.=<<___; + mov $b_org, $b_ptr + mov 8*0($b_org), %rax + mov 8*0($a_ptr), $acc1 + mov 8*1($a_ptr), $acc2 + mov 8*2($a_ptr), $acc3 + mov 8*3($a_ptr), $acc4 + + call __ecp_nistz256_mul_montq +___ +$code.=<<___ if ($addx); + jmp .Lmul_mont_done + +.align 32 +.Lmul_montx: + mov $b_org, $b_ptr + mov 8*0($b_org), %rdx + mov 8*0($a_ptr), $acc1 + mov 8*1($a_ptr), $acc2 + mov 8*2($a_ptr), $acc3 + mov 8*3($a_ptr), $acc4 + lea -128($a_ptr), $a_ptr # control u-op density + + call __ecp_nistz256_mul_montx +___ +$code.=<<___; +.Lmul_mont_done: + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + pop %rbp + ret +.size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont + +.type __ecp_nistz256_mul_montq,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_montq: + ######################################################################## + # Multiply a by b[0] + mov %rax, $t1 + mulq $acc1 + mov .Lpoly+8*1(%rip),$poly1 + mov %rax, $acc0 + mov $t1, %rax + mov %rdx, $acc1 + + mulq $acc2 + mov .Lpoly+8*3(%rip),$poly3 + add %rax, $acc1 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $acc2 + + mulq $acc3 + add %rax, $acc2 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $acc3 + + mulq $acc4 + add %rax, $acc3 + mov $acc0, %rax + adc \$0, %rdx + xor $acc5, $acc5 + mov %rdx, $acc4 + + ######################################################################## + # First reduction step + # Basically now we want to multiply acc[0] by p256, + # and add the result to the acc. + # Due to the special form of p256 we do some optimizations + # + # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0] + # then we add acc[0] and get acc[0] x 2^96 + + mov $acc0, $t1 + shl \$32, $acc0 + mulq $poly3 + shr \$32, $t1 + add $acc0, $acc1 # +=acc[0]<<96 + adc $t1, $acc2 + adc %rax, $acc3 + mov 8*1($b_ptr), %rax + adc %rdx, $acc4 + adc \$0, $acc5 + xor $acc0, $acc0 + + ######################################################################## + # Multiply by b[1] + mov %rax, $t1 + mulq 8*0($a_ptr) + add %rax, $acc1 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*1($a_ptr) + add $t0, $acc2 + adc \$0, %rdx + add %rax, $acc2 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*2($a_ptr) + add $t0, $acc3 + adc \$0, %rdx + add %rax, $acc3 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*3($a_ptr) + add $t0, $acc4 + adc \$0, %rdx + add %rax, $acc4 + mov $acc1, %rax + adc %rdx, $acc5 + adc \$0, $acc0 + + ######################################################################## + # Second reduction step + mov $acc1, $t1 + shl \$32, $acc1 + mulq $poly3 + shr \$32, $t1 + add $acc1, $acc2 + adc $t1, $acc3 + adc %rax, $acc4 + mov 8*2($b_ptr), %rax + adc %rdx, $acc5 + adc \$0, $acc0 + xor $acc1, $acc1 + + ######################################################################## + # Multiply by b[2] + mov %rax, $t1 + mulq 8*0($a_ptr) + add %rax, $acc2 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*1($a_ptr) + add $t0, $acc3 + adc \$0, %rdx + add %rax, $acc3 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*2($a_ptr) + add $t0, $acc4 + adc \$0, %rdx + add %rax, $acc4 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*3($a_ptr) + add $t0, $acc5 + adc \$0, %rdx + add %rax, $acc5 + mov $acc2, %rax + adc %rdx, $acc0 + adc \$0, $acc1 + + ######################################################################## + # Third reduction step + mov $acc2, $t1 + shl \$32, $acc2 + mulq $poly3 + shr \$32, $t1 + add $acc2, $acc3 + adc $t1, $acc4 + adc %rax, $acc5 + mov 8*3($b_ptr), %rax + adc %rdx, $acc0 + adc \$0, $acc1 + xor $acc2, $acc2 + + ######################################################################## + # Multiply by b[3] + mov %rax, $t1 + mulq 8*0($a_ptr) + add %rax, $acc3 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*1($a_ptr) + add $t0, $acc4 + adc \$0, %rdx + add %rax, $acc4 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*2($a_ptr) + add $t0, $acc5 + adc \$0, %rdx + add %rax, $acc5 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*3($a_ptr) + add $t0, $acc0 + adc \$0, %rdx + add %rax, $acc0 + mov $acc3, %rax + adc %rdx, $acc1 + adc \$0, $acc2 + + ######################################################################## + # Final reduction step + mov $acc3, $t1 + shl \$32, $acc3 + mulq $poly3 + shr \$32, $t1 + add $acc3, $acc4 + adc $t1, $acc5 + mov $acc4, $t0 + adc %rax, $acc0 + adc %rdx, $acc1 + mov $acc5, $t1 + adc \$0, $acc2 + + ######################################################################## + # Branch-less conditional subtraction of P + sub \$-1, $acc4 # .Lpoly[0] + mov $acc0, $t2 + sbb $poly1, $acc5 # .Lpoly[1] + sbb \$0, $acc0 # .Lpoly[2] + mov $acc1, $t3 + sbb $poly3, $acc1 # .Lpoly[3] + sbb \$0, $acc2 + + cmovc $t0, $acc4 + cmovc $t1, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $t2, $acc0 + mov $acc5, 8*1($r_ptr) + cmovc $t3, $acc1 + mov $acc0, 8*2($r_ptr) + mov $acc1, 8*3($r_ptr) + + ret +.size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq + +################################################################################ +# void ecp_nistz256_sqr_mont( +# uint64_t res[4], +# uint64_t a[4]); + +# we optimize the square according to S.Gueron and V.Krasnov, +# "Speeding up Big-Number Squaring" +.globl ecp_nistz256_sqr_mont +.type ecp_nistz256_sqr_mont,\@function,2 +.align 32 +ecp_nistz256_sqr_mont: +___ +$code.=<<___ if ($addx); + mov \$0x80100, %ecx + and OPENSSL_ia32cap_P+8(%rip), %ecx +___ +$code.=<<___; + push %rbp + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($addx); + cmp \$0x80100, %ecx + je .Lsqr_montx +___ +$code.=<<___; + mov 8*0($a_ptr), %rax + mov 8*1($a_ptr), $acc6 + mov 8*2($a_ptr), $acc7 + mov 8*3($a_ptr), $acc0 + + call __ecp_nistz256_sqr_montq +___ +$code.=<<___ if ($addx); + jmp .Lsqr_mont_done + +.align 32 +.Lsqr_montx: + mov 8*0($a_ptr), %rdx + mov 8*1($a_ptr), $acc6 + mov 8*2($a_ptr), $acc7 + mov 8*3($a_ptr), $acc0 + lea -128($a_ptr), $a_ptr # control u-op density + + call __ecp_nistz256_sqr_montx +___ +$code.=<<___; +.Lsqr_mont_done: + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + pop %rbp + ret +.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont + +.type __ecp_nistz256_sqr_montq,\@abi-omnipotent +.align 32 +__ecp_nistz256_sqr_montq: + mov %rax, $acc5 + mulq $acc6 # a[1]*a[0] + mov %rax, $acc1 + mov $acc7, %rax + mov %rdx, $acc2 + + mulq $acc5 # a[0]*a[2] + add %rax, $acc2 + mov $acc0, %rax + adc \$0, %rdx + mov %rdx, $acc3 + + mulq $acc5 # a[0]*a[3] + add %rax, $acc3 + mov $acc7, %rax + adc \$0, %rdx + mov %rdx, $acc4 + + ################################# + mulq $acc6 # a[1]*a[2] + add %rax, $acc3 + mov $acc0, %rax + adc \$0, %rdx + mov %rdx, $t1 + + mulq $acc6 # a[1]*a[3] + add %rax, $acc4 + mov $acc0, %rax + adc \$0, %rdx + add $t1, $acc4 + mov %rdx, $acc5 + adc \$0, $acc5 + + ################################# + mulq $acc7 # a[2]*a[3] + xor $acc7, $acc7 + add %rax, $acc5 + mov 8*0($a_ptr), %rax + mov %rdx, $acc6 + adc \$0, $acc6 + + add $acc1, $acc1 # acc1:6<<1 + adc $acc2, $acc2 + adc $acc3, $acc3 + adc $acc4, $acc4 + adc $acc5, $acc5 + adc $acc6, $acc6 + adc \$0, $acc7 + + mulq %rax + mov %rax, $acc0 + mov 8*1($a_ptr), %rax + mov %rdx, $t0 + + mulq %rax + add $t0, $acc1 + adc %rax, $acc2 + mov 8*2($a_ptr), %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq %rax + add $t0, $acc3 + adc %rax, $acc4 + mov 8*3($a_ptr), %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq %rax + add $t0, $acc5 + adc %rax, $acc6 + mov $acc0, %rax + adc %rdx, $acc7 + + mov .Lpoly+8*1(%rip), $a_ptr + mov .Lpoly+8*3(%rip), $t1 + + ########################################## + # Now the reduction + # First iteration + mov $acc0, $t0 + shl \$32, $acc0 + mulq $t1 + shr \$32, $t0 + add $acc0, $acc1 # +=acc[0]<<96 + adc $t0, $acc2 + adc %rax, $acc3 + mov $acc1, %rax + adc \$0, %rdx + + ########################################## + # Second iteration + mov $acc1, $t0 + shl \$32, $acc1 + mov %rdx, $acc0 + mulq $t1 + shr \$32, $t0 + add $acc1, $acc2 + adc $t0, $acc3 + adc %rax, $acc0 + mov $acc2, %rax + adc \$0, %rdx + + ########################################## + # Third iteration + mov $acc2, $t0 + shl \$32, $acc2 + mov %rdx, $acc1 + mulq $t1 + shr \$32, $t0 + add $acc2, $acc3 + adc $t0, $acc0 + adc %rax, $acc1 + mov $acc3, %rax + adc \$0, %rdx + + ########################################### + # Last iteration + mov $acc3, $t0 + shl \$32, $acc3 + mov %rdx, $acc2 + mulq $t1 + shr \$32, $t0 + add $acc3, $acc0 + adc $t0, $acc1 + adc %rax, $acc2 + adc \$0, %rdx + xor $acc3, $acc3 + + ############################################ + # Add the rest of the acc + add $acc0, $acc4 + adc $acc1, $acc5 + mov $acc4, $acc0 + adc $acc2, $acc6 + adc %rdx, $acc7 + mov $acc5, $acc1 + adc \$0, $acc3 + + sub \$-1, $acc4 # .Lpoly[0] + mov $acc6, $acc2 + sbb $a_ptr, $acc5 # .Lpoly[1] + sbb \$0, $acc6 # .Lpoly[2] + mov $acc7, $t0 + sbb $t1, $acc7 # .Lpoly[3] + sbb \$0, $acc3 + + cmovc $acc0, $acc4 + cmovc $acc1, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $acc2, $acc6 + mov $acc5, 8*1($r_ptr) + cmovc $t0, $acc7 + mov $acc6, 8*2($r_ptr) + mov $acc7, 8*3($r_ptr) + + ret +.size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq +___ + +if ($addx) { +$code.=<<___; +.type __ecp_nistz256_mul_montx,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_montx: + ######################################################################## + # Multiply by b[0] + mulx $acc1, $acc0, $acc1 + mulx $acc2, $t0, $acc2 + mov \$32, $poly1 + xor $acc5, $acc5 # cf=0 + mulx $acc3, $t1, $acc3 + mov .Lpoly+8*3(%rip), $poly3 + adc $t0, $acc1 + mulx $acc4, $t0, $acc4 + mov $acc0, %rdx + adc $t1, $acc2 + shlx $poly1,$acc0,$t1 + adc $t0, $acc3 + shrx $poly1,$acc0,$t0 + adc \$0, $acc4 + + ######################################################################## + # First reduction step + add $t1, $acc1 + adc $t0, $acc2 + + mulx $poly3, $t0, $t1 + mov 8*1($b_ptr), %rdx + adc $t0, $acc3 + adc $t1, $acc4 + adc \$0, $acc5 + xor $acc0, $acc0 # $acc0=0,cf=0,of=0 + + ######################################################################## + # Multiply by b[1] + mulx 8*0+128($a_ptr), $t0, $t1 + adcx $t0, $acc1 + adox $t1, $acc2 + + mulx 8*1+128($a_ptr), $t0, $t1 + adcx $t0, $acc2 + adox $t1, $acc3 + + mulx 8*2+128($a_ptr), $t0, $t1 + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx 8*3+128($a_ptr), $t0, $t1 + mov $acc1, %rdx + adcx $t0, $acc4 + shlx $poly1, $acc1, $t0 + adox $t1, $acc5 + shrx $poly1, $acc1, $t1 + + adcx $acc0, $acc5 + adox $acc0, $acc0 + adc \$0, $acc0 + + ######################################################################## + # Second reduction step + add $t0, $acc2 + adc $t1, $acc3 + + mulx $poly3, $t0, $t1 + mov 8*2($b_ptr), %rdx + adc $t0, $acc4 + adc $t1, $acc5 + adc \$0, $acc0 + xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0 + + ######################################################################## + # Multiply by b[2] + mulx 8*0+128($a_ptr), $t0, $t1 + adcx $t0, $acc2 + adox $t1, $acc3 + + mulx 8*1+128($a_ptr), $t0, $t1 + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx 8*2+128($a_ptr), $t0, $t1 + adcx $t0, $acc4 + adox $t1, $acc5 + + mulx 8*3+128($a_ptr), $t0, $t1 + mov $acc2, %rdx + adcx $t0, $acc5 + shlx $poly1, $acc2, $t0 + adox $t1, $acc0 + shrx $poly1, $acc2, $t1 + + adcx $acc1, $acc0 + adox $acc1, $acc1 + adc \$0, $acc1 + + ######################################################################## + # Third reduction step + add $t0, $acc3 + adc $t1, $acc4 + + mulx $poly3, $t0, $t1 + mov 8*3($b_ptr), %rdx + adc $t0, $acc5 + adc $t1, $acc0 + adc \$0, $acc1 + xor $acc2, $acc2 # $acc2=0,cf=0,of=0 + + ######################################################################## + # Multiply by b[3] + mulx 8*0+128($a_ptr), $t0, $t1 + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx 8*1+128($a_ptr), $t0, $t1 + adcx $t0, $acc4 + adox $t1, $acc5 + + mulx 8*2+128($a_ptr), $t0, $t1 + adcx $t0, $acc5 + adox $t1, $acc0 + + mulx 8*3+128($a_ptr), $t0, $t1 + mov $acc3, %rdx + adcx $t0, $acc0 + shlx $poly1, $acc3, $t0 + adox $t1, $acc1 + shrx $poly1, $acc3, $t1 + + adcx $acc2, $acc1 + adox $acc2, $acc2 + adc \$0, $acc2 + + ######################################################################## + # Fourth reduction step + add $t0, $acc4 + adc $t1, $acc5 + + mulx $poly3, $t0, $t1 + mov $acc4, $t2 + mov .Lpoly+8*1(%rip), $poly1 + adc $t0, $acc0 + mov $acc5, $t3 + adc $t1, $acc1 + adc \$0, $acc2 + + ######################################################################## + # Branch-less conditional subtraction of P + xor %eax, %eax + mov $acc0, $t0 + sbb \$-1, $acc4 # .Lpoly[0] + sbb $poly1, $acc5 # .Lpoly[1] + sbb \$0, $acc0 # .Lpoly[2] + mov $acc1, $t1 + sbb $poly3, $acc1 # .Lpoly[3] + sbb \$0, $acc2 + + cmovc $t2, $acc4 + cmovc $t3, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $t0, $acc0 + mov $acc5, 8*1($r_ptr) + cmovc $t1, $acc1 + mov $acc0, 8*2($r_ptr) + mov $acc1, 8*3($r_ptr) + + ret +.size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx + +.type __ecp_nistz256_sqr_montx,\@abi-omnipotent +.align 32 +__ecp_nistz256_sqr_montx: + mulx $acc6, $acc1, $acc2 # a[0]*a[1] + mulx $acc7, $t0, $acc3 # a[0]*a[2] + xor %eax, %eax + adc $t0, $acc2 + mulx $acc0, $t1, $acc4 # a[0]*a[3] + mov $acc6, %rdx + adc $t1, $acc3 + adc \$0, $acc4 + xor $acc5, $acc5 # $acc5=0,cf=0,of=0 + + ################################# + mulx $acc7, $t0, $t1 # a[1]*a[2] + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx $acc0, $t0, $t1 # a[1]*a[3] + mov $acc7, %rdx + adcx $t0, $acc4 + adox $t1, $acc5 + adc \$0, $acc5 + + ################################# + mulx $acc0, $t0, $acc6 # a[2]*a[3] + mov 8*0+128($a_ptr), %rdx + xor $acc7, $acc7 # $acc7=0,cf=0,of=0 + adcx $acc1, $acc1 # acc1:6<<1 + adox $t0, $acc5 + adcx $acc2, $acc2 + adox $acc7, $acc6 # of=0 + + mulx %rdx, $acc0, $t1 + mov 8*1+128($a_ptr), %rdx + adcx $acc3, $acc3 + adox $t1, $acc1 + adcx $acc4, $acc4 + mulx %rdx, $t0, $t4 + mov 8*2+128($a_ptr), %rdx + adcx $acc5, $acc5 + adox $t0, $acc2 + adcx $acc6, $acc6 + .byte 0x67 + mulx %rdx, $t0, $t1 + mov 8*3+128($a_ptr), %rdx + adox $t4, $acc3 + adcx $acc7, $acc7 + adox $t0, $acc4 + mov \$32, $a_ptr + adox $t1, $acc5 + .byte 0x67,0x67 + mulx %rdx, $t0, $t4 + mov $acc0, %rdx + adox $t0, $acc6 + shlx $a_ptr, $acc0, $t0 + adox $t4, $acc7 + shrx $a_ptr, $acc0, $t4 + mov .Lpoly+8*3(%rip), $t1 + + # reduction step 1 + add $t0, $acc1 + adc $t4, $acc2 + + mulx $t1, $t0, $acc0 + mov $acc1, %rdx + adc $t0, $acc3 + shlx $a_ptr, $acc1, $t0 + adc \$0, $acc0 + shrx $a_ptr, $acc1, $t4 + + # reduction step 2 + add $t0, $acc2 + adc $t4, $acc3 + + mulx $t1, $t0, $acc1 + mov $acc2, %rdx + adc $t0, $acc0 + shlx $a_ptr, $acc2, $t0 + adc \$0, $acc1 + shrx $a_ptr, $acc2, $t4 + + # reduction step 3 + add $t0, $acc3 + adc $t4, $acc0 + + mulx $t1, $t0, $acc2 + mov $acc3, %rdx + adc $t0, $acc1 + shlx $a_ptr, $acc3, $t0 + adc \$0, $acc2 + shrx $a_ptr, $acc3, $t4 + + # reduction step 4 + add $t0, $acc0 + adc $t4, $acc1 + + mulx $t1, $t0, $acc3 + adc $t0, $acc2 + adc \$0, $acc3 + + xor $t3, $t3 # cf=0 + adc $acc0, $acc4 # accumulate upper half + mov .Lpoly+8*1(%rip), $a_ptr + adc $acc1, $acc5 + mov $acc4, $acc0 + adc $acc2, $acc6 + adc $acc3, $acc7 + mov $acc5, $acc1 + adc \$0, $t3 + + xor %eax, %eax # cf=0 + sbb \$-1, $acc4 # .Lpoly[0] + mov $acc6, $acc2 + sbb $a_ptr, $acc5 # .Lpoly[1] + sbb \$0, $acc6 # .Lpoly[2] + mov $acc7, $acc3 + sbb $t1, $acc7 # .Lpoly[3] + sbb \$0, $t3 + + cmovc $acc0, $acc4 + cmovc $acc1, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $acc2, $acc6 + mov $acc5, 8*1($r_ptr) + cmovc $acc3, $acc7 + mov $acc6, 8*2($r_ptr) + mov $acc7, 8*3($r_ptr) + + ret +.size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx +___ +} +} +{ +my ($r_ptr,$in_ptr)=("%rdi","%rsi"); +my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11)); +my ($t0,$t1,$t2)=("%rcx","%r12","%r13"); + +$code.=<<___; +################################################################################ +# void ecp_nistz256_from_mont( +# uint64_t res[4], +# uint64_t in[4]); +# This one performs Montgomery multiplication by 1, so we only need the reduction + +.globl ecp_nistz256_from_mont +.type ecp_nistz256_from_mont,\@function,2 +.align 32 +ecp_nistz256_from_mont: + push %r12 + push %r13 + + mov 8*0($in_ptr), %rax + mov .Lpoly+8*3(%rip), $t2 + mov 8*1($in_ptr), $acc1 + mov 8*2($in_ptr), $acc2 + mov 8*3($in_ptr), $acc3 + mov %rax, $acc0 + mov .Lpoly+8*1(%rip), $t1 + + ######################################### + # First iteration + mov %rax, $t0 + shl \$32, $acc0 + mulq $t2 + shr \$32, $t0 + add $acc0, $acc1 + adc $t0, $acc2 + adc %rax, $acc3 + mov $acc1, %rax + adc \$0, %rdx + + ######################################### + # Second iteration + mov $acc1, $t0 + shl \$32, $acc1 + mov %rdx, $acc0 + mulq $t2 + shr \$32, $t0 + add $acc1, $acc2 + adc $t0, $acc3 + adc %rax, $acc0 + mov $acc2, %rax + adc \$0, %rdx + + ########################################## + # Third iteration + mov $acc2, $t0 + shl \$32, $acc2 + mov %rdx, $acc1 + mulq $t2 + shr \$32, $t0 + add $acc2, $acc3 + adc $t0, $acc0 + adc %rax, $acc1 + mov $acc3, %rax + adc \$0, %rdx + + ########################################### + # Last iteration + mov $acc3, $t0 + shl \$32, $acc3 + mov %rdx, $acc2 + mulq $t2 + shr \$32, $t0 + add $acc3, $acc0 + adc $t0, $acc1 + mov $acc0, $t0 + adc %rax, $acc2 + mov $acc1, $in_ptr + adc \$0, %rdx + + ########################################### + # Branch-less conditional subtraction + sub \$-1, $acc0 + mov $acc2, %rax + sbb $t1, $acc1 + sbb \$0, $acc2 + mov %rdx, $acc3 + sbb $t2, %rdx + sbb $t2, $t2 + + cmovnz $t0, $acc0 + cmovnz $in_ptr, $acc1 + mov $acc0, 8*0($r_ptr) + cmovnz %rax, $acc2 + mov $acc1, 8*1($r_ptr) + cmovz %rdx, $acc3 + mov $acc2, 8*2($r_ptr) + mov $acc3, 8*3($r_ptr) + + pop %r13 + pop %r12 + ret +.size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont +___ +} +{ +my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); +my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7)); +my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15)); +my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15)); + +$code.=<<___; +################################################################################ +# void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w5 +.type ecp_nistz256_select_w5,\@abi-omnipotent +.align 32 +ecp_nistz256_select_w5: +___ +$code.=<<___ if ($avx>1); + mov OPENSSL_ia32cap_P+8(%rip), %eax + test \$`1<<5`, %eax + jnz .Lavx2_select_w5 +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp), %rax +.LSEH_begin_ecp_nistz256_select_w5: + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp + .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax) + .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax) + .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax) + .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax) + .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax) + .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax) + .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax) + .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax) + .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax) + .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + movdqa .LOne(%rip), $ONE + movd $index, $INDEX + + pxor $Ra, $Ra + pxor $Rb, $Rb + pxor $Rc, $Rc + pxor $Rd, $Rd + pxor $Re, $Re + pxor $Rf, $Rf + + movdqa $ONE, $M0 + pshufd \$0, $INDEX, $INDEX + + mov \$16, %rax +.Lselect_loop_sse_w5: + + movdqa $M0, $TMP0 + paddd $ONE, $M0 + pcmpeqd $INDEX, $TMP0 + + movdqa 16*0($in_t), $T0a + movdqa 16*1($in_t), $T0b + movdqa 16*2($in_t), $T0c + movdqa 16*3($in_t), $T0d + movdqa 16*4($in_t), $T0e + movdqa 16*5($in_t), $T0f + lea 16*6($in_t), $in_t + + pand $TMP0, $T0a + pand $TMP0, $T0b + por $T0a, $Ra + pand $TMP0, $T0c + por $T0b, $Rb + pand $TMP0, $T0d + por $T0c, $Rc + pand $TMP0, $T0e + por $T0d, $Rd + pand $TMP0, $T0f + por $T0e, $Re + por $T0f, $Rf + + dec %rax + jnz .Lselect_loop_sse_w5 + + movdqu $Ra, 16*0($val) + movdqu $Rb, 16*1($val) + movdqu $Rc, 16*2($val) + movdqu $Rd, 16*3($val) + movdqu $Re, 16*4($val) + movdqu $Rf, 16*5($val) +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea 0xa8(%rsp), %rsp +.LSEH_end_ecp_nistz256_select_w5: +___ +$code.=<<___; + ret +.size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5 + +################################################################################ +# void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w7 +.type ecp_nistz256_select_w7,\@abi-omnipotent +.align 32 +ecp_nistz256_select_w7: +___ +$code.=<<___ if ($avx>1); + mov OPENSSL_ia32cap_P+8(%rip), %eax + test \$`1<<5`, %eax + jnz .Lavx2_select_w7 +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp), %rax +.LSEH_begin_ecp_nistz256_select_w7: + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp + .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax) + .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax) + .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax) + .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax) + .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax) + .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax) + .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax) + .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax) + .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax) + .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + movdqa .LOne(%rip), $M0 + movd $index, $INDEX + + pxor $Ra, $Ra + pxor $Rb, $Rb + pxor $Rc, $Rc + pxor $Rd, $Rd + + movdqa $M0, $ONE + pshufd \$0, $INDEX, $INDEX + mov \$64, %rax + +.Lselect_loop_sse_w7: + movdqa $M0, $TMP0 + paddd $ONE, $M0 + movdqa 16*0($in_t), $T0a + movdqa 16*1($in_t), $T0b + pcmpeqd $INDEX, $TMP0 + movdqa 16*2($in_t), $T0c + movdqa 16*3($in_t), $T0d + lea 16*4($in_t), $in_t + + pand $TMP0, $T0a + pand $TMP0, $T0b + por $T0a, $Ra + pand $TMP0, $T0c + por $T0b, $Rb + pand $TMP0, $T0d + por $T0c, $Rc + prefetcht0 255($in_t) + por $T0d, $Rd + + dec %rax + jnz .Lselect_loop_sse_w7 + + movdqu $Ra, 16*0($val) + movdqu $Rb, 16*1($val) + movdqu $Rc, 16*2($val) + movdqu $Rd, 16*3($val) +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea 0xa8(%rsp), %rsp +.LSEH_end_ecp_nistz256_select_w7: +___ +$code.=<<___; + ret +.size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7 +___ +} +if ($avx>1) { +my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); +my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4)); +my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9)); +my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14)); + +$code.=<<___; +################################################################################ +# void ecp_nistz256_avx2_select_w5(uint64_t *val, uint64_t *in_t, int index); +.type ecp_nistz256_avx2_select_w5,\@abi-omnipotent +.align 32 +ecp_nistz256_avx2_select_w5: +.Lavx2_select_w5: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp), %rax +.LSEH_begin_ecp_nistz256_avx2_select_w5: + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp + .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax) + .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax) + .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax) + .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax) + .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax) + .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax) + .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax) + .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax) + .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax) + .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + vmovdqa .LTwo(%rip), $TWO + + vpxor $Ra, $Ra, $Ra + vpxor $Rb, $Rb, $Rb + vpxor $Rc, $Rc, $Rc + + vmovdqa .LOne(%rip), $M0 + vmovdqa .LTwo(%rip), $M1 + + vmovd $index, %xmm1 + vpermd $INDEX, $Ra, $INDEX + + mov \$8, %rax +.Lselect_loop_avx2_w5: + + vmovdqa 32*0($in_t), $T0a + vmovdqa 32*1($in_t), $T0b + vmovdqa 32*2($in_t), $T0c + + vmovdqa 32*3($in_t), $T1a + vmovdqa 32*4($in_t), $T1b + vmovdqa 32*5($in_t), $T1c + + vpcmpeqd $INDEX, $M0, $TMP0 + vpcmpeqd $INDEX, $M1, $TMP1 + + vpaddd $TWO, $M0, $M0 + vpaddd $TWO, $M1, $M1 + lea 32*6($in_t), $in_t + + vpand $TMP0, $T0a, $T0a + vpand $TMP0, $T0b, $T0b + vpand $TMP0, $T0c, $T0c + vpand $TMP1, $T1a, $T1a + vpand $TMP1, $T1b, $T1b + vpand $TMP1, $T1c, $T1c + + vpxor $T0a, $Ra, $Ra + vpxor $T0b, $Rb, $Rb + vpxor $T0c, $Rc, $Rc + vpxor $T1a, $Ra, $Ra + vpxor $T1b, $Rb, $Rb + vpxor $T1c, $Rc, $Rc + + dec %rax + jnz .Lselect_loop_avx2_w5 + + vmovdqu $Ra, 32*0($val) + vmovdqu $Rb, 32*1($val) + vmovdqu $Rc, 32*2($val) + vzeroupper +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea 0xa8(%rsp), %rsp +.LSEH_end_ecp_nistz256_avx2_select_w5: +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_select_w5,.-ecp_nistz256_avx2_select_w5 +___ +} +if ($avx>1) { +my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); +my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3)); +my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7)); +my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11)); +my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15)); + +$code.=<<___; + +################################################################################ +# void ecp_nistz256_avx2_select_w7(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_avx2_select_w7 +.type ecp_nistz256_avx2_select_w7,\@abi-omnipotent +.align 32 +ecp_nistz256_avx2_select_w7: +.Lavx2_select_w7: + vzeroupper +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp), %rax +.LSEH_begin_ecp_nistz256_avx2_select_w7: + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp + .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax) + .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax) + .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax) + .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax) + .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax) + .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax) + .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax) + .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax) + .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax) + .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + vmovdqa .LThree(%rip), $THREE + + vpxor $Ra, $Ra, $Ra + vpxor $Rb, $Rb, $Rb + + vmovdqa .LOne(%rip), $M0 + vmovdqa .LTwo(%rip), $M1 + vmovdqa .LThree(%rip), $M2 + + vmovd $index, %xmm1 + vpermd $INDEX, $Ra, $INDEX + # Skip index = 0, because it is implicitly the point at infinity + + mov \$21, %rax +.Lselect_loop_avx2_w7: + + vmovdqa 32*0($in_t), $T0a + vmovdqa 32*1($in_t), $T0b + + vmovdqa 32*2($in_t), $T1a + vmovdqa 32*3($in_t), $T1b + + vmovdqa 32*4($in_t), $T2a + vmovdqa 32*5($in_t), $T2b + + vpcmpeqd $INDEX, $M0, $TMP0 + vpcmpeqd $INDEX, $M1, $TMP1 + vpcmpeqd $INDEX, $M2, $TMP2 + + vpaddd $THREE, $M0, $M0 + vpaddd $THREE, $M1, $M1 + vpaddd $THREE, $M2, $M2 + lea 32*6($in_t), $in_t + + vpand $TMP0, $T0a, $T0a + vpand $TMP0, $T0b, $T0b + vpand $TMP1, $T1a, $T1a + vpand $TMP1, $T1b, $T1b + vpand $TMP2, $T2a, $T2a + vpand $TMP2, $T2b, $T2b + + vpxor $T0a, $Ra, $Ra + vpxor $T0b, $Rb, $Rb + vpxor $T1a, $Ra, $Ra + vpxor $T1b, $Rb, $Rb + vpxor $T2a, $Ra, $Ra + vpxor $T2b, $Rb, $Rb + + dec %rax + jnz .Lselect_loop_avx2_w7 + + + vmovdqa 32*0($in_t), $T0a + vmovdqa 32*1($in_t), $T0b + + vpcmpeqd $INDEX, $M0, $TMP0 + + vpand $TMP0, $T0a, $T0a + vpand $TMP0, $T0b, $T0b + + vpxor $T0a, $Ra, $Ra + vpxor $T0b, $Rb, $Rb + + vmovdqu $Ra, 32*0($val) + vmovdqu $Rb, 32*1($val) + vzeroupper +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea 0xa8(%rsp), %rsp +.LSEH_end_ecp_nistz256_avx2_select_w7: +___ +$code.=<<___; + ret +.size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7 +___ +} else { +$code.=<<___; +.globl ecp_nistz256_avx2_select_w7 +.type ecp_nistz256_avx2_select_w7,\@function,3 +.align 32 +ecp_nistz256_avx2_select_w7: + .byte 0x0f,0x0b # ud2 + ret +.size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7 +___ +} +{{{ +######################################################################## +# This block implements higher level point_double, point_add and +# point_add_affine. The key to performance in this case is to allow +# out-of-order execution logic to overlap computations from next step +# with tail processing from current step. By using tailored calling +# sequence we minimize inter-step overhead to give processor better +# shot at overlapping operations... +# +# You will notice that input data is copied to stack. Trouble is that +# there are no registers to spare for holding original pointers and +# reloading them, pointers, would create undesired dependencies on +# effective addresses calculation paths. In other words it's too done +# to favour out-of-order execution logic. +# + +my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx"); +my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15)); +my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4); +my ($poly1,$poly3)=($acc6,$acc7); + +sub load_for_mul () { +my ($a,$b,$src0) = @_; +my $bias = $src0 eq "%rax" ? 0 : -128; + +" mov $b, $src0 + lea $b, $b_ptr + mov 8*0+$a, $acc1 + mov 8*1+$a, $acc2 + lea $bias+$a, $a_ptr + mov 8*2+$a, $acc3 + mov 8*3+$a, $acc4" +} + +sub load_for_sqr () { +my ($a,$src0) = @_; +my $bias = $src0 eq "%rax" ? 0 : -128; + +" mov 8*0+$a, $src0 + mov 8*1+$a, $acc6 + lea $bias+$a, $a_ptr + mov 8*2+$a, $acc7 + mov 8*3+$a, $acc0" +} + + { +######################################################################## +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); + +$code.=<<___; +.type __ecp_nistz256_add_toq,\@abi-omnipotent +.align 32 +__ecp_nistz256_add_toq: + add 8*0($b_ptr), $a0 + adc 8*1($b_ptr), $a1 + mov $a0, $t0 + adc 8*2($b_ptr), $a2 + adc 8*3($b_ptr), $a3 + mov $a1, $t1 + sbb $t4, $t4 + + sub \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq + +.type __ecp_nistz256_sub_fromq,\@abi-omnipotent +.align 32 +__ecp_nistz256_sub_fromq: + sub 8*0($b_ptr), $a0 + sbb 8*1($b_ptr), $a1 + mov $a0, $t0 + sbb 8*2($b_ptr), $a2 + sbb 8*3($b_ptr), $a3 + mov $a1, $t1 + sbb $t4, $t4 + + add \$-1, $a0 + mov $a2, $t2 + adc $poly1, $a1 + adc \$0, $a2 + mov $a3, $t3 + adc $poly3, $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq + +.type __ecp_nistz256_subq,\@abi-omnipotent +.align 32 +__ecp_nistz256_subq: + sub $a0, $t0 + sbb $a1, $t1 + mov $t0, $a0 + sbb $a2, $t2 + sbb $a3, $t3 + mov $t1, $a1 + sbb $t4, $t4 + + add \$-1, $t0 + mov $t2, $a2 + adc $poly1, $t1 + adc \$0, $t2 + mov $t3, $a3 + adc $poly3, $t3 + test $t4, $t4 + + cmovnz $t0, $a0 + cmovnz $t1, $a1 + cmovnz $t2, $a2 + cmovnz $t3, $a3 + + ret +.size __ecp_nistz256_subq,.-__ecp_nistz256_subq + +.type __ecp_nistz256_mul_by_2q,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_by_2q: + add $a0, $a0 # a0:a3+a0:a3 + adc $a1, $a1 + mov $a0, $t0 + adc $a2, $a2 + adc $a3, $a3 + mov $a1, $t1 + sbb $t4, $t4 + + sub \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q +___ + } +sub gen_double () { + my $x = shift; + my ($src0,$sfx,$bias); + my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4)); + + if ($x ne "x") { + $src0 = "%rax"; + $sfx = ""; + $bias = 0; + +$code.=<<___; +.globl ecp_nistz256_point_double +.type ecp_nistz256_point_double,\@function,2 +.align 32 +ecp_nistz256_point_double: +___ +$code.=<<___ if ($addx); + mov \$0x80100, %ecx + and OPENSSL_ia32cap_P+8(%rip), %ecx + cmp \$0x80100, %ecx + je .Lpoint_doublex +___ + } else { + $src0 = "%rdx"; + $sfx = "x"; + $bias = 128; + +$code.=<<___; +.type ecp_nistz256_point_doublex,\@function,2 +.align 32 +ecp_nistz256_point_doublex: +.Lpoint_doublex: +___ + } +$code.=<<___; + push %rbp + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 + sub \$32*5+8, %rsp + + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x + mov $a_ptr, $b_ptr # backup copy + movdqu 0x10($a_ptr), %xmm1 + mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order + mov 0x20+8*1($a_ptr), $acc5 + mov 0x20+8*2($a_ptr), $acc0 + mov 0x20+8*3($a_ptr), $acc1 + mov .Lpoly+8*1(%rip), $poly1 + mov .Lpoly+8*3(%rip), $poly3 + movdqa %xmm0, $in_x(%rsp) + movdqa %xmm1, $in_x+0x10(%rsp) + lea 0x20($r_ptr), $acc2 + lea 0x40($r_ptr), $acc3 + movq $r_ptr, %xmm0 + movq $acc2, %xmm1 + movq $acc3, %xmm2 + + lea $S(%rsp), $r_ptr + call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y); + + mov 0x40+8*0($a_ptr), $src0 + mov 0x40+8*1($a_ptr), $acc6 + mov 0x40+8*2($a_ptr), $acc7 + mov 0x40+8*3($a_ptr), $acc0 + lea 0x40-$bias($a_ptr), $a_ptr + lea $Zsqr(%rsp), $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z); + + `&load_for_sqr("$S(%rsp)", "$src0")` + lea $S(%rsp), $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S); + + mov 0x20($b_ptr), $src0 # $b_ptr is still valid + mov 0x40+8*0($b_ptr), $acc1 + mov 0x40+8*1($b_ptr), $acc2 + mov 0x40+8*2($b_ptr), $acc3 + mov 0x40+8*3($b_ptr), $acc4 + lea 0x40-$bias($b_ptr), $a_ptr + lea 0x20($b_ptr), $b_ptr + movq %xmm2, $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z); + + mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order + mov $in_x+8*1(%rsp), $acc5 + lea $Zsqr(%rsp), $b_ptr + mov $in_x+8*2(%rsp), $acc0 + mov $in_x+8*3(%rsp), $acc1 + lea $M(%rsp), $r_ptr + call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr); + + mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order + mov $in_x+8*1(%rsp), $acc5 + lea $Zsqr(%rsp), $b_ptr + mov $in_x+8*2(%rsp), $acc0 + mov $in_x+8*3(%rsp), $acc1 + lea $Zsqr(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr); + + `&load_for_sqr("$S(%rsp)", "$src0")` + movq %xmm1, $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S); +___ +{ +######## ecp_nistz256_div_by_2(res_y, res_y); ########################## +# operate in 4-5-6-7 "name space" that matches squaring output +# +my ($poly1,$poly3)=($a_ptr,$t1); +my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2); + +$code.=<<___; + xor $t4, $t4 + mov $a0, $t0 + add \$-1, $a0 + mov $a1, $t1 + adc $poly1, $a1 + mov $a2, $t2 + adc \$0, $a2 + mov $a3, $t3 + adc $poly3, $a3 + adc \$0, $t4 + xor $a_ptr, $a_ptr # borrow $a_ptr + test \$1, $t0 + + cmovz $t0, $a0 + cmovz $t1, $a1 + cmovz $t2, $a2 + cmovz $t3, $a3 + cmovz $a_ptr, $t4 + + mov $a1, $t0 # a0:a3>>1 + shr \$1, $a0 + shl \$63, $t0 + mov $a2, $t1 + shr \$1, $a1 + or $t0, $a0 + shl \$63, $t1 + mov $a3, $t2 + shr \$1, $a2 + or $t1, $a1 + shl \$63, $t2 + mov $a0, 8*0($r_ptr) + shr \$1, $a3 + mov $a1, 8*1($r_ptr) + shl \$63, $t4 + or $t2, $a2 + or $t4, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) +___ +} +$code.=<<___; + `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")` + lea $M(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr); + + lea $tmp0(%rsp), $r_ptr + call __ecp_nistz256_mul_by_2$x + + lea $M(%rsp), $b_ptr + lea $M(%rsp), $r_ptr + call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M); + + `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")` + lea $S(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x); + + lea $tmp0(%rsp), $r_ptr + call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S); + + `&load_for_sqr("$M(%rsp)", "$src0")` + movq %xmm0, $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M); + + lea $tmp0(%rsp), $b_ptr + mov $acc6, $acc0 # harmonize sqr output and sub input + mov $acc7, $acc1 + mov $a_ptr, $poly1 + mov $t1, $poly3 + call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0); + + mov $S+8*0(%rsp), $t0 + mov $S+8*1(%rsp), $t1 + mov $S+8*2(%rsp), $t2 + mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order + lea $S(%rsp), $r_ptr + call __ecp_nistz256_sub$x # p256_sub(S, S, res_x); + + mov $M(%rsp), $src0 + lea $M(%rsp), $b_ptr + mov $acc4, $acc6 # harmonize sub output and mul input + xor %ecx, %ecx + mov $acc4, $S+8*0(%rsp) # have to save:-( + mov $acc5, $acc2 + mov $acc5, $S+8*1(%rsp) + cmovz $acc0, $acc3 + mov $acc0, $S+8*2(%rsp) + lea $S-$bias(%rsp), $a_ptr + cmovz $acc1, $acc4 + mov $acc1, $S+8*3(%rsp) + mov $acc6, $acc1 + lea $S(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M); + + movq %xmm1, $b_ptr + movq %xmm1, $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y); + + add \$32*5+8, %rsp + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + pop %rbp + ret +.size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx +___ +} +&gen_double("q"); + +sub gen_add () { + my $x = shift; + my ($src0,$sfx,$bias); + my ($H,$Hsqr,$R,$Rsqr,$Hcub, + $U1,$U2,$S1,$S2, + $res_x,$res_y,$res_z, + $in1_x,$in1_y,$in1_z, + $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17)); + my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr); + + if ($x ne "x") { + $src0 = "%rax"; + $sfx = ""; + $bias = 0; + +$code.=<<___; +.globl ecp_nistz256_point_add +.type ecp_nistz256_point_add,\@function,3 +.align 32 +ecp_nistz256_point_add: +___ +$code.=<<___ if ($addx); + mov \$0x80100, %ecx + and OPENSSL_ia32cap_P+8(%rip), %ecx + cmp \$0x80100, %ecx + je .Lpoint_addx +___ + } else { + $src0 = "%rdx"; + $sfx = "x"; + $bias = 128; + +$code.=<<___; +.type ecp_nistz256_point_addx,\@function,3 +.align 32 +ecp_nistz256_point_addx: +.Lpoint_addx: +___ + } +$code.=<<___; + push %rbp + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 + sub \$32*18+8, %rsp + + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr + movdqu 0x10($a_ptr), %xmm1 + movdqu 0x20($a_ptr), %xmm2 + movdqu 0x30($a_ptr), %xmm3 + movdqu 0x40($a_ptr), %xmm4 + movdqu 0x50($a_ptr), %xmm5 + mov $a_ptr, $b_ptr # reassign + mov $b_org, $a_ptr # reassign + movdqa %xmm0, $in1_x(%rsp) + movdqa %xmm1, $in1_x+0x10(%rsp) + por %xmm0, %xmm1 + movdqa %xmm2, $in1_y(%rsp) + movdqa %xmm3, $in1_y+0x10(%rsp) + por %xmm2, %xmm3 + movdqa %xmm4, $in1_z(%rsp) + movdqa %xmm5, $in1_z+0x10(%rsp) + por %xmm1, %xmm3 + + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr + pshufd \$0xb1, %xmm3, %xmm5 + movdqu 0x10($a_ptr), %xmm1 + movdqu 0x20($a_ptr), %xmm2 + por %xmm3, %xmm5 + movdqu 0x30($a_ptr), %xmm3 + mov 0x40+8*0($a_ptr), $src0 # load original in2_z + mov 0x40+8*1($a_ptr), $acc6 + mov 0x40+8*2($a_ptr), $acc7 + mov 0x40+8*3($a_ptr), $acc0 + movdqa %xmm0, $in2_x(%rsp) + pshufd \$0x1e, %xmm5, %xmm4 + movdqa %xmm1, $in2_x+0x10(%rsp) + por %xmm0, %xmm1 + movq $r_ptr, %xmm0 # save $r_ptr + movdqa %xmm2, $in2_y(%rsp) + movdqa %xmm3, $in2_y+0x10(%rsp) + por %xmm2, %xmm3 + por %xmm4, %xmm5 + pxor %xmm4, %xmm4 + por %xmm1, %xmm3 + + lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid + mov $src0, $in2_z+8*0(%rsp) # make in2_z copy + mov $acc6, $in2_z+8*1(%rsp) + mov $acc7, $in2_z+8*2(%rsp) + mov $acc0, $in2_z+8*3(%rsp) + lea $Z2sqr(%rsp), $r_ptr # Z2^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z); + + pcmpeqd %xmm4, %xmm5 + pshufd \$0xb1, %xmm3, %xmm4 + por %xmm3, %xmm4 + pshufd \$0, %xmm5, %xmm5 # in1infty + pshufd \$0x1e, %xmm4, %xmm3 + por %xmm3, %xmm4 + pxor %xmm3, %xmm3 + pcmpeqd %xmm3, %xmm4 + pshufd \$0, %xmm4, %xmm4 # in2infty + mov 0x40+8*0($b_ptr), $src0 # load original in1_z + mov 0x40+8*1($b_ptr), $acc6 + mov 0x40+8*2($b_ptr), $acc7 + mov 0x40+8*3($b_ptr), $acc0 + + lea 0x40-$bias($b_ptr), $a_ptr + lea $Z1sqr(%rsp), $r_ptr # Z1^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z); + + `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")` + lea $S1(%rsp), $r_ptr # S1 = Z2^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z); + + `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z); + + `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")` + lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y); + + `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y); + + lea $S1(%rsp), $b_ptr + lea $R(%rsp), $r_ptr # R = S2 - S1 + call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1); + + or $acc5, $acc4 # see if result is zero + movdqa %xmm4, %xmm2 + or $acc0, $acc4 + or $acc1, $acc4 + por %xmm5, %xmm2 # in1infty || in2infty + movq $acc4, %xmm3 + + `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")` + lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr); + + `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")` + lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr); + + lea $U1(%rsp), $b_ptr + lea $H(%rsp), $r_ptr # H = U2 - U1 + call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1); + + or $acc5, $acc4 # see if result is zero + or $acc0, $acc4 + or $acc1, $acc4 + + .byte 0x3e # predict taken + jnz .Ladd_proceed$x # is_equal(U1,U2)? + movq %xmm2, $acc0 + movq %xmm3, $acc1 + test $acc0, $acc0 + jnz .Ladd_proceed$x # (in1infty || in2infty)? + test $acc1, $acc1 + jz .Ladd_proceed$x # is_equal(S1,S2)? + + movq %xmm0, $r_ptr # restore $r_ptr + pxor %xmm0, %xmm0 + movdqu %xmm0, 0x00($r_ptr) + movdqu %xmm0, 0x10($r_ptr) + movdqu %xmm0, 0x20($r_ptr) + movdqu %xmm0, 0x30($r_ptr) + movdqu %xmm0, 0x40($r_ptr) + movdqu %xmm0, 0x50($r_ptr) + jmp .Ladd_done$x + +.align 32 +.Ladd_proceed$x: + `&load_for_sqr("$R(%rsp)", "$src0")` + lea $Rsqr(%rsp), $r_ptr # R^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R); + + `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z); + + `&load_for_sqr("$H(%rsp)", "$src0")` + lea $Hsqr(%rsp), $r_ptr # H^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H); + + `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")` + lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z); + + `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")` + lea $Hcub(%rsp), $r_ptr # H^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H); + + `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")` + lea $U2(%rsp), $r_ptr # U1*H^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr); +___ +{ +####################################################################### +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); +my ($poly1, $poly3)=($acc6,$acc7); + +$code.=<<___; + #lea $U2(%rsp), $a_ptr + #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2 + #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); + + add $acc0, $acc0 # a0:a3+a0:a3 + lea $Rsqr(%rsp), $a_ptr + adc $acc1, $acc1 + mov $acc0, $t0 + adc $acc2, $acc2 + adc $acc3, $acc3 + mov $acc1, $t1 + sbb $t4, $t4 + + sub \$-1, $acc0 + mov $acc2, $t2 + sbb $poly1, $acc1 + sbb \$0, $acc2 + mov $acc3, $t3 + sbb $poly3, $acc3 + test $t4, $t4 + + cmovz $t0, $acc0 + mov 8*0($a_ptr), $t0 + cmovz $t1, $acc1 + mov 8*1($a_ptr), $t1 + cmovz $t2, $acc2 + mov 8*2($a_ptr), $t2 + cmovz $t3, $acc3 + mov 8*3($a_ptr), $t3 + + call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr); + + lea $Hcub(%rsp), $b_ptr + lea $res_x(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub); + + mov $U2+8*0(%rsp), $t0 + mov $U2+8*1(%rsp), $t1 + mov $U2+8*2(%rsp), $t2 + mov $U2+8*3(%rsp), $t3 + lea $res_y(%rsp), $r_ptr + + call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x); + + mov $acc0, 8*0($r_ptr) # save the result, as + mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't + mov $acc2, 8*2($r_ptr) + mov $acc3, 8*3($r_ptr) +___ +} +$code.=<<___; + `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub); + + `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")` + lea $res_y(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y); + + lea $S2(%rsp), $b_ptr + lea $res_y(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2); + + movq %xmm0, $r_ptr # restore $r_ptr + + movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_z(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_z+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_z(%rsp), %xmm2 + pand $in2_z+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_z(%rsp), %xmm2 + pand $in1_z+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x40($r_ptr) + movdqu %xmm3, 0x50($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_x(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_x+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_x(%rsp), %xmm2 + pand $in2_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_x(%rsp), %xmm2 + pand $in1_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x00($r_ptr) + movdqu %xmm3, 0x10($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_y(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_y+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_y(%rsp), %xmm2 + pand $in2_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_y(%rsp), %xmm2 + pand $in1_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x20($r_ptr) + movdqu %xmm3, 0x30($r_ptr) + +.Ladd_done$x: + add \$32*18+8, %rsp + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + pop %rbp + ret +.size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx +___ +} +&gen_add("q"); + +sub gen_add_affine () { + my $x = shift; + my ($src0,$sfx,$bias); + my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr, + $res_x,$res_y,$res_z, + $in1_x,$in1_y,$in1_z, + $in2_x,$in2_y)=map(32*$_,(0..14)); + my $Z1sqr = $S2; + + if ($x ne "x") { + $src0 = "%rax"; + $sfx = ""; + $bias = 0; + +$code.=<<___; +.globl ecp_nistz256_point_add_affine +.type ecp_nistz256_point_add_affine,\@function,3 +.align 32 +ecp_nistz256_point_add_affine: +___ +$code.=<<___ if ($addx); + mov \$0x80100, %ecx + and OPENSSL_ia32cap_P+8(%rip), %ecx + cmp \$0x80100, %ecx + je .Lpoint_add_affinex +___ + } else { + $src0 = "%rdx"; + $sfx = "x"; + $bias = 128; + +$code.=<<___; +.type ecp_nistz256_point_add_affinex,\@function,3 +.align 32 +ecp_nistz256_point_add_affinex: +.Lpoint_add_affinex: +___ + } +$code.=<<___; + push %rbp + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 + sub \$32*15+8, %rsp + + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr + mov $b_org, $b_ptr # reassign + movdqu 0x10($a_ptr), %xmm1 + movdqu 0x20($a_ptr), %xmm2 + movdqu 0x30($a_ptr), %xmm3 + movdqu 0x40($a_ptr), %xmm4 + movdqu 0x50($a_ptr), %xmm5 + mov 0x40+8*0($a_ptr), $src0 # load original in1_z + mov 0x40+8*1($a_ptr), $acc6 + mov 0x40+8*2($a_ptr), $acc7 + mov 0x40+8*3($a_ptr), $acc0 + movdqa %xmm0, $in1_x(%rsp) + movdqa %xmm1, $in1_x+0x10(%rsp) + por %xmm0, %xmm1 + movdqa %xmm2, $in1_y(%rsp) + movdqa %xmm3, $in1_y+0x10(%rsp) + por %xmm2, %xmm3 + movdqa %xmm4, $in1_z(%rsp) + movdqa %xmm5, $in1_z+0x10(%rsp) + por %xmm1, %xmm3 + + movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr + pshufd \$0xb1, %xmm3, %xmm5 + movdqu 0x10($b_ptr), %xmm1 + movdqu 0x20($b_ptr), %xmm2 + por %xmm3, %xmm5 + movdqu 0x30($b_ptr), %xmm3 + movdqa %xmm0, $in2_x(%rsp) + pshufd \$0x1e, %xmm5, %xmm4 + movdqa %xmm1, $in2_x+0x10(%rsp) + por %xmm0, %xmm1 + movq $r_ptr, %xmm0 # save $r_ptr + movdqa %xmm2, $in2_y(%rsp) + movdqa %xmm3, $in2_y+0x10(%rsp) + por %xmm2, %xmm3 + por %xmm4, %xmm5 + pxor %xmm4, %xmm4 + por %xmm1, %xmm3 + + lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid + lea $Z1sqr(%rsp), $r_ptr # Z1^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z); + + pcmpeqd %xmm4, %xmm5 + pshufd \$0xb1, %xmm3, %xmm4 + mov 0x00($b_ptr), $src0 # $b_ptr is still valid + #lea 0x00($b_ptr), $b_ptr + mov $acc4, $acc1 # harmonize sqr output and mul input + por %xmm3, %xmm4 + pshufd \$0, %xmm5, %xmm5 # in1infty + pshufd \$0x1e, %xmm4, %xmm3 + mov $acc5, $acc2 + por %xmm3, %xmm4 + pxor %xmm3, %xmm3 + mov $acc6, $acc3 + pcmpeqd %xmm3, %xmm4 + pshufd \$0, %xmm4, %xmm4 # in2infty + + lea $Z1sqr-$bias(%rsp), $a_ptr + mov $acc7, $acc4 + lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x); + + lea $in1_x(%rsp), $b_ptr + lea $H(%rsp), $r_ptr # H = U2 - U1 + call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x); + + `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z); + + `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z); + + `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y); + + lea $in1_y(%rsp), $b_ptr + lea $R(%rsp), $r_ptr # R = S2 - S1 + call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y); + + `&load_for_sqr("$H(%rsp)", "$src0")` + lea $Hsqr(%rsp), $r_ptr # H^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H); + + `&load_for_sqr("$R(%rsp)", "$src0")` + lea $Rsqr(%rsp), $r_ptr # R^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R); + + `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")` + lea $Hcub(%rsp), $r_ptr # H^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H); + + `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")` + lea $U2(%rsp), $r_ptr # U1*H^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr); +___ +{ +####################################################################### +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); +my ($poly1, $poly3)=($acc6,$acc7); + +$code.=<<___; + #lea $U2(%rsp), $a_ptr + #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2 + #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); + + add $acc0, $acc0 # a0:a3+a0:a3 + lea $Rsqr(%rsp), $a_ptr + adc $acc1, $acc1 + mov $acc0, $t0 + adc $acc2, $acc2 + adc $acc3, $acc3 + mov $acc1, $t1 + sbb $t4, $t4 + + sub \$-1, $acc0 + mov $acc2, $t2 + sbb $poly1, $acc1 + sbb \$0, $acc2 + mov $acc3, $t3 + sbb $poly3, $acc3 + test $t4, $t4 + + cmovz $t0, $acc0 + mov 8*0($a_ptr), $t0 + cmovz $t1, $acc1 + mov 8*1($a_ptr), $t1 + cmovz $t2, $acc2 + mov 8*2($a_ptr), $t2 + cmovz $t3, $acc3 + mov 8*3($a_ptr), $t3 + + call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr); + + lea $Hcub(%rsp), $b_ptr + lea $res_x(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub); + + mov $U2+8*0(%rsp), $t0 + mov $U2+8*1(%rsp), $t1 + mov $U2+8*2(%rsp), $t2 + mov $U2+8*3(%rsp), $t3 + lea $H(%rsp), $r_ptr + + call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x); + + mov $acc0, 8*0($r_ptr) # save the result, as + mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't + mov $acc2, 8*2($r_ptr) + mov $acc3, 8*3($r_ptr) +___ +} +$code.=<<___; + `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y); + + `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")` + lea $H(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R); + + lea $S2(%rsp), $b_ptr + lea $res_y(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2); + + movq %xmm0, $r_ptr # restore $r_ptr + + movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_z(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_z+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand .LONE_mont(%rip), %xmm2 + pand .LONE_mont+0x10(%rip), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_z(%rsp), %xmm2 + pand $in1_z+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x40($r_ptr) + movdqu %xmm3, 0x50($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_x(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_x+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_x(%rsp), %xmm2 + pand $in2_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_x(%rsp), %xmm2 + pand $in1_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x00($r_ptr) + movdqu %xmm3, 0x10($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_y(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_y+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_y(%rsp), %xmm2 + pand $in2_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_y(%rsp), %xmm2 + pand $in1_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x20($r_ptr) + movdqu %xmm3, 0x30($r_ptr) + + add \$32*15+8, %rsp + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + pop %rbp + ret +.size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx +___ +} +&gen_add_affine("q"); + +######################################################################## +# AD*X magic +# +if ($addx) { { +######################################################################## +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); + +$code.=<<___; +.type __ecp_nistz256_add_tox,\@abi-omnipotent +.align 32 +__ecp_nistz256_add_tox: + xor $t4, $t4 + adc 8*0($b_ptr), $a0 + adc 8*1($b_ptr), $a1 + mov $a0, $t0 + adc 8*2($b_ptr), $a2 + adc 8*3($b_ptr), $a3 + mov $a1, $t1 + adc \$0, $t4 + + xor $t3, $t3 + sbb \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + + bt \$0, $t4 + cmovnc $t0, $a0 + cmovnc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovnc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovnc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox + +.type __ecp_nistz256_sub_fromx,\@abi-omnipotent +.align 32 +__ecp_nistz256_sub_fromx: + xor $t4, $t4 + sbb 8*0($b_ptr), $a0 + sbb 8*1($b_ptr), $a1 + mov $a0, $t0 + sbb 8*2($b_ptr), $a2 + sbb 8*3($b_ptr), $a3 + mov $a1, $t1 + sbb \$0, $t4 + + xor $t3, $t3 + adc \$-1, $a0 + mov $a2, $t2 + adc $poly1, $a1 + adc \$0, $a2 + mov $a3, $t3 + adc $poly3, $a3 + + bt \$0, $t4 + cmovnc $t0, $a0 + cmovnc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovnc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovnc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx + +.type __ecp_nistz256_subx,\@abi-omnipotent +.align 32 +__ecp_nistz256_subx: + xor $t4, $t4 + sbb $a0, $t0 + sbb $a1, $t1 + mov $t0, $a0 + sbb $a2, $t2 + sbb $a3, $t3 + mov $t1, $a1 + sbb \$0, $t4 + + xor $a3 ,$a3 + adc \$-1, $t0 + mov $t2, $a2 + adc $poly1, $t1 + adc \$0, $t2 + mov $t3, $a3 + adc $poly3, $t3 + + bt \$0, $t4 + cmovc $t0, $a0 + cmovc $t1, $a1 + cmovc $t2, $a2 + cmovc $t3, $a3 + + ret +.size __ecp_nistz256_subx,.-__ecp_nistz256_subx + +.type __ecp_nistz256_mul_by_2x,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_by_2x: + xor $t4, $t4 + adc $a0, $a0 # a0:a3+a0:a3 + adc $a1, $a1 + mov $a0, $t0 + adc $a2, $a2 + adc $a3, $a3 + mov $a1, $t1 + adc \$0, $t4 + + xor $t3, $t3 + sbb \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + + bt \$0, $t4 + cmovnc $t0, $a0 + cmovnc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovnc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovnc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x +___ + } +&gen_double("x"); +&gen_add("x"); +&gen_add_affine("x"); +} +}}} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/ec/ec.h b/crypto/ec/ec.h index c4e7aea938c2..6d3178f609f4 100644 --- a/crypto/ec/ec.h +++ b/crypto/ec/ec.h @@ -240,6 +240,12 @@ int EC_GROUP_set_generator(EC_GROUP *group, const EC_POINT *generator, */ const EC_POINT *EC_GROUP_get0_generator(const EC_GROUP *group); +/** Returns the montgomery data for order(Generator) + * \param group EC_GROUP object + * \return the currently used generator (possibly NULL). +*/ +BN_MONT_CTX *EC_GROUP_get_mont_data(const EC_GROUP *group); + /** Gets the order of a EC_GROUP * \param group EC_GROUP object * \param order BIGNUM to which the order is copied @@ -404,6 +410,9 @@ typedef struct { */ size_t EC_get_builtin_curves(EC_builtin_curve *r, size_t nitems); +const char *EC_curve_nid2nist(int nid); +int EC_curve_nist2nid(const char *name); + /********************************************************************/ /* EC_POINT functions */ /********************************************************************/ @@ -986,10 +995,78 @@ int EC_KEY_print_fp(FILE *fp, const EC_KEY *key, int off); # endif # define EVP_PKEY_CTX_set_ec_paramgen_curve_nid(ctx, nid) \ - EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, EVP_PKEY_OP_PARAMGEN, \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_PARAMGEN|EVP_PKEY_OP_KEYGEN, \ EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID, nid, NULL) +# define EVP_PKEY_CTX_set_ec_param_enc(ctx, flag) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_PARAMGEN|EVP_PKEY_OP_KEYGEN, \ + EVP_PKEY_CTRL_EC_PARAM_ENC, flag, NULL) + +# define EVP_PKEY_CTX_set_ecdh_cofactor_mode(ctx, flag) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_EC_ECDH_COFACTOR, flag, NULL) + +# define EVP_PKEY_CTX_get_ecdh_cofactor_mode(ctx) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_EC_ECDH_COFACTOR, -2, NULL) + +# define EVP_PKEY_CTX_set_ecdh_kdf_type(ctx, kdf) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_EC_KDF_TYPE, kdf, NULL) + +# define EVP_PKEY_CTX_get_ecdh_kdf_type(ctx) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_EC_KDF_TYPE, -2, NULL) + +# define EVP_PKEY_CTX_set_ecdh_kdf_md(ctx, md) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_EC_KDF_MD, 0, (void *)md) + +# define EVP_PKEY_CTX_get_ecdh_kdf_md(ctx, pmd) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_GET_EC_KDF_MD, 0, (void *)pmd) + +# define EVP_PKEY_CTX_set_ecdh_kdf_outlen(ctx, len) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_EC_KDF_OUTLEN, len, NULL) + +# define EVP_PKEY_CTX_get_ecdh_kdf_outlen(ctx, plen) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_GET_EC_KDF_OUTLEN, 0, (void *)plen) + +# define EVP_PKEY_CTX_set0_ecdh_kdf_ukm(ctx, p, plen) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_EC_KDF_UKM, plen, (void *)p) + +# define EVP_PKEY_CTX_get0_ecdh_kdf_ukm(ctx, p) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, \ + EVP_PKEY_OP_DERIVE, \ + EVP_PKEY_CTRL_GET_EC_KDF_UKM, 0, (void *)p) + # define EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID (EVP_PKEY_ALG_CTRL + 1) +# define EVP_PKEY_CTRL_EC_PARAM_ENC (EVP_PKEY_ALG_CTRL + 2) +# define EVP_PKEY_CTRL_EC_ECDH_COFACTOR (EVP_PKEY_ALG_CTRL + 3) +# define EVP_PKEY_CTRL_EC_KDF_TYPE (EVP_PKEY_ALG_CTRL + 4) +# define EVP_PKEY_CTRL_EC_KDF_MD (EVP_PKEY_ALG_CTRL + 5) +# define EVP_PKEY_CTRL_GET_EC_KDF_MD (EVP_PKEY_ALG_CTRL + 6) +# define EVP_PKEY_CTRL_EC_KDF_OUTLEN (EVP_PKEY_ALG_CTRL + 7) +# define EVP_PKEY_CTRL_GET_EC_KDF_OUTLEN (EVP_PKEY_ALG_CTRL + 8) +# define EVP_PKEY_CTRL_EC_KDF_UKM (EVP_PKEY_ALG_CTRL + 9) +# define EVP_PKEY_CTRL_GET_EC_KDF_UKM (EVP_PKEY_ALG_CTRL + 10) +/* KDF types */ +# define EVP_PKEY_ECDH_KDF_NONE 1 +# define EVP_PKEY_ECDH_KDF_X9_62 2 /* BEGIN ERROR CODES */ /* @@ -1007,6 +1084,8 @@ void ERR_load_EC_strings(void); # define EC_F_D2I_ECPKPARAMETERS 145 # define EC_F_D2I_ECPRIVATEKEY 146 # define EC_F_DO_EC_KEY_PRINT 221 +# define EC_F_ECDH_CMS_DECRYPT 238 +# define EC_F_ECDH_CMS_SET_SHARED_INFO 239 # define EC_F_ECKEY_PARAM2TYPE 223 # define EC_F_ECKEY_PARAM_DECODE 212 # define EC_F_ECKEY_PRIV_DECODE 213 @@ -1018,6 +1097,12 @@ void ERR_load_EC_strings(void); # define EC_F_ECPARAMETERS_PRINT_FP 148 # define EC_F_ECPKPARAMETERS_PRINT 149 # define EC_F_ECPKPARAMETERS_PRINT_FP 150 +# define EC_F_ECP_NISTZ256_GET_AFFINE 240 +# define EC_F_ECP_NISTZ256_MULT_PRECOMPUTE 243 +# define EC_F_ECP_NISTZ256_POINTS_MUL 241 +# define EC_F_ECP_NISTZ256_PRE_COMP_NEW 244 +# define EC_F_ECP_NISTZ256_SET_WORDS 245 +# define EC_F_ECP_NISTZ256_WINDOWED_MUL 242 # define EC_F_ECP_NIST_MOD_192 203 # define EC_F_ECP_NIST_MOD_224 204 # define EC_F_ECP_NIST_MOD_256 205 @@ -1157,6 +1242,7 @@ void ERR_load_EC_strings(void); # define EC_R_INVALID_COMPRESSED_POINT 110 # define EC_R_INVALID_COMPRESSION_BIT 109 # define EC_R_INVALID_CURVE 141 +# define EC_R_INVALID_DIGEST 151 # define EC_R_INVALID_DIGEST_TYPE 138 # define EC_R_INVALID_ENCODING 102 # define EC_R_INVALID_FIELD 103 @@ -1165,6 +1251,7 @@ void ERR_load_EC_strings(void); # define EC_R_INVALID_PENTANOMIAL_BASIS 132 # define EC_R_INVALID_PRIVATE_KEY 123 # define EC_R_INVALID_TRINOMIAL_BASIS 137 +# define EC_R_KDF_PARAMETER_ERROR 148 # define EC_R_KEYS_NOT_SET 140 # define EC_R_MISSING_PARAMETERS 124 # define EC_R_MISSING_PRIVATE_KEY 125 @@ -1175,9 +1262,11 @@ void ERR_load_EC_strings(void); # define EC_R_NO_FIELD_MOD 133 # define EC_R_NO_PARAMETERS_SET 139 # define EC_R_PASSED_NULL_PARAMETER 134 +# define EC_R_PEER_KEY_ERROR 149 # define EC_R_PKPARAMETERS2GROUP_FAILURE 127 # define EC_R_POINT_AT_INFINITY 106 # define EC_R_POINT_IS_NOT_ON_CURVE 107 +# define EC_R_SHARED_INFO_ERROR 150 # define EC_R_SLOT_FULL 108 # define EC_R_UNDEFINED_GENERATOR 113 # define EC_R_UNDEFINED_ORDER 128 diff --git a/crypto/ec/ec_ameth.c b/crypto/ec/ec_ameth.c index 5cefb5ad303a..83e208cfe491 100644 --- a/crypto/ec/ec_ameth.c +++ b/crypto/ec/ec_ameth.c @@ -64,8 +64,12 @@ #ifndef OPENSSL_NO_CMS # include #endif +#include #include "asn1_locl.h" +static int ecdh_cms_decrypt(CMS_RecipientInfo *ri); +static int ecdh_cms_encrypt(CMS_RecipientInfo *ri); + static int eckey_param2type(int *pptype, void **ppval, EC_KEY *ec_key) { const EC_GROUP *group; @@ -580,10 +584,21 @@ static int ec_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2) X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0); } return 1; + + case ASN1_PKEY_CTRL_CMS_ENVELOPE: + if (arg1 == 1) + return ecdh_cms_decrypt(arg2); + else if (arg1 == 0) + return ecdh_cms_encrypt(arg2); + return -2; + + case ASN1_PKEY_CTRL_CMS_RI_TYPE: + *(int *)arg2 = CMS_RECIPINFO_AGREE; + return 1; #endif case ASN1_PKEY_CTRL_DEFAULT_MD_NID: - *(int *)arg2 = NID_sha1; + *(int *)arg2 = NID_sha256; return 2; default: @@ -625,3 +640,326 @@ const EVP_PKEY_ASN1_METHOD eckey_asn1_meth = { old_ec_priv_decode, old_ec_priv_encode }; + +#ifndef OPENSSL_NO_CMS + +static int ecdh_cms_set_peerkey(EVP_PKEY_CTX *pctx, + X509_ALGOR *alg, ASN1_BIT_STRING *pubkey) +{ + ASN1_OBJECT *aoid; + int atype; + void *aval; + int rv = 0; + EVP_PKEY *pkpeer = NULL; + EC_KEY *ecpeer = NULL; + const unsigned char *p; + int plen; + X509_ALGOR_get0(&aoid, &atype, &aval, alg); + if (OBJ_obj2nid(aoid) != NID_X9_62_id_ecPublicKey) + goto err; + /* If absent parameters get group from main key */ + if (atype == V_ASN1_UNDEF || atype == V_ASN1_NULL) { + const EC_GROUP *grp; + EVP_PKEY *pk; + pk = EVP_PKEY_CTX_get0_pkey(pctx); + if (!pk) + goto err; + grp = EC_KEY_get0_group(pk->pkey.ec); + ecpeer = EC_KEY_new(); + if (!ecpeer) + goto err; + if (!EC_KEY_set_group(ecpeer, grp)) + goto err; + } else { + ecpeer = eckey_type2param(atype, aval); + if (!ecpeer) + goto err; + } + /* We have parameters now set public key */ + plen = ASN1_STRING_length(pubkey); + p = ASN1_STRING_data(pubkey); + if (!p || !plen) + goto err; + if (!o2i_ECPublicKey(&ecpeer, &p, plen)) + goto err; + pkpeer = EVP_PKEY_new(); + if (!pkpeer) + goto err; + EVP_PKEY_set1_EC_KEY(pkpeer, ecpeer); + if (EVP_PKEY_derive_set_peer(pctx, pkpeer) > 0) + rv = 1; + err: + if (ecpeer) + EC_KEY_free(ecpeer); + if (pkpeer) + EVP_PKEY_free(pkpeer); + return rv; +} + +/* Set KDF parameters based on KDF NID */ +static int ecdh_cms_set_kdf_param(EVP_PKEY_CTX *pctx, int eckdf_nid) +{ + int kdf_nid, kdfmd_nid, cofactor; + const EVP_MD *kdf_md; + if (eckdf_nid == NID_undef) + return 0; + + /* Lookup KDF type, cofactor mode and digest */ + if (!OBJ_find_sigid_algs(eckdf_nid, &kdfmd_nid, &kdf_nid)) + return 0; + + if (kdf_nid == NID_dh_std_kdf) + cofactor = 0; + else if (kdf_nid == NID_dh_cofactor_kdf) + cofactor = 1; + else + return 0; + + if (EVP_PKEY_CTX_set_ecdh_cofactor_mode(pctx, cofactor) <= 0) + return 0; + + if (EVP_PKEY_CTX_set_ecdh_kdf_type(pctx, EVP_PKEY_ECDH_KDF_X9_62) <= 0) + return 0; + + kdf_md = EVP_get_digestbynid(kdfmd_nid); + if (!kdf_md) + return 0; + + if (EVP_PKEY_CTX_set_ecdh_kdf_md(pctx, kdf_md) <= 0) + return 0; + return 1; +} + +static int ecdh_cms_set_shared_info(EVP_PKEY_CTX *pctx, CMS_RecipientInfo *ri) +{ + int rv = 0; + + X509_ALGOR *alg, *kekalg = NULL; + ASN1_OCTET_STRING *ukm; + const unsigned char *p; + unsigned char *der = NULL; + int plen, keylen; + const EVP_CIPHER *kekcipher; + EVP_CIPHER_CTX *kekctx; + + if (!CMS_RecipientInfo_kari_get0_alg(ri, &alg, &ukm)) + return 0; + + if (!ecdh_cms_set_kdf_param(pctx, OBJ_obj2nid(alg->algorithm))) { + ECerr(EC_F_ECDH_CMS_SET_SHARED_INFO, EC_R_KDF_PARAMETER_ERROR); + return 0; + } + + if (alg->parameter->type != V_ASN1_SEQUENCE) + return 0; + + p = alg->parameter->value.sequence->data; + plen = alg->parameter->value.sequence->length; + kekalg = d2i_X509_ALGOR(NULL, &p, plen); + if (!kekalg) + goto err; + kekctx = CMS_RecipientInfo_kari_get0_ctx(ri); + if (!kekctx) + goto err; + kekcipher = EVP_get_cipherbyobj(kekalg->algorithm); + if (!kekcipher || EVP_CIPHER_mode(kekcipher) != EVP_CIPH_WRAP_MODE) + goto err; + if (!EVP_EncryptInit_ex(kekctx, kekcipher, NULL, NULL, NULL)) + goto err; + if (EVP_CIPHER_asn1_to_param(kekctx, kekalg->parameter) <= 0) + goto err; + + keylen = EVP_CIPHER_CTX_key_length(kekctx); + if (EVP_PKEY_CTX_set_ecdh_kdf_outlen(pctx, keylen) <= 0) + goto err; + + plen = CMS_SharedInfo_encode(&der, kekalg, ukm, keylen); + + if (!plen) + goto err; + + if (EVP_PKEY_CTX_set0_ecdh_kdf_ukm(pctx, der, plen) <= 0) + goto err; + der = NULL; + + rv = 1; + err: + if (kekalg) + X509_ALGOR_free(kekalg); + if (der) + OPENSSL_free(der); + return rv; +} + +static int ecdh_cms_decrypt(CMS_RecipientInfo *ri) +{ + EVP_PKEY_CTX *pctx; + pctx = CMS_RecipientInfo_get0_pkey_ctx(ri); + if (!pctx) + return 0; + /* See if we need to set peer key */ + if (!EVP_PKEY_CTX_get0_peerkey(pctx)) { + X509_ALGOR *alg; + ASN1_BIT_STRING *pubkey; + if (!CMS_RecipientInfo_kari_get0_orig_id(ri, &alg, &pubkey, + NULL, NULL, NULL)) + return 0; + if (!alg || !pubkey) + return 0; + if (!ecdh_cms_set_peerkey(pctx, alg, pubkey)) { + ECerr(EC_F_ECDH_CMS_DECRYPT, EC_R_PEER_KEY_ERROR); + return 0; + } + } + /* Set ECDH derivation parameters and initialise unwrap context */ + if (!ecdh_cms_set_shared_info(pctx, ri)) { + ECerr(EC_F_ECDH_CMS_DECRYPT, EC_R_SHARED_INFO_ERROR); + return 0; + } + return 1; +} + +static int ecdh_cms_encrypt(CMS_RecipientInfo *ri) +{ + EVP_PKEY_CTX *pctx; + EVP_PKEY *pkey; + EVP_CIPHER_CTX *ctx; + int keylen; + X509_ALGOR *talg, *wrap_alg = NULL; + ASN1_OBJECT *aoid; + ASN1_BIT_STRING *pubkey; + ASN1_STRING *wrap_str; + ASN1_OCTET_STRING *ukm; + unsigned char *penc = NULL; + int penclen; + int rv = 0; + int ecdh_nid, kdf_type, kdf_nid, wrap_nid; + const EVP_MD *kdf_md; + pctx = CMS_RecipientInfo_get0_pkey_ctx(ri); + if (!pctx) + return 0; + /* Get ephemeral key */ + pkey = EVP_PKEY_CTX_get0_pkey(pctx); + if (!CMS_RecipientInfo_kari_get0_orig_id(ri, &talg, &pubkey, + NULL, NULL, NULL)) + goto err; + X509_ALGOR_get0(&aoid, NULL, NULL, talg); + /* Is everything uninitialised? */ + if (aoid == OBJ_nid2obj(NID_undef)) { + + EC_KEY *eckey = pkey->pkey.ec; + /* Set the key */ + unsigned char *p; + + penclen = i2o_ECPublicKey(eckey, NULL); + if (penclen <= 0) + goto err; + penc = OPENSSL_malloc(penclen); + if (!penc) + goto err; + p = penc; + penclen = i2o_ECPublicKey(eckey, &p); + if (penclen <= 0) + goto err; + ASN1_STRING_set0(pubkey, penc, penclen); + pubkey->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + pubkey->flags |= ASN1_STRING_FLAG_BITS_LEFT; + + penc = NULL; + X509_ALGOR_set0(talg, OBJ_nid2obj(NID_X9_62_id_ecPublicKey), + V_ASN1_UNDEF, NULL); + } + + /* See if custom paraneters set */ + kdf_type = EVP_PKEY_CTX_get_ecdh_kdf_type(pctx); + if (kdf_type <= 0) + goto err; + if (!EVP_PKEY_CTX_get_ecdh_kdf_md(pctx, &kdf_md)) + goto err; + ecdh_nid = EVP_PKEY_CTX_get_ecdh_cofactor_mode(pctx); + if (ecdh_nid < 0) + goto err; + else if (ecdh_nid == 0) + ecdh_nid = NID_dh_std_kdf; + else if (ecdh_nid == 1) + ecdh_nid = NID_dh_cofactor_kdf; + + if (kdf_type == EVP_PKEY_ECDH_KDF_NONE) { + kdf_type = EVP_PKEY_ECDH_KDF_X9_62; + if (EVP_PKEY_CTX_set_ecdh_kdf_type(pctx, kdf_type) <= 0) + goto err; + } else + /* Uknown KDF */ + goto err; + if (kdf_md == NULL) { + /* Fixme later for better MD */ + kdf_md = EVP_sha1(); + if (EVP_PKEY_CTX_set_ecdh_kdf_md(pctx, kdf_md) <= 0) + goto err; + } + + if (!CMS_RecipientInfo_kari_get0_alg(ri, &talg, &ukm)) + goto err; + + /* Lookup NID for KDF+cofactor+digest */ + + if (!OBJ_find_sigid_by_algs(&kdf_nid, EVP_MD_type(kdf_md), ecdh_nid)) + goto err; + /* Get wrap NID */ + ctx = CMS_RecipientInfo_kari_get0_ctx(ri); + wrap_nid = EVP_CIPHER_CTX_type(ctx); + keylen = EVP_CIPHER_CTX_key_length(ctx); + + /* Package wrap algorithm in an AlgorithmIdentifier */ + + wrap_alg = X509_ALGOR_new(); + if (!wrap_alg) + goto err; + wrap_alg->algorithm = OBJ_nid2obj(wrap_nid); + wrap_alg->parameter = ASN1_TYPE_new(); + if (!wrap_alg->parameter) + goto err; + if (EVP_CIPHER_param_to_asn1(ctx, wrap_alg->parameter) <= 0) + goto err; + if (ASN1_TYPE_get(wrap_alg->parameter) == NID_undef) { + ASN1_TYPE_free(wrap_alg->parameter); + wrap_alg->parameter = NULL; + } + + if (EVP_PKEY_CTX_set_ecdh_kdf_outlen(pctx, keylen) <= 0) + goto err; + + penclen = CMS_SharedInfo_encode(&penc, wrap_alg, ukm, keylen); + + if (!penclen) + goto err; + + if (EVP_PKEY_CTX_set0_ecdh_kdf_ukm(pctx, penc, penclen) <= 0) + goto err; + penc = NULL; + + /* + * Now need to wrap encoding of wrap AlgorithmIdentifier into parameter + * of another AlgorithmIdentifier. + */ + penclen = i2d_X509_ALGOR(wrap_alg, &penc); + if (!penc || !penclen) + goto err; + wrap_str = ASN1_STRING_new(); + if (!wrap_str) + goto err; + ASN1_STRING_set0(wrap_str, penc, penclen); + penc = NULL; + X509_ALGOR_set0(talg, OBJ_nid2obj(kdf_nid), V_ASN1_SEQUENCE, wrap_str); + + rv = 1; + + err: + if (penc) + OPENSSL_free(penc); + if (wrap_alg) + X509_ALGOR_free(wrap_alg); + return rv; +} + +#endif diff --git a/crypto/ec/ec_curve.c b/crypto/ec/ec_curve.c index e4bcb5dd81dc..6dbe9d8258de 100644 --- a/crypto/ec/ec_curve.c +++ b/crypto/ec/ec_curve.c @@ -69,11 +69,16 @@ * */ +#include #include "ec_lcl.h" #include #include #include +#ifdef OPENSSL_FIPS +# include +#endif + typedef struct { int field_type, /* either NID_X9_62_prime_field or * NID_X9_62_characteristic_two_field */ @@ -2282,6 +2287,554 @@ static const struct { #endif +/* + * These curves were added by Annie Yousar + * For the definition of RFC 5639 curves see + * http://www.ietf.org/rfc/rfc5639.txt These curves are generated verifiable + * at random, nevertheless the seed is omitted as parameter because the + * generation mechanism is different from those defined in ANSI X9.62. + */ + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 20 * 6]; +} _EC_brainpoolP160r1 = { + { + NID_X9_62_prime_field, 0, 20, 1 + }, + { + /* no seed */ + /* p */ + 0xE9, 0x5E, 0x4A, 0x5F, 0x73, 0x70, 0x59, 0xDC, 0x60, 0xDF, 0xC7, 0xAD, + 0x95, 0xB3, 0xD8, 0x13, 0x95, 0x15, 0x62, 0x0F, + /* a */ + 0x34, 0x0E, 0x7B, 0xE2, 0xA2, 0x80, 0xEB, 0x74, 0xE2, 0xBE, 0x61, 0xBA, + 0xDA, 0x74, 0x5D, 0x97, 0xE8, 0xF7, 0xC3, 0x00, + /* b */ + 0x1E, 0x58, 0x9A, 0x85, 0x95, 0x42, 0x34, 0x12, 0x13, 0x4F, 0xAA, 0x2D, + 0xBD, 0xEC, 0x95, 0xC8, 0xD8, 0x67, 0x5E, 0x58, + /* x */ + 0xBE, 0xD5, 0xAF, 0x16, 0xEA, 0x3F, 0x6A, 0x4F, 0x62, 0x93, 0x8C, 0x46, + 0x31, 0xEB, 0x5A, 0xF7, 0xBD, 0xBC, 0xDB, 0xC3, + /* y */ + 0x16, 0x67, 0xCB, 0x47, 0x7A, 0x1A, 0x8E, 0xC3, 0x38, 0xF9, 0x47, 0x41, + 0x66, 0x9C, 0x97, 0x63, 0x16, 0xDA, 0x63, 0x21, + /* order */ + 0xE9, 0x5E, 0x4A, 0x5F, 0x73, 0x70, 0x59, 0xDC, 0x60, 0xDF, 0x59, 0x91, + 0xD4, 0x50, 0x29, 0x40, 0x9E, 0x60, 0xFC, 0x09 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 20 * 6]; +} _EC_brainpoolP160t1 = { + { + NID_X9_62_prime_field, 0, 20, 1 + }, + { + /* no seed */ + /* p */ + 0xE9, 0x5E, 0x4A, 0x5F, 0x73, 0x70, 0x59, 0xDC, 0x60, 0xDF, 0xC7, 0xAD, + 0x95, 0xB3, 0xD8, 0x13, 0x95, 0x15, 0x62, 0x0F, + /* a */ + 0xE9, 0x5E, 0x4A, 0x5F, 0x73, 0x70, 0x59, 0xDC, 0x60, 0xDF, 0xC7, 0xAD, + 0x95, 0xB3, 0xD8, 0x13, 0x95, 0x15, 0x62, 0x0C, + /* b */ + 0x7A, 0x55, 0x6B, 0x6D, 0xAE, 0x53, 0x5B, 0x7B, 0x51, 0xED, 0x2C, 0x4D, + 0x7D, 0xAA, 0x7A, 0x0B, 0x5C, 0x55, 0xF3, 0x80, + /* x */ + 0xB1, 0x99, 0xB1, 0x3B, 0x9B, 0x34, 0xEF, 0xC1, 0x39, 0x7E, 0x64, 0xBA, + 0xEB, 0x05, 0xAC, 0xC2, 0x65, 0xFF, 0x23, 0x78, + /* y */ + 0xAD, 0xD6, 0x71, 0x8B, 0x7C, 0x7C, 0x19, 0x61, 0xF0, 0x99, 0x1B, 0x84, + 0x24, 0x43, 0x77, 0x21, 0x52, 0xC9, 0xE0, 0xAD, + /* order */ + 0xE9, 0x5E, 0x4A, 0x5F, 0x73, 0x70, 0x59, 0xDC, 0x60, 0xDF, 0x59, 0x91, + 0xD4, 0x50, 0x29, 0x40, 0x9E, 0x60, 0xFC, 0x09 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 24 * 6]; +} _EC_brainpoolP192r1 = { + { + NID_X9_62_prime_field, 0, 24, 1 + }, + { + /* no seed */ + /* p */ + 0xC3, 0x02, 0xF4, 0x1D, 0x93, 0x2A, 0x36, 0xCD, 0xA7, 0xA3, 0x46, 0x30, + 0x93, 0xD1, 0x8D, 0xB7, 0x8F, 0xCE, 0x47, 0x6D, 0xE1, 0xA8, 0x62, 0x97, + /* a */ + 0x6A, 0x91, 0x17, 0x40, 0x76, 0xB1, 0xE0, 0xE1, 0x9C, 0x39, 0xC0, 0x31, + 0xFE, 0x86, 0x85, 0xC1, 0xCA, 0xE0, 0x40, 0xE5, 0xC6, 0x9A, 0x28, 0xEF, + /* b */ + 0x46, 0x9A, 0x28, 0xEF, 0x7C, 0x28, 0xCC, 0xA3, 0xDC, 0x72, 0x1D, 0x04, + 0x4F, 0x44, 0x96, 0xBC, 0xCA, 0x7E, 0xF4, 0x14, 0x6F, 0xBF, 0x25, 0xC9, + /* x */ + 0xC0, 0xA0, 0x64, 0x7E, 0xAA, 0xB6, 0xA4, 0x87, 0x53, 0xB0, 0x33, 0xC5, + 0x6C, 0xB0, 0xF0, 0x90, 0x0A, 0x2F, 0x5C, 0x48, 0x53, 0x37, 0x5F, 0xD6, + /* y */ + 0x14, 0xB6, 0x90, 0x86, 0x6A, 0xBD, 0x5B, 0xB8, 0x8B, 0x5F, 0x48, 0x28, + 0xC1, 0x49, 0x00, 0x02, 0xE6, 0x77, 0x3F, 0xA2, 0xFA, 0x29, 0x9B, 0x8F, + /* order */ + 0xC3, 0x02, 0xF4, 0x1D, 0x93, 0x2A, 0x36, 0xCD, 0xA7, 0xA3, 0x46, 0x2F, + 0x9E, 0x9E, 0x91, 0x6B, 0x5B, 0xE8, 0xF1, 0x02, 0x9A, 0xC4, 0xAC, 0xC1 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 24 * 6]; +} _EC_brainpoolP192t1 = { + { + NID_X9_62_prime_field, 0, 24, 1 + }, + { + /* no seed */ + /* p */ + 0xC3, 0x02, 0xF4, 0x1D, 0x93, 0x2A, 0x36, 0xCD, 0xA7, 0xA3, 0x46, 0x30, + 0x93, 0xD1, 0x8D, 0xB7, 0x8F, 0xCE, 0x47, 0x6D, 0xE1, 0xA8, 0x62, 0x97, + /* a */ + 0xC3, 0x02, 0xF4, 0x1D, 0x93, 0x2A, 0x36, 0xCD, 0xA7, 0xA3, 0x46, 0x30, + 0x93, 0xD1, 0x8D, 0xB7, 0x8F, 0xCE, 0x47, 0x6D, 0xE1, 0xA8, 0x62, 0x94, + /* b */ + 0x13, 0xD5, 0x6F, 0xFA, 0xEC, 0x78, 0x68, 0x1E, 0x68, 0xF9, 0xDE, 0xB4, + 0x3B, 0x35, 0xBE, 0xC2, 0xFB, 0x68, 0x54, 0x2E, 0x27, 0x89, 0x7B, 0x79, + /* x */ + 0x3A, 0xE9, 0xE5, 0x8C, 0x82, 0xF6, 0x3C, 0x30, 0x28, 0x2E, 0x1F, 0xE7, + 0xBB, 0xF4, 0x3F, 0xA7, 0x2C, 0x44, 0x6A, 0xF6, 0xF4, 0x61, 0x81, 0x29, + /* y */ + 0x09, 0x7E, 0x2C, 0x56, 0x67, 0xC2, 0x22, 0x3A, 0x90, 0x2A, 0xB5, 0xCA, + 0x44, 0x9D, 0x00, 0x84, 0xB7, 0xE5, 0xB3, 0xDE, 0x7C, 0xCC, 0x01, 0xC9, + /* order */ + 0xC3, 0x02, 0xF4, 0x1D, 0x93, 0x2A, 0x36, 0xCD, 0xA7, 0xA3, 0x46, 0x2F, + 0x9E, 0x9E, 0x91, 0x6B, 0x5B, 0xE8, 0xF1, 0x02, 0x9A, 0xC4, 0xAC, 0xC1 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 28 * 6]; +} _EC_brainpoolP224r1 = { + { + NID_X9_62_prime_field, 0, 28, 1 + }, + { + /* no seed */ + /* p */ + 0xD7, 0xC1, 0x34, 0xAA, 0x26, 0x43, 0x66, 0x86, 0x2A, 0x18, 0x30, 0x25, + 0x75, 0xD1, 0xD7, 0x87, 0xB0, 0x9F, 0x07, 0x57, 0x97, 0xDA, 0x89, 0xF5, + 0x7E, 0xC8, 0xC0, 0xFF, + /* a */ + 0x68, 0xA5, 0xE6, 0x2C, 0xA9, 0xCE, 0x6C, 0x1C, 0x29, 0x98, 0x03, 0xA6, + 0xC1, 0x53, 0x0B, 0x51, 0x4E, 0x18, 0x2A, 0xD8, 0xB0, 0x04, 0x2A, 0x59, + 0xCA, 0xD2, 0x9F, 0x43, + /* b */ + 0x25, 0x80, 0xF6, 0x3C, 0xCF, 0xE4, 0x41, 0x38, 0x87, 0x07, 0x13, 0xB1, + 0xA9, 0x23, 0x69, 0xE3, 0x3E, 0x21, 0x35, 0xD2, 0x66, 0xDB, 0xB3, 0x72, + 0x38, 0x6C, 0x40, 0x0B, + /* x */ + 0x0D, 0x90, 0x29, 0xAD, 0x2C, 0x7E, 0x5C, 0xF4, 0x34, 0x08, 0x23, 0xB2, + 0xA8, 0x7D, 0xC6, 0x8C, 0x9E, 0x4C, 0xE3, 0x17, 0x4C, 0x1E, 0x6E, 0xFD, + 0xEE, 0x12, 0xC0, 0x7D, + /* y */ + 0x58, 0xAA, 0x56, 0xF7, 0x72, 0xC0, 0x72, 0x6F, 0x24, 0xC6, 0xB8, 0x9E, + 0x4E, 0xCD, 0xAC, 0x24, 0x35, 0x4B, 0x9E, 0x99, 0xCA, 0xA3, 0xF6, 0xD3, + 0x76, 0x14, 0x02, 0xCD, + /* order */ + 0xD7, 0xC1, 0x34, 0xAA, 0x26, 0x43, 0x66, 0x86, 0x2A, 0x18, 0x30, 0x25, + 0x75, 0xD0, 0xFB, 0x98, 0xD1, 0x16, 0xBC, 0x4B, 0x6D, 0xDE, 0xBC, 0xA3, + 0xA5, 0xA7, 0x93, 0x9F + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 28 * 6]; +} _EC_brainpoolP224t1 = { + { + NID_X9_62_prime_field, 0, 28, 1 + }, + { + /* no seed */ + /* p */ + 0xD7, 0xC1, 0x34, 0xAA, 0x26, 0x43, 0x66, 0x86, 0x2A, 0x18, 0x30, 0x25, + 0x75, 0xD1, 0xD7, 0x87, 0xB0, 0x9F, 0x07, 0x57, 0x97, 0xDA, 0x89, 0xF5, + 0x7E, 0xC8, 0xC0, 0xFF, + /* a */ + 0xD7, 0xC1, 0x34, 0xAA, 0x26, 0x43, 0x66, 0x86, 0x2A, 0x18, 0x30, 0x25, + 0x75, 0xD1, 0xD7, 0x87, 0xB0, 0x9F, 0x07, 0x57, 0x97, 0xDA, 0x89, 0xF5, + 0x7E, 0xC8, 0xC0, 0xFC, + /* b */ + 0x4B, 0x33, 0x7D, 0x93, 0x41, 0x04, 0xCD, 0x7B, 0xEF, 0x27, 0x1B, 0xF6, + 0x0C, 0xED, 0x1E, 0xD2, 0x0D, 0xA1, 0x4C, 0x08, 0xB3, 0xBB, 0x64, 0xF1, + 0x8A, 0x60, 0x88, 0x8D, + /* x */ + 0x6A, 0xB1, 0xE3, 0x44, 0xCE, 0x25, 0xFF, 0x38, 0x96, 0x42, 0x4E, 0x7F, + 0xFE, 0x14, 0x76, 0x2E, 0xCB, 0x49, 0xF8, 0x92, 0x8A, 0xC0, 0xC7, 0x60, + 0x29, 0xB4, 0xD5, 0x80, + /* y */ + 0x03, 0x74, 0xE9, 0xF5, 0x14, 0x3E, 0x56, 0x8C, 0xD2, 0x3F, 0x3F, 0x4D, + 0x7C, 0x0D, 0x4B, 0x1E, 0x41, 0xC8, 0xCC, 0x0D, 0x1C, 0x6A, 0xBD, 0x5F, + 0x1A, 0x46, 0xDB, 0x4C, + /* order */ + 0xD7, 0xC1, 0x34, 0xAA, 0x26, 0x43, 0x66, 0x86, 0x2A, 0x18, 0x30, 0x25, + 0x75, 0xD0, 0xFB, 0x98, 0xD1, 0x16, 0xBC, 0x4B, 0x6D, 0xDE, 0xBC, 0xA3, + 0xA5, 0xA7, 0x93, 0x9F + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 32 * 6]; +} _EC_brainpoolP256r1 = { + { + NID_X9_62_prime_field, 0, 32, 1 + }, + { + /* no seed */ + /* p */ + 0xA9, 0xFB, 0x57, 0xDB, 0xA1, 0xEE, 0xA9, 0xBC, 0x3E, 0x66, 0x0A, 0x90, + 0x9D, 0x83, 0x8D, 0x72, 0x6E, 0x3B, 0xF6, 0x23, 0xD5, 0x26, 0x20, 0x28, + 0x20, 0x13, 0x48, 0x1D, 0x1F, 0x6E, 0x53, 0x77, + /* a */ + 0x7D, 0x5A, 0x09, 0x75, 0xFC, 0x2C, 0x30, 0x57, 0xEE, 0xF6, 0x75, 0x30, + 0x41, 0x7A, 0xFF, 0xE7, 0xFB, 0x80, 0x55, 0xC1, 0x26, 0xDC, 0x5C, 0x6C, + 0xE9, 0x4A, 0x4B, 0x44, 0xF3, 0x30, 0xB5, 0xD9, + /* b */ + 0x26, 0xDC, 0x5C, 0x6C, 0xE9, 0x4A, 0x4B, 0x44, 0xF3, 0x30, 0xB5, 0xD9, + 0xBB, 0xD7, 0x7C, 0xBF, 0x95, 0x84, 0x16, 0x29, 0x5C, 0xF7, 0xE1, 0xCE, + 0x6B, 0xCC, 0xDC, 0x18, 0xFF, 0x8C, 0x07, 0xB6, + /* x */ + 0x8B, 0xD2, 0xAE, 0xB9, 0xCB, 0x7E, 0x57, 0xCB, 0x2C, 0x4B, 0x48, 0x2F, + 0xFC, 0x81, 0xB7, 0xAF, 0xB9, 0xDE, 0x27, 0xE1, 0xE3, 0xBD, 0x23, 0xC2, + 0x3A, 0x44, 0x53, 0xBD, 0x9A, 0xCE, 0x32, 0x62, + /* y */ + 0x54, 0x7E, 0xF8, 0x35, 0xC3, 0xDA, 0xC4, 0xFD, 0x97, 0xF8, 0x46, 0x1A, + 0x14, 0x61, 0x1D, 0xC9, 0xC2, 0x77, 0x45, 0x13, 0x2D, 0xED, 0x8E, 0x54, + 0x5C, 0x1D, 0x54, 0xC7, 0x2F, 0x04, 0x69, 0x97, + /* order */ + 0xA9, 0xFB, 0x57, 0xDB, 0xA1, 0xEE, 0xA9, 0xBC, 0x3E, 0x66, 0x0A, 0x90, + 0x9D, 0x83, 0x8D, 0x71, 0x8C, 0x39, 0x7A, 0xA3, 0xB5, 0x61, 0xA6, 0xF7, + 0x90, 0x1E, 0x0E, 0x82, 0x97, 0x48, 0x56, 0xA7 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 32 * 6]; +} _EC_brainpoolP256t1 = { + { + NID_X9_62_prime_field, 0, 32, 1 + }, + { + /* no seed */ + /* p */ + 0xA9, 0xFB, 0x57, 0xDB, 0xA1, 0xEE, 0xA9, 0xBC, 0x3E, 0x66, 0x0A, 0x90, + 0x9D, 0x83, 0x8D, 0x72, 0x6E, 0x3B, 0xF6, 0x23, 0xD5, 0x26, 0x20, 0x28, + 0x20, 0x13, 0x48, 0x1D, 0x1F, 0x6E, 0x53, 0x77, + /* a */ + 0xA9, 0xFB, 0x57, 0xDB, 0xA1, 0xEE, 0xA9, 0xBC, 0x3E, 0x66, 0x0A, 0x90, + 0x9D, 0x83, 0x8D, 0x72, 0x6E, 0x3B, 0xF6, 0x23, 0xD5, 0x26, 0x20, 0x28, + 0x20, 0x13, 0x48, 0x1D, 0x1F, 0x6E, 0x53, 0x74, + /* b */ + 0x66, 0x2C, 0x61, 0xC4, 0x30, 0xD8, 0x4E, 0xA4, 0xFE, 0x66, 0xA7, 0x73, + 0x3D, 0x0B, 0x76, 0xB7, 0xBF, 0x93, 0xEB, 0xC4, 0xAF, 0x2F, 0x49, 0x25, + 0x6A, 0xE5, 0x81, 0x01, 0xFE, 0xE9, 0x2B, 0x04, + /* x */ + 0xA3, 0xE8, 0xEB, 0x3C, 0xC1, 0xCF, 0xE7, 0xB7, 0x73, 0x22, 0x13, 0xB2, + 0x3A, 0x65, 0x61, 0x49, 0xAF, 0xA1, 0x42, 0xC4, 0x7A, 0xAF, 0xBC, 0x2B, + 0x79, 0xA1, 0x91, 0x56, 0x2E, 0x13, 0x05, 0xF4, + /* y */ + 0x2D, 0x99, 0x6C, 0x82, 0x34, 0x39, 0xC5, 0x6D, 0x7F, 0x7B, 0x22, 0xE1, + 0x46, 0x44, 0x41, 0x7E, 0x69, 0xBC, 0xB6, 0xDE, 0x39, 0xD0, 0x27, 0x00, + 0x1D, 0xAB, 0xE8, 0xF3, 0x5B, 0x25, 0xC9, 0xBE, + /* order */ + 0xA9, 0xFB, 0x57, 0xDB, 0xA1, 0xEE, 0xA9, 0xBC, 0x3E, 0x66, 0x0A, 0x90, + 0x9D, 0x83, 0x8D, 0x71, 0x8C, 0x39, 0x7A, 0xA3, 0xB5, 0x61, 0xA6, 0xF7, + 0x90, 0x1E, 0x0E, 0x82, 0x97, 0x48, 0x56, 0xA7 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 40 * 6]; +} _EC_brainpoolP320r1 = { + { + NID_X9_62_prime_field, 0, 40, 1 + }, + { + /* no seed */ + /* p */ + 0xD3, 0x5E, 0x47, 0x20, 0x36, 0xBC, 0x4F, 0xB7, 0xE1, 0x3C, 0x78, 0x5E, + 0xD2, 0x01, 0xE0, 0x65, 0xF9, 0x8F, 0xCF, 0xA6, 0xF6, 0xF4, 0x0D, 0xEF, + 0x4F, 0x92, 0xB9, 0xEC, 0x78, 0x93, 0xEC, 0x28, 0xFC, 0xD4, 0x12, 0xB1, + 0xF1, 0xB3, 0x2E, 0x27, + /* a */ + 0x3E, 0xE3, 0x0B, 0x56, 0x8F, 0xBA, 0xB0, 0xF8, 0x83, 0xCC, 0xEB, 0xD4, + 0x6D, 0x3F, 0x3B, 0xB8, 0xA2, 0xA7, 0x35, 0x13, 0xF5, 0xEB, 0x79, 0xDA, + 0x66, 0x19, 0x0E, 0xB0, 0x85, 0xFF, 0xA9, 0xF4, 0x92, 0xF3, 0x75, 0xA9, + 0x7D, 0x86, 0x0E, 0xB4, + /* b */ + 0x52, 0x08, 0x83, 0x94, 0x9D, 0xFD, 0xBC, 0x42, 0xD3, 0xAD, 0x19, 0x86, + 0x40, 0x68, 0x8A, 0x6F, 0xE1, 0x3F, 0x41, 0x34, 0x95, 0x54, 0xB4, 0x9A, + 0xCC, 0x31, 0xDC, 0xCD, 0x88, 0x45, 0x39, 0x81, 0x6F, 0x5E, 0xB4, 0xAC, + 0x8F, 0xB1, 0xF1, 0xA6, + /* x */ + 0x43, 0xBD, 0x7E, 0x9A, 0xFB, 0x53, 0xD8, 0xB8, 0x52, 0x89, 0xBC, 0xC4, + 0x8E, 0xE5, 0xBF, 0xE6, 0xF2, 0x01, 0x37, 0xD1, 0x0A, 0x08, 0x7E, 0xB6, + 0xE7, 0x87, 0x1E, 0x2A, 0x10, 0xA5, 0x99, 0xC7, 0x10, 0xAF, 0x8D, 0x0D, + 0x39, 0xE2, 0x06, 0x11, + /* y */ + 0x14, 0xFD, 0xD0, 0x55, 0x45, 0xEC, 0x1C, 0xC8, 0xAB, 0x40, 0x93, 0x24, + 0x7F, 0x77, 0x27, 0x5E, 0x07, 0x43, 0xFF, 0xED, 0x11, 0x71, 0x82, 0xEA, + 0xA9, 0xC7, 0x78, 0x77, 0xAA, 0xAC, 0x6A, 0xC7, 0xD3, 0x52, 0x45, 0xD1, + 0x69, 0x2E, 0x8E, 0xE1, + /* order */ + 0xD3, 0x5E, 0x47, 0x20, 0x36, 0xBC, 0x4F, 0xB7, 0xE1, 0x3C, 0x78, 0x5E, + 0xD2, 0x01, 0xE0, 0x65, 0xF9, 0x8F, 0xCF, 0xA5, 0xB6, 0x8F, 0x12, 0xA3, + 0x2D, 0x48, 0x2E, 0xC7, 0xEE, 0x86, 0x58, 0xE9, 0x86, 0x91, 0x55, 0x5B, + 0x44, 0xC5, 0x93, 0x11 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 40 * 6]; +} _EC_brainpoolP320t1 = { + { + NID_X9_62_prime_field, 0, 40, 1 + }, + { + /* no seed */ + /* p */ + 0xD3, 0x5E, 0x47, 0x20, 0x36, 0xBC, 0x4F, 0xB7, 0xE1, 0x3C, 0x78, 0x5E, + 0xD2, 0x01, 0xE0, 0x65, 0xF9, 0x8F, 0xCF, 0xA6, 0xF6, 0xF4, 0x0D, 0xEF, + 0x4F, 0x92, 0xB9, 0xEC, 0x78, 0x93, 0xEC, 0x28, 0xFC, 0xD4, 0x12, 0xB1, + 0xF1, 0xB3, 0x2E, 0x27, + /* a */ + 0xD3, 0x5E, 0x47, 0x20, 0x36, 0xBC, 0x4F, 0xB7, 0xE1, 0x3C, 0x78, 0x5E, + 0xD2, 0x01, 0xE0, 0x65, 0xF9, 0x8F, 0xCF, 0xA6, 0xF6, 0xF4, 0x0D, 0xEF, + 0x4F, 0x92, 0xB9, 0xEC, 0x78, 0x93, 0xEC, 0x28, 0xFC, 0xD4, 0x12, 0xB1, + 0xF1, 0xB3, 0x2E, 0x24, + /* b */ + 0xA7, 0xF5, 0x61, 0xE0, 0x38, 0xEB, 0x1E, 0xD5, 0x60, 0xB3, 0xD1, 0x47, + 0xDB, 0x78, 0x20, 0x13, 0x06, 0x4C, 0x19, 0xF2, 0x7E, 0xD2, 0x7C, 0x67, + 0x80, 0xAA, 0xF7, 0x7F, 0xB8, 0xA5, 0x47, 0xCE, 0xB5, 0xB4, 0xFE, 0xF4, + 0x22, 0x34, 0x03, 0x53, + /* x */ + 0x92, 0x5B, 0xE9, 0xFB, 0x01, 0xAF, 0xC6, 0xFB, 0x4D, 0x3E, 0x7D, 0x49, + 0x90, 0x01, 0x0F, 0x81, 0x34, 0x08, 0xAB, 0x10, 0x6C, 0x4F, 0x09, 0xCB, + 0x7E, 0xE0, 0x78, 0x68, 0xCC, 0x13, 0x6F, 0xFF, 0x33, 0x57, 0xF6, 0x24, + 0xA2, 0x1B, 0xED, 0x52, + /* y */ + 0x63, 0xBA, 0x3A, 0x7A, 0x27, 0x48, 0x3E, 0xBF, 0x66, 0x71, 0xDB, 0xEF, + 0x7A, 0xBB, 0x30, 0xEB, 0xEE, 0x08, 0x4E, 0x58, 0xA0, 0xB0, 0x77, 0xAD, + 0x42, 0xA5, 0xA0, 0x98, 0x9D, 0x1E, 0xE7, 0x1B, 0x1B, 0x9B, 0xC0, 0x45, + 0x5F, 0xB0, 0xD2, 0xC3, + /* order */ + 0xD3, 0x5E, 0x47, 0x20, 0x36, 0xBC, 0x4F, 0xB7, 0xE1, 0x3C, 0x78, 0x5E, + 0xD2, 0x01, 0xE0, 0x65, 0xF9, 0x8F, 0xCF, 0xA5, 0xB6, 0x8F, 0x12, 0xA3, + 0x2D, 0x48, 0x2E, 0xC7, 0xEE, 0x86, 0x58, 0xE9, 0x86, 0x91, 0x55, 0x5B, + 0x44, 0xC5, 0x93, 0x11 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 48 * 6]; +} _EC_brainpoolP384r1 = { + { + NID_X9_62_prime_field, 0, 48, 1 + }, + { + /* no seed */ + /* p */ + 0x8C, 0xB9, 0x1E, 0x82, 0xA3, 0x38, 0x6D, 0x28, 0x0F, 0x5D, 0x6F, 0x7E, + 0x50, 0xE6, 0x41, 0xDF, 0x15, 0x2F, 0x71, 0x09, 0xED, 0x54, 0x56, 0xB4, + 0x12, 0xB1, 0xDA, 0x19, 0x7F, 0xB7, 0x11, 0x23, 0xAC, 0xD3, 0xA7, 0x29, + 0x90, 0x1D, 0x1A, 0x71, 0x87, 0x47, 0x00, 0x13, 0x31, 0x07, 0xEC, 0x53, + /* a */ + 0x7B, 0xC3, 0x82, 0xC6, 0x3D, 0x8C, 0x15, 0x0C, 0x3C, 0x72, 0x08, 0x0A, + 0xCE, 0x05, 0xAF, 0xA0, 0xC2, 0xBE, 0xA2, 0x8E, 0x4F, 0xB2, 0x27, 0x87, + 0x13, 0x91, 0x65, 0xEF, 0xBA, 0x91, 0xF9, 0x0F, 0x8A, 0xA5, 0x81, 0x4A, + 0x50, 0x3A, 0xD4, 0xEB, 0x04, 0xA8, 0xC7, 0xDD, 0x22, 0xCE, 0x28, 0x26, + /* b */ + 0x04, 0xA8, 0xC7, 0xDD, 0x22, 0xCE, 0x28, 0x26, 0x8B, 0x39, 0xB5, 0x54, + 0x16, 0xF0, 0x44, 0x7C, 0x2F, 0xB7, 0x7D, 0xE1, 0x07, 0xDC, 0xD2, 0xA6, + 0x2E, 0x88, 0x0E, 0xA5, 0x3E, 0xEB, 0x62, 0xD5, 0x7C, 0xB4, 0x39, 0x02, + 0x95, 0xDB, 0xC9, 0x94, 0x3A, 0xB7, 0x86, 0x96, 0xFA, 0x50, 0x4C, 0x11, + /* x */ + 0x1D, 0x1C, 0x64, 0xF0, 0x68, 0xCF, 0x45, 0xFF, 0xA2, 0xA6, 0x3A, 0x81, + 0xB7, 0xC1, 0x3F, 0x6B, 0x88, 0x47, 0xA3, 0xE7, 0x7E, 0xF1, 0x4F, 0xE3, + 0xDB, 0x7F, 0xCA, 0xFE, 0x0C, 0xBD, 0x10, 0xE8, 0xE8, 0x26, 0xE0, 0x34, + 0x36, 0xD6, 0x46, 0xAA, 0xEF, 0x87, 0xB2, 0xE2, 0x47, 0xD4, 0xAF, 0x1E, + /* y */ + 0x8A, 0xBE, 0x1D, 0x75, 0x20, 0xF9, 0xC2, 0xA4, 0x5C, 0xB1, 0xEB, 0x8E, + 0x95, 0xCF, 0xD5, 0x52, 0x62, 0xB7, 0x0B, 0x29, 0xFE, 0xEC, 0x58, 0x64, + 0xE1, 0x9C, 0x05, 0x4F, 0xF9, 0x91, 0x29, 0x28, 0x0E, 0x46, 0x46, 0x21, + 0x77, 0x91, 0x81, 0x11, 0x42, 0x82, 0x03, 0x41, 0x26, 0x3C, 0x53, 0x15, + /* order */ + 0x8C, 0xB9, 0x1E, 0x82, 0xA3, 0x38, 0x6D, 0x28, 0x0F, 0x5D, 0x6F, 0x7E, + 0x50, 0xE6, 0x41, 0xDF, 0x15, 0x2F, 0x71, 0x09, 0xED, 0x54, 0x56, 0xB3, + 0x1F, 0x16, 0x6E, 0x6C, 0xAC, 0x04, 0x25, 0xA7, 0xCF, 0x3A, 0xB6, 0xAF, + 0x6B, 0x7F, 0xC3, 0x10, 0x3B, 0x88, 0x32, 0x02, 0xE9, 0x04, 0x65, 0x65 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 48 * 6]; +} _EC_brainpoolP384t1 = { + { + NID_X9_62_prime_field, 0, 48, 1 + }, + { + /* no seed */ + /* p */ + 0x8C, 0xB9, 0x1E, 0x82, 0xA3, 0x38, 0x6D, 0x28, 0x0F, 0x5D, 0x6F, 0x7E, + 0x50, 0xE6, 0x41, 0xDF, 0x15, 0x2F, 0x71, 0x09, 0xED, 0x54, 0x56, 0xB4, + 0x12, 0xB1, 0xDA, 0x19, 0x7F, 0xB7, 0x11, 0x23, 0xAC, 0xD3, 0xA7, 0x29, + 0x90, 0x1D, 0x1A, 0x71, 0x87, 0x47, 0x00, 0x13, 0x31, 0x07, 0xEC, 0x53, + /* a */ + 0x8C, 0xB9, 0x1E, 0x82, 0xA3, 0x38, 0x6D, 0x28, 0x0F, 0x5D, 0x6F, 0x7E, + 0x50, 0xE6, 0x41, 0xDF, 0x15, 0x2F, 0x71, 0x09, 0xED, 0x54, 0x56, 0xB4, + 0x12, 0xB1, 0xDA, 0x19, 0x7F, 0xB7, 0x11, 0x23, 0xAC, 0xD3, 0xA7, 0x29, + 0x90, 0x1D, 0x1A, 0x71, 0x87, 0x47, 0x00, 0x13, 0x31, 0x07, 0xEC, 0x50, + /* b */ + 0x7F, 0x51, 0x9E, 0xAD, 0xA7, 0xBD, 0xA8, 0x1B, 0xD8, 0x26, 0xDB, 0xA6, + 0x47, 0x91, 0x0F, 0x8C, 0x4B, 0x93, 0x46, 0xED, 0x8C, 0xCD, 0xC6, 0x4E, + 0x4B, 0x1A, 0xBD, 0x11, 0x75, 0x6D, 0xCE, 0x1D, 0x20, 0x74, 0xAA, 0x26, + 0x3B, 0x88, 0x80, 0x5C, 0xED, 0x70, 0x35, 0x5A, 0x33, 0xB4, 0x71, 0xEE, + /* x */ + 0x18, 0xDE, 0x98, 0xB0, 0x2D, 0xB9, 0xA3, 0x06, 0xF2, 0xAF, 0xCD, 0x72, + 0x35, 0xF7, 0x2A, 0x81, 0x9B, 0x80, 0xAB, 0x12, 0xEB, 0xD6, 0x53, 0x17, + 0x24, 0x76, 0xFE, 0xCD, 0x46, 0x2A, 0xAB, 0xFF, 0xC4, 0xFF, 0x19, 0x1B, + 0x94, 0x6A, 0x5F, 0x54, 0xD8, 0xD0, 0xAA, 0x2F, 0x41, 0x88, 0x08, 0xCC, + /* y */ + 0x25, 0xAB, 0x05, 0x69, 0x62, 0xD3, 0x06, 0x51, 0xA1, 0x14, 0xAF, 0xD2, + 0x75, 0x5A, 0xD3, 0x36, 0x74, 0x7F, 0x93, 0x47, 0x5B, 0x7A, 0x1F, 0xCA, + 0x3B, 0x88, 0xF2, 0xB6, 0xA2, 0x08, 0xCC, 0xFE, 0x46, 0x94, 0x08, 0x58, + 0x4D, 0xC2, 0xB2, 0x91, 0x26, 0x75, 0xBF, 0x5B, 0x9E, 0x58, 0x29, 0x28, + /* order */ + 0x8C, 0xB9, 0x1E, 0x82, 0xA3, 0x38, 0x6D, 0x28, 0x0F, 0x5D, 0x6F, 0x7E, + 0x50, 0xE6, 0x41, 0xDF, 0x15, 0x2F, 0x71, 0x09, 0xED, 0x54, 0x56, 0xB3, + 0x1F, 0x16, 0x6E, 0x6C, 0xAC, 0x04, 0x25, 0xA7, 0xCF, 0x3A, 0xB6, 0xAF, + 0x6B, 0x7F, 0xC3, 0x10, 0x3B, 0x88, 0x32, 0x02, 0xE9, 0x04, 0x65, 0x65 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 64 * 6]; +} _EC_brainpoolP512r1 = { + { + NID_X9_62_prime_field, 0, 64, 1 + }, + { + /* no seed */ + /* p */ + 0xAA, 0xDD, 0x9D, 0xB8, 0xDB, 0xE9, 0xC4, 0x8B, 0x3F, 0xD4, 0xE6, 0xAE, + 0x33, 0xC9, 0xFC, 0x07, 0xCB, 0x30, 0x8D, 0xB3, 0xB3, 0xC9, 0xD2, 0x0E, + 0xD6, 0x63, 0x9C, 0xCA, 0x70, 0x33, 0x08, 0x71, 0x7D, 0x4D, 0x9B, 0x00, + 0x9B, 0xC6, 0x68, 0x42, 0xAE, 0xCD, 0xA1, 0x2A, 0xE6, 0xA3, 0x80, 0xE6, + 0x28, 0x81, 0xFF, 0x2F, 0x2D, 0x82, 0xC6, 0x85, 0x28, 0xAA, 0x60, 0x56, + 0x58, 0x3A, 0x48, 0xF3, + /* a */ + 0x78, 0x30, 0xA3, 0x31, 0x8B, 0x60, 0x3B, 0x89, 0xE2, 0x32, 0x71, 0x45, + 0xAC, 0x23, 0x4C, 0xC5, 0x94, 0xCB, 0xDD, 0x8D, 0x3D, 0xF9, 0x16, 0x10, + 0xA8, 0x34, 0x41, 0xCA, 0xEA, 0x98, 0x63, 0xBC, 0x2D, 0xED, 0x5D, 0x5A, + 0xA8, 0x25, 0x3A, 0xA1, 0x0A, 0x2E, 0xF1, 0xC9, 0x8B, 0x9A, 0xC8, 0xB5, + 0x7F, 0x11, 0x17, 0xA7, 0x2B, 0xF2, 0xC7, 0xB9, 0xE7, 0xC1, 0xAC, 0x4D, + 0x77, 0xFC, 0x94, 0xCA, + /* b */ + 0x3D, 0xF9, 0x16, 0x10, 0xA8, 0x34, 0x41, 0xCA, 0xEA, 0x98, 0x63, 0xBC, + 0x2D, 0xED, 0x5D, 0x5A, 0xA8, 0x25, 0x3A, 0xA1, 0x0A, 0x2E, 0xF1, 0xC9, + 0x8B, 0x9A, 0xC8, 0xB5, 0x7F, 0x11, 0x17, 0xA7, 0x2B, 0xF2, 0xC7, 0xB9, + 0xE7, 0xC1, 0xAC, 0x4D, 0x77, 0xFC, 0x94, 0xCA, 0xDC, 0x08, 0x3E, 0x67, + 0x98, 0x40, 0x50, 0xB7, 0x5E, 0xBA, 0xE5, 0xDD, 0x28, 0x09, 0xBD, 0x63, + 0x80, 0x16, 0xF7, 0x23, + /* x */ + 0x81, 0xAE, 0xE4, 0xBD, 0xD8, 0x2E, 0xD9, 0x64, 0x5A, 0x21, 0x32, 0x2E, + 0x9C, 0x4C, 0x6A, 0x93, 0x85, 0xED, 0x9F, 0x70, 0xB5, 0xD9, 0x16, 0xC1, + 0xB4, 0x3B, 0x62, 0xEE, 0xF4, 0xD0, 0x09, 0x8E, 0xFF, 0x3B, 0x1F, 0x78, + 0xE2, 0xD0, 0xD4, 0x8D, 0x50, 0xD1, 0x68, 0x7B, 0x93, 0xB9, 0x7D, 0x5F, + 0x7C, 0x6D, 0x50, 0x47, 0x40, 0x6A, 0x5E, 0x68, 0x8B, 0x35, 0x22, 0x09, + 0xBC, 0xB9, 0xF8, 0x22, + /* y */ + 0x7D, 0xDE, 0x38, 0x5D, 0x56, 0x63, 0x32, 0xEC, 0xC0, 0xEA, 0xBF, 0xA9, + 0xCF, 0x78, 0x22, 0xFD, 0xF2, 0x09, 0xF7, 0x00, 0x24, 0xA5, 0x7B, 0x1A, + 0xA0, 0x00, 0xC5, 0x5B, 0x88, 0x1F, 0x81, 0x11, 0xB2, 0xDC, 0xDE, 0x49, + 0x4A, 0x5F, 0x48, 0x5E, 0x5B, 0xCA, 0x4B, 0xD8, 0x8A, 0x27, 0x63, 0xAE, + 0xD1, 0xCA, 0x2B, 0x2F, 0xA8, 0xF0, 0x54, 0x06, 0x78, 0xCD, 0x1E, 0x0F, + 0x3A, 0xD8, 0x08, 0x92, + /* order */ + 0xAA, 0xDD, 0x9D, 0xB8, 0xDB, 0xE9, 0xC4, 0x8B, 0x3F, 0xD4, 0xE6, 0xAE, + 0x33, 0xC9, 0xFC, 0x07, 0xCB, 0x30, 0x8D, 0xB3, 0xB3, 0xC9, 0xD2, 0x0E, + 0xD6, 0x63, 0x9C, 0xCA, 0x70, 0x33, 0x08, 0x70, 0x55, 0x3E, 0x5C, 0x41, + 0x4C, 0xA9, 0x26, 0x19, 0x41, 0x86, 0x61, 0x19, 0x7F, 0xAC, 0x10, 0x47, + 0x1D, 0xB1, 0xD3, 0x81, 0x08, 0x5D, 0xDA, 0xDD, 0xB5, 0x87, 0x96, 0x82, + 0x9C, 0xA9, 0x00, 0x69 + } +}; + +static const struct { + EC_CURVE_DATA h; + unsigned char data[0 + 64 * 6]; +} _EC_brainpoolP512t1 = { + { + NID_X9_62_prime_field, 0, 64, 1 + }, + { + /* no seed */ + /* p */ + 0xAA, 0xDD, 0x9D, 0xB8, 0xDB, 0xE9, 0xC4, 0x8B, 0x3F, 0xD4, 0xE6, 0xAE, + 0x33, 0xC9, 0xFC, 0x07, 0xCB, 0x30, 0x8D, 0xB3, 0xB3, 0xC9, 0xD2, 0x0E, + 0xD6, 0x63, 0x9C, 0xCA, 0x70, 0x33, 0x08, 0x71, 0x7D, 0x4D, 0x9B, 0x00, + 0x9B, 0xC6, 0x68, 0x42, 0xAE, 0xCD, 0xA1, 0x2A, 0xE6, 0xA3, 0x80, 0xE6, + 0x28, 0x81, 0xFF, 0x2F, 0x2D, 0x82, 0xC6, 0x85, 0x28, 0xAA, 0x60, 0x56, + 0x58, 0x3A, 0x48, 0xF3, + /* a */ + 0xAA, 0xDD, 0x9D, 0xB8, 0xDB, 0xE9, 0xC4, 0x8B, 0x3F, 0xD4, 0xE6, 0xAE, + 0x33, 0xC9, 0xFC, 0x07, 0xCB, 0x30, 0x8D, 0xB3, 0xB3, 0xC9, 0xD2, 0x0E, + 0xD6, 0x63, 0x9C, 0xCA, 0x70, 0x33, 0x08, 0x71, 0x7D, 0x4D, 0x9B, 0x00, + 0x9B, 0xC6, 0x68, 0x42, 0xAE, 0xCD, 0xA1, 0x2A, 0xE6, 0xA3, 0x80, 0xE6, + 0x28, 0x81, 0xFF, 0x2F, 0x2D, 0x82, 0xC6, 0x85, 0x28, 0xAA, 0x60, 0x56, + 0x58, 0x3A, 0x48, 0xF0, + /* b */ + 0x7C, 0xBB, 0xBC, 0xF9, 0x44, 0x1C, 0xFA, 0xB7, 0x6E, 0x18, 0x90, 0xE4, + 0x68, 0x84, 0xEA, 0xE3, 0x21, 0xF7, 0x0C, 0x0B, 0xCB, 0x49, 0x81, 0x52, + 0x78, 0x97, 0x50, 0x4B, 0xEC, 0x3E, 0x36, 0xA6, 0x2B, 0xCD, 0xFA, 0x23, + 0x04, 0x97, 0x65, 0x40, 0xF6, 0x45, 0x00, 0x85, 0xF2, 0xDA, 0xE1, 0x45, + 0xC2, 0x25, 0x53, 0xB4, 0x65, 0x76, 0x36, 0x89, 0x18, 0x0E, 0xA2, 0x57, + 0x18, 0x67, 0x42, 0x3E, + /* x */ + 0x64, 0x0E, 0xCE, 0x5C, 0x12, 0x78, 0x87, 0x17, 0xB9, 0xC1, 0xBA, 0x06, + 0xCB, 0xC2, 0xA6, 0xFE, 0xBA, 0x85, 0x84, 0x24, 0x58, 0xC5, 0x6D, 0xDE, + 0x9D, 0xB1, 0x75, 0x8D, 0x39, 0xC0, 0x31, 0x3D, 0x82, 0xBA, 0x51, 0x73, + 0x5C, 0xDB, 0x3E, 0xA4, 0x99, 0xAA, 0x77, 0xA7, 0xD6, 0x94, 0x3A, 0x64, + 0xF7, 0xA3, 0xF2, 0x5F, 0xE2, 0x6F, 0x06, 0xB5, 0x1B, 0xAA, 0x26, 0x96, + 0xFA, 0x90, 0x35, 0xDA, + /* y */ + 0x5B, 0x53, 0x4B, 0xD5, 0x95, 0xF5, 0xAF, 0x0F, 0xA2, 0xC8, 0x92, 0x37, + 0x6C, 0x84, 0xAC, 0xE1, 0xBB, 0x4E, 0x30, 0x19, 0xB7, 0x16, 0x34, 0xC0, + 0x11, 0x31, 0x15, 0x9C, 0xAE, 0x03, 0xCE, 0xE9, 0xD9, 0x93, 0x21, 0x84, + 0xBE, 0xEF, 0x21, 0x6B, 0xD7, 0x1D, 0xF2, 0xDA, 0xDF, 0x86, 0xA6, 0x27, + 0x30, 0x6E, 0xCF, 0xF9, 0x6D, 0xBB, 0x8B, 0xAC, 0xE1, 0x98, 0xB6, 0x1E, + 0x00, 0xF8, 0xB3, 0x32, + /* order */ + 0xAA, 0xDD, 0x9D, 0xB8, 0xDB, 0xE9, 0xC4, 0x8B, 0x3F, 0xD4, 0xE6, 0xAE, + 0x33, 0xC9, 0xFC, 0x07, 0xCB, 0x30, 0x8D, 0xB3, 0xB3, 0xC9, 0xD2, 0x0E, + 0xD6, 0x63, 0x9C, 0xCA, 0x70, 0x33, 0x08, 0x70, 0x55, 0x3E, 0x5C, 0x41, + 0x4C, 0xA9, 0x26, 0x19, 0x41, 0x86, 0x61, 0x19, 0x7F, 0xAC, 0x10, 0x47, + 0x1D, 0xB1, 0xD3, 0x81, 0x08, 0x5D, 0xDA, 0xDD, 0xB5, 0x87, 0x96, 0x82, + 0x9C, 0xA9, 0x00, 0x69 + } +}; + typedef struct _ec_list_element_st { int nid; const EC_CURVE_DATA *data; @@ -2343,13 +2896,15 @@ static const ec_list_element curve_list[] = { "X9.62 curve over a 239 bit prime field"}, {NID_X9_62_prime239v3, &_EC_X9_62_PRIME_239V3.h, 0, "X9.62 curve over a 239 bit prime field"}, -#ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 - {NID_X9_62_prime256v1, &_EC_X9_62_PRIME_256V1.h, EC_GFp_nistp256_method, - "X9.62/SECG curve over a 256 bit prime field"}, + {NID_X9_62_prime256v1, &_EC_X9_62_PRIME_256V1.h, +#if defined(ECP_NISTZ256_ASM) + EC_GFp_nistz256_method, +#elif !defined(OPENSSL_NO_EC_NISTP_64_GCC_128) + EC_GFp_nistp256_method, #else - {NID_X9_62_prime256v1, &_EC_X9_62_PRIME_256V1.h, 0, - "X9.62/SECG curve over a 256 bit prime field"}, + 0, #endif + "X9.62/SECG curve over a 256 bit prime field"}, #ifndef OPENSSL_NO_EC2M /* characteristic two field curves */ /* NIST/SECG curves */ @@ -2460,6 +3015,35 @@ static const ec_list_element curve_list[] = { "\n\tIPSec/IKE/Oakley curve #4 over a 185 bit binary field.\n" "\tNot suitable for ECDSA.\n\tQuestionable extension field!"}, #endif + /* brainpool curves */ + {NID_brainpoolP160r1, &_EC_brainpoolP160r1.h, 0, + "RFC 5639 curve over a 160 bit prime field"}, + {NID_brainpoolP160t1, &_EC_brainpoolP160t1.h, 0, + "RFC 5639 curve over a 160 bit prime field"}, + {NID_brainpoolP192r1, &_EC_brainpoolP192r1.h, 0, + "RFC 5639 curve over a 192 bit prime field"}, + {NID_brainpoolP192t1, &_EC_brainpoolP192t1.h, 0, + "RFC 5639 curve over a 192 bit prime field"}, + {NID_brainpoolP224r1, &_EC_brainpoolP224r1.h, 0, + "RFC 5639 curve over a 224 bit prime field"}, + {NID_brainpoolP224t1, &_EC_brainpoolP224t1.h, 0, + "RFC 5639 curve over a 224 bit prime field"}, + {NID_brainpoolP256r1, &_EC_brainpoolP256r1.h, 0, + "RFC 5639 curve over a 256 bit prime field"}, + {NID_brainpoolP256t1, &_EC_brainpoolP256t1.h, 0, + "RFC 5639 curve over a 256 bit prime field"}, + {NID_brainpoolP320r1, &_EC_brainpoolP320r1.h, 0, + "RFC 5639 curve over a 320 bit prime field"}, + {NID_brainpoolP320t1, &_EC_brainpoolP320t1.h, 0, + "RFC 5639 curve over a 320 bit prime field"}, + {NID_brainpoolP384r1, &_EC_brainpoolP384r1.h, 0, + "RFC 5639 curve over a 384 bit prime field"}, + {NID_brainpoolP384t1, &_EC_brainpoolP384t1.h, 0, + "RFC 5639 curve over a 384 bit prime field"}, + {NID_brainpoolP512r1, &_EC_brainpoolP512r1.h, 0, + "RFC 5639 curve over a 512 bit prime field"}, + {NID_brainpoolP512t1, &_EC_brainpoolP512t1.h, 0, + "RFC 5639 curve over a 512 bit prime field"}, }; #define curve_list_length (sizeof(curve_list)/sizeof(ec_list_element)) @@ -2578,6 +3162,10 @@ EC_GROUP *EC_GROUP_new_by_curve_name(int nid) size_t i; EC_GROUP *ret = NULL; +#ifdef OPENSSL_FIPS + if (FIPS_mode()) + return FIPS_ec_group_new_by_curve_name(nid); +#endif if (nid <= 0) return NULL; @@ -2613,3 +3201,48 @@ size_t EC_get_builtin_curves(EC_builtin_curve *r, size_t nitems) return curve_list_length; } + +/* Functions to translate between common NIST curve names and NIDs */ + +typedef struct { + const char *name; /* NIST Name of curve */ + int nid; /* Curve NID */ +} EC_NIST_NAME; + +static EC_NIST_NAME nist_curves[] = { + {"B-163", NID_sect163r2}, + {"B-233", NID_sect233r1}, + {"B-283", NID_sect283r1}, + {"B-409", NID_sect409r1}, + {"B-571", NID_sect571r1}, + {"K-163", NID_sect163k1}, + {"K-233", NID_sect233k1}, + {"K-283", NID_sect283k1}, + {"K-409", NID_sect409k1}, + {"K-571", NID_sect571k1}, + {"P-192", NID_X9_62_prime192v1}, + {"P-224", NID_secp224r1}, + {"P-256", NID_X9_62_prime256v1}, + {"P-384", NID_secp384r1}, + {"P-521", NID_secp521r1} +}; + +const char *EC_curve_nid2nist(int nid) +{ + size_t i; + for (i = 0; i < sizeof(nist_curves) / sizeof(EC_NIST_NAME); i++) { + if (nist_curves[i].nid == nid) + return nist_curves[i].name; + } + return NULL; +} + +int EC_curve_nist2nid(const char *name) +{ + size_t i; + for (i = 0; i < sizeof(nist_curves) / sizeof(EC_NIST_NAME); i++) { + if (!strcmp(nist_curves[i].name, name)) + return nist_curves[i].nid; + } + return NID_undef; +} diff --git a/crypto/ec/ec_cvt.c b/crypto/ec/ec_cvt.c index 487d727407e0..5a832ba1cfa4 100644 --- a/crypto/ec/ec_cvt.c +++ b/crypto/ec/ec_cvt.c @@ -72,12 +72,20 @@ #include #include "ec_lcl.h" +#ifdef OPENSSL_FIPS +# include +#endif + EC_GROUP *EC_GROUP_new_curve_GFp(const BIGNUM *p, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx) { const EC_METHOD *meth; EC_GROUP *ret; +#ifdef OPENSSL_FIPS + if (FIPS_mode()) + return FIPS_ec_group_new_curve_gfp(p, a, b, ctx); +#endif #if defined(OPENSSL_BN_ASM_MONT) /* * This might appear controversial, but the fact is that generic @@ -152,6 +160,10 @@ EC_GROUP *EC_GROUP_new_curve_GF2m(const BIGNUM *p, const BIGNUM *a, const EC_METHOD *meth; EC_GROUP *ret; +# ifdef OPENSSL_FIPS + if (FIPS_mode()) + return FIPS_ec_group_new_curve_gf2m(p, a, b, ctx); +# endif meth = EC_GF2m_simple_method(); ret = EC_GROUP_new(meth); diff --git a/crypto/ec/ec_err.c b/crypto/ec/ec_err.c index 58eae7c6668e..6fe5baafd4b3 100644 --- a/crypto/ec/ec_err.c +++ b/crypto/ec/ec_err.c @@ -1,6 +1,6 @@ /* crypto/ec/ec_err.c */ /* ==================================================================== - * Copyright (c) 1999-2011 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2015 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -76,6 +76,8 @@ static ERR_STRING_DATA EC_str_functs[] = { {ERR_FUNC(EC_F_D2I_ECPKPARAMETERS), "d2i_ECPKParameters"}, {ERR_FUNC(EC_F_D2I_ECPRIVATEKEY), "d2i_ECPrivateKey"}, {ERR_FUNC(EC_F_DO_EC_KEY_PRINT), "DO_EC_KEY_PRINT"}, + {ERR_FUNC(EC_F_ECDH_CMS_DECRYPT), "ECDH_CMS_DECRYPT"}, + {ERR_FUNC(EC_F_ECDH_CMS_SET_SHARED_INFO), "ECDH_CMS_SET_SHARED_INFO"}, {ERR_FUNC(EC_F_ECKEY_PARAM2TYPE), "ECKEY_PARAM2TYPE"}, {ERR_FUNC(EC_F_ECKEY_PARAM_DECODE), "ECKEY_PARAM_DECODE"}, {ERR_FUNC(EC_F_ECKEY_PRIV_DECODE), "ECKEY_PRIV_DECODE"}, @@ -87,6 +89,13 @@ static ERR_STRING_DATA EC_str_functs[] = { {ERR_FUNC(EC_F_ECPARAMETERS_PRINT_FP), "ECParameters_print_fp"}, {ERR_FUNC(EC_F_ECPKPARAMETERS_PRINT), "ECPKParameters_print"}, {ERR_FUNC(EC_F_ECPKPARAMETERS_PRINT_FP), "ECPKParameters_print_fp"}, + {ERR_FUNC(EC_F_ECP_NISTZ256_GET_AFFINE), "ecp_nistz256_get_affine"}, + {ERR_FUNC(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE), + "ecp_nistz256_mult_precompute"}, + {ERR_FUNC(EC_F_ECP_NISTZ256_POINTS_MUL), "ecp_nistz256_points_mul"}, + {ERR_FUNC(EC_F_ECP_NISTZ256_PRE_COMP_NEW), "ecp_nistz256_pre_comp_new"}, + {ERR_FUNC(EC_F_ECP_NISTZ256_SET_WORDS), "ecp_nistz256_set_words"}, + {ERR_FUNC(EC_F_ECP_NISTZ256_WINDOWED_MUL), "ecp_nistz256_windowed_mul"}, {ERR_FUNC(EC_F_ECP_NIST_MOD_192), "ECP_NIST_MOD_192"}, {ERR_FUNC(EC_F_ECP_NIST_MOD_224), "ECP_NIST_MOD_224"}, {ERR_FUNC(EC_F_ECP_NIST_MOD_256), "ECP_NIST_MOD_256"}, @@ -271,6 +280,7 @@ static ERR_STRING_DATA EC_str_reasons[] = { {ERR_REASON(EC_R_INVALID_COMPRESSED_POINT), "invalid compressed point"}, {ERR_REASON(EC_R_INVALID_COMPRESSION_BIT), "invalid compression bit"}, {ERR_REASON(EC_R_INVALID_CURVE), "invalid curve"}, + {ERR_REASON(EC_R_INVALID_DIGEST), "invalid digest"}, {ERR_REASON(EC_R_INVALID_DIGEST_TYPE), "invalid digest type"}, {ERR_REASON(EC_R_INVALID_ENCODING), "invalid encoding"}, {ERR_REASON(EC_R_INVALID_FIELD), "invalid field"}, @@ -279,6 +289,7 @@ static ERR_STRING_DATA EC_str_reasons[] = { {ERR_REASON(EC_R_INVALID_PENTANOMIAL_BASIS), "invalid pentanomial basis"}, {ERR_REASON(EC_R_INVALID_PRIVATE_KEY), "invalid private key"}, {ERR_REASON(EC_R_INVALID_TRINOMIAL_BASIS), "invalid trinomial basis"}, + {ERR_REASON(EC_R_KDF_PARAMETER_ERROR), "kdf parameter error"}, {ERR_REASON(EC_R_KEYS_NOT_SET), "keys not set"}, {ERR_REASON(EC_R_MISSING_PARAMETERS), "missing parameters"}, {ERR_REASON(EC_R_MISSING_PRIVATE_KEY), "missing private key"}, @@ -290,10 +301,12 @@ static ERR_STRING_DATA EC_str_reasons[] = { {ERR_REASON(EC_R_NO_FIELD_MOD), "no field mod"}, {ERR_REASON(EC_R_NO_PARAMETERS_SET), "no parameters set"}, {ERR_REASON(EC_R_PASSED_NULL_PARAMETER), "passed null parameter"}, + {ERR_REASON(EC_R_PEER_KEY_ERROR), "peer key error"}, {ERR_REASON(EC_R_PKPARAMETERS2GROUP_FAILURE), "pkparameters2group failure"}, {ERR_REASON(EC_R_POINT_AT_INFINITY), "point at infinity"}, {ERR_REASON(EC_R_POINT_IS_NOT_ON_CURVE), "point is not on curve"}, + {ERR_REASON(EC_R_SHARED_INFO_ERROR), "shared info error"}, {ERR_REASON(EC_R_SLOT_FULL), "slot full"}, {ERR_REASON(EC_R_UNDEFINED_GENERATOR), "undefined generator"}, {ERR_REASON(EC_R_UNDEFINED_ORDER), "undefined order"}, diff --git a/crypto/ec/ec_lcl.h b/crypto/ec/ec_lcl.h index d79ed1e4d92d..969fd147ef93 100644 --- a/crypto/ec/ec_lcl.h +++ b/crypto/ec/ec_lcl.h @@ -212,6 +212,13 @@ struct ec_group_st { BIGNUM order, cofactor; int curve_name; /* optional NID for named curve */ int asn1_flag; /* flag to control the asn1 encoding */ + /* + * Kludge: upper bit of ans1_flag is used to denote structure + * version. Is set, then last field is present. This is done + * for interoperation with FIPS code. + */ +#define EC_GROUP_ASN1_FLAG_MASK 0x7fffffff +#define EC_GROUP_VERSION(p) (p->asn1_flag&~EC_GROUP_ASN1_FLAG_MASK) point_conversion_form_t asn1_form; unsigned char *seed; /* optional seed for parameters (appears in * ASN1) */ @@ -252,6 +259,7 @@ struct ec_group_st { /* method-specific */ int (*field_mod_func) (BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *); + BN_MONT_CTX *mont_data; /* data for ECDSA inverse */ } /* EC_GROUP */ ; struct ec_key_st { @@ -541,3 +549,20 @@ void ec_GFp_nistp_points_make_affine_internal(size_t num, void *point_array, void ec_GFp_nistp_recode_scalar_bits(unsigned char *sign, unsigned char *digit, unsigned char in); #endif +int ec_precompute_mont_data(EC_GROUP *); + +#ifdef ECP_NISTZ256_ASM +/** Returns GFp methods using montgomery multiplication, with x86-64 optimized + * P256. See http://eprint.iacr.org/2013/816. + * \return EC_METHOD object + */ +const EC_METHOD *EC_GFp_nistz256_method(void); +#endif + +#ifdef OPENSSL_FIPS +EC_GROUP *FIPS_ec_group_new_curve_gfp(const BIGNUM *p, const BIGNUM *a, + const BIGNUM *b, BN_CTX *ctx); +EC_GROUP *FIPS_ec_group_new_curve_gf2m(const BIGNUM *p, const BIGNUM *a, + const BIGNUM *b, BN_CTX *ctx); +EC_GROUP *FIPS_ec_group_new_by_curve_name(int nid); +#endif diff --git a/crypto/ec/ec_lib.c b/crypto/ec/ec_lib.c index e2275207ed22..3ffa112cc306 100644 --- a/crypto/ec/ec_lib.c +++ b/crypto/ec/ec_lib.c @@ -94,13 +94,14 @@ EC_GROUP *EC_GROUP_new(const EC_METHOD *meth) ret->meth = meth; ret->extra_data = NULL; + ret->mont_data = NULL; ret->generator = NULL; BN_init(&ret->order); BN_init(&ret->cofactor); ret->curve_name = 0; - ret->asn1_flag = 0; + ret->asn1_flag = ~EC_GROUP_ASN1_FLAG_MASK; ret->asn1_form = POINT_CONVERSION_UNCOMPRESSED; ret->seed = NULL; @@ -124,6 +125,9 @@ void EC_GROUP_free(EC_GROUP *group) EC_EX_DATA_free_all_data(&group->extra_data); + if (EC_GROUP_VERSION(group) && group->mont_data) + BN_MONT_CTX_free(group->mont_data); + if (group->generator != NULL) EC_POINT_free(group->generator); BN_free(&group->order); @@ -147,6 +151,9 @@ void EC_GROUP_clear_free(EC_GROUP *group) EC_EX_DATA_clear_free_all_data(&group->extra_data); + if (EC_GROUP_VERSION(group) && group->mont_data) + BN_MONT_CTX_free(group->mont_data); + if (group->generator != NULL) EC_POINT_clear_free(group->generator); BN_clear_free(&group->order); @@ -189,6 +196,22 @@ int EC_GROUP_copy(EC_GROUP *dest, const EC_GROUP *src) return 0; } + if (EC_GROUP_VERSION(src) && src->mont_data != NULL) { + if (dest->mont_data == NULL) { + dest->mont_data = BN_MONT_CTX_new(); + if (dest->mont_data == NULL) + return 0; + } + if (!BN_MONT_CTX_copy(dest->mont_data, src->mont_data)) + return 0; + } else { + /* src->generator == NULL */ + if (EC_GROUP_VERSION(dest) && dest->mont_data != NULL) { + BN_MONT_CTX_free(dest->mont_data); + dest->mont_data = NULL; + } + } + if (src->generator != NULL) { if (dest->generator == NULL) { dest->generator = EC_POINT_new(dest); @@ -295,6 +318,13 @@ int EC_GROUP_set_generator(EC_GROUP *group, const EC_POINT *generator, } else BN_zero(&group->cofactor); + /* + * We ignore the return value because some groups have an order with + * factors of two, which makes the Montgomery setup fail. + * |group->mont_data| will be NULL in this case. + */ + ec_precompute_mont_data(group); + return 1; } @@ -303,6 +333,11 @@ const EC_POINT *EC_GROUP_get0_generator(const EC_GROUP *group) return group->generator; } +BN_MONT_CTX *EC_GROUP_get_mont_data(const EC_GROUP *group) +{ + return EC_GROUP_VERSION(group) ? group->mont_data : NULL; +} + int EC_GROUP_get_order(const EC_GROUP *group, BIGNUM *order, BN_CTX *ctx) { if (!BN_copy(order, &group->order)) @@ -332,12 +367,13 @@ int EC_GROUP_get_curve_name(const EC_GROUP *group) void EC_GROUP_set_asn1_flag(EC_GROUP *group, int flag) { - group->asn1_flag = flag; + group->asn1_flag &= ~EC_GROUP_ASN1_FLAG_MASK; + group->asn1_flag |= flag & EC_GROUP_ASN1_FLAG_MASK; } int EC_GROUP_get_asn1_flag(const EC_GROUP *group) { - return group->asn1_flag; + return group->asn1_flag & EC_GROUP_ASN1_FLAG_MASK; } void EC_GROUP_set_point_conversion_form(EC_GROUP *group, @@ -1057,3 +1093,42 @@ int EC_GROUP_have_precompute_mult(const EC_GROUP *group) return 0; /* cannot tell whether precomputation has * been performed */ } + +/* + * ec_precompute_mont_data sets |group->mont_data| from |group->order| and + * returns one on success. On error it returns zero. + */ +int ec_precompute_mont_data(EC_GROUP *group) +{ + BN_CTX *ctx = BN_CTX_new(); + int ret = 0; + + if (!EC_GROUP_VERSION(group)) + goto err; + + if (group->mont_data) { + BN_MONT_CTX_free(group->mont_data); + group->mont_data = NULL; + } + + if (ctx == NULL) + goto err; + + group->mont_data = BN_MONT_CTX_new(); + if (!group->mont_data) + goto err; + + if (!BN_MONT_CTX_set(group->mont_data, &group->order, ctx)) { + BN_MONT_CTX_free(group->mont_data); + group->mont_data = NULL; + goto err; + } + + ret = 1; + + err: + + if (ctx) + BN_CTX_free(ctx); + return ret; +} diff --git a/crypto/ec/ec_pmeth.c b/crypto/ec/ec_pmeth.c index c189d3ffbf2a..b76749010c93 100644 --- a/crypto/ec/ec_pmeth.c +++ b/crypto/ec/ec_pmeth.c @@ -61,6 +61,7 @@ #include #include #include +#include "ec_lcl.h" #include #include #include "evp_locl.h" @@ -72,6 +73,19 @@ typedef struct { EC_GROUP *gen_group; /* message digest */ const EVP_MD *md; + /* Duplicate key if custom cofactor needed */ + EC_KEY *co_key; + /* Cofactor mode */ + signed char cofactor_mode; + /* KDF (if any) to use for ECDH */ + char kdf_type; + /* Message digest to use for key derivation */ + const EVP_MD *kdf_md; + /* User key material */ + unsigned char *kdf_ukm; + size_t kdf_ukmlen; + /* KDF output length */ + size_t kdf_outlen; } EC_PKEY_CTX; static int pkey_ec_init(EVP_PKEY_CTX *ctx) @@ -83,6 +97,14 @@ static int pkey_ec_init(EVP_PKEY_CTX *ctx) dctx->gen_group = NULL; dctx->md = NULL; + dctx->cofactor_mode = -1; + dctx->co_key = NULL; + dctx->kdf_type = EVP_PKEY_ECDH_KDF_NONE; + dctx->kdf_md = NULL; + dctx->kdf_outlen = 0; + dctx->kdf_ukm = NULL; + dctx->kdf_ukmlen = 0; + ctx->data = dctx; return 1; @@ -101,6 +123,22 @@ static int pkey_ec_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src) return 0; } dctx->md = sctx->md; + + if (sctx->co_key) { + dctx->co_key = EC_KEY_dup(sctx->co_key); + if (!dctx->co_key) + return 0; + } + dctx->kdf_type = sctx->kdf_type; + dctx->kdf_md = sctx->kdf_md; + dctx->kdf_outlen = sctx->kdf_outlen; + if (sctx->kdf_ukm) { + dctx->kdf_ukm = BUF_memdup(sctx->kdf_ukm, sctx->kdf_ukmlen); + if (!dctx->kdf_ukm) + return 0; + } else + dctx->kdf_ukm = NULL; + dctx->kdf_ukmlen = sctx->kdf_ukmlen; return 1; } @@ -110,6 +148,10 @@ static void pkey_ec_cleanup(EVP_PKEY_CTX *ctx) if (dctx) { if (dctx->gen_group) EC_GROUP_free(dctx->gen_group); + if (dctx->co_key) + EC_KEY_free(dctx->co_key); + if (dctx->kdf_ukm) + OPENSSL_free(dctx->kdf_ukm); OPENSSL_free(dctx); } } @@ -168,18 +210,21 @@ static int pkey_ec_derive(EVP_PKEY_CTX *ctx, unsigned char *key, int ret; size_t outlen; const EC_POINT *pubkey = NULL; + EC_KEY *eckey; + EC_PKEY_CTX *dctx = ctx->data; if (!ctx->pkey || !ctx->peerkey) { ECerr(EC_F_PKEY_EC_DERIVE, EC_R_KEYS_NOT_SET); return 0; } + eckey = dctx->co_key ? dctx->co_key : ctx->pkey->pkey.ec; + if (!key) { const EC_GROUP *group; - group = EC_KEY_get0_group(ctx->pkey->pkey.ec); + group = EC_KEY_get0_group(eckey); *keylen = (EC_GROUP_get_degree(group) + 7) / 8; return 1; } - pubkey = EC_KEY_get0_public_key(ctx->peerkey->pkey.ec); /* @@ -189,12 +234,48 @@ static int pkey_ec_derive(EVP_PKEY_CTX *ctx, unsigned char *key, outlen = *keylen; - ret = ECDH_compute_key(key, outlen, pubkey, ctx->pkey->pkey.ec, 0); - if (ret < 0) - return ret; + ret = ECDH_compute_key(key, outlen, pubkey, eckey, 0); + if (ret <= 0) + return 0; *keylen = ret; return 1; } + +static int pkey_ec_kdf_derive(EVP_PKEY_CTX *ctx, + unsigned char *key, size_t *keylen) +{ + EC_PKEY_CTX *dctx = ctx->data; + unsigned char *ktmp = NULL; + size_t ktmplen; + int rv = 0; + if (dctx->kdf_type == EVP_PKEY_ECDH_KDF_NONE) + return pkey_ec_derive(ctx, key, keylen); + if (!key) { + *keylen = dctx->kdf_outlen; + return 1; + } + if (*keylen != dctx->kdf_outlen) + return 0; + if (!pkey_ec_derive(ctx, NULL, &ktmplen)) + return 0; + ktmp = OPENSSL_malloc(ktmplen); + if (!ktmp) + return 0; + if (!pkey_ec_derive(ctx, ktmp, &ktmplen)) + goto err; + /* Do KDF stuff */ + if (!ECDH_KDF_X9_62(key, *keylen, ktmp, ktmplen, + dctx->kdf_ukm, dctx->kdf_ukmlen, dctx->kdf_md)) + goto err; + rv = 1; + + err: + if (ktmp) { + OPENSSL_cleanse(ktmp, ktmplen); + OPENSSL_free(ktmp); + } + return rv; +} #endif static int pkey_ec_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) @@ -213,6 +294,90 @@ static int pkey_ec_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) dctx->gen_group = group; return 1; + case EVP_PKEY_CTRL_EC_PARAM_ENC: + if (!dctx->gen_group) { + ECerr(EC_F_PKEY_EC_CTRL, EC_R_NO_PARAMETERS_SET); + return 0; + } + EC_GROUP_set_asn1_flag(dctx->gen_group, p1); + return 1; + +#ifndef OPENSSL_NO_ECDH + case EVP_PKEY_CTRL_EC_ECDH_COFACTOR: + if (p1 == -2) { + if (dctx->cofactor_mode != -1) + return dctx->cofactor_mode; + else { + EC_KEY *ec_key = ctx->pkey->pkey.ec; + return EC_KEY_get_flags(ec_key) & EC_FLAG_COFACTOR_ECDH ? 1 : + 0; + } + } else if (p1 < -1 || p1 > 1) + return -2; + dctx->cofactor_mode = p1; + if (p1 != -1) { + EC_KEY *ec_key = ctx->pkey->pkey.ec; + if (!ec_key->group) + return -2; + /* If cofactor is 1 cofactor mode does nothing */ + if (BN_is_one(&ec_key->group->cofactor)) + return 1; + if (!dctx->co_key) { + dctx->co_key = EC_KEY_dup(ec_key); + if (!dctx->co_key) + return 0; + } + if (p1) + EC_KEY_set_flags(dctx->co_key, EC_FLAG_COFACTOR_ECDH); + else + EC_KEY_clear_flags(dctx->co_key, EC_FLAG_COFACTOR_ECDH); + } else if (dctx->co_key) { + EC_KEY_free(dctx->co_key); + dctx->co_key = NULL; + } + return 1; +#endif + + case EVP_PKEY_CTRL_EC_KDF_TYPE: + if (p1 == -2) + return dctx->kdf_type; + if (p1 != EVP_PKEY_ECDH_KDF_NONE && p1 != EVP_PKEY_ECDH_KDF_X9_62) + return -2; + dctx->kdf_type = p1; + return 1; + + case EVP_PKEY_CTRL_EC_KDF_MD: + dctx->kdf_md = p2; + return 1; + + case EVP_PKEY_CTRL_GET_EC_KDF_MD: + *(const EVP_MD **)p2 = dctx->kdf_md; + return 1; + + case EVP_PKEY_CTRL_EC_KDF_OUTLEN: + if (p1 <= 0) + return -2; + dctx->kdf_outlen = (size_t)p1; + return 1; + + case EVP_PKEY_CTRL_GET_EC_KDF_OUTLEN: + *(int *)p2 = dctx->kdf_outlen; + return 1; + + case EVP_PKEY_CTRL_EC_KDF_UKM: + if (dctx->kdf_ukm) + OPENSSL_free(dctx->kdf_ukm); + dctx->kdf_ukm = p2; + if (p2) + dctx->kdf_ukmlen = p1; + else + dctx->kdf_ukmlen = 0; + return 1; + + case EVP_PKEY_CTRL_GET_EC_KDF_UKM: + *(unsigned char **)p2 = dctx->kdf_ukm; + return dctx->kdf_ukmlen; + case EVP_PKEY_CTRL_MD: if (EVP_MD_type((const EVP_MD *)p2) != NID_sha1 && EVP_MD_type((const EVP_MD *)p2) != NID_ecdsa_with_SHA1 && @@ -226,6 +391,10 @@ static int pkey_ec_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) dctx->md = p2; return 1; + case EVP_PKEY_CTRL_GET_MD: + *(const EVP_MD **)p2 = dctx->md; + return 1; + case EVP_PKEY_CTRL_PEER_KEY: /* Default behaviour is OK */ case EVP_PKEY_CTRL_DIGESTINIT: @@ -244,7 +413,9 @@ static int pkey_ec_ctrl_str(EVP_PKEY_CTX *ctx, { if (!strcmp(type, "ec_paramgen_curve")) { int nid; - nid = OBJ_sn2nid(value); + nid = EC_curve_nist2nid(value); + if (nid == NID_undef) + nid = OBJ_sn2nid(value); if (nid == NID_undef) nid = OBJ_ln2nid(value); if (nid == NID_undef) { @@ -252,7 +423,28 @@ static int pkey_ec_ctrl_str(EVP_PKEY_CTX *ctx, return 0; } return EVP_PKEY_CTX_set_ec_paramgen_curve_nid(ctx, nid); + } else if (!strcmp(type, "ec_param_enc")) { + int param_enc; + if (!strcmp(value, "explicit")) + param_enc = 0; + else if (!strcmp(value, "named_curve")) + param_enc = OPENSSL_EC_NAMED_CURVE; + else + return -2; + return EVP_PKEY_CTX_set_ec_param_enc(ctx, param_enc); + } else if (!strcmp(type, "ecdh_kdf_md")) { + const EVP_MD *md; + if (!(md = EVP_get_digestbyname(value))) { + ECerr(EC_F_PKEY_EC_CTRL_STR, EC_R_INVALID_DIGEST); + return 0; + } + return EVP_PKEY_CTX_set_ecdh_kdf_md(ctx, md); + } else if (!strcmp(type, "ecdh_cofactor_mode")) { + int co_mode; + co_mode = atoi(value); + return EVP_PKEY_CTX_set_ecdh_cofactor_mode(ctx, co_mode); } + return -2; } @@ -279,7 +471,8 @@ static int pkey_ec_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) static int pkey_ec_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) { EC_KEY *ec = NULL; - if (ctx->pkey == NULL) { + EC_PKEY_CTX *dctx = ctx->data; + if (ctx->pkey == NULL && dctx->gen_group == NULL) { ECerr(EC_F_PKEY_EC_KEYGEN, EC_R_NO_PARAMETERS_SET); return 0; } @@ -287,9 +480,14 @@ static int pkey_ec_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) if (!ec) return 0; EVP_PKEY_assign_EC_KEY(pkey, ec); - /* Note: if error return, pkey is freed by parent routine */ - if (!EVP_PKEY_copy_parameters(pkey, ctx->pkey)) - return 0; + if (ctx->pkey) { + /* Note: if error return, pkey is freed by parent routine */ + if (!EVP_PKEY_copy_parameters(pkey, ctx->pkey)) + return 0; + } else { + if (!EC_KEY_set_group(ec, dctx->gen_group)) + return 0; + } return EC_KEY_generate_key(pkey->pkey.ec); } @@ -322,7 +520,7 @@ const EVP_PKEY_METHOD ec_pkey_meth = { 0, #ifndef OPENSSL_NO_ECDH - pkey_ec_derive, + pkey_ec_kdf_derive, #else 0, #endif diff --git a/crypto/ec/eck_prn.c b/crypto/ec/eck_prn.c index 5ef12ec02444..df9b37a750d6 100644 --- a/crypto/ec/eck_prn.c +++ b/crypto/ec/eck_prn.c @@ -171,6 +171,7 @@ int ECPKParameters_print(BIO *bp, const EC_GROUP *x, int off) if (EC_GROUP_get_asn1_flag(x)) { /* the curve parameter are given by an asn1 OID */ int nid; + const char *nname; if (!BIO_indent(bp, off, 128)) goto err; @@ -183,6 +184,13 @@ int ECPKParameters_print(BIO *bp, const EC_GROUP *x, int off) goto err; if (BIO_printf(bp, "\n") <= 0) goto err; + nname = EC_curve_nid2nist(nid); + if (nname) { + if (!BIO_indent(bp, off, 128)) + goto err; + if (BIO_printf(bp, "NIST CURVE: %s\n", nname) <= 0) + goto err; + } } else { /* explicit parameters */ int is_char_two = 0; diff --git a/crypto/ec/ecp_nistp521.c b/crypto/ec/ecp_nistp521.c index cc7634512d44..360b9a3516f6 100644 --- a/crypto/ec/ecp_nistp521.c +++ b/crypto/ec/ecp_nistp521.c @@ -1282,11 +1282,11 @@ static void point_add(felem x3, felem y3, felem z3, felem_scalar128(tmp2, 2); /* tmp2[i] < 17*2^121 */ felem_diff128(tmp, tmp2); - /*- - * tmp[i] < 2^127 - 2^69 + 17*2^122 - * = 2^126 - 2^122 - 2^6 - 2^2 - 1 - * < 2^127 - */ + /*- + * tmp[i] < 2^127 - 2^69 + 17*2^122 + * = 2^126 - 2^122 - 2^6 - 2^2 - 1 + * < 2^127 + */ felem_reduce(y_out, tmp); copy_conditional(x_out, x2, z1_is_zero); diff --git a/crypto/ec/ecp_nistz256.c b/crypto/ec/ecp_nistz256.c new file mode 100644 index 000000000000..ca44d0aaeec4 --- /dev/null +++ b/crypto/ec/ecp_nistz256.c @@ -0,0 +1,1521 @@ +/****************************************************************************** + * * + * Copyright 2014 Intel Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + * * + ****************************************************************************** + * * + * Developers and authors: * + * Shay Gueron (1, 2), and Vlad Krasnov (1) * + * (1) Intel Corporation, Israel Development Center * + * (2) University of Haifa * + * Reference: * + * S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with * + * 256 Bit Primes" * + * * + ******************************************************************************/ + +#include + +#include +#include +#include +#include "cryptlib.h" + +#include "ec_lcl.h" + +#if BN_BITS2 != 64 +# define TOBN(hi,lo) lo,hi +#else +# define TOBN(hi,lo) ((BN_ULONG)hi<<32|lo) +#endif + +#if defined(__GNUC__) +# define ALIGN32 __attribute((aligned(32))) +#elif defined(_MSC_VER) +# define ALIGN32 __declspec(align(32)) +#else +# define ALIGN32 +#endif + +#define ALIGNPTR(p,N) ((unsigned char *)p+N-(size_t)p%N) +#define P256_LIMBS (256/BN_BITS2) + +typedef unsigned short u16; + +typedef struct { + BN_ULONG X[P256_LIMBS]; + BN_ULONG Y[P256_LIMBS]; + BN_ULONG Z[P256_LIMBS]; +} P256_POINT; + +typedef struct { + BN_ULONG X[P256_LIMBS]; + BN_ULONG Y[P256_LIMBS]; +} P256_POINT_AFFINE; + +typedef P256_POINT_AFFINE PRECOMP256_ROW[64]; + +/* structure for precomputed multiples of the generator */ +typedef struct ec_pre_comp_st { + const EC_GROUP *group; /* Parent EC_GROUP object */ + size_t w; /* Window size */ + /* + * Constant time access to the X and Y coordinates of the pre-computed, + * generator multiplies, in the Montgomery domain. Pre-calculated + * multiplies are stored in affine form. + */ + PRECOMP256_ROW *precomp; + void *precomp_storage; + int references; +} EC_PRE_COMP; + +/* Functions implemented in assembly */ +/* Modular mul by 2: res = 2*a mod P */ +void ecp_nistz256_mul_by_2(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +/* Modular div by 2: res = a/2 mod P */ +void ecp_nistz256_div_by_2(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +/* Modular mul by 3: res = 3*a mod P */ +void ecp_nistz256_mul_by_3(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +/* Modular add: res = a+b mod P */ +void ecp_nistz256_add(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +/* Modular sub: res = a-b mod P */ +void ecp_nistz256_sub(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +/* Modular neg: res = -a mod P */ +void ecp_nistz256_neg(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); +/* Montgomery mul: res = a*b*2^-256 mod P */ +void ecp_nistz256_mul_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +/* Montgomery sqr: res = a*a*2^-256 mod P */ +void ecp_nistz256_sqr_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +/* Convert a number from Montgomery domain, by multiplying with 1 */ +void ecp_nistz256_from_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG in[P256_LIMBS]); +/* Convert a number to Montgomery domain, by multiplying with 2^512 mod P*/ +void ecp_nistz256_to_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG in[P256_LIMBS]); +/* Functions that perform constant time access to the precomputed tables */ +void ecp_nistz256_select_w5(P256_POINT * val, + const P256_POINT * in_t, int index); +void ecp_nistz256_select_w7(P256_POINT_AFFINE * val, + const P256_POINT_AFFINE * in_t, int index); + +/* One converted into the Montgomery domain */ +static const BN_ULONG ONE[P256_LIMBS] = { + TOBN(0x00000000, 0x00000001), TOBN(0xffffffff, 0x00000000), + TOBN(0xffffffff, 0xffffffff), TOBN(0x00000000, 0xfffffffe) +}; + +static void *ecp_nistz256_pre_comp_dup(void *); +static void ecp_nistz256_pre_comp_free(void *); +static void ecp_nistz256_pre_comp_clear_free(void *); +static EC_PRE_COMP *ecp_nistz256_pre_comp_new(const EC_GROUP *group); + +/* Precomputed tables for the default generator */ +#include "ecp_nistz256_table.c" + +/* Recode window to a signed digit, see ecp_nistputil.c for details */ +static unsigned int _booth_recode_w5(unsigned int in) +{ + unsigned int s, d; + + s = ~((in >> 5) - 1); + d = (1 << 6) - in - 1; + d = (d & s) | (in & ~s); + d = (d >> 1) + (d & 1); + + return (d << 1) + (s & 1); +} + +static unsigned int _booth_recode_w7(unsigned int in) +{ + unsigned int s, d; + + s = ~((in >> 7) - 1); + d = (1 << 8) - in - 1; + d = (d & s) | (in & ~s); + d = (d >> 1) + (d & 1); + + return (d << 1) + (s & 1); +} + +static void copy_conditional(BN_ULONG dst[P256_LIMBS], + const BN_ULONG src[P256_LIMBS], BN_ULONG move) +{ + BN_ULONG mask1 = -move; + BN_ULONG mask2 = ~mask1; + + dst[0] = (src[0] & mask1) ^ (dst[0] & mask2); + dst[1] = (src[1] & mask1) ^ (dst[1] & mask2); + dst[2] = (src[2] & mask1) ^ (dst[2] & mask2); + dst[3] = (src[3] & mask1) ^ (dst[3] & mask2); + if (P256_LIMBS == 8) { + dst[4] = (src[4] & mask1) ^ (dst[4] & mask2); + dst[5] = (src[5] & mask1) ^ (dst[5] & mask2); + dst[6] = (src[6] & mask1) ^ (dst[6] & mask2); + dst[7] = (src[7] & mask1) ^ (dst[7] & mask2); + } +} + +static BN_ULONG is_zero(BN_ULONG in) +{ + in |= (0 - in); + in = ~in; + in &= BN_MASK2; + in >>= BN_BITS2 - 1; + return in; +} + +static BN_ULONG is_equal(const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]) +{ + BN_ULONG res; + + res = a[0] ^ b[0]; + res |= a[1] ^ b[1]; + res |= a[2] ^ b[2]; + res |= a[3] ^ b[3]; + if (P256_LIMBS == 8) { + res |= a[4] ^ b[4]; + res |= a[5] ^ b[5]; + res |= a[6] ^ b[6]; + res |= a[7] ^ b[7]; + } + + return is_zero(res); +} + +static BN_ULONG is_one(const BN_ULONG a[P256_LIMBS]) +{ + BN_ULONG res; + + res = a[0] ^ ONE[0]; + res |= a[1] ^ ONE[1]; + res |= a[2] ^ ONE[2]; + res |= a[3] ^ ONE[3]; + if (P256_LIMBS == 8) { + res |= a[4] ^ ONE[4]; + res |= a[5] ^ ONE[5]; + res |= a[6] ^ ONE[6]; + } + + return is_zero(res); +} + +static int ecp_nistz256_set_words(BIGNUM *a, BN_ULONG words[P256_LIMBS]) + { + if (bn_wexpand(a, P256_LIMBS) == NULL) { + ECerr(EC_F_ECP_NISTZ256_SET_WORDS, ERR_R_MALLOC_FAILURE); + return 0; + } + memcpy(a->d, words, sizeof(BN_ULONG) * P256_LIMBS); + a->top = P256_LIMBS; + bn_correct_top(a); + return 1; +} + +#ifndef ECP_NISTZ256_REFERENCE_IMPLEMENTATION +void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a); +void ecp_nistz256_point_add(P256_POINT *r, + const P256_POINT *a, const P256_POINT *b); +void ecp_nistz256_point_add_affine(P256_POINT *r, + const P256_POINT *a, + const P256_POINT_AFFINE *b); +#else +/* Point double: r = 2*a */ +static void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a) +{ + BN_ULONG S[P256_LIMBS]; + BN_ULONG M[P256_LIMBS]; + BN_ULONG Zsqr[P256_LIMBS]; + BN_ULONG tmp0[P256_LIMBS]; + + const BN_ULONG *in_x = a->X; + const BN_ULONG *in_y = a->Y; + const BN_ULONG *in_z = a->Z; + + BN_ULONG *res_x = r->X; + BN_ULONG *res_y = r->Y; + BN_ULONG *res_z = r->Z; + + ecp_nistz256_mul_by_2(S, in_y); + + ecp_nistz256_sqr_mont(Zsqr, in_z); + + ecp_nistz256_sqr_mont(S, S); + + ecp_nistz256_mul_mont(res_z, in_z, in_y); + ecp_nistz256_mul_by_2(res_z, res_z); + + ecp_nistz256_add(M, in_x, Zsqr); + ecp_nistz256_sub(Zsqr, in_x, Zsqr); + + ecp_nistz256_sqr_mont(res_y, S); + ecp_nistz256_div_by_2(res_y, res_y); + + ecp_nistz256_mul_mont(M, M, Zsqr); + ecp_nistz256_mul_by_3(M, M); + + ecp_nistz256_mul_mont(S, S, in_x); + ecp_nistz256_mul_by_2(tmp0, S); + + ecp_nistz256_sqr_mont(res_x, M); + + ecp_nistz256_sub(res_x, res_x, tmp0); + ecp_nistz256_sub(S, S, res_x); + + ecp_nistz256_mul_mont(S, S, M); + ecp_nistz256_sub(res_y, S, res_y); +} + +/* Point addition: r = a+b */ +static void ecp_nistz256_point_add(P256_POINT *r, + const P256_POINT *a, const P256_POINT *b) +{ + BN_ULONG U2[P256_LIMBS], S2[P256_LIMBS]; + BN_ULONG U1[P256_LIMBS], S1[P256_LIMBS]; + BN_ULONG Z1sqr[P256_LIMBS]; + BN_ULONG Z2sqr[P256_LIMBS]; + BN_ULONG H[P256_LIMBS], R[P256_LIMBS]; + BN_ULONG Hsqr[P256_LIMBS]; + BN_ULONG Rsqr[P256_LIMBS]; + BN_ULONG Hcub[P256_LIMBS]; + + BN_ULONG res_x[P256_LIMBS]; + BN_ULONG res_y[P256_LIMBS]; + BN_ULONG res_z[P256_LIMBS]; + + BN_ULONG in1infty, in2infty; + + const BN_ULONG *in1_x = a->X; + const BN_ULONG *in1_y = a->Y; + const BN_ULONG *in1_z = a->Z; + + const BN_ULONG *in2_x = b->X; + const BN_ULONG *in2_y = b->Y; + const BN_ULONG *in2_z = b->Z; + + /* We encode infinity as (0,0), which is not on the curve, + * so it is OK. */ + in1infty = (in1_x[0] | in1_x[1] | in1_x[2] | in1_x[3] | + in1_y[0] | in1_y[1] | in1_y[2] | in1_y[3]); + if (P256_LIMBS == 8) + in1infty |= (in1_x[4] | in1_x[5] | in1_x[6] | in1_x[7] | + in1_y[4] | in1_y[5] | in1_y[6] | in1_y[7]); + + in2infty = (in2_x[0] | in2_x[1] | in2_x[2] | in2_x[3] | + in2_y[0] | in2_y[1] | in2_y[2] | in2_y[3]); + if (P256_LIMBS == 8) + in2infty |= (in2_x[4] | in2_x[5] | in2_x[6] | in2_x[7] | + in2_y[4] | in2_y[5] | in2_y[6] | in2_y[7]); + + in1infty = is_zero(in1infty); + in2infty = is_zero(in2infty); + + ecp_nistz256_sqr_mont(Z2sqr, in2_z); /* Z2^2 */ + ecp_nistz256_sqr_mont(Z1sqr, in1_z); /* Z1^2 */ + + ecp_nistz256_mul_mont(S1, Z2sqr, in2_z); /* S1 = Z2^3 */ + ecp_nistz256_mul_mont(S2, Z1sqr, in1_z); /* S2 = Z1^3 */ + + ecp_nistz256_mul_mont(S1, S1, in1_y); /* S1 = Y1*Z2^3 */ + ecp_nistz256_mul_mont(S2, S2, in2_y); /* S2 = Y2*Z1^3 */ + ecp_nistz256_sub(R, S2, S1); /* R = S2 - S1 */ + + ecp_nistz256_mul_mont(U1, in1_x, Z2sqr); /* U1 = X1*Z2^2 */ + ecp_nistz256_mul_mont(U2, in2_x, Z1sqr); /* U2 = X2*Z1^2 */ + ecp_nistz256_sub(H, U2, U1); /* H = U2 - U1 */ + + /* + * This should not happen during sign/ecdh, so no constant time violation + */ + if (is_equal(U1, U2) && !in1infty && !in2infty) { + if (is_equal(S1, S2)) { + ecp_nistz256_point_double(r, a); + return; + } else { + memset(r, 0, sizeof(*r)); + return; + } + } + + ecp_nistz256_sqr_mont(Rsqr, R); /* R^2 */ + ecp_nistz256_mul_mont(res_z, H, in1_z); /* Z3 = H*Z1*Z2 */ + ecp_nistz256_sqr_mont(Hsqr, H); /* H^2 */ + ecp_nistz256_mul_mont(res_z, res_z, in2_z); /* Z3 = H*Z1*Z2 */ + ecp_nistz256_mul_mont(Hcub, Hsqr, H); /* H^3 */ + + ecp_nistz256_mul_mont(U2, U1, Hsqr); /* U1*H^2 */ + ecp_nistz256_mul_by_2(Hsqr, U2); /* 2*U1*H^2 */ + + ecp_nistz256_sub(res_x, Rsqr, Hsqr); + ecp_nistz256_sub(res_x, res_x, Hcub); + + ecp_nistz256_sub(res_y, U2, res_x); + + ecp_nistz256_mul_mont(S2, S1, Hcub); + ecp_nistz256_mul_mont(res_y, R, res_y); + ecp_nistz256_sub(res_y, res_y, S2); + + copy_conditional(res_x, in2_x, in1infty); + copy_conditional(res_y, in2_y, in1infty); + copy_conditional(res_z, in2_z, in1infty); + + copy_conditional(res_x, in1_x, in2infty); + copy_conditional(res_y, in1_y, in2infty); + copy_conditional(res_z, in1_z, in2infty); + + memcpy(r->X, res_x, sizeof(res_x)); + memcpy(r->Y, res_y, sizeof(res_y)); + memcpy(r->Z, res_z, sizeof(res_z)); +} + +/* Point addition when b is known to be affine: r = a+b */ +static void ecp_nistz256_point_add_affine(P256_POINT *r, + const P256_POINT *a, + const P256_POINT_AFFINE *b) +{ + BN_ULONG U2[P256_LIMBS], S2[P256_LIMBS]; + BN_ULONG Z1sqr[P256_LIMBS]; + BN_ULONG H[P256_LIMBS], R[P256_LIMBS]; + BN_ULONG Hsqr[P256_LIMBS]; + BN_ULONG Rsqr[P256_LIMBS]; + BN_ULONG Hcub[P256_LIMBS]; + + BN_ULONG res_x[P256_LIMBS]; + BN_ULONG res_y[P256_LIMBS]; + BN_ULONG res_z[P256_LIMBS]; + + BN_ULONG in1infty, in2infty; + + const BN_ULONG *in1_x = a->X; + const BN_ULONG *in1_y = a->Y; + const BN_ULONG *in1_z = a->Z; + + const BN_ULONG *in2_x = b->X; + const BN_ULONG *in2_y = b->Y; + + /* + * In affine representation we encode infty as (0,0), which is not on the + * curve, so it is OK + */ + in1infty = (in1_x[0] | in1_x[1] | in1_x[2] | in1_x[3] | + in1_y[0] | in1_y[1] | in1_y[2] | in1_y[3]); + if (P256_LIMBS == 8) + in1infty |= (in1_x[4] | in1_x[5] | in1_x[6] | in1_x[7] | + in1_y[4] | in1_y[5] | in1_y[6] | in1_y[7]); + + in2infty = (in2_x[0] | in2_x[1] | in2_x[2] | in2_x[3] | + in2_y[0] | in2_y[1] | in2_y[2] | in2_y[3]); + if (P256_LIMBS == 8) + in2infty |= (in2_x[4] | in2_x[5] | in2_x[6] | in2_x[7] | + in2_y[4] | in2_y[5] | in2_y[6] | in2_y[7]); + + in1infty = is_zero(in1infty); + in2infty = is_zero(in2infty); + + ecp_nistz256_sqr_mont(Z1sqr, in1_z); /* Z1^2 */ + + ecp_nistz256_mul_mont(U2, in2_x, Z1sqr); /* U2 = X2*Z1^2 */ + ecp_nistz256_sub(H, U2, in1_x); /* H = U2 - U1 */ + + ecp_nistz256_mul_mont(S2, Z1sqr, in1_z); /* S2 = Z1^3 */ + + ecp_nistz256_mul_mont(res_z, H, in1_z); /* Z3 = H*Z1*Z2 */ + + ecp_nistz256_mul_mont(S2, S2, in2_y); /* S2 = Y2*Z1^3 */ + ecp_nistz256_sub(R, S2, in1_y); /* R = S2 - S1 */ + + ecp_nistz256_sqr_mont(Hsqr, H); /* H^2 */ + ecp_nistz256_sqr_mont(Rsqr, R); /* R^2 */ + ecp_nistz256_mul_mont(Hcub, Hsqr, H); /* H^3 */ + + ecp_nistz256_mul_mont(U2, in1_x, Hsqr); /* U1*H^2 */ + ecp_nistz256_mul_by_2(Hsqr, U2); /* 2*U1*H^2 */ + + ecp_nistz256_sub(res_x, Rsqr, Hsqr); + ecp_nistz256_sub(res_x, res_x, Hcub); + ecp_nistz256_sub(H, U2, res_x); + + ecp_nistz256_mul_mont(S2, in1_y, Hcub); + ecp_nistz256_mul_mont(H, H, R); + ecp_nistz256_sub(res_y, H, S2); + + copy_conditional(res_x, in2_x, in1infty); + copy_conditional(res_x, in1_x, in2infty); + + copy_conditional(res_y, in2_y, in1infty); + copy_conditional(res_y, in1_y, in2infty); + + copy_conditional(res_z, ONE, in1infty); + copy_conditional(res_z, in1_z, in2infty); + + memcpy(r->X, res_x, sizeof(res_x)); + memcpy(r->Y, res_y, sizeof(res_y)); + memcpy(r->Z, res_z, sizeof(res_z)); +} +#endif + +/* r = in^-1 mod p */ +static void ecp_nistz256_mod_inverse(BN_ULONG r[P256_LIMBS], + const BN_ULONG in[P256_LIMBS]) +{ + /* + * The poly is ffffffff 00000001 00000000 00000000 00000000 ffffffff + * ffffffff ffffffff We use FLT and used poly-2 as exponent + */ + BN_ULONG p2[P256_LIMBS]; + BN_ULONG p4[P256_LIMBS]; + BN_ULONG p8[P256_LIMBS]; + BN_ULONG p16[P256_LIMBS]; + BN_ULONG p32[P256_LIMBS]; + BN_ULONG res[P256_LIMBS]; + int i; + + ecp_nistz256_sqr_mont(res, in); + ecp_nistz256_mul_mont(p2, res, in); /* 3*p */ + + ecp_nistz256_sqr_mont(res, p2); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(p4, res, p2); /* f*p */ + + ecp_nistz256_sqr_mont(res, p4); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(p8, res, p4); /* ff*p */ + + ecp_nistz256_sqr_mont(res, p8); + for (i = 0; i < 7; i++) + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(p16, res, p8); /* ffff*p */ + + ecp_nistz256_sqr_mont(res, p16); + for (i = 0; i < 15; i++) + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(p32, res, p16); /* ffffffff*p */ + + ecp_nistz256_sqr_mont(res, p32); + for (i = 0; i < 31; i++) + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, in); + + for (i = 0; i < 32 * 4; i++) + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, p32); + + for (i = 0; i < 32; i++) + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, p32); + + for (i = 0; i < 16; i++) + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, p16); + + for (i = 0; i < 8; i++) + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, p8); + + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, p4); + + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, p2); + + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_sqr_mont(res, res); + ecp_nistz256_mul_mont(res, res, in); + + memcpy(r, res, sizeof(res)); +} + +/* + * ecp_nistz256_bignum_to_field_elem copies the contents of |in| to |out| and + * returns one if it fits. Otherwise it returns zero. + */ +static int ecp_nistz256_bignum_to_field_elem(BN_ULONG out[P256_LIMBS], + const BIGNUM *in) +{ + if (in->top > P256_LIMBS) + return 0; + + memset(out, 0, sizeof(BN_ULONG) * P256_LIMBS); + memcpy(out, in->d, sizeof(BN_ULONG) * in->top); + return 1; +} + +/* r = sum(scalar[i]*point[i]) */ +static int ecp_nistz256_windowed_mul(const EC_GROUP *group, + P256_POINT *r, + const BIGNUM **scalar, + const EC_POINT **point, + int num, BN_CTX *ctx) +{ + + int i, j, ret = 0; + unsigned int index; + unsigned char (*p_str)[33] = NULL; + const unsigned int window_size = 5; + const unsigned int mask = (1 << (window_size + 1)) - 1; + unsigned int wvalue; + BN_ULONG tmp[P256_LIMBS]; + ALIGN32 P256_POINT h; + const BIGNUM **scalars = NULL; + P256_POINT (*table)[16] = NULL; + void *table_storage = NULL; + + if ((table_storage = + OPENSSL_malloc(num * 16 * sizeof(P256_POINT) + 64)) == NULL + || (p_str = + OPENSSL_malloc(num * 33 * sizeof(unsigned char))) == NULL + || (scalars = OPENSSL_malloc(num * sizeof(BIGNUM *))) == NULL) { + ECerr(EC_F_ECP_NISTZ256_WINDOWED_MUL, ERR_R_MALLOC_FAILURE); + goto err; + } else { + table = (void *)ALIGNPTR(table_storage, 64); + } + + for (i = 0; i < num; i++) { + P256_POINT *row = table[i]; + + /* This is an unusual input, we don't guarantee constant-timeness. */ + if ((BN_num_bits(scalar[i]) > 256) || BN_is_negative(scalar[i])) { + BIGNUM *mod; + + if ((mod = BN_CTX_get(ctx)) == NULL) + goto err; + if (!BN_nnmod(mod, scalar[i], &group->order, ctx)) { + ECerr(EC_F_ECP_NISTZ256_WINDOWED_MUL, ERR_R_BN_LIB); + goto err; + } + scalars[i] = mod; + } else + scalars[i] = scalar[i]; + + for (j = 0; j < scalars[i]->top * BN_BYTES; j += BN_BYTES) { + BN_ULONG d = scalars[i]->d[j / BN_BYTES]; + + p_str[i][j + 0] = d & 0xff; + p_str[i][j + 1] = (d >> 8) & 0xff; + p_str[i][j + 2] = (d >> 16) & 0xff; + p_str[i][j + 3] = (d >>= 24) & 0xff; + if (BN_BYTES == 8) { + d >>= 8; + p_str[i][j + 4] = d & 0xff; + p_str[i][j + 5] = (d >> 8) & 0xff; + p_str[i][j + 6] = (d >> 16) & 0xff; + p_str[i][j + 7] = (d >> 24) & 0xff; + } + } + for (; j < 33; j++) + p_str[i][j] = 0; + + /* table[0] is implicitly (0,0,0) (the point at infinity), + * therefore it is not stored. All other values are actually + * stored with an offset of -1 in table. + */ + + if (!ecp_nistz256_bignum_to_field_elem(row[1 - 1].X, &point[i]->X) + || !ecp_nistz256_bignum_to_field_elem(row[1 - 1].Y, &point[i]->Y) + || !ecp_nistz256_bignum_to_field_elem(row[1 - 1].Z, &point[i]->Z)) { + ECerr(EC_F_ECP_NISTZ256_WINDOWED_MUL, EC_R_COORDINATES_OUT_OF_RANGE); + goto err; + } + + ecp_nistz256_point_double(&row[ 2 - 1], &row[ 1 - 1]); + ecp_nistz256_point_add (&row[ 3 - 1], &row[ 2 - 1], &row[1 - 1]); + ecp_nistz256_point_double(&row[ 4 - 1], &row[ 2 - 1]); + ecp_nistz256_point_double(&row[ 6 - 1], &row[ 3 - 1]); + ecp_nistz256_point_double(&row[ 8 - 1], &row[ 4 - 1]); + ecp_nistz256_point_double(&row[12 - 1], &row[ 6 - 1]); + ecp_nistz256_point_add (&row[ 5 - 1], &row[ 4 - 1], &row[1 - 1]); + ecp_nistz256_point_add (&row[ 7 - 1], &row[ 6 - 1], &row[1 - 1]); + ecp_nistz256_point_add (&row[ 9 - 1], &row[ 8 - 1], &row[1 - 1]); + ecp_nistz256_point_add (&row[13 - 1], &row[12 - 1], &row[1 - 1]); + ecp_nistz256_point_double(&row[14 - 1], &row[ 7 - 1]); + ecp_nistz256_point_double(&row[10 - 1], &row[ 5 - 1]); + ecp_nistz256_point_add (&row[15 - 1], &row[14 - 1], &row[1 - 1]); + ecp_nistz256_point_add (&row[11 - 1], &row[10 - 1], &row[1 - 1]); + ecp_nistz256_point_add (&row[16 - 1], &row[15 - 1], &row[1 - 1]); + } + + index = 255; + + wvalue = p_str[0][(index - 1) / 8]; + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + + ecp_nistz256_select_w5(r, table[0], _booth_recode_w5(wvalue) >> 1); + + while (index >= 5) { + for (i = (index == 255 ? 1 : 0); i < num; i++) { + unsigned int off = (index - 1) / 8; + + wvalue = p_str[i][off] | p_str[i][off + 1] << 8; + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + + wvalue = _booth_recode_w5(wvalue); + + ecp_nistz256_select_w5(&h, table[i], wvalue >> 1); + + ecp_nistz256_neg(tmp, h.Y); + copy_conditional(h.Y, tmp, (wvalue & 1)); + + ecp_nistz256_point_add(r, r, &h); + } + + index -= window_size; + + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + } + + /* Final window */ + for (i = 0; i < num; i++) { + wvalue = p_str[i][0]; + wvalue = (wvalue << 1) & mask; + + wvalue = _booth_recode_w5(wvalue); + + ecp_nistz256_select_w5(&h, table[i], wvalue >> 1); + + ecp_nistz256_neg(tmp, h.Y); + copy_conditional(h.Y, tmp, wvalue & 1); + + ecp_nistz256_point_add(r, r, &h); + } + + ret = 1; + err: + if (table_storage) + OPENSSL_free(table_storage); + if (p_str) + OPENSSL_free(p_str); + if (scalars) + OPENSSL_free(scalars); + return ret; +} + +/* Coordinates of G, for which we have precomputed tables */ +const static BN_ULONG def_xG[P256_LIMBS] = { + TOBN(0x79e730d4, 0x18a9143c), TOBN(0x75ba95fc, 0x5fedb601), + TOBN(0x79fb732b, 0x77622510), TOBN(0x18905f76, 0xa53755c6) +}; + +const static BN_ULONG def_yG[P256_LIMBS] = { + TOBN(0xddf25357, 0xce95560a), TOBN(0x8b4ab8e4, 0xba19e45c), + TOBN(0xd2e88688, 0xdd21f325), TOBN(0x8571ff18, 0x25885d85) +}; + +/* + * ecp_nistz256_is_affine_G returns one if |generator| is the standard, P-256 + * generator. + */ +static int ecp_nistz256_is_affine_G(const EC_POINT *generator) +{ + return (generator->X.top == P256_LIMBS) && + (generator->Y.top == P256_LIMBS) && + (generator->Z.top == (P256_LIMBS - P256_LIMBS / 8)) && + is_equal(generator->X.d, def_xG) && + is_equal(generator->Y.d, def_yG) && is_one(generator->Z.d); +} + +static int ecp_nistz256_mult_precompute(EC_GROUP *group, BN_CTX *ctx) +{ + /* + * We precompute a table for a Booth encoded exponent (wNAF) based + * computation. Each table holds 64 values for safe access, with an + * implicit value of infinity at index zero. We use window of size 7, and + * therefore require ceil(256/7) = 37 tables. + */ + BIGNUM *order; + EC_POINT *P = NULL, *T = NULL; + const EC_POINT *generator; + EC_PRE_COMP *pre_comp; + BN_CTX *new_ctx = NULL; + int i, j, k, ret = 0; + size_t w; + + PRECOMP256_ROW *preComputedTable = NULL; + unsigned char *precomp_storage = NULL; + + /* if there is an old EC_PRE_COMP object, throw it away */ + EC_EX_DATA_free_data(&group->extra_data, ecp_nistz256_pre_comp_dup, + ecp_nistz256_pre_comp_free, + ecp_nistz256_pre_comp_clear_free); + + generator = EC_GROUP_get0_generator(group); + if (generator == NULL) { + ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE, EC_R_UNDEFINED_GENERATOR); + return 0; + } + + if (ecp_nistz256_is_affine_G(generator)) { + /* + * No need to calculate tables for the standard generator because we + * have them statically. + */ + return 1; + } + + if ((pre_comp = ecp_nistz256_pre_comp_new(group)) == NULL) + return 0; + + if (ctx == NULL) { + ctx = new_ctx = BN_CTX_new(); + if (ctx == NULL) + goto err; + } + + BN_CTX_start(ctx); + order = BN_CTX_get(ctx); + + if (order == NULL) + goto err; + + if (!EC_GROUP_get_order(group, order, ctx)) + goto err; + + if (BN_is_zero(order)) { + ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE, EC_R_UNKNOWN_ORDER); + goto err; + } + + w = 7; + + if ((precomp_storage = + OPENSSL_malloc(37 * 64 * sizeof(P256_POINT_AFFINE) + 64)) == NULL) { + ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE, ERR_R_MALLOC_FAILURE); + goto err; + } else { + preComputedTable = (void *)ALIGNPTR(precomp_storage, 64); + } + + P = EC_POINT_new(group); + T = EC_POINT_new(group); + if (P == NULL || T == NULL) + goto err; + + /* + * The zero entry is implicitly infinity, and we skip it, storing other + * values with -1 offset. + */ + if (!EC_POINT_copy(T, generator)) + goto err; + + for (k = 0; k < 64; k++) { + if (!EC_POINT_copy(P, T)) + goto err; + for (j = 0; j < 37; j++) { + /* + * It would be faster to use EC_POINTs_make_affine and + * make multiple points affine at the same time. + */ + if (!EC_POINT_make_affine(group, P, ctx)) + goto err; + if (!ecp_nistz256_bignum_to_field_elem(preComputedTable[j][k].X, + &P->X) || + !ecp_nistz256_bignum_to_field_elem(preComputedTable[j][k].Y, + &P->Y)) { + ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE, + EC_R_COORDINATES_OUT_OF_RANGE); + goto err; + } + for (i = 0; i < 7; i++) { + if (!EC_POINT_dbl(group, P, P, ctx)) + goto err; + } + } + if (!EC_POINT_add(group, T, T, generator, ctx)) + goto err; + } + + pre_comp->group = group; + pre_comp->w = w; + pre_comp->precomp = preComputedTable; + pre_comp->precomp_storage = precomp_storage; + + precomp_storage = NULL; + + if (!EC_EX_DATA_set_data(&group->extra_data, pre_comp, + ecp_nistz256_pre_comp_dup, + ecp_nistz256_pre_comp_free, + ecp_nistz256_pre_comp_clear_free)) { + goto err; + } + + pre_comp = NULL; + + ret = 1; + + err: + if (ctx != NULL) + BN_CTX_end(ctx); + BN_CTX_free(new_ctx); + + if (pre_comp) + ecp_nistz256_pre_comp_free(pre_comp); + if (precomp_storage) + OPENSSL_free(precomp_storage); + if (P) + EC_POINT_free(P); + if (T) + EC_POINT_free(T); + return ret; +} + +/* + * Note that by default ECP_NISTZ256_AVX2 is undefined. While it's great + * code processing 4 points in parallel, corresponding serial operation + * is several times slower, because it uses 29x29=58-bit multiplication + * as opposite to 64x64=128-bit in integer-only scalar case. As result + * it doesn't provide *significant* performance improvement. Note that + * just defining ECP_NISTZ256_AVX2 is not sufficient to make it work, + * you'd need to compile even asm/ecp_nistz256-avx.pl module. + */ +#if defined(ECP_NISTZ256_AVX2) +# if !(defined(__x86_64) || defined(__x86_64__)) || \ + defined(_M_AMD64) || defined(_MX64)) || \ + !(defined(__GNUC__) || defined(_MSC_VER)) /* this is for ALIGN32 */ +# undef ECP_NISTZ256_AVX2 +# else +/* Constant time access, loading four values, from four consecutive tables */ +void ecp_nistz256_avx2_select_w7(P256_POINT_AFFINE * val, + const P256_POINT_AFFINE * in_t, int index); +void ecp_nistz256_avx2_multi_select_w7(void *result, const void *in, int index0, + int index1, int index2, int index3); +void ecp_nistz256_avx2_transpose_convert(void *RESULTx4, const void *in); +void ecp_nistz256_avx2_convert_transpose_back(void *result, const void *Ax4); +void ecp_nistz256_avx2_point_add_affine_x4(void *RESULTx4, const void *Ax4, + const void *Bx4); +void ecp_nistz256_avx2_point_add_affines_x4(void *RESULTx4, const void *Ax4, + const void *Bx4); +void ecp_nistz256_avx2_to_mont(void *RESULTx4, const void *Ax4); +void ecp_nistz256_avx2_from_mont(void *RESULTx4, const void *Ax4); +void ecp_nistz256_avx2_set1(void *RESULTx4); +int ecp_nistz_avx2_eligible(void); + +static void booth_recode_w7(unsigned char *sign, + unsigned char *digit, unsigned char in) +{ + unsigned char s, d; + + s = ~((in >> 7) - 1); + d = (1 << 8) - in - 1; + d = (d & s) | (in & ~s); + d = (d >> 1) + (d & 1); + + *sign = s & 1; + *digit = d; +} + +/* + * ecp_nistz256_avx2_mul_g performs multiplication by G, using only the + * precomputed table. It does 4 affine point additions in parallel, + * significantly speeding up point multiplication for a fixed value. + */ +static void ecp_nistz256_avx2_mul_g(P256_POINT *r, + unsigned char p_str[33], + const P256_POINT_AFFINE(*preComputedTable)[64]) +{ + const unsigned int window_size = 7; + const unsigned int mask = (1 << (window_size + 1)) - 1; + unsigned int wvalue; + /* Using 4 windows at a time */ + unsigned char sign0, digit0; + unsigned char sign1, digit1; + unsigned char sign2, digit2; + unsigned char sign3, digit3; + unsigned int index = 0; + BN_ULONG tmp[P256_LIMBS]; + int i; + + ALIGN32 BN_ULONG aX4[4 * 9 * 3] = { 0 }; + ALIGN32 BN_ULONG bX4[4 * 9 * 2] = { 0 }; + ALIGN32 P256_POINT_AFFINE point_arr[P256_LIMBS]; + ALIGN32 P256_POINT res_point_arr[P256_LIMBS]; + + /* Initial four windows */ + wvalue = *((u16 *) & p_str[0]); + wvalue = (wvalue << 1) & mask; + index += window_size; + booth_recode_w7(&sign0, &digit0, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign1, &digit1, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign2, &digit2, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign3, &digit3, wvalue); + + ecp_nistz256_avx2_multi_select_w7(point_arr, preComputedTable[0], + digit0, digit1, digit2, digit3); + + ecp_nistz256_neg(tmp, point_arr[0].Y); + copy_conditional(point_arr[0].Y, tmp, sign0); + ecp_nistz256_neg(tmp, point_arr[1].Y); + copy_conditional(point_arr[1].Y, tmp, sign1); + ecp_nistz256_neg(tmp, point_arr[2].Y); + copy_conditional(point_arr[2].Y, tmp, sign2); + ecp_nistz256_neg(tmp, point_arr[3].Y); + copy_conditional(point_arr[3].Y, tmp, sign3); + + ecp_nistz256_avx2_transpose_convert(aX4, point_arr); + ecp_nistz256_avx2_to_mont(aX4, aX4); + ecp_nistz256_avx2_to_mont(&aX4[4 * 9], &aX4[4 * 9]); + ecp_nistz256_avx2_set1(&aX4[4 * 9 * 2]); + + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign0, &digit0, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign1, &digit1, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign2, &digit2, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign3, &digit3, wvalue); + + ecp_nistz256_avx2_multi_select_w7(point_arr, preComputedTable[4 * 1], + digit0, digit1, digit2, digit3); + + ecp_nistz256_neg(tmp, point_arr[0].Y); + copy_conditional(point_arr[0].Y, tmp, sign0); + ecp_nistz256_neg(tmp, point_arr[1].Y); + copy_conditional(point_arr[1].Y, tmp, sign1); + ecp_nistz256_neg(tmp, point_arr[2].Y); + copy_conditional(point_arr[2].Y, tmp, sign2); + ecp_nistz256_neg(tmp, point_arr[3].Y); + copy_conditional(point_arr[3].Y, tmp, sign3); + + ecp_nistz256_avx2_transpose_convert(bX4, point_arr); + ecp_nistz256_avx2_to_mont(bX4, bX4); + ecp_nistz256_avx2_to_mont(&bX4[4 * 9], &bX4[4 * 9]); + /* Optimized when both inputs are affine */ + ecp_nistz256_avx2_point_add_affines_x4(aX4, aX4, bX4); + + for (i = 2; i < 9; i++) { + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign0, &digit0, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign1, &digit1, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign2, &digit2, wvalue); + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + booth_recode_w7(&sign3, &digit3, wvalue); + + ecp_nistz256_avx2_multi_select_w7(point_arr, + preComputedTable[4 * i], + digit0, digit1, digit2, digit3); + + ecp_nistz256_neg(tmp, point_arr[0].Y); + copy_conditional(point_arr[0].Y, tmp, sign0); + ecp_nistz256_neg(tmp, point_arr[1].Y); + copy_conditional(point_arr[1].Y, tmp, sign1); + ecp_nistz256_neg(tmp, point_arr[2].Y); + copy_conditional(point_arr[2].Y, tmp, sign2); + ecp_nistz256_neg(tmp, point_arr[3].Y); + copy_conditional(point_arr[3].Y, tmp, sign3); + + ecp_nistz256_avx2_transpose_convert(bX4, point_arr); + ecp_nistz256_avx2_to_mont(bX4, bX4); + ecp_nistz256_avx2_to_mont(&bX4[4 * 9], &bX4[4 * 9]); + + ecp_nistz256_avx2_point_add_affine_x4(aX4, aX4, bX4); + } + + ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 0], &aX4[4 * 9 * 0]); + ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 1], &aX4[4 * 9 * 1]); + ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 2], &aX4[4 * 9 * 2]); + + ecp_nistz256_avx2_convert_transpose_back(res_point_arr, aX4); + /* Last window is performed serially */ + wvalue = *((u16 *) & p_str[(index - 1) / 8]); + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + booth_recode_w7(&sign0, &digit0, wvalue); + ecp_nistz256_avx2_select_w7((P256_POINT_AFFINE *) r, + preComputedTable[36], digit0); + ecp_nistz256_neg(tmp, r->Y); + copy_conditional(r->Y, tmp, sign0); + memcpy(r->Z, ONE, sizeof(ONE)); + /* Sum the four windows */ + ecp_nistz256_point_add(r, r, &res_point_arr[0]); + ecp_nistz256_point_add(r, r, &res_point_arr[1]); + ecp_nistz256_point_add(r, r, &res_point_arr[2]); + ecp_nistz256_point_add(r, r, &res_point_arr[3]); +} +# endif +#endif + +static int ecp_nistz256_set_from_affine(EC_POINT *out, const EC_GROUP *group, + const P256_POINT_AFFINE *in, + BN_CTX *ctx) +{ + BIGNUM x, y; + BN_ULONG d_x[P256_LIMBS], d_y[P256_LIMBS]; + int ret = 0; + + memcpy(d_x, in->X, sizeof(d_x)); + x.d = d_x; + x.dmax = x.top = P256_LIMBS; + x.neg = 0; + x.flags = BN_FLG_STATIC_DATA; + + memcpy(d_y, in->Y, sizeof(d_y)); + y.d = d_y; + y.dmax = y.top = P256_LIMBS; + y.neg = 0; + y.flags = BN_FLG_STATIC_DATA; + + ret = EC_POINT_set_affine_coordinates_GFp(group, out, &x, &y, ctx); + + return ret; +} + +/* r = scalar*G + sum(scalars[i]*points[i]) */ +static int ecp_nistz256_points_mul(const EC_GROUP *group, + EC_POINT *r, + const BIGNUM *scalar, + size_t num, + const EC_POINT *points[], + const BIGNUM *scalars[], BN_CTX *ctx) +{ + int i = 0, ret = 0, no_precomp_for_generator = 0, p_is_infinity = 0; + size_t j; + unsigned char p_str[33] = { 0 }; + const PRECOMP256_ROW *preComputedTable = NULL; + const EC_PRE_COMP *pre_comp = NULL; + const EC_POINT *generator = NULL; + unsigned int index = 0; + BN_CTX *new_ctx = NULL; + const BIGNUM **new_scalars = NULL; + const EC_POINT **new_points = NULL; + const unsigned int window_size = 7; + const unsigned int mask = (1 << (window_size + 1)) - 1; + unsigned int wvalue; + ALIGN32 union { + P256_POINT p; + P256_POINT_AFFINE a; + } t, p; + BIGNUM *tmp_scalar; + + if (group->meth != r->meth) { + ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, EC_R_INCOMPATIBLE_OBJECTS); + return 0; + } + + if ((scalar == NULL) && (num == 0)) + return EC_POINT_set_to_infinity(group, r); + + for (j = 0; j < num; j++) { + if (group->meth != points[j]->meth) { + ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, EC_R_INCOMPATIBLE_OBJECTS); + return 0; + } + } + + if (ctx == NULL) { + ctx = new_ctx = BN_CTX_new(); + if (ctx == NULL) + goto err; + } + + BN_CTX_start(ctx); + + if (scalar) { + generator = EC_GROUP_get0_generator(group); + if (generator == NULL) { + ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, EC_R_UNDEFINED_GENERATOR); + goto err; + } + + /* look if we can use precomputed multiples of generator */ + pre_comp = + EC_EX_DATA_get_data(group->extra_data, ecp_nistz256_pre_comp_dup, + ecp_nistz256_pre_comp_free, + ecp_nistz256_pre_comp_clear_free); + + if (pre_comp) { + /* + * If there is a precomputed table for the generator, check that + * it was generated with the same generator. + */ + EC_POINT *pre_comp_generator = EC_POINT_new(group); + if (pre_comp_generator == NULL) + goto err; + + if (!ecp_nistz256_set_from_affine + (pre_comp_generator, group, pre_comp->precomp[0], ctx)) { + EC_POINT_free(pre_comp_generator); + goto err; + } + + if (0 == EC_POINT_cmp(group, generator, pre_comp_generator, ctx)) + preComputedTable = (const PRECOMP256_ROW *)pre_comp->precomp; + + EC_POINT_free(pre_comp_generator); + } + + if (preComputedTable == NULL && ecp_nistz256_is_affine_G(generator)) { + /* + * If there is no precomputed data, but the generator + * is the default, a hardcoded table of precomputed + * data is used. This is because applications, such as + * Apache, do not use EC_KEY_precompute_mult. + */ + preComputedTable = (const PRECOMP256_ROW *)ecp_nistz256_precomputed; + } + + if (preComputedTable) { + if ((BN_num_bits(scalar) > 256) + || BN_is_negative(scalar)) { + if ((tmp_scalar = BN_CTX_get(ctx)) == NULL) + goto err; + + if (!BN_nnmod(tmp_scalar, scalar, &group->order, ctx)) { + ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, ERR_R_BN_LIB); + goto err; + } + scalar = tmp_scalar; + } + + for (i = 0; i < scalar->top * BN_BYTES; i += BN_BYTES) { + BN_ULONG d = scalar->d[i / BN_BYTES]; + + p_str[i + 0] = d & 0xff; + p_str[i + 1] = (d >> 8) & 0xff; + p_str[i + 2] = (d >> 16) & 0xff; + p_str[i + 3] = (d >>= 24) & 0xff; + if (BN_BYTES == 8) { + d >>= 8; + p_str[i + 4] = d & 0xff; + p_str[i + 5] = (d >> 8) & 0xff; + p_str[i + 6] = (d >> 16) & 0xff; + p_str[i + 7] = (d >> 24) & 0xff; + } + } + + for (; i < 33; i++) + p_str[i] = 0; + +#if defined(ECP_NISTZ256_AVX2) + if (ecp_nistz_avx2_eligible()) { + ecp_nistz256_avx2_mul_g(&p.p, p_str, preComputedTable); + } else +#endif + { + /* First window */ + wvalue = (p_str[0] << 1) & mask; + index += window_size; + + wvalue = _booth_recode_w7(wvalue); + + ecp_nistz256_select_w7(&p.a, preComputedTable[0], wvalue >> 1); + + ecp_nistz256_neg(p.p.Z, p.p.Y); + copy_conditional(p.p.Y, p.p.Z, wvalue & 1); + + memcpy(p.p.Z, ONE, sizeof(ONE)); + + for (i = 1; i < 37; i++) { + unsigned int off = (index - 1) / 8; + wvalue = p_str[off] | p_str[off + 1] << 8; + wvalue = (wvalue >> ((index - 1) % 8)) & mask; + index += window_size; + + wvalue = _booth_recode_w7(wvalue); + + ecp_nistz256_select_w7(&t.a, + preComputedTable[i], wvalue >> 1); + + ecp_nistz256_neg(t.p.Z, t.a.Y); + copy_conditional(t.a.Y, t.p.Z, wvalue & 1); + + ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a); + } + } + } else { + p_is_infinity = 1; + no_precomp_for_generator = 1; + } + } else + p_is_infinity = 1; + + if (no_precomp_for_generator) { + /* + * Without a precomputed table for the generator, it has to be + * handled like a normal point. + */ + new_scalars = OPENSSL_malloc((num + 1) * sizeof(BIGNUM *)); + if (!new_scalars) { + ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, ERR_R_MALLOC_FAILURE); + goto err; + } + + new_points = OPENSSL_malloc((num + 1) * sizeof(EC_POINT *)); + if (!new_points) { + ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, ERR_R_MALLOC_FAILURE); + goto err; + } + + memcpy(new_scalars, scalars, num * sizeof(BIGNUM *)); + new_scalars[num] = scalar; + memcpy(new_points, points, num * sizeof(EC_POINT *)); + new_points[num] = generator; + + scalars = new_scalars; + points = new_points; + num++; + } + + if (num) { + P256_POINT *out = &t.p; + if (p_is_infinity) + out = &p.p; + + if (!ecp_nistz256_windowed_mul(group, out, scalars, points, num, ctx)) + goto err; + + if (!p_is_infinity) + ecp_nistz256_point_add(&p.p, &p.p, out); + } + + /* Not constant-time, but we're only operating on the public output. */ + if (!ecp_nistz256_set_words(&r->X, p.p.X) || + !ecp_nistz256_set_words(&r->Y, p.p.Y) || + !ecp_nistz256_set_words(&r->Z, p.p.Z)) { + goto err; + } + r->Z_is_one = is_one(p.p.Z) & 1; + + ret = 1; + +err: + if (ctx) + BN_CTX_end(ctx); + BN_CTX_free(new_ctx); + if (new_points) + OPENSSL_free(new_points); + if (new_scalars) + OPENSSL_free(new_scalars); + return ret; +} + +static int ecp_nistz256_get_affine(const EC_GROUP *group, + const EC_POINT *point, + BIGNUM *x, BIGNUM *y, BN_CTX *ctx) +{ + BN_ULONG z_inv2[P256_LIMBS]; + BN_ULONG z_inv3[P256_LIMBS]; + BN_ULONG x_aff[P256_LIMBS]; + BN_ULONG y_aff[P256_LIMBS]; + BN_ULONG point_x[P256_LIMBS], point_y[P256_LIMBS], point_z[P256_LIMBS]; + BN_ULONG x_ret[P256_LIMBS], y_ret[P256_LIMBS]; + + if (EC_POINT_is_at_infinity(group, point)) { + ECerr(EC_F_ECP_NISTZ256_GET_AFFINE, EC_R_POINT_AT_INFINITY); + return 0; + } + + if (!ecp_nistz256_bignum_to_field_elem(point_x, &point->X) || + !ecp_nistz256_bignum_to_field_elem(point_y, &point->Y) || + !ecp_nistz256_bignum_to_field_elem(point_z, &point->Z)) { + ECerr(EC_F_ECP_NISTZ256_GET_AFFINE, EC_R_COORDINATES_OUT_OF_RANGE); + return 0; + } + + ecp_nistz256_mod_inverse(z_inv3, point_z); + ecp_nistz256_sqr_mont(z_inv2, z_inv3); + ecp_nistz256_mul_mont(x_aff, z_inv2, point_x); + + if (x != NULL) { + ecp_nistz256_from_mont(x_ret, x_aff); + if (!ecp_nistz256_set_words(x, x_ret)) + return 0; + } + + if (y != NULL) { + ecp_nistz256_mul_mont(z_inv3, z_inv3, z_inv2); + ecp_nistz256_mul_mont(y_aff, z_inv3, point_y); + ecp_nistz256_from_mont(y_ret, y_aff); + if (!ecp_nistz256_set_words(y, y_ret)) + return 0; + } + + return 1; +} + +static EC_PRE_COMP *ecp_nistz256_pre_comp_new(const EC_GROUP *group) +{ + EC_PRE_COMP *ret = NULL; + + if (!group) + return NULL; + + ret = (EC_PRE_COMP *)OPENSSL_malloc(sizeof(EC_PRE_COMP)); + + if (!ret) { + ECerr(EC_F_ECP_NISTZ256_PRE_COMP_NEW, ERR_R_MALLOC_FAILURE); + return ret; + } + + ret->group = group; + ret->w = 6; /* default */ + ret->precomp = NULL; + ret->precomp_storage = NULL; + ret->references = 1; + return ret; +} + +static void *ecp_nistz256_pre_comp_dup(void *src_) +{ + EC_PRE_COMP *src = src_; + + /* no need to actually copy, these objects never change! */ + CRYPTO_add(&src->references, 1, CRYPTO_LOCK_EC_PRE_COMP); + + return src_; +} + +static void ecp_nistz256_pre_comp_free(void *pre_) +{ + int i; + EC_PRE_COMP *pre = pre_; + + if (!pre) + return; + + i = CRYPTO_add(&pre->references, -1, CRYPTO_LOCK_EC_PRE_COMP); + if (i > 0) + return; + + if (pre->precomp_storage) + OPENSSL_free(pre->precomp_storage); + + OPENSSL_free(pre); +} + +static void ecp_nistz256_pre_comp_clear_free(void *pre_) +{ + int i; + EC_PRE_COMP *pre = pre_; + + if (!pre) + return; + + i = CRYPTO_add(&pre->references, -1, CRYPTO_LOCK_EC_PRE_COMP); + if (i > 0) + return; + + if (pre->precomp_storage) { + OPENSSL_cleanse(pre->precomp, + 32 * sizeof(unsigned char) * (1 << pre->w) * 2 * 37); + OPENSSL_free(pre->precomp_storage); + } + OPENSSL_cleanse(pre, sizeof *pre); + OPENSSL_free(pre); +} + +static int ecp_nistz256_window_have_precompute_mult(const EC_GROUP *group) +{ + /* There is a hard-coded table for the default generator. */ + const EC_POINT *generator = EC_GROUP_get0_generator(group); + if (generator != NULL && ecp_nistz256_is_affine_G(generator)) { + /* There is a hard-coded table for the default generator. */ + return 1; + } + + return EC_EX_DATA_get_data(group->extra_data, ecp_nistz256_pre_comp_dup, + ecp_nistz256_pre_comp_free, + ecp_nistz256_pre_comp_clear_free) != NULL; +} + +const EC_METHOD *EC_GFp_nistz256_method(void) +{ + static const EC_METHOD ret = { + EC_FLAGS_DEFAULT_OCT, + NID_X9_62_prime_field, + ec_GFp_mont_group_init, + ec_GFp_mont_group_finish, + ec_GFp_mont_group_clear_finish, + ec_GFp_mont_group_copy, + ec_GFp_mont_group_set_curve, + ec_GFp_simple_group_get_curve, + ec_GFp_simple_group_get_degree, + ec_GFp_simple_group_check_discriminant, + ec_GFp_simple_point_init, + ec_GFp_simple_point_finish, + ec_GFp_simple_point_clear_finish, + ec_GFp_simple_point_copy, + ec_GFp_simple_point_set_to_infinity, + ec_GFp_simple_set_Jprojective_coordinates_GFp, + ec_GFp_simple_get_Jprojective_coordinates_GFp, + ec_GFp_simple_point_set_affine_coordinates, + ecp_nistz256_get_affine, + 0, 0, 0, + ec_GFp_simple_add, + ec_GFp_simple_dbl, + ec_GFp_simple_invert, + ec_GFp_simple_is_at_infinity, + ec_GFp_simple_is_on_curve, + ec_GFp_simple_cmp, + ec_GFp_simple_make_affine, + ec_GFp_simple_points_make_affine, + ecp_nistz256_points_mul, /* mul */ + ecp_nistz256_mult_precompute, /* precompute_mult */ + ecp_nistz256_window_have_precompute_mult, /* have_precompute_mult */ + ec_GFp_mont_field_mul, + ec_GFp_mont_field_sqr, + 0, /* field_div */ + ec_GFp_mont_field_encode, + ec_GFp_mont_field_decode, + ec_GFp_mont_field_set_to_one + }; + + return &ret; +} diff --git a/crypto/ec/ecp_nistz256_table.c b/crypto/ec/ecp_nistz256_table.c new file mode 100644 index 000000000000..216d024e0120 --- /dev/null +++ b/crypto/ec/ecp_nistz256_table.c @@ -0,0 +1,9533 @@ +/* + * This is the precomputed constant time access table for the code in + * ecp_montp256.c, for the default generator. The table consists of 37 + * subtables, each subtable contains 64 affine points. The affine points are + * encoded as eight uint64's, four for the x coordinate and four for the y. + * Both values are in little-endian order. There are 37 tables because a + * signed, 6-bit wNAF form of the scalar is used and ceil(256/(6 + 1)) = 37. + * Within each table there are 64 values because the 6-bit wNAF value can + * take 64 values, ignoring the sign bit, which is implemented by performing + * a negation of the affine point when required. We would like to align it + * to 2MB in order to increase the chances of using a large page but that + * appears to lead to invalid ELF files being produced. + */ + +#if defined(__GNUC__) +__attribute((aligned(4096))) +#elif defined(_MSC_VER) +__declspec(align(4096)) +#elif defined(__SUNPRO_C) +# pragma align 4096(ecp_nistz256_precomputed) +#endif +static const BN_ULONG ecp_nistz256_precomputed[37][64 * + sizeof(P256_POINT_AFFINE) / + sizeof(BN_ULONG)] = { + {TOBN(0x79e730d4, 0x18a9143c), TOBN(0x75ba95fc, 0x5fedb601), + TOBN(0x79fb732b, 0x77622510), TOBN(0x18905f76, 0xa53755c6), + TOBN(0xddf25357, 0xce95560a), TOBN(0x8b4ab8e4, 0xba19e45c), + TOBN(0xd2e88688, 0xdd21f325), TOBN(0x8571ff18, 0x25885d85), + TOBN(0x850046d4, 0x10ddd64d), TOBN(0xaa6ae3c1, 0xa433827d), + TOBN(0x73220503, 0x8d1490d9), TOBN(0xf6bb32e4, 0x3dcf3a3b), + TOBN(0x2f3648d3, 0x61bee1a5), TOBN(0x152cd7cb, 0xeb236ff8), + TOBN(0x19a8fb0e, 0x92042dbe), TOBN(0x78c57751, 0x0a5b8a3b), + TOBN(0xffac3f90, 0x4eebc127), TOBN(0xb027f84a, 0x087d81fb), + TOBN(0x66ad77dd, 0x87cbbc98), TOBN(0x26936a3f, 0xb6ff747e), + TOBN(0xb04c5c1f, 0xc983a7eb), TOBN(0x583e47ad, 0x0861fe1a), + TOBN(0x78820831, 0x1a2ee98e), TOBN(0xd5f06a29, 0xe587cc07), + TOBN(0x74b0b50d, 0x46918dcc), TOBN(0x4650a6ed, 0xc623c173), + TOBN(0x0cdaacac, 0xe8100af2), TOBN(0x577362f5, 0x41b0176b), + TOBN(0x2d96f24c, 0xe4cbaba6), TOBN(0x17628471, 0xfad6f447), + TOBN(0x6b6c36de, 0xe5ddd22e), TOBN(0x84b14c39, 0x4c5ab863), + TOBN(0xbe1b8aae, 0xc45c61f5), TOBN(0x90ec649a, 0x94b9537d), + TOBN(0x941cb5aa, 0xd076c20c), TOBN(0xc9079605, 0x890523c8), + TOBN(0xeb309b4a, 0xe7ba4f10), TOBN(0x73c568ef, 0xe5eb882b), + TOBN(0x3540a987, 0x7e7a1f68), TOBN(0x73a076bb, 0x2dd1e916), + TOBN(0x40394737, 0x3e77664a), TOBN(0x55ae744f, 0x346cee3e), + TOBN(0xd50a961a, 0x5b17a3ad), TOBN(0x13074b59, 0x54213673), + TOBN(0x93d36220, 0xd377e44b), TOBN(0x299c2b53, 0xadff14b5), + TOBN(0xf424d44c, 0xef639f11), TOBN(0xa4c9916d, 0x4a07f75f), + TOBN(0x0746354e, 0xa0173b4f), TOBN(0x2bd20213, 0xd23c00f7), + TOBN(0xf43eaab5, 0x0c23bb08), TOBN(0x13ba5119, 0xc3123e03), + TOBN(0x2847d030, 0x3f5b9d4d), TOBN(0x6742f2f2, 0x5da67bdd), + TOBN(0xef933bdc, 0x77c94195), TOBN(0xeaedd915, 0x6e240867), + TOBN(0x27f14cd1, 0x9499a78f), TOBN(0x462ab5c5, 0x6f9b3455), + TOBN(0x8f90f02a, 0xf02cfc6b), TOBN(0xb763891e, 0xb265230d), + TOBN(0xf59da3a9, 0x532d4977), TOBN(0x21e3327d, 0xcf9eba15), + TOBN(0x123c7b84, 0xbe60bbf0), TOBN(0x56ec12f2, 0x7706df76), + TOBN(0x75c96e8f, 0x264e20e8), TOBN(0xabe6bfed, 0x59a7a841), + TOBN(0x2cc09c04, 0x44c8eb00), TOBN(0xe05b3080, 0xf0c4e16b), + TOBN(0x1eb7777a, 0xa45f3314), TOBN(0x56af7bed, 0xce5d45e3), + TOBN(0x2b6e019a, 0x88b12f1a), TOBN(0x086659cd, 0xfd835f9b), + TOBN(0x2c18dbd1, 0x9dc21ec8), TOBN(0x98f9868a, 0x0fcf8139), + TOBN(0x737d2cd6, 0x48250b49), TOBN(0xcc61c947, 0x24b3428f), + TOBN(0x0c2b4078, 0x80dd9e76), TOBN(0xc43a8991, 0x383fbe08), + TOBN(0x5f7d2d65, 0x779be5d2), TOBN(0x78719a54, 0xeb3b4ab5), + TOBN(0xea7d260a, 0x6245e404), TOBN(0x9de40795, 0x6e7fdfe0), + TOBN(0x1ff3a415, 0x8dac1ab5), TOBN(0x3e7090f1, 0x649c9073), + TOBN(0x1a768561, 0x2b944e88), TOBN(0x250f939e, 0xe57f61c8), + TOBN(0x0c0daa89, 0x1ead643d), TOBN(0x68930023, 0xe125b88e), + TOBN(0x04b71aa7, 0xd2697768), TOBN(0xabdedef5, 0xca345a33), + TOBN(0x2409d29d, 0xee37385e), TOBN(0x4ee1df77, 0xcb83e156), + TOBN(0x0cac12d9, 0x1cbb5b43), TOBN(0x170ed2f6, 0xca895637), + TOBN(0x28228cfa, 0x8ade6d66), TOBN(0x7ff57c95, 0x53238aca), + TOBN(0xccc42563, 0x4b2ed709), TOBN(0x0e356769, 0x856fd30d), + TOBN(0xbcbcd43f, 0x559e9811), TOBN(0x738477ac, 0x5395b759), + TOBN(0x35752b90, 0xc00ee17f), TOBN(0x68748390, 0x742ed2e3), + TOBN(0x7cd06422, 0xbd1f5bc1), TOBN(0xfbc08769, 0xc9e7b797), + TOBN(0xa242a35b, 0xb0cf664a), TOBN(0x126e48f7, 0x7f9707e3), + TOBN(0x1717bf54, 0xc6832660), TOBN(0xfaae7332, 0xfd12c72e), + TOBN(0x27b52db7, 0x995d586b), TOBN(0xbe29569e, 0x832237c2), + TOBN(0xe8e4193e, 0x2a65e7db), TOBN(0x152706dc, 0x2eaa1bbb), + TOBN(0x72bcd8b7, 0xbc60055b), TOBN(0x03cc23ee, 0x56e27e4b), + TOBN(0xee337424, 0xe4819370), TOBN(0xe2aa0e43, 0x0ad3da09), + TOBN(0x40b8524f, 0x6383c45d), TOBN(0xd7663554, 0x42a41b25), + TOBN(0x64efa6de, 0x778a4797), TOBN(0x2042170a, 0x7079adf4), + TOBN(0x808b0b65, 0x0bc6fb80), TOBN(0x5882e075, 0x3ffe2e6b), + TOBN(0xd5ef2f7c, 0x2c83f549), TOBN(0x54d63c80, 0x9103b723), + TOBN(0xf2f11bd6, 0x52a23f9b), TOBN(0x3670c319, 0x4b0b6587), + TOBN(0x55c4623b, 0xb1580e9e), TOBN(0x64edf7b2, 0x01efe220), + TOBN(0x97091dcb, 0xd53c5c9d), TOBN(0xf17624b6, 0xac0a177b), + TOBN(0xb0f13975, 0x2cfe2dff), TOBN(0xc1a35c0a, 0x6c7a574e), + TOBN(0x227d3146, 0x93e79987), TOBN(0x0575bf30, 0xe89cb80e), + TOBN(0x2f4e247f, 0x0d1883bb), TOBN(0xebd51226, 0x3274c3d0), + TOBN(0x5f3e51c8, 0x56ada97a), TOBN(0x4afc964d, 0x8f8b403e), + TOBN(0xa6f247ab, 0x412e2979), TOBN(0x675abd1b, 0x6f80ebda), + TOBN(0x66a2bd72, 0x5e485a1d), TOBN(0x4b2a5caf, 0x8f4f0b3c), + TOBN(0x2626927f, 0x1b847bba), TOBN(0x6c6fc7d9, 0x0502394d), + TOBN(0xfea912ba, 0xa5659ae8), TOBN(0x68363aba, 0x25e1a16e), + TOBN(0xb8842277, 0x752c41ac), TOBN(0xfe545c28, 0x2897c3fc), + TOBN(0x2d36e9e7, 0xdc4c696b), TOBN(0x5806244a, 0xfba977c5), + TOBN(0x85665e9b, 0xe39508c1), TOBN(0xf720ee25, 0x6d12597b), + TOBN(0x8a979129, 0xd2337a31), TOBN(0x5916868f, 0x0f862bdc), + TOBN(0x048099d9, 0x5dd283ba), TOBN(0xe2d1eeb6, 0xfe5bfb4e), + TOBN(0x82ef1c41, 0x7884005d), TOBN(0xa2d4ec17, 0xffffcbae), + TOBN(0x9161c53f, 0x8aa95e66), TOBN(0x5ee104e1, 0xc5fee0d0), + TOBN(0x562e4cec, 0xc135b208), TOBN(0x74e1b265, 0x4783f47d), + TOBN(0x6d2a506c, 0x5a3f3b30), TOBN(0xecead9f4, 0xc16762fc), + TOBN(0xf29dd4b2, 0xe286e5b9), TOBN(0x1b0fadc0, 0x83bb3c61), + TOBN(0x7a75023e, 0x7fac29a4), TOBN(0xc086d5f1, 0xc9477fa3), + TOBN(0x0fc61135, 0x2f6f3076), TOBN(0xc99ffa23, 0xe3912a9a), + TOBN(0x6a0b0685, 0xd2f8ba3d), TOBN(0xfdc777e8, 0xe93358a4), + TOBN(0x94a787bb, 0x35415f04), TOBN(0x640c2d6a, 0x4d23fea4), + TOBN(0x9de917da, 0x153a35b5), TOBN(0x793e8d07, 0x5d5cd074), + TOBN(0xf4f87653, 0x2de45068), TOBN(0x37c7a7e8, 0x9e2e1f6e), + TOBN(0xd0825fa2, 0xa3584069), TOBN(0xaf2cea7c, 0x1727bf42), + TOBN(0x0360a4fb, 0x9e4785a9), TOBN(0xe5fda49c, 0x27299f4a), + TOBN(0x48068e13, 0x71ac2f71), TOBN(0x83d0687b, 0x9077666f), + TOBN(0x6d3883b2, 0x15d02819), TOBN(0x6d0d7550, 0x40dd9a35), + TOBN(0x61d7cbf9, 0x1d2b469f), TOBN(0xf97b232f, 0x2efc3115), + TOBN(0xa551d750, 0xb24bcbc7), TOBN(0x11ea4949, 0x88a1e356), + TOBN(0x7669f031, 0x93cb7501), TOBN(0x595dc55e, 0xca737b8a), + TOBN(0xa4a319ac, 0xd837879f), TOBN(0x6fc1b49e, 0xed6b67b0), + TOBN(0xe3959933, 0x32f1f3af), TOBN(0x966742eb, 0x65432a2e), + TOBN(0x4b8dc9fe, 0xb4966228), TOBN(0x96cc6312, 0x43f43950), + TOBN(0x12068859, 0xc9b731ee), TOBN(0x7b948dc3, 0x56f79968), + TOBN(0x61e4ad32, 0xed1f8008), TOBN(0xe6c9267a, 0xd8b17538), + TOBN(0x1ac7c5eb, 0x857ff6fb), TOBN(0x994baaa8, 0x55f2fb10), + TOBN(0x84cf14e1, 0x1d248018), TOBN(0x5a39898b, 0x628ac508), + TOBN(0x14fde97b, 0x5fa944f5), TOBN(0xed178030, 0xd12e5ac7), + TOBN(0x042c2af4, 0x97e2feb4), TOBN(0xd36a42d7, 0xaebf7313), + TOBN(0x49d2c9eb, 0x084ffdd7), TOBN(0x9f8aa54b, 0x2ef7c76a), + TOBN(0x9200b7ba, 0x09895e70), TOBN(0x3bd0c66f, 0xddb7fb58), + TOBN(0x2d97d108, 0x78eb4cbb), TOBN(0x2d431068, 0xd84bde31), + TOBN(0x4b523eb7, 0x172ccd1f), TOBN(0x7323cb28, 0x30a6a892), + TOBN(0x97082ec0, 0xcfe153eb), TOBN(0xe97f6b6a, 0xf2aadb97), + TOBN(0x1d3d393e, 0xd1a83da1), TOBN(0xa6a7f9c7, 0x804b2a68), + TOBN(0x4a688b48, 0x2d0cb71e), TOBN(0xa9b4cc5f, 0x40585278), + TOBN(0x5e5db46a, 0xcb66e132), TOBN(0xf1be963a, 0x0d925880), + TOBN(0x944a7027, 0x0317b9e2), TOBN(0xe266f959, 0x48603d48), + TOBN(0x98db6673, 0x5c208899), TOBN(0x90472447, 0xa2fb18a3), + TOBN(0x8a966939, 0x777c619f), TOBN(0x3798142a, 0x2a3be21b), + TOBN(0xb4241cb1, 0x3298b343), TOBN(0xa3a14e49, 0xb44f65a1), + TOBN(0xc5f4d6cd, 0x3ac77acd), TOBN(0xd0288cb5, 0x52b6fc3c), + TOBN(0xd5cc8c2f, 0x1c040abc), TOBN(0xb675511e, 0x06bf9b4a), + TOBN(0xd667da37, 0x9b3aa441), TOBN(0x460d45ce, 0x51601f72), + TOBN(0xe2f73c69, 0x6755ff89), TOBN(0xdd3cf7e7, 0x473017e6), + TOBN(0x8ef5689d, 0x3cf7600d), TOBN(0x948dc4f8, 0xb1fc87b4), + TOBN(0xd9e9fe81, 0x4ea53299), TOBN(0x2d921ca2, 0x98eb6028), + TOBN(0xfaecedfd, 0x0c9803fc), TOBN(0xf38ae891, 0x4d7b4745), + TOBN(0xd8c5fccf, 0xc5e3a3d8), TOBN(0xbefd904c, 0x4079dfbf), + TOBN(0xbc6d6a58, 0xfead0197), TOBN(0x39227077, 0x695532a4), + TOBN(0x09e23e6d, 0xdbef42f5), TOBN(0x7e449b64, 0x480a9908), + TOBN(0x7b969c1a, 0xad9a2e40), TOBN(0x6231d792, 0x9591c2a4), + TOBN(0x87151456, 0x0f664534), TOBN(0x85ceae7c, 0x4b68f103), + TOBN(0xac09c4ae, 0x65578ab9), TOBN(0x33ec6868, 0xf044b10c), + TOBN(0x6ac4832b, 0x3a8ec1f1), TOBN(0x5509d128, 0x5847d5ef), + TOBN(0xf909604f, 0x763f1574), TOBN(0xb16c4303, 0xc32f63c4), + TOBN(0xb6ab2014, 0x7ca23cd3), TOBN(0xcaa7a5c6, 0xa391849d), + TOBN(0x5b0673a3, 0x75678d94), TOBN(0xc982ddd4, 0xdd303e64), + TOBN(0xfd7b000b, 0x5db6f971), TOBN(0xbba2cb1f, 0x6f876f92), + TOBN(0xc77332a3, 0x3c569426), TOBN(0xa159100c, 0x570d74f8), + TOBN(0xfd16847f, 0xdec67ef5), TOBN(0x742ee464, 0x233e76b7), + TOBN(0x0b8e4134, 0xefc2b4c8), TOBN(0xca640b86, 0x42a3e521), + TOBN(0x653a0190, 0x8ceb6aa9), TOBN(0x313c300c, 0x547852d5), + TOBN(0x24e4ab12, 0x6b237af7), TOBN(0x2ba90162, 0x8bb47af8), + TOBN(0x3d5e58d6, 0xa8219bb7), TOBN(0xc691d0bd, 0x1b06c57f), + TOBN(0x0ae4cb10, 0xd257576e), TOBN(0x3569656c, 0xd54a3dc3), + TOBN(0xe5ebaebd, 0x94cda03a), TOBN(0x934e82d3, 0x162bfe13), + TOBN(0x450ac0ba, 0xe251a0c6), TOBN(0x480b9e11, 0xdd6da526), + TOBN(0x00467bc5, 0x8cce08b5), TOBN(0xb636458c, 0x7f178d55), + TOBN(0xc5748bae, 0xa677d806), TOBN(0x2763a387, 0xdfa394eb), + TOBN(0xa12b448a, 0x7d3cebb6), TOBN(0xe7adda3e, 0x6f20d850), + TOBN(0xf63ebce5, 0x1558462c), TOBN(0x58b36143, 0x620088a8), + TOBN(0x8a2cc3ca, 0x4d63c0ee), TOBN(0x51233117, 0x0fe948ce), + TOBN(0x7463fd85, 0x222ef33b), TOBN(0xadf0c7dc, 0x7c603d6c), + TOBN(0x0ec32d3b, 0xfe7765e5), TOBN(0xccaab359, 0xbf380409), + TOBN(0xbdaa84d6, 0x8e59319c), TOBN(0xd9a4c280, 0x9c80c34d), + TOBN(0xa9d89488, 0xa059c142), TOBN(0x6f5ae714, 0xff0b9346), + TOBN(0x068f237d, 0x16fb3664), TOBN(0x5853e4c4, 0x363186ac), + TOBN(0xe2d87d23, 0x63c52f98), TOBN(0x2ec4a766, 0x81828876), + TOBN(0x47b864fa, 0xe14e7b1c), TOBN(0x0c0bc0e5, 0x69192408), + TOBN(0xe4d7681d, 0xb82e9f3e), TOBN(0x83200f0b, 0xdf25e13c), + TOBN(0x8909984c, 0x66f27280), TOBN(0x462d7b00, 0x75f73227), + TOBN(0xd90ba188, 0xf2651798), TOBN(0x74c6e18c, 0x36ab1c34), + TOBN(0xab256ea3, 0x5ef54359), TOBN(0x03466612, 0xd1aa702f), + TOBN(0x624d6049, 0x2ed22e91), TOBN(0x6fdfe0b5, 0x6f072822), + TOBN(0xeeca1115, 0x39ce2271), TOBN(0x98100a4f, 0xdb01614f), + TOBN(0xb6b0daa2, 0xa35c628f), TOBN(0xb6f94d2e, 0xc87e9a47), + TOBN(0xc6773259, 0x1d57d9ce), TOBN(0xf70bfeec, 0x03884a7b), + TOBN(0x5fb35ccf, 0xed2bad01), TOBN(0xa155cbe3, 0x1da6a5c7), + TOBN(0xc2e2594c, 0x30a92f8f), TOBN(0x649c89ce, 0x5bfafe43), + TOBN(0xd158667d, 0xe9ff257a), TOBN(0x9b359611, 0xf32c50ae), + TOBN(0x4b00b20b, 0x906014cf), TOBN(0xf3a8cfe3, 0x89bc7d3d), + TOBN(0x4ff23ffd, 0x248a7d06), TOBN(0x80c5bfb4, 0x878873fa), + TOBN(0xb7d9ad90, 0x05745981), TOBN(0x179c85db, 0x3db01994), + TOBN(0xba41b062, 0x61a6966c), TOBN(0x4d82d052, 0xeadce5a8), + TOBN(0x9e91cd3b, 0xa5e6a318), TOBN(0x47795f4f, 0x95b2dda0), + TOBN(0xecfd7c1f, 0xd55a897c), TOBN(0x009194ab, 0xb29110fb), + TOBN(0x5f0e2046, 0xe381d3b0), TOBN(0x5f3425f6, 0xa98dd291), + TOBN(0xbfa06687, 0x730d50da), TOBN(0x0423446c, 0x4b083b7f), + TOBN(0x397a247d, 0xd69d3417), TOBN(0xeb629f90, 0x387ba42a), + TOBN(0x1ee426cc, 0xd5cd79bf), TOBN(0x0032940b, 0x946c6e18), + TOBN(0x1b1e8ae0, 0x57477f58), TOBN(0xe94f7d34, 0x6d823278), + TOBN(0xc747cb96, 0x782ba21a), TOBN(0xc5254469, 0xf72b33a5), + TOBN(0x772ef6de, 0xc7f80c81), TOBN(0xd73acbfe, 0x2cd9e6b5), + TOBN(0x4075b5b1, 0x49ee90d9), TOBN(0x785c339a, 0xa06e9eba), + TOBN(0xa1030d5b, 0xabf825e0), TOBN(0xcec684c3, 0xa42931dc), + TOBN(0x42ab62c9, 0xc1586e63), TOBN(0x45431d66, 0x5ab43f2b), + TOBN(0x57c8b2c0, 0x55f7835d), TOBN(0x033da338, 0xc1b7f865), + TOBN(0x283c7513, 0xcaa76097), TOBN(0x0a624fa9, 0x36c83906), + TOBN(0x6b20afec, 0x715af2c7), TOBN(0x4b969974, 0xeba78bfd), + TOBN(0x220755cc, 0xd921d60e), TOBN(0x9b944e10, 0x7baeca13), + TOBN(0x04819d51, 0x5ded93d4), TOBN(0x9bbff86e, 0x6dddfd27), + TOBN(0x6b344130, 0x77adc612), TOBN(0xa7496529, 0xbbd803a0), + TOBN(0x1a1baaa7, 0x6d8805bd), TOBN(0xc8403902, 0x470343ad), + TOBN(0x39f59f66, 0x175adff1), TOBN(0x0b26d7fb, 0xb7d8c5b7), + TOBN(0xa875f5ce, 0x529d75e3), TOBN(0x85efc7e9, 0x41325cc2), + TOBN(0x21950b42, 0x1ff6acd3), TOBN(0xffe70484, 0x53dc6909), + TOBN(0xff4cd0b2, 0x28766127), TOBN(0xabdbe608, 0x4fb7db2b), + TOBN(0x837c9228, 0x5e1109e8), TOBN(0x26147d27, 0xf4645b5a), + TOBN(0x4d78f592, 0xf7818ed8), TOBN(0xd394077e, 0xf247fa36), + TOBN(0x0fb9c2d0, 0x488c171a), TOBN(0xa78bfbaa, 0x13685278), + TOBN(0xedfbe268, 0xd5b1fa6a), TOBN(0x0dceb8db, 0x2b7eaba7), + TOBN(0xbf9e8089, 0x9ae2b710), TOBN(0xefde7ae6, 0xa4449c96), + TOBN(0x43b7716b, 0xcc143a46), TOBN(0xd7d34194, 0xc3628c13), + TOBN(0x508cec1c, 0x3b3f64c9), TOBN(0xe20bc0ba, 0x1e5edf3f), + TOBN(0xda1deb85, 0x2f4318d4), TOBN(0xd20ebe0d, 0x5c3fa443), + TOBN(0x370b4ea7, 0x73241ea3), TOBN(0x61f1511c, 0x5e1a5f65), + TOBN(0x99a5e23d, 0x82681c62), TOBN(0xd731e383, 0xa2f54c2d), + TOBN(0x2692f36e, 0x83445904), TOBN(0x2e0ec469, 0xaf45f9c0), + TOBN(0x905a3201, 0xc67528b7), TOBN(0x88f77f34, 0xd0e5e542), + TOBN(0xf67a8d29, 0x5864687c), TOBN(0x23b92eae, 0x22df3562), + TOBN(0x5c27014b, 0x9bbec39e), TOBN(0x7ef2f226, 0x9c0f0f8d), + TOBN(0x97359638, 0x546c4d8d), TOBN(0x5f9c3fc4, 0x92f24679), + TOBN(0x912e8bed, 0xa8c8acd9), TOBN(0xec3a318d, 0x306634b0), + TOBN(0x80167f41, 0xc31cb264), TOBN(0x3db82f6f, 0x522113f2), + TOBN(0xb155bcd2, 0xdcafe197), TOBN(0xfba1da59, 0x43465283), + TOBN(0xa0425b8e, 0xb212cf53), TOBN(0x4f2e512e, 0xf8557c5f), + TOBN(0xc1286ff9, 0x25c4d56c), TOBN(0xbb8a0fea, 0xee26c851), + TOBN(0xc28f70d2, 0xe7d6107e), TOBN(0x7ee0c444, 0xe76265aa), + TOBN(0x3df277a4, 0x1d1936b1), TOBN(0x1a556e3f, 0xea9595eb), + TOBN(0x258bbbf9, 0xe7305683), TOBN(0x31eea5bf, 0x07ef5be6), + TOBN(0x0deb0e4a, 0x46c814c1), TOBN(0x5cee8449, 0xa7b730dd), + TOBN(0xeab495c5, 0xa0182bde), TOBN(0xee759f87, 0x9e27a6b4), + TOBN(0xc2cf6a68, 0x80e518ca), TOBN(0x25e8013f, 0xf14cf3f4), + TOBN(0x8fc44140, 0x7e8d7a14), TOBN(0xbb1ff3ca, 0x9556f36a), + TOBN(0x6a844385, 0x14600044), TOBN(0xba3f0c4a, 0x7451ae63), + TOBN(0xdfcac25b, 0x1f9af32a), TOBN(0x01e0db86, 0xb1f2214b), + TOBN(0x4e9a5bc2, 0xa4b596ac), TOBN(0x83927681, 0x026c2c08), + TOBN(0x3ec832e7, 0x7acaca28), TOBN(0x1bfeea57, 0xc7385b29), + TOBN(0x068212e3, 0xfd1eaf38), TOBN(0xc1329830, 0x6acf8ccc), + TOBN(0xb909f2db, 0x2aac9e59), TOBN(0x5748060d, 0xb661782a), + TOBN(0xc5ab2632, 0xc79b7a01), TOBN(0xda44c6c6, 0x00017626), + TOBN(0xf26c00e8, 0xa7ea82f0), TOBN(0x99cac80d, 0xe4299aaf), + TOBN(0xd66fe3b6, 0x7ed78be1), TOBN(0x305f725f, 0x648d02cd), + TOBN(0x33ed1bc4, 0x623fb21b), TOBN(0xfa70533e, 0x7a6319ad), + TOBN(0x17ab562d, 0xbe5ffb3e), TOBN(0x06374994, 0x56674741), + TOBN(0x69d44ed6, 0x5c46aa8e), TOBN(0x2100d5d3, 0xa8d063d1), + TOBN(0xcb9727ea, 0xa2d17c36), TOBN(0x4c2bab1b, 0x8add53b7), + TOBN(0xa084e90c, 0x15426704), TOBN(0x778afcd3, 0xa837ebea), + TOBN(0x6651f701, 0x7ce477f8), TOBN(0xa0624998, 0x46fb7a8b), + TOBN(0xdc1e6828, 0xed8a6e19), TOBN(0x33fc2336, 0x4189d9c7), + TOBN(0x026f8fe2, 0x671c39bc), TOBN(0xd40c4ccd, 0xbc6f9915), + TOBN(0xafa135bb, 0xf80e75ca), TOBN(0x12c651a0, 0x22adff2c), + TOBN(0xc40a04bd, 0x4f51ad96), TOBN(0x04820109, 0xbbe4e832), + TOBN(0x3667eb1a, 0x7f4c04cc), TOBN(0x59556621, 0xa9404f84), + TOBN(0x71cdf653, 0x7eceb50a), TOBN(0x994a44a6, 0x9b8335fa), + TOBN(0xd7faf819, 0xdbeb9b69), TOBN(0x473c5680, 0xeed4350d), + TOBN(0xb6658466, 0xda44bba2), TOBN(0x0d1bc780, 0x872bdbf3), + TOBN(0xe535f175, 0xa1962f91), TOBN(0x6ed7e061, 0xed58f5a7), + TOBN(0x177aa4c0, 0x2089a233), TOBN(0x0dbcb03a, 0xe539b413), + TOBN(0xe3dc424e, 0xbb32e38e), TOBN(0x6472e5ef, 0x6806701e), + TOBN(0xdd47ff98, 0x814be9ee), TOBN(0x6b60cfff, 0x35ace009), + TOBN(0xb8d3d931, 0x9ff91fe5), TOBN(0x039c4800, 0xf0518eed), + TOBN(0x95c37632, 0x9182cb26), TOBN(0x0763a434, 0x82fc568d), + TOBN(0x707c04d5, 0x383e76ba), TOBN(0xac98b930, 0x824e8197), + TOBN(0x92bf7c8f, 0x91230de0), TOBN(0x90876a01, 0x40959b70), + TOBN(0xdb6d96f3, 0x05968b80), TOBN(0x380a0913, 0x089f73b9), + TOBN(0x7da70b83, 0xc2c61e01), TOBN(0x95fb8394, 0x569b38c7), + TOBN(0x9a3c6512, 0x80edfe2f), TOBN(0x8f726bb9, 0x8faeaf82), + TOBN(0x8010a4a0, 0x78424bf8), TOBN(0x29672044, 0x0e844970)} + , + {TOBN(0x63c5cb81, 0x7a2ad62a), TOBN(0x7ef2b6b9, 0xac62ff54), + TOBN(0x3749bba4, 0xb3ad9db5), TOBN(0xad311f2c, 0x46d5a617), + TOBN(0xb77a8087, 0xc2ff3b6d), TOBN(0xb46feaf3, 0x367834ff), + TOBN(0xf8aa266d, 0x75d6b138), TOBN(0xfa38d320, 0xec008188), + TOBN(0x486d8ffa, 0x696946fc), TOBN(0x50fbc6d8, 0xb9cba56d), + TOBN(0x7e3d423e, 0x90f35a15), TOBN(0x7c3da195, 0xc0dd962c), + TOBN(0xe673fdb0, 0x3cfd5d8b), TOBN(0x0704b7c2, 0x889dfca5), + TOBN(0xf6ce581f, 0xf52305aa), TOBN(0x399d49eb, 0x914d5e53), + TOBN(0x380a496d, 0x6ec293cd), TOBN(0x733dbda7, 0x8e7051f5), + TOBN(0x037e388d, 0xb849140a), TOBN(0xee4b32b0, 0x5946dbf6), + TOBN(0xb1c4fda9, 0xcae368d1), TOBN(0x5001a7b0, 0xfdb0b2f3), + TOBN(0x6df59374, 0x2e3ac46e), TOBN(0x4af675f2, 0x39b3e656), + TOBN(0x44e38110, 0x39949296), TOBN(0x5b63827b, 0x361db1b5), + TOBN(0x3e5323ed, 0x206eaff5), TOBN(0x942370d2, 0xc21f4290), + TOBN(0xf2caaf2e, 0xe0d985a1), TOBN(0x192cc64b, 0x7239846d), + TOBN(0x7c0b8f47, 0xae6312f8), TOBN(0x7dc61f91, 0x96620108), + TOBN(0xb830fb5b, 0xc2da7de9), TOBN(0xd0e643df, 0x0ff8d3be), + TOBN(0x31ee77ba, 0x188a9641), TOBN(0x4e8aa3aa, 0xbcf6d502), + TOBN(0xf9fb6532, 0x9a49110f), TOBN(0xd18317f6, 0x2dd6b220), + TOBN(0x7e3ced41, 0x52c3ea5a), TOBN(0x0d296a14, 0x7d579c4a), + TOBN(0x35d6a53e, 0xed4c3717), TOBN(0x9f8240cf, 0x3d0ed2a3), + TOBN(0x8c0d4d05, 0xe5543aa5), TOBN(0x45d5bbfb, 0xdd33b4b4), + TOBN(0xfa04cc73, 0x137fd28e), TOBN(0x862ac6ef, 0xc73b3ffd), + TOBN(0x403ff9f5, 0x31f51ef2), TOBN(0x34d5e0fc, 0xbc73f5a2), + TOBN(0xf2526820, 0x08913f4f), TOBN(0xea20ed61, 0xeac93d95), + TOBN(0x51ed38b4, 0x6ca6b26c), TOBN(0x8662dcbc, 0xea4327b0), + TOBN(0x6daf295c, 0x725d2aaa), TOBN(0xbad2752f, 0x8e52dcda), + TOBN(0x2210e721, 0x0b17dacc), TOBN(0xa37f7912, 0xd51e8232), + TOBN(0x4f7081e1, 0x44cc3add), TOBN(0xd5ffa1d6, 0x87be82cf), + TOBN(0x89890b6c, 0x0edd6472), TOBN(0xada26e1a, 0x3ed17863), + TOBN(0x276f2715, 0x63483caa), TOBN(0xe6924cd9, 0x2f6077fd), + TOBN(0x05a7fe98, 0x0a466e3c), TOBN(0xf1c794b0, 0xb1902d1f), + TOBN(0xe5213688, 0x82a8042c), TOBN(0xd931cfaf, 0xcd278298), + TOBN(0x069a0ae0, 0xf597a740), TOBN(0x0adbb3f3, 0xeb59107c), + TOBN(0x983e951e, 0x5eaa8eb8), TOBN(0xe663a8b5, 0x11b48e78), + TOBN(0x1631cc0d, 0x8a03f2c5), TOBN(0x7577c11e, 0x11e271e2), + TOBN(0x33b2385c, 0x08369a90), TOBN(0x2990c59b, 0x190eb4f8), + TOBN(0x819a6145, 0xc68eac80), TOBN(0x7a786d62, 0x2ec4a014), + TOBN(0x33faadbe, 0x20ac3a8d), TOBN(0x31a21781, 0x5aba2d30), + TOBN(0x209d2742, 0xdba4f565), TOBN(0xdb2ce9e3, 0x55aa0fbb), + TOBN(0x8cef334b, 0x168984df), TOBN(0xe81dce17, 0x33879638), + TOBN(0xf6e6949c, 0x263720f0), TOBN(0x5c56feaf, 0xf593cbec), + TOBN(0x8bff5601, 0xfde58c84), TOBN(0x74e24117, 0x2eccb314), + TOBN(0xbcf01b61, 0x4c9a8a78), TOBN(0xa233e35e, 0x544c9868), + TOBN(0xb3156bf3, 0x8bd7aff1), TOBN(0x1b5ee4cb, 0x1d81b146), + TOBN(0x7ba1ac41, 0xd628a915), TOBN(0x8f3a8f9c, 0xfd89699e), + TOBN(0x7329b9c9, 0xa0748be7), TOBN(0x1d391c95, 0xa92e621f), + TOBN(0xe51e6b21, 0x4d10a837), TOBN(0xd255f53a, 0x4947b435), + TOBN(0x07669e04, 0xf1788ee3), TOBN(0xc14f27af, 0xa86938a2), + TOBN(0x8b47a334, 0xe93a01c0), TOBN(0xff627438, 0xd9366808), + TOBN(0x7a0985d8, 0xca2a5965), TOBN(0x3d9a5542, 0xd6e9b9b3), + TOBN(0xc23eb80b, 0x4cf972e8), TOBN(0x5c1c33bb, 0x4fdf72fd), + TOBN(0x0c4a58d4, 0x74a86108), TOBN(0xf8048a8f, 0xee4c5d90), + TOBN(0xe3c7c924, 0xe86d4c80), TOBN(0x28c889de, 0x056a1e60), + TOBN(0x57e2662e, 0xb214a040), TOBN(0xe8c48e98, 0x37e10347), + TOBN(0x87742862, 0x80ac748a), TOBN(0xf1c24022, 0x186b06f2), + TOBN(0xac2dd4c3, 0x5f74040a), TOBN(0x409aeb71, 0xfceac957), + TOBN(0x4fbad782, 0x55c4ec23), TOBN(0xb359ed61, 0x8a7b76ec), + TOBN(0x12744926, 0xed6f4a60), TOBN(0xe21e8d7f, 0x4b912de3), + TOBN(0xe2575a59, 0xfc705a59), TOBN(0x72f1d4de, 0xed2dbc0e), + TOBN(0x3d2b24b9, 0xeb7926b8), TOBN(0xbff88cb3, 0xcdbe5509), + TOBN(0xd0f399af, 0xe4dd640b), TOBN(0x3c5fe130, 0x2f76ed45), + TOBN(0x6f3562f4, 0x3764fb3d), TOBN(0x7b5af318, 0x3151b62d), + TOBN(0xd5bd0bc7, 0xd79ce5f3), TOBN(0xfdaf6b20, 0xec66890f), + TOBN(0x735c67ec, 0x6063540c), TOBN(0x50b259c2, 0xe5f9cb8f), + TOBN(0xb8734f9a, 0x3f99c6ab), TOBN(0xf8cc13d5, 0xa3a7bc85), + TOBN(0x80c1b305, 0xc5217659), TOBN(0xfe5364d4, 0x4ec12a54), + TOBN(0xbd87045e, 0x681345fe), TOBN(0x7f8efeb1, 0x582f897f), + TOBN(0xe8cbf1e5, 0xd5923359), TOBN(0xdb0cea9d, 0x539b9fb0), + TOBN(0x0c5b34cf, 0x49859b98), TOBN(0x5e583c56, 0xa4403cc6), + TOBN(0x11fc1a2d, 0xd48185b7), TOBN(0xc93fbc7e, 0x6e521787), + TOBN(0x47e7a058, 0x05105b8b), TOBN(0x7b4d4d58, 0xdb8260c8), + TOBN(0xe33930b0, 0x46eb842a), TOBN(0x8e844a9a, 0x7bdae56d), + TOBN(0x34ef3a9e, 0x13f7fdfc), TOBN(0xb3768f82, 0x636ca176), + TOBN(0x2821f4e0, 0x4e09e61c), TOBN(0x414dc3a1, 0xa0c7cddc), + TOBN(0xd5379437, 0x54945fcd), TOBN(0x151b6eef, 0xb3555ff1), + TOBN(0xb31bd613, 0x6339c083), TOBN(0x39ff8155, 0xdfb64701), + TOBN(0x7c3388d2, 0xe29604ab), TOBN(0x1e19084b, 0xa6b10442), + TOBN(0x17cf54c0, 0xeccd47ef), TOBN(0x89693385, 0x4a5dfb30), + TOBN(0x69d023fb, 0x47daf9f6), TOBN(0x9222840b, 0x7d91d959), + TOBN(0x439108f5, 0x803bac62), TOBN(0x0b7dd91d, 0x379bd45f), + TOBN(0xd651e827, 0xca63c581), TOBN(0x5c5d75f6, 0x509c104f), + TOBN(0x7d5fc738, 0x1f2dc308), TOBN(0x20faa7bf, 0xd98454be), + TOBN(0x95374bee, 0xa517b031), TOBN(0xf036b9b1, 0x642692ac), + TOBN(0xc5106109, 0x39842194), TOBN(0xb7e2353e, 0x49d05295), + TOBN(0xfc8c1d5c, 0xefb42ee0), TOBN(0xe04884eb, 0x08ce811c), + TOBN(0xf1f75d81, 0x7419f40e), TOBN(0x5b0ac162, 0xa995c241), + TOBN(0x120921bb, 0xc4c55646), TOBN(0x713520c2, 0x8d33cf97), + TOBN(0xb4a65a5c, 0xe98c5100), TOBN(0x6cec871d, 0x2ddd0f5a), + TOBN(0x251f0b7f, 0x9ba2e78b), TOBN(0x224a8434, 0xce3a2a5f), + TOBN(0x26827f61, 0x25f5c46f), TOBN(0x6a22bedc, 0x48545ec0), + TOBN(0x25ae5fa0, 0xb1bb5cdc), TOBN(0xd693682f, 0xfcb9b98f), + TOBN(0x32027fe8, 0x91e5d7d3), TOBN(0xf14b7d17, 0x73a07678), + TOBN(0xf88497b3, 0xc0dfdd61), TOBN(0xf7c2eec0, 0x2a8c4f48), + TOBN(0xaa5573f4, 0x3756e621), TOBN(0xc013a240, 0x1825b948), + TOBN(0x1c03b345, 0x63878572), TOBN(0xa0472bea, 0x653a4184), + TOBN(0xf4222e27, 0x0ac69a80), TOBN(0x34096d25, 0xf51e54f6), + TOBN(0x00a648cb, 0x8fffa591), TOBN(0x4e87acdc, 0x69b6527f), + TOBN(0x0575e037, 0xe285ccb4), TOBN(0x188089e4, 0x50ddcf52), + TOBN(0xaa96c9a8, 0x870ff719), TOBN(0x74a56cd8, 0x1fc7e369), + TOBN(0x41d04ee2, 0x1726931a), TOBN(0x0bbbb2c8, 0x3660ecfd), + TOBN(0xa6ef6de5, 0x24818e18), TOBN(0xe421cc51, 0xe7d57887), + TOBN(0xf127d208, 0xbea87be6), TOBN(0x16a475d3, 0xb1cdd682), + TOBN(0x9db1b684, 0x439b63f7), TOBN(0x5359b3db, 0xf0f113b6), + TOBN(0xdfccf1de, 0x8bf06e31), TOBN(0x1fdf8f44, 0xdd383901), + TOBN(0x10775cad, 0x5017e7d2), TOBN(0xdfc3a597, 0x58d11eef), + TOBN(0x6ec9c8a0, 0xb1ecff10), TOBN(0xee6ed6cc, 0x28400549), + TOBN(0xb5ad7bae, 0x1b4f8d73), TOBN(0x61b4f11d, 0xe00aaab9), + TOBN(0x7b32d69b, 0xd4eff2d7), TOBN(0x88ae6771, 0x4288b60f), + TOBN(0x159461b4, 0x37a1e723), TOBN(0x1f3d4789, 0x570aae8c), + TOBN(0x869118c0, 0x7f9871da), TOBN(0x35fbda78, 0xf635e278), + TOBN(0x738f3641, 0xe1541dac), TOBN(0x6794b13a, 0xc0dae45f), + TOBN(0x065064ac, 0x09cc0917), TOBN(0x27c53729, 0xc68540fd), + TOBN(0x0d2d4c8e, 0xef227671), TOBN(0xd23a9f80, 0xa1785a04), + TOBN(0x98c59528, 0x52650359), TOBN(0xfa09ad01, 0x74a1acad), + TOBN(0x082d5a29, 0x0b55bf5c), TOBN(0xa40f1c67, 0x419b8084), + TOBN(0x3a5c752e, 0xdcc18770), TOBN(0x4baf1f2f, 0x8825c3a5), + TOBN(0xebd63f74, 0x21b153ed), TOBN(0xa2383e47, 0xb2f64723), + TOBN(0xe7bf620a, 0x2646d19a), TOBN(0x56cb44ec, 0x03c83ffd), + TOBN(0xaf7267c9, 0x4f6be9f1), TOBN(0x8b2dfd7b, 0xc06bb5e9), + TOBN(0xb87072f2, 0xa672c5c7), TOBN(0xeacb11c8, 0x0d53c5e2), + TOBN(0x22dac29d, 0xff435932), TOBN(0x37bdb99d, 0x4408693c), + TOBN(0xf6e62fb6, 0x2899c20f), TOBN(0x3535d512, 0x447ece24), + TOBN(0xfbdc6b88, 0xff577ce3), TOBN(0x726693bd, 0x190575f2), + TOBN(0x6772b0e5, 0xab4b35a2), TOBN(0x1d8b6001, 0xf5eeaacf), + TOBN(0x728f7ce4, 0x795b9580), TOBN(0x4a20ed2a, 0x41fb81da), + TOBN(0x9f685cd4, 0x4fec01e6), TOBN(0x3ed7ddcc, 0xa7ff50ad), + TOBN(0x460fd264, 0x0c2d97fd), TOBN(0x3a241426, 0xeb82f4f9), + TOBN(0x17d1df2c, 0x6a8ea820), TOBN(0xb2b50d3b, 0xf22cc254), + TOBN(0x03856cba, 0xb7291426), TOBN(0x87fd26ae, 0x04f5ee39), + TOBN(0x9cb696cc, 0x02bee4ba), TOBN(0x53121804, 0x06820fd6), + TOBN(0xa5dfc269, 0x0212e985), TOBN(0x666f7ffa, 0x160f9a09), + TOBN(0xc503cd33, 0xbccd9617), TOBN(0x365dede4, 0xba7730a3), + TOBN(0x798c6355, 0x5ddb0786), TOBN(0xa6c3200e, 0xfc9cd3bc), + TOBN(0x060ffb2c, 0xe5e35efd), TOBN(0x99a4e25b, 0x5555a1c1), + TOBN(0x11d95375, 0xf70b3751), TOBN(0x0a57354a, 0x160e1bf6), + TOBN(0xecb3ae4b, 0xf8e4b065), TOBN(0x07a834c4, 0x2e53022b), + TOBN(0x1cd300b3, 0x8692ed96), TOBN(0x16a6f792, 0x61ee14ec), + TOBN(0x8f1063c6, 0x6a8649ed), TOBN(0xfbcdfcfe, 0x869f3e14), + TOBN(0x2cfb97c1, 0x00a7b3ec), TOBN(0xcea49b3c, 0x7130c2f1), + TOBN(0x462d044f, 0xe9d96488), TOBN(0x4b53d52e, 0x8182a0c1), + TOBN(0x84b6ddd3, 0x0391e9e9), TOBN(0x80ab7b48, 0xb1741a09), + TOBN(0xec0e15d4, 0x27d3317f), TOBN(0x8dfc1ddb, 0x1a64671e), + TOBN(0x93cc5d5f, 0xd49c5b92), TOBN(0xc995d53d, 0x3674a331), + TOBN(0x302e41ec, 0x090090ae), TOBN(0x2278a0cc, 0xedb06830), + TOBN(0x1d025932, 0xfbc99690), TOBN(0x0c32fbd2, 0xb80d68da), + TOBN(0xd79146da, 0xf341a6c1), TOBN(0xae0ba139, 0x1bef68a0), + TOBN(0xc6b8a563, 0x8d774b3a), TOBN(0x1cf307bd, 0x880ba4d7), + TOBN(0xc033bdc7, 0x19803511), TOBN(0xa9f97b3b, 0x8888c3be), + TOBN(0x3d68aebc, 0x85c6d05e), TOBN(0xc3b88a9d, 0x193919eb), + TOBN(0x2d300748, 0xc48b0ee3), TOBN(0x7506bc7c, 0x07a746c1), + TOBN(0xfc48437c, 0x6e6d57f3), TOBN(0x5bd71587, 0xcfeaa91a), + TOBN(0xa4ed0408, 0xc1bc5225), TOBN(0xd0b946db, 0x2719226d), + TOBN(0x109ecd62, 0x758d2d43), TOBN(0x75c8485a, 0x2751759b), + TOBN(0xb0b75f49, 0x9ce4177a), TOBN(0x4fa61a1e, 0x79c10c3d), + TOBN(0xc062d300, 0xa167fcd7), TOBN(0x4df3874c, 0x750f0fa8), + TOBN(0x29ae2cf9, 0x83dfedc9), TOBN(0xf8437134, 0x8d87631a), + TOBN(0xaf571711, 0x7429c8d2), TOBN(0x18d15867, 0x146d9272), + TOBN(0x83053ecf, 0x69769bb7), TOBN(0xc55eb856, 0xc479ab82), + TOBN(0x5ef7791c, 0x21b0f4b2), TOBN(0xaa5956ba, 0x3d491525), + TOBN(0x407a96c2, 0x9fe20eba), TOBN(0xf27168bb, 0xe52a5ad3), + TOBN(0x43b60ab3, 0xbf1d9d89), TOBN(0xe45c51ef, 0x710e727a), + TOBN(0xdfca5276, 0x099b4221), TOBN(0x8dc6407c, 0x2557a159), + TOBN(0x0ead8335, 0x91035895), TOBN(0x0a9db957, 0x9c55dc32), + TOBN(0xe40736d3, 0xdf61bc76), TOBN(0x13a619c0, 0x3f778cdb), + TOBN(0x6dd921a4, 0xc56ea28f), TOBN(0x76a52433, 0x2fa647b4), + TOBN(0x23591891, 0xac5bdc5d), TOBN(0xff4a1a72, 0xbac7dc01), + TOBN(0x9905e261, 0x62df8453), TOBN(0x3ac045df, 0xe63b265f), + TOBN(0x8a3f341b, 0xad53dba7), TOBN(0x8ec269cc, 0x837b625a), + TOBN(0xd71a2782, 0x3ae31189), TOBN(0x8fb4f9a3, 0x55e96120), + TOBN(0x804af823, 0xff9875cf), TOBN(0x23224f57, 0x5d442a9b), + TOBN(0x1c4d3b9e, 0xecc62679), TOBN(0x91da22fb, 0xa0e7ddb1), + TOBN(0xa370324d, 0x6c04a661), TOBN(0x9710d3b6, 0x5e376d17), + TOBN(0xed8c98f0, 0x3044e357), TOBN(0xc364ebbe, 0x6422701c), + TOBN(0x347f5d51, 0x7733d61c), TOBN(0xd55644b9, 0xcea826c3), + TOBN(0x80c6e0ad, 0x55a25548), TOBN(0x0aa7641d, 0x844220a7), + TOBN(0x1438ec81, 0x31810660), TOBN(0x9dfa6507, 0xde4b4043), + TOBN(0x10b515d8, 0xcc3e0273), TOBN(0x1b6066dd, 0x28d8cfb2), + TOBN(0xd3b04591, 0x9c9efebd), TOBN(0x425d4bdf, 0xa21c1ff4), + TOBN(0x5fe5af19, 0xd57607d3), TOBN(0xbbf773f7, 0x54481084), + TOBN(0x8435bd69, 0x94b03ed1), TOBN(0xd9ad1de3, 0x634cc546), + TOBN(0x2cf423fc, 0x00e420ca), TOBN(0xeed26d80, 0xa03096dd), + TOBN(0xd7f60be7, 0xa4db09d2), TOBN(0xf47f569d, 0x960622f7), + TOBN(0xe5925fd7, 0x7296c729), TOBN(0xeff2db26, 0x26ca2715), + TOBN(0xa6fcd014, 0xb913e759), TOBN(0x53da4786, 0x8ff4de93), + TOBN(0x14616d79, 0xc32068e1), TOBN(0xb187d664, 0xccdf352e), + TOBN(0xf7afb650, 0x1dc90b59), TOBN(0x8170e943, 0x7daa1b26), + TOBN(0xc8e3bdd8, 0x700c0a84), TOBN(0x6e8d345f, 0x6482bdfa), + TOBN(0x84cfbfa1, 0xc5c5ea50), TOBN(0xd3baf14c, 0x67960681), + TOBN(0x26398403, 0x0dd50942), TOBN(0xe4b7839c, 0x4716a663), + TOBN(0xd5f1f794, 0xe7de6dc0), TOBN(0x5cd0f4d4, 0x622aa7ce), + TOBN(0x5295f3f1, 0x59acfeec), TOBN(0x8d933552, 0x953e0607), + TOBN(0xc7db8ec5, 0x776c5722), TOBN(0xdc467e62, 0x2b5f290c), + TOBN(0xd4297e70, 0x4ff425a9), TOBN(0x4be924c1, 0x0cf7bb72), + TOBN(0x0d5dc5ae, 0xa1892131), TOBN(0x8bf8a8e3, 0xa705c992), + TOBN(0x73a0b064, 0x7a305ac5), TOBN(0x00c9ca4e, 0x9a8c77a8), + TOBN(0x5dfee80f, 0x83774bdd), TOBN(0x63131602, 0x85734485), + TOBN(0xa1b524ae, 0x914a69a9), TOBN(0xebc2ffaf, 0xd4e300d7), + TOBN(0x52c93db7, 0x7cfa46a5), TOBN(0x71e6161f, 0x21653b50), + TOBN(0x3574fc57, 0xa4bc580a), TOBN(0xc09015dd, 0xe1bc1253), + TOBN(0x4b7b47b2, 0xd174d7aa), TOBN(0x4072d8e8, 0xf3a15d04), + TOBN(0xeeb7d47f, 0xd6fa07ed), TOBN(0x6f2b9ff9, 0xedbdafb1), + TOBN(0x18c51615, 0x3760fe8a), TOBN(0x7a96e6bf, 0xf06c6c13), + TOBN(0x4d7a0410, 0x0ea2d071), TOBN(0xa1914e9b, 0x0be2a5ce), + TOBN(0x5726e357, 0xd8a3c5cf), TOBN(0x1197ecc3, 0x2abb2b13), + TOBN(0x6c0d7f7f, 0x31ae88dd), TOBN(0x15b20d1a, 0xfdbb3efe), + TOBN(0xcd06aa26, 0x70584039), TOBN(0x2277c969, 0xa7dc9747), + TOBN(0xbca69587, 0x7855d815), TOBN(0x899ea238, 0x5188b32a), + TOBN(0x37d9228b, 0x760c1c9d), TOBN(0xc7efbb11, 0x9b5c18da), + TOBN(0x7f0d1bc8, 0x19f6dbc5), TOBN(0x4875384b, 0x07e6905b), + TOBN(0xc7c50baa, 0x3ba8cd86), TOBN(0xb0ce40fb, 0xc2905de0), + TOBN(0x70840673, 0x7a231952), TOBN(0xa912a262, 0xcf43de26), + TOBN(0x9c38ddcc, 0xeb5b76c1), TOBN(0x746f5285, 0x26fc0ab4), + TOBN(0x52a63a50, 0xd62c269f), TOBN(0x60049c55, 0x99458621), + TOBN(0xe7f48f82, 0x3c2f7c9e), TOBN(0x6bd99043, 0x917d5cf3), + TOBN(0xeb1317a8, 0x8701f469), TOBN(0xbd3fe2ed, 0x9a449fe0), + TOBN(0x421e79ca, 0x12ef3d36), TOBN(0x9ee3c36c, 0x3e7ea5de), + TOBN(0xe48198b5, 0xcdff36f7), TOBN(0xaff4f967, 0xc6b82228), + TOBN(0x15e19dd0, 0xc47adb7e), TOBN(0x45699b23, 0x032e7dfa), + TOBN(0x40680c8b, 0x1fae026a), TOBN(0x5a347a48, 0x550dbf4d), + TOBN(0xe652533b, 0x3cef0d7d), TOBN(0xd94f7b18, 0x2bbb4381), + TOBN(0x838752be, 0x0e80f500), TOBN(0x8e6e2488, 0x9e9c9bfb), + TOBN(0xc9751697, 0x16caca6a), TOBN(0x866c49d8, 0x38531ad9), + TOBN(0xc917e239, 0x7151ade1), TOBN(0x2d016ec1, 0x6037c407), + TOBN(0xa407ccc9, 0x00eac3f9), TOBN(0x835f6280, 0xe2ed4748), + TOBN(0xcc54c347, 0x1cc98e0d), TOBN(0x0e969937, 0xdcb572eb), + TOBN(0x1b16c8e8, 0x8f30c9cb), TOBN(0xa606ae75, 0x373c4661), + TOBN(0x47aa689b, 0x35502cab), TOBN(0xf89014ae, 0x4d9bb64f), + TOBN(0x202f6a9c, 0x31c71f7b), TOBN(0x01f95aa3, 0x296ffe5c), + TOBN(0x5fc06014, 0x53cec3a3), TOBN(0xeb991237, 0x5f498a45), + TOBN(0xae9a935e, 0x5d91ba87), TOBN(0xc6ac6281, 0x0b564a19), + TOBN(0x8a8fe81c, 0x3bd44e69), TOBN(0x7c8b467f, 0x9dd11d45), + TOBN(0xf772251f, 0xea5b8e69), TOBN(0xaeecb3bd, 0xc5b75fbc), + TOBN(0x1aca3331, 0x887ff0e5), TOBN(0xbe5d49ff, 0x19f0a131), + TOBN(0x582c13aa, 0xe5c8646f), TOBN(0xdbaa12e8, 0x20e19980), + TOBN(0x8f40f31a, 0xf7abbd94), TOBN(0x1f13f5a8, 0x1dfc7663), + TOBN(0x5d81f1ee, 0xaceb4fc0), TOBN(0x36256002, 0x5e6f0f42), + TOBN(0x4b67d6d7, 0x751370c8), TOBN(0x2608b698, 0x03e80589), + TOBN(0xcfc0d2fc, 0x05268301), TOBN(0xa6943d39, 0x40309212), + TOBN(0x192a90c2, 0x1fd0e1c2), TOBN(0xb209f113, 0x37f1dc76), + TOBN(0xefcc5e06, 0x97bf1298), TOBN(0xcbdb6730, 0x219d639e), + TOBN(0xd009c116, 0xb81e8c6f), TOBN(0xa3ffdde3, 0x1a7ce2e5), + TOBN(0xc53fbaaa, 0xa914d3ba), TOBN(0x836d500f, 0x88df85ee), + TOBN(0xd98dc71b, 0x66ee0751), TOBN(0x5a3d7005, 0x714516fd), + TOBN(0x21d3634d, 0x39eedbba), TOBN(0x35cd2e68, 0x0455a46d), + TOBN(0xc8cafe65, 0xf9d7eb0c), TOBN(0xbda3ce9e, 0x00cefb3e), + TOBN(0xddc17a60, 0x2c9cf7a4), TOBN(0x01572ee4, 0x7bcb8773), + TOBN(0xa92b2b01, 0x8c7548df), TOBN(0x732fd309, 0xa84600e3), + TOBN(0xe22109c7, 0x16543a40), TOBN(0x9acafd36, 0xfede3c6c), + TOBN(0xfb206852, 0x6824e614), TOBN(0x2a4544a9, 0xda25dca0), + TOBN(0x25985262, 0x91d60b06), TOBN(0x281b7be9, 0x28753545), + TOBN(0xec667b1a, 0x90f13b27), TOBN(0x33a83aff, 0x940e2eb4), + TOBN(0x80009862, 0xd5d721d5), TOBN(0x0c3357a3, 0x5bd3a182), + TOBN(0x27f3a83b, 0x7aa2cda4), TOBN(0xb58ae74e, 0xf6f83085), + TOBN(0x2a911a81, 0x2e6dad6b), TOBN(0xde286051, 0xf43d6c5b), + TOBN(0x4bdccc41, 0xf996c4d8), TOBN(0xe7312ec0, 0x0ae1e24e)} + , + {TOBN(0xf8d112e7, 0x6e6485b3), TOBN(0x4d3e24db, 0x771c52f8), + TOBN(0x48e3ee41, 0x684a2f6d), TOBN(0x7161957d, 0x21d95551), + TOBN(0x19631283, 0xcdb12a6c), TOBN(0xbf3fa882, 0x2e50e164), + TOBN(0xf6254b63, 0x3166cc73), TOBN(0x3aefa7ae, 0xaee8cc38), + TOBN(0x79b0fe62, 0x3b36f9fd), TOBN(0x26543b23, 0xfde19fc0), + TOBN(0x136e64a0, 0x958482ef), TOBN(0x23f63771, 0x9b095825), + TOBN(0x14cfd596, 0xb6a1142e), TOBN(0x5ea6aac6, 0x335aac0b), + TOBN(0x86a0e8bd, 0xf3081dd5), TOBN(0x5fb89d79, 0x003dc12a), + TOBN(0xf615c33a, 0xf72e34d4), TOBN(0x0bd9ea40, 0x110eec35), + TOBN(0x1c12bc5b, 0xc1dea34e), TOBN(0x686584c9, 0x49ae4699), + TOBN(0x13ad95d3, 0x8c97b942), TOBN(0x4609561a, 0x4e5c7562), + TOBN(0x9e94a4ae, 0xf2737f89), TOBN(0xf57594c6, 0x371c78b6), + TOBN(0x0f0165fc, 0xe3779ee3), TOBN(0xe00e7f9d, 0xbd495d9e), + TOBN(0x1fa4efa2, 0x20284e7a), TOBN(0x4564bade, 0x47ac6219), + TOBN(0x90e6312a, 0xc4708e8e), TOBN(0x4f5725fb, 0xa71e9adf), + TOBN(0xe95f55ae, 0x3d684b9f), TOBN(0x47f7ccb1, 0x1e94b415), + TOBN(0x7322851b, 0x8d946581), TOBN(0xf0d13133, 0xbdf4a012), + TOBN(0xa3510f69, 0x6584dae0), TOBN(0x03a7c171, 0x3c9f6c6d), + TOBN(0x5be97f38, 0xe475381a), TOBN(0xca1ba422, 0x85823334), + TOBN(0xf83cc5c7, 0x0be17dda), TOBN(0x158b1494, 0x0b918c0f), + TOBN(0xda3a77e5, 0x522e6b69), TOBN(0x69c908c3, 0xbbcd6c18), + TOBN(0x1f1b9e48, 0xd924fd56), TOBN(0x37c64e36, 0xaa4bb3f7), + TOBN(0x5a4fdbdf, 0xee478d7d), TOBN(0xba75c8bc, 0x0193f7a0), + TOBN(0x84bc1e84, 0x56cd16df), TOBN(0x1fb08f08, 0x46fad151), + TOBN(0x8a7cabf9, 0x842e9f30), TOBN(0xa331d4bf, 0x5eab83af), + TOBN(0xd272cfba, 0x017f2a6a), TOBN(0x27560abc, 0x83aba0e3), + TOBN(0x94b83387, 0x0e3a6b75), TOBN(0x25c6aea2, 0x6b9f50f5), + TOBN(0x803d691d, 0xb5fdf6d0), TOBN(0x03b77509, 0xe6333514), + TOBN(0x36178903, 0x61a341c1), TOBN(0x3604dc60, 0x0cfd6142), + TOBN(0x022295eb, 0x8533316c), TOBN(0x3dbde4ac, 0x44af2922), + TOBN(0x898afc5d, 0x1c7eef69), TOBN(0x58896805, 0xd14f4fa1), + TOBN(0x05002160, 0x203c21ca), TOBN(0x6f0d1f30, 0x40ef730b), + TOBN(0x8e8c44d4, 0x196224f8), TOBN(0x75a4ab95, 0x374d079d), + TOBN(0x79085ecc, 0x7d48f123), TOBN(0x56f04d31, 0x1bf65ad8), + TOBN(0xe220bf1c, 0xbda602b2), TOBN(0x73ee1742, 0xf9612c69), + TOBN(0x76008fc8, 0x084fd06b), TOBN(0x4000ef9f, 0xf11380d1), + TOBN(0x48201b4b, 0x12cfe297), TOBN(0x3eee129c, 0x292f74e5), + TOBN(0xe1fe114e, 0xc9e874e8), TOBN(0x899b055c, 0x92c5fc41), + TOBN(0x4e477a64, 0x3a39c8cf), TOBN(0x82f09efe, 0x78963cc9), + TOBN(0x6fd3fd8f, 0xd333f863), TOBN(0x85132b2a, 0xdc949c63), + TOBN(0x7e06a3ab, 0x516eb17b), TOBN(0x73bec06f, 0xd2c7372b), + TOBN(0xe4f74f55, 0xba896da6), TOBN(0xbb4afef8, 0x8e9eb40f), + TOBN(0x2d75bec8, 0xe61d66b0), TOBN(0x02bda4b4, 0xef29300b), + TOBN(0x8bbaa8de, 0x026baa5a), TOBN(0xff54befd, 0xa07f4440), + TOBN(0xbd9b8b1d, 0xbe7a2af3), TOBN(0xec51caa9, 0x4fb74a72), + TOBN(0xb9937a4b, 0x63879697), TOBN(0x7c9a9d20, 0xec2687d5), + TOBN(0x1773e44f, 0x6ef5f014), TOBN(0x8abcf412, 0xe90c6900), + TOBN(0x387bd022, 0x8142161e), TOBN(0x50393755, 0xfcb6ff2a), + TOBN(0x9813fd56, 0xed6def63), TOBN(0x53cf6482, 0x7d53106c), + TOBN(0x991a35bd, 0x431f7ac1), TOBN(0xf1e274dd, 0x63e65faf), + TOBN(0xf63ffa3c, 0x44cc7880), TOBN(0x411a426b, 0x7c256981), + TOBN(0xb698b9fd, 0x93a420e0), TOBN(0x89fdddc0, 0xae53f8fe), + TOBN(0x766e0722, 0x32398baa), TOBN(0x205fee42, 0x5cfca031), + TOBN(0xa49f5341, 0x7a029cf2), TOBN(0xa88c68b8, 0x4023890d), + TOBN(0xbc275041, 0x7337aaa8), TOBN(0x9ed364ad, 0x0eb384f4), + TOBN(0xe0816f85, 0x29aba92f), TOBN(0x2e9e1941, 0x04e38a88), + TOBN(0x57eef44a, 0x3dafd2d5), TOBN(0x35d1fae5, 0x97ed98d8), + TOBN(0x50628c09, 0x2307f9b1), TOBN(0x09d84aae, 0xd6cba5c6), + TOBN(0x67071bc7, 0x88aaa691), TOBN(0x2dea57a9, 0xafe6cb03), + TOBN(0xdfe11bb4, 0x3d78ac01), TOBN(0x7286418c, 0x7fd7aa51), + TOBN(0xfabf7709, 0x77f7195a), TOBN(0x8ec86167, 0xadeb838f), + TOBN(0xea1285a8, 0xbb4f012d), TOBN(0xd6883503, 0x9a3eab3f), + TOBN(0xee5d24f8, 0x309004c2), TOBN(0xa96e4b76, 0x13ffe95e), + TOBN(0x0cdffe12, 0xbd223ea4), TOBN(0x8f5c2ee5, 0xb6739a53), + TOBN(0x5cb4aaa5, 0xdd968198), TOBN(0xfa131c52, 0x72413a6c), + TOBN(0x53d46a90, 0x9536d903), TOBN(0xb270f0d3, 0x48606d8e), + TOBN(0x518c7564, 0xa053a3bc), TOBN(0x088254b7, 0x1a86caef), + TOBN(0xb3ba8cb4, 0x0ab5efd0), TOBN(0x5c59900e, 0x4605945d), + TOBN(0xecace1dd, 0xa1887395), TOBN(0x40960f36, 0x932a65de), + TOBN(0x9611ff5c, 0x3aa95529), TOBN(0xc58215b0, 0x7c1e5a36), + TOBN(0xd48c9b58, 0xf0e1a524), TOBN(0xb406856b, 0xf590dfb8), + TOBN(0xc7605e04, 0x9cd95662), TOBN(0x0dd036ee, 0xa33ecf82), + TOBN(0xa50171ac, 0xc33156b3), TOBN(0xf09d24ea, 0x4a80172e), + TOBN(0x4e1f72c6, 0x76dc8eef), TOBN(0xe60caadc, 0x5e3d44ee), + TOBN(0x006ef8a6, 0x979b1d8f), TOBN(0x60908a1c, 0x97788d26), + TOBN(0x6e08f95b, 0x266feec0), TOBN(0x618427c2, 0x22e8c94e), + TOBN(0x3d613339, 0x59145a65), TOBN(0xcd9bc368, 0xfa406337), + TOBN(0x82d11be3, 0x2d8a52a0), TOBN(0xf6877b27, 0x97a1c590), + TOBN(0x837a819b, 0xf5cbdb25), TOBN(0x2a4fd1d8, 0xde090249), + TOBN(0x622a7de7, 0x74990e5f), TOBN(0x840fa5a0, 0x7945511b), + TOBN(0x30b974be, 0x6558842d), TOBN(0x70df8c64, 0x17f3d0a6), + TOBN(0x7c803520, 0x7542e46d), TOBN(0x7251fe7f, 0xe4ecc823), + TOBN(0xe59134cb, 0x5e9aac9a), TOBN(0x11bb0934, 0xf0045d71), + TOBN(0x53e5d9b5, 0xdbcb1d4e), TOBN(0x8d97a905, 0x92defc91), + TOBN(0xfe289327, 0x7946d3f9), TOBN(0xe132bd24, 0x07472273), + TOBN(0xeeeb510c, 0x1eb6ae86), TOBN(0x777708c5, 0xf0595067), + TOBN(0x18e2c8cd, 0x1297029e), TOBN(0x2c61095c, 0xbbf9305e), + TOBN(0xe466c258, 0x6b85d6d9), TOBN(0x8ac06c36, 0xda1ea530), + TOBN(0xa365dc39, 0xa1304668), TOBN(0xe4a9c885, 0x07f89606), + TOBN(0x65a4898f, 0xacc7228d), TOBN(0x3e2347ff, 0x84ca8303), + TOBN(0xa5f6fb77, 0xea7d23a3), TOBN(0x2fac257d, 0x672a71cd), + TOBN(0x6908bef8, 0x7e6a44d3), TOBN(0x8ff87566, 0x891d3d7a), + TOBN(0xe58e90b3, 0x6b0cf82e), TOBN(0x6438d246, 0x2615b5e7), + TOBN(0x07b1f8fc, 0x669c145a), TOBN(0xb0d8b2da, 0x36f1e1cb), + TOBN(0x54d5dadb, 0xd9184c4d), TOBN(0x3dbb18d5, 0xf93d9976), + TOBN(0x0a3e0f56, 0xd1147d47), TOBN(0x2afa8c8d, 0xa0a48609), + TOBN(0x275353e8, 0xbc36742c), TOBN(0x898f427e, 0xeea0ed90), + TOBN(0x26f4947e, 0x3e477b00), TOBN(0x8ad8848a, 0x308741e3), + TOBN(0x6c703c38, 0xd74a2a46), TOBN(0x5e3e05a9, 0x9ba17ba2), + TOBN(0xc1fa6f66, 0x4ab9a9e4), TOBN(0x474a2d9a, 0x3841d6ec), + TOBN(0x871239ad, 0x653ae326), TOBN(0x14bcf72a, 0xa74cbb43), + TOBN(0x8737650e, 0x20d4c083), TOBN(0x3df86536, 0x110ed4af), + TOBN(0xd2d86fe7, 0xb53ca555), TOBN(0x688cb00d, 0xabd5d538), + TOBN(0xcf81bda3, 0x1ad38468), TOBN(0x7ccfe3cc, 0xf01167b6), + TOBN(0xcf4f47e0, 0x6c4c1fe6), TOBN(0x557e1f1a, 0x298bbb79), + TOBN(0xf93b974f, 0x30d45a14), TOBN(0x174a1d2d, 0x0baf97c4), + TOBN(0x7a003b30, 0xc51fbf53), TOBN(0xd8940991, 0xee68b225), + TOBN(0x5b0aa7b7, 0x1c0f4173), TOBN(0x975797c9, 0xa20a7153), + TOBN(0x26e08c07, 0xe3533d77), TOBN(0xd7222e6a, 0x2e341c99), + TOBN(0x9d60ec3d, 0x8d2dc4ed), TOBN(0xbdfe0d8f, 0x7c476cf8), + TOBN(0x1fe59ab6, 0x1d056605), TOBN(0xa9ea9df6, 0x86a8551f), + TOBN(0x8489941e, 0x47fb8d8c), TOBN(0xfeb874eb, 0x4a7f1b10), + TOBN(0xfe5fea86, 0x7ee0d98f), TOBN(0x201ad34b, 0xdbf61864), + TOBN(0x45d8fe47, 0x37c031d4), TOBN(0xd5f49fae, 0x795f0822), + TOBN(0xdb0fb291, 0xc7f4a40c), TOBN(0x2e69d9c1, 0x730ddd92), + TOBN(0x754e1054, 0x49d76987), TOBN(0x8a24911d, 0x7662db87), + TOBN(0x61fc1810, 0x60a71676), TOBN(0xe852d1a8, 0xf66a8ad1), + TOBN(0x172bbd65, 0x6417231e), TOBN(0x0d6de7bd, 0x3babb11f), + TOBN(0x6fde6f88, 0xc8e347f8), TOBN(0x1c587547, 0x9bd99cc3), + TOBN(0x78e54ed0, 0x34076950), TOBN(0x97f0f334, 0x796e83ba), + TOBN(0xe4dbe1ce, 0x4924867a), TOBN(0xbd5f51b0, 0x60b84917), + TOBN(0x37530040, 0x3cb09a79), TOBN(0xdb3fe0f8, 0xff1743d8), + TOBN(0xed7894d8, 0x556fa9db), TOBN(0xfa262169, 0x23412fbf), + TOBN(0x563be0db, 0xba7b9291), TOBN(0x6ca8b8c0, 0x0c9fb234), + TOBN(0xed406aa9, 0xbd763802), TOBN(0xc21486a0, 0x65303da1), + TOBN(0x61ae291e, 0xc7e62ec4), TOBN(0x622a0492, 0xdf99333e), + TOBN(0x7fd80c9d, 0xbb7a8ee0), TOBN(0xdc2ed3bc, 0x6c01aedb), + TOBN(0x35c35a12, 0x08be74ec), TOBN(0xd540cb1a, 0x469f671f), + TOBN(0xd16ced4e, 0xcf84f6c7), TOBN(0x8561fb9c, 0x2d090f43), + TOBN(0x7e693d79, 0x6f239db4), TOBN(0xa736f928, 0x77bd0d94), + TOBN(0x07b4d929, 0x2c1950ee), TOBN(0xda177543, 0x56dc11b3), + TOBN(0xa5dfbbaa, 0x7a6a878e), TOBN(0x1c70cb29, 0x4decb08a), + TOBN(0xfba28c8b, 0x6f0f7c50), TOBN(0xa8eba2b8, 0x854dcc6d), + TOBN(0x5ff8e89a, 0x36b78642), TOBN(0x070c1c8e, 0xf6873adf), + TOBN(0xbbd3c371, 0x6484d2e4), TOBN(0xfb78318f, 0x0d414129), + TOBN(0x2621a39c, 0x6ad93b0b), TOBN(0x979d74c2, 0xa9e917f7), + TOBN(0xfc195647, 0x61fb0428), TOBN(0x4d78954a, 0xbee624d4), + TOBN(0xb94896e0, 0xb8ae86fd), TOBN(0x6667ac0c, 0xc91c8b13), + TOBN(0x9f180512, 0x43bcf832), TOBN(0xfbadf8b7, 0xa0010137), + TOBN(0xc69b4089, 0xb3ba8aa7), TOBN(0xfac4bacd, 0xe687ce85), + TOBN(0x9164088d, 0x977eab40), TOBN(0x51f4c5b6, 0x2760b390), + TOBN(0xd238238f, 0x340dd553), TOBN(0x358566c3, 0xdb1d31c9), + TOBN(0x3a5ad69e, 0x5068f5ff), TOBN(0xf31435fc, 0xdaff6b06), + TOBN(0xae549a5b, 0xd6debff0), TOBN(0x59e5f0b7, 0x75e01331), + TOBN(0x5d492fb8, 0x98559acf), TOBN(0x96018c2e, 0x4db79b50), + TOBN(0x55f4a48f, 0x609f66aa), TOBN(0x1943b3af, 0x4900a14f), + TOBN(0xc22496df, 0x15a40d39), TOBN(0xb2a44684, 0x4c20f7c5), + TOBN(0x76a35afa, 0x3b98404c), TOBN(0xbec75725, 0xff5d1b77), + TOBN(0xb67aa163, 0xbea06444), TOBN(0x27e95bb2, 0xf724b6f2), + TOBN(0x3c20e3e9, 0xd238c8ab), TOBN(0x1213754e, 0xddd6ae17), + TOBN(0x8c431020, 0x716e0f74), TOBN(0x6679c82e, 0xffc095c2), + TOBN(0x2eb3adf4, 0xd0ac2932), TOBN(0x2cc970d3, 0x01bb7a76), + TOBN(0x70c71f2f, 0x740f0e66), TOBN(0x545c616b, 0x2b6b23cc), + TOBN(0x4528cfcb, 0xb40a8bd7), TOBN(0xff839633, 0x2ab27722), + TOBN(0x049127d9, 0x025ac99a), TOBN(0xd314d4a0, 0x2b63e33b), + TOBN(0xc8c310e7, 0x28d84519), TOBN(0x0fcb8983, 0xb3bc84ba), + TOBN(0x2cc52261, 0x38634818), TOBN(0x501814f4, 0xb44c2e0b), + TOBN(0xf7e181aa, 0x54dfdba3), TOBN(0xcfd58ff0, 0xe759718c), + TOBN(0xf90cdb14, 0xd3b507a8), TOBN(0x57bd478e, 0xc50bdad8), + TOBN(0x29c197e2, 0x50e5f9aa), TOBN(0x4db6eef8, 0xe40bc855), + TOBN(0x2cc8f21a, 0xd1fc0654), TOBN(0xc71cc963, 0x81269d73), + TOBN(0xecfbb204, 0x077f49f9), TOBN(0xdde92571, 0xca56b793), + TOBN(0x9abed6a3, 0xf97ad8f7), TOBN(0xe6c19d3f, 0x924de3bd), + TOBN(0x8dce92f4, 0xa140a800), TOBN(0x85f44d1e, 0x1337af07), + TOBN(0x5953c08b, 0x09d64c52), TOBN(0xa1b5e49f, 0xf5df9749), + TOBN(0x336a8fb8, 0x52735f7d), TOBN(0xb332b6db, 0x9add676b), + TOBN(0x558b88a0, 0xb4511aa4), TOBN(0x09788752, 0xdbd5cc55), + TOBN(0x16b43b9c, 0xd8cd52bd), TOBN(0x7f0bc5a0, 0xc2a2696b), + TOBN(0x146e12d4, 0xc11f61ef), TOBN(0x9ce10754, 0x3a83e79e), + TOBN(0x08ec73d9, 0x6cbfca15), TOBN(0x09ff29ad, 0x5b49653f), + TOBN(0xe31b72bd, 0xe7da946e), TOBN(0xebf9eb3b, 0xee80a4f2), + TOBN(0xd1aabd08, 0x17598ce4), TOBN(0x18b5fef4, 0x53f37e80), + TOBN(0xd5d5cdd3, 0x5958cd79), TOBN(0x3580a1b5, 0x1d373114), + TOBN(0xa36e4c91, 0xfa935726), TOBN(0xa38c534d, 0xef20d760), + TOBN(0x7088e40a, 0x2ff5845b), TOBN(0xe5bb40bd, 0xbd78177f), + TOBN(0x4f06a7a8, 0x857f9920), TOBN(0xe3cc3e50, 0xe968f05d), + TOBN(0x1d68b7fe, 0xe5682d26), TOBN(0x5206f76f, 0xaec7f87c), + TOBN(0x41110530, 0x041951ab), TOBN(0x58ec52c1, 0xd4b5a71a), + TOBN(0xf3488f99, 0x0f75cf9a), TOBN(0xf411951f, 0xba82d0d5), + TOBN(0x27ee75be, 0x618895ab), TOBN(0xeae060d4, 0x6d8aab14), + TOBN(0x9ae1df73, 0x7fb54dc2), TOBN(0x1f3e391b, 0x25963649), + TOBN(0x242ec32a, 0xfe055081), TOBN(0x5bd450ef, 0x8491c9bd), + TOBN(0x367efc67, 0x981eb389), TOBN(0xed7e1928, 0x3a0550d5), + TOBN(0x362e776b, 0xab3ce75c), TOBN(0xe890e308, 0x1f24c523), + TOBN(0xb961b682, 0xfeccef76), TOBN(0x8b8e11f5, 0x8bba6d92), + TOBN(0x8f2ccc4c, 0x2b2375c4), TOBN(0x0d7f7a52, 0xe2f86cfa), + TOBN(0xfd94d30a, 0x9efe5633), TOBN(0x2d8d246b, 0x5451f934), + TOBN(0x2234c6e3, 0x244e6a00), TOBN(0xde2b5b0d, 0xddec8c50), + TOBN(0x2ce53c5a, 0xbf776f5b), TOBN(0x6f724071, 0x60357b05), + TOBN(0xb2593717, 0x71bf3f7a), TOBN(0x87d2501c, 0x440c4a9f), + TOBN(0x440552e1, 0x87b05340), TOBN(0xb7bf7cc8, 0x21624c32), + TOBN(0x4155a6ce, 0x22facddb), TOBN(0x5a4228cb, 0x889837ef), + TOBN(0xef87d6d6, 0xfd4fd671), TOBN(0xa233687e, 0xc2daa10e), + TOBN(0x75622244, 0x03c0eb96), TOBN(0x7632d184, 0x8bf19be6), + TOBN(0x05d0f8e9, 0x40735ff4), TOBN(0x3a3e6e13, 0xc00931f1), + TOBN(0x31ccde6a, 0xdafe3f18), TOBN(0xf381366a, 0xcfe51207), + TOBN(0x24c222a9, 0x60167d92), TOBN(0x62f9d6f8, 0x7529f18c), + TOBN(0x412397c0, 0x0353b114), TOBN(0x334d89dc, 0xef808043), + TOBN(0xd9ec63ba, 0x2a4383ce), TOBN(0xcec8e937, 0x5cf92ba0), + TOBN(0xfb8b4288, 0xc8be74c0), TOBN(0x67d6912f, 0x105d4391), + TOBN(0x7b996c46, 0x1b913149), TOBN(0x36aae2ef, 0x3a4e02da), + TOBN(0xb68aa003, 0x972de594), TOBN(0x284ec70d, 0x4ec6d545), + TOBN(0xf3d2b2d0, 0x61391d54), TOBN(0x69c5d5d6, 0xfe114e92), + TOBN(0xbe0f00b5, 0xb4482dff), TOBN(0xe1596fa5, 0xf5bf33c5), + TOBN(0x10595b56, 0x96a71cba), TOBN(0x944938b2, 0xfdcadeb7), + TOBN(0xa282da4c, 0xfccd8471), TOBN(0x98ec05f3, 0x0d37bfe1), + TOBN(0xe171ce1b, 0x0698304a), TOBN(0x2d691444, 0x21bdf79b), + TOBN(0xd0cd3b74, 0x1b21dec1), TOBN(0x712ecd8b, 0x16a15f71), + TOBN(0x8d4c00a7, 0x00fd56e1), TOBN(0x02ec9692, 0xf9527c18), + TOBN(0x21c44937, 0x4a3e42e1), TOBN(0x9176fbab, 0x1392ae0a), + TOBN(0x8726f1ba, 0x44b7b618), TOBN(0xb4d7aae9, 0xf1de491c), + TOBN(0xf91df7b9, 0x07b582c0), TOBN(0x7e116c30, 0xef60aa3a), + TOBN(0x99270f81, 0x466265d7), TOBN(0xb15b6fe2, 0x4df7adf0), + TOBN(0xfe33b2d3, 0xf9738f7f), TOBN(0x48553ab9, 0xd6d70f95), + TOBN(0x2cc72ac8, 0xc21e94db), TOBN(0x795ac38d, 0xbdc0bbee), + TOBN(0x0a1be449, 0x2e40478f), TOBN(0x81bd3394, 0x052bde55), + TOBN(0x63c8dbe9, 0x56b3c4f2), TOBN(0x017a99cf, 0x904177cc), + TOBN(0x947bbddb, 0x4d010fc1), TOBN(0xacf9b00b, 0xbb2c9b21), + TOBN(0x2970bc8d, 0x47173611), TOBN(0x1a4cbe08, 0xac7d756f), + TOBN(0x06d9f4aa, 0x67d541a2), TOBN(0xa3e8b689, 0x59c2cf44), + TOBN(0xaad066da, 0x4d88f1dd), TOBN(0xc604f165, 0x7ad35dea), + TOBN(0x7edc0720, 0x4478ca67), TOBN(0xa10dfae0, 0xba02ce06), + TOBN(0xeceb1c76, 0xaf36f4e4), TOBN(0x994b2292, 0xaf3f8f48), + TOBN(0xbf9ed77b, 0x77c8a68c), TOBN(0x74f544ea, 0x51744c9d), + TOBN(0x82d05bb9, 0x8113a757), TOBN(0x4ef2d2b4, 0x8a9885e4), + TOBN(0x1e332be5, 0x1aa7865f), TOBN(0x22b76b18, 0x290d1a52), + TOBN(0x308a2310, 0x44351683), TOBN(0x9d861896, 0xa3f22840), + TOBN(0x5959ddcd, 0x841ed947), TOBN(0x0def0c94, 0x154b73bf), + TOBN(0xf0105417, 0x4c7c15e0), TOBN(0x539bfb02, 0x3a277c32), + TOBN(0xe699268e, 0xf9dccf5f), TOBN(0x9f5796a5, 0x0247a3bd), + TOBN(0x8b839de8, 0x4f157269), TOBN(0xc825c1e5, 0x7a30196b), + TOBN(0x6ef0aabc, 0xdc8a5a91), TOBN(0xf4a8ce6c, 0x498b7fe6), + TOBN(0x1cce35a7, 0x70cbac78), TOBN(0x83488e9b, 0xf6b23958), + TOBN(0x0341a070, 0xd76cb011), TOBN(0xda6c9d06, 0xae1b2658), + TOBN(0xb701fb30, 0xdd648c52), TOBN(0x994ca02c, 0x52fb9fd1), + TOBN(0x06933117, 0x6f563086), TOBN(0x3d2b8100, 0x17856bab), + TOBN(0xe89f48c8, 0x5963a46e), TOBN(0x658ab875, 0xa99e61c7), + TOBN(0x6e296f87, 0x4b8517b4), TOBN(0x36c4fcdc, 0xfc1bc656), + TOBN(0xde5227a1, 0xa3906def), TOBN(0x9fe95f57, 0x62418945), + TOBN(0x20c91e81, 0xfdd96cde), TOBN(0x5adbe47e, 0xda4480de), + TOBN(0xa009370f, 0x396de2b6), TOBN(0x98583d4b, 0xf0ecc7bd), + TOBN(0xf44f6b57, 0xe51d0672), TOBN(0x03d6b078, 0x556b1984), + TOBN(0x27dbdd93, 0xb0b64912), TOBN(0x9b3a3434, 0x15687b09), + TOBN(0x0dba6461, 0x51ec20a9), TOBN(0xec93db7f, 0xff28187c), + TOBN(0x00ff8c24, 0x66e48bdd), TOBN(0x2514f2f9, 0x11ccd78e), + TOBN(0xeba11f4f, 0xe1250603), TOBN(0x8a22cd41, 0x243fa156), + TOBN(0xa4e58df4, 0xb283e4c6), TOBN(0x78c29859, 0x8b39783f), + TOBN(0x5235aee2, 0xa5259809), TOBN(0xc16284b5, 0x0e0227dd), + TOBN(0xa5f57916, 0x1338830d), TOBN(0x6d4b8a6b, 0xd2123fca), + TOBN(0x236ea68a, 0xf9c546f8), TOBN(0xc1d36873, 0xfa608d36), + TOBN(0xcd76e495, 0x8d436d13), TOBN(0xd4d9c221, 0x8fb080af), + TOBN(0x665c1728, 0xe8ad3fb5), TOBN(0xcf1ebe4d, 0xb3d572e0), + TOBN(0xa7a8746a, 0x584c5e20), TOBN(0x267e4ea1, 0xb9dc7035), + TOBN(0x593a15cf, 0xb9548c9b), TOBN(0x5e6e2135, 0x4bd012f3), + TOBN(0xdf31cc6a, 0x8c8f936e), TOBN(0x8af84d04, 0xb5c241dc), + TOBN(0x63990a6f, 0x345efb86), TOBN(0x6fef4e61, 0xb9b962cb)} + , + {TOBN(0xf6368f09, 0x25722608), TOBN(0x131260db, 0x131cf5c6), + TOBN(0x40eb353b, 0xfab4f7ac), TOBN(0x85c78880, 0x37eee829), + TOBN(0x4c1581ff, 0xc3bdf24e), TOBN(0x5bff75cb, 0xf5c3c5a8), + TOBN(0x35e8c83f, 0xa14e6f40), TOBN(0xb81d1c0f, 0x0295e0ca), + TOBN(0xfcde7cc8, 0xf43a730f), TOBN(0xe89b6f3c, 0x33ab590e), + TOBN(0xc823f529, 0xad03240b), TOBN(0x82b79afe, 0x98bea5db), + TOBN(0x568f2856, 0x962fe5de), TOBN(0x0c590adb, 0x60c591f3), + TOBN(0x1fc74a14, 0x4a28a858), TOBN(0x3b662498, 0xb3203f4c), + TOBN(0x91e3cf0d, 0x6c39765a), TOBN(0xa2db3acd, 0xac3cca0b), + TOBN(0x288f2f08, 0xcb953b50), TOBN(0x2414582c, 0xcf43cf1a), + TOBN(0x8dec8bbc, 0x60eee9a8), TOBN(0x54c79f02, 0x729aa042), + TOBN(0xd81cd5ec, 0x6532f5d5), TOBN(0xa672303a, 0xcf82e15f), + TOBN(0x376aafa8, 0x719c0563), TOBN(0xcd8ad2dc, 0xbc5fc79f), + TOBN(0x303fdb9f, 0xcb750cd3), TOBN(0x14ff052f, 0x4418b08e), + TOBN(0xf75084cf, 0x3e2d6520), TOBN(0x7ebdf0f8, 0x144ed509), + TOBN(0xf43bf0f2, 0xd3f25b98), TOBN(0x86ad71cf, 0xa354d837), + TOBN(0xb827fe92, 0x26f43572), TOBN(0xdfd3ab5b, 0x5d824758), + TOBN(0x315dd23a, 0x539094c1), TOBN(0x85c0e37a, 0x66623d68), + TOBN(0x575c7972, 0x7be19ae0), TOBN(0x616a3396, 0xdf0d36b5), + TOBN(0xa1ebb3c8, 0x26b1ff7e), TOBN(0x635b9485, 0x140ad453), + TOBN(0x92bf3cda, 0xda430c0b), TOBN(0x4702850e, 0x3a96dac6), + TOBN(0xc91cf0a5, 0x15ac326a), TOBN(0x95de4f49, 0xab8c25e4), + TOBN(0xb01bad09, 0xe265c17c), TOBN(0x24e45464, 0x087b3881), + TOBN(0xd43e583c, 0xe1fac5ca), TOBN(0xe17cb318, 0x6ead97a6), + TOBN(0x6cc39243, 0x74dcec46), TOBN(0x33cfc02d, 0x54c2b73f), + TOBN(0x82917844, 0xf26cd99c), TOBN(0x8819dd95, 0xd1773f89), + TOBN(0x09572aa6, 0x0871f427), TOBN(0x8e0cf365, 0xf6f01c34), + TOBN(0x7fa52988, 0xbff1f5af), TOBN(0x4eb357ea, 0xe75e8e50), + TOBN(0xd9d0c8c4, 0x868af75d), TOBN(0xd7325cff, 0x45c8c7ea), + TOBN(0xab471996, 0xcc81ecb0), TOBN(0xff5d55f3, 0x611824ed), + TOBN(0xbe314541, 0x1977a0ee), TOBN(0x5085c4c5, 0x722038c6), + TOBN(0x2d5335bf, 0xf94bb495), TOBN(0x894ad8a6, 0xc8e2a082), + TOBN(0x5c3e2341, 0xada35438), TOBN(0xf4a9fc89, 0x049b8c4e), + TOBN(0xbeeb355a, 0x9f17cf34), TOBN(0x3f311e0e, 0x6c91fe10), + TOBN(0xc2d20038, 0x92ab9891), TOBN(0x257bdcc1, 0x3e8ce9a9), + TOBN(0x1b2d9789, 0x88c53bee), TOBN(0x927ce89a, 0xcdba143a), + TOBN(0xb0a32cca, 0x523db280), TOBN(0x5c889f8a, 0x50d43783), + TOBN(0x503e04b3, 0x4897d16f), TOBN(0x8cdb6e78, 0x08f5f2e8), + TOBN(0x6ab91cf0, 0x179c8e74), TOBN(0xd8874e52, 0x48211d60), + TOBN(0xf948d4d5, 0xea851200), TOBN(0x4076d41e, 0xe6f9840a), + TOBN(0xc20e263c, 0x47b517ea), TOBN(0x79a448fd, 0x30685e5e), + TOBN(0xe55f6f78, 0xf90631a0), TOBN(0x88a790b1, 0xa79e6346), + TOBN(0x62160c7d, 0x80969fe8), TOBN(0x54f92fd4, 0x41491bb9), + TOBN(0xa6645c23, 0x5c957526), TOBN(0xf44cc5ae, 0xbea3ce7b), + TOBN(0xf7628327, 0x8b1e68b7), TOBN(0xc731ad7a, 0x303f29d3), + TOBN(0xfe5a9ca9, 0x57d03ecb), TOBN(0x96c0d50c, 0x41bc97a7), + TOBN(0xc4669fe7, 0x9b4f7f24), TOBN(0xfdd781d8, 0x3d9967ef), + TOBN(0x7892c7c3, 0x5d2c208d), TOBN(0x8bf64f7c, 0xae545cb3), + TOBN(0xc01f862c, 0x467be912), TOBN(0xf4c85ee9, 0xc73d30cc), + TOBN(0x1fa6f4be, 0x6ab83ec7), TOBN(0xa07a3c1c, 0x4e3e3cf9), + TOBN(0x87f8ef45, 0x0c00beb3), TOBN(0x30e2c2b3, 0x000d4c3e), + TOBN(0x1aa00b94, 0xfe08bf5b), TOBN(0x32c133aa, 0x9224ef52), + TOBN(0x38df16bb, 0x32e5685d), TOBN(0x68a9e069, 0x58e6f544), + TOBN(0x495aaff7, 0xcdc5ebc6), TOBN(0xf894a645, 0x378b135f), + TOBN(0xf316350a, 0x09e27ecf), TOBN(0xeced201e, 0x58f7179d), + TOBN(0x2eec273c, 0xe97861ba), TOBN(0x47ec2cae, 0xd693be2e), + TOBN(0xfa4c97c4, 0xf68367ce), TOBN(0xe4f47d0b, 0xbe5a5755), + TOBN(0x17de815d, 0xb298a979), TOBN(0xd7eca659, 0xc177dc7d), + TOBN(0x20fdbb71, 0x49ded0a3), TOBN(0x4cb2aad4, 0xfb34d3c5), + TOBN(0x2cf31d28, 0x60858a33), TOBN(0x3b6873ef, 0xa24aa40f), + TOBN(0x540234b2, 0x2c11bb37), TOBN(0x2d0366dd, 0xed4c74a3), + TOBN(0xf9a968da, 0xeec5f25d), TOBN(0x36601068, 0x67b63142), + TOBN(0x07cd6d2c, 0x68d7b6d4), TOBN(0xa8f74f09, 0x0c842942), + TOBN(0xe2751404, 0x7768b1ee), TOBN(0x4b5f7e89, 0xfe62aee4), + TOBN(0xc6a77177, 0x89070d26), TOBN(0xa1f28e4e, 0xdd1c8bc7), + TOBN(0xea5f4f06, 0x469e1f17), TOBN(0x78fc242a, 0xfbdb78e0), + TOBN(0xc9c7c592, 0x8b0588f1), TOBN(0xb6b7a0fd, 0x1535921e), + TOBN(0xcc5bdb91, 0xbde5ae35), TOBN(0xb42c485e, 0x12ff1864), + TOBN(0xa1113e13, 0xdbab98aa), TOBN(0xde9d469b, 0xa17b1024), + TOBN(0x23f48b37, 0xc0462d3a), TOBN(0x3752e537, 0x7c5c078d), + TOBN(0xe3a86add, 0x15544eb9), TOBN(0xf013aea7, 0x80fba279), + TOBN(0x8b5bb76c, 0xf22001b5), TOBN(0xe617ba14, 0xf02891ab), + TOBN(0xd39182a6, 0x936219d3), TOBN(0x5ce1f194, 0xae51cb19), + TOBN(0xc78f8598, 0xbf07a74c), TOBN(0x6d7158f2, 0x22cbf1bc), + TOBN(0x3b846b21, 0xe300ce18), TOBN(0x35fba630, 0x2d11275d), + TOBN(0x5fe25c36, 0xa0239b9b), TOBN(0xd8beb35d, 0xdf05d940), + TOBN(0x4db02bb0, 0x1f7e320d), TOBN(0x0641c364, 0x6da320ea), + TOBN(0x6d95fa5d, 0x821389a3), TOBN(0x92699748, 0x8fcd8e3d), + TOBN(0x316fef17, 0xceb6c143), TOBN(0x67fcb841, 0xd933762b), + TOBN(0xbb837e35, 0x118b17f8), TOBN(0x4b92552f, 0x9fd24821), + TOBN(0xae6bc70e, 0x46aca793), TOBN(0x1cf0b0e4, 0xe579311b), + TOBN(0x8dc631be, 0x5802f716), TOBN(0x099bdc6f, 0xbddbee4d), + TOBN(0xcc352bb2, 0x0caf8b05), TOBN(0xf74d505a, 0x72d63df2), + TOBN(0xb9876d4b, 0x91c4f408), TOBN(0x1ce18473, 0x9e229b2d), + TOBN(0x49507597, 0x83abdb4a), TOBN(0x850fbcb6, 0xdee84b18), + TOBN(0x6325236e, 0x609e67dc), TOBN(0x04d831d9, 0x9336c6d8), + TOBN(0x8deaae3b, 0xfa12d45d), TOBN(0xe425f8ce, 0x4746e246), + TOBN(0x8004c175, 0x24f5f31e), TOBN(0xaca16d8f, 0xad62c3b7), + TOBN(0x0dc15a6a, 0x9152f934), TOBN(0xf1235e5d, 0xed0e12c1), + TOBN(0xc33c06ec, 0xda477dac), TOBN(0x76be8732, 0xb2ea0006), + TOBN(0xcf3f7831, 0x0c0cd313), TOBN(0x3c524553, 0xa614260d), + TOBN(0x31a756f8, 0xcab22d15), TOBN(0x03ee10d1, 0x77827a20), + TOBN(0xd1e059b2, 0x1994ef20), TOBN(0x2a653b69, 0x638ae318), + TOBN(0x70d5eb58, 0x2f699010), TOBN(0x279739f7, 0x09f5f84a), + TOBN(0x5da4663c, 0x8b799336), TOBN(0xfdfdf14d, 0x203c37eb), + TOBN(0x32d8a9dc, 0xa1dbfb2d), TOBN(0xab40cff0, 0x77d48f9b), + TOBN(0xc018b383, 0xd20b42d5), TOBN(0xf9a810ef, 0x9f78845f), + TOBN(0x40af3753, 0xbdba9df0), TOBN(0xb90bdcfc, 0x131dfdf9), + TOBN(0x18720591, 0xf01ab782), TOBN(0xc823f211, 0x6af12a88), + TOBN(0xa51b80f3, 0x0dc14401), TOBN(0xde248f77, 0xfb2dfbe3), + TOBN(0xef5a44e5, 0x0cafe751), TOBN(0x73997c9c, 0xd4dcd221), + TOBN(0x32fd86d1, 0xde854024), TOBN(0xd5b53adc, 0xa09b84bb), + TOBN(0x008d7a11, 0xdcedd8d1), TOBN(0x406bd1c8, 0x74b32c84), + TOBN(0x5d4472ff, 0x05dde8b1), TOBN(0x2e25f2cd, 0xfce2b32f), + TOBN(0xbec0dd5e, 0x29dfc254), TOBN(0x4455fcf6, 0x2b98b267), + TOBN(0x0b4d43a5, 0xc72df2ad), TOBN(0xea70e6be, 0x48a75397), + TOBN(0x2aad6169, 0x5820f3bf), TOBN(0xf410d2dd, 0x9e37f68f), + TOBN(0x70fb7dba, 0x7be5ac83), TOBN(0x636bb645, 0x36ec3eec), + TOBN(0x27104ea3, 0x9754e21c), TOBN(0xbc87a3e6, 0x8d63c373), + TOBN(0x483351d7, 0x4109db9a), TOBN(0x0fa724e3, 0x60134da7), + TOBN(0x9ff44c29, 0xb0720b16), TOBN(0x2dd0cf13, 0x06aceead), + TOBN(0x5942758c, 0xe26929a6), TOBN(0x96c5db92, 0xb766a92b), + TOBN(0xcec7d4c0, 0x5f18395e), TOBN(0xd3f22744, 0x1f80d032), + TOBN(0x7a68b37a, 0xcb86075b), TOBN(0x074764dd, 0xafef92db), + TOBN(0xded1e950, 0x7bc7f389), TOBN(0xc580c850, 0xb9756460), + TOBN(0xaeeec2a4, 0x7da48157), TOBN(0x3f0b4e7f, 0x82c587b3), + TOBN(0x231c6de8, 0xa9f19c53), TOBN(0x5717bd73, 0x6974e34e), + TOBN(0xd9e1d216, 0xf1508fa9), TOBN(0x9f112361, 0xdadaa124), + TOBN(0x80145e31, 0x823b7348), TOBN(0x4dd8f0d5, 0xac634069), + TOBN(0xe3d82fc7, 0x2297c258), TOBN(0x276fcfee, 0x9cee7431), + TOBN(0x8eb61b5e, 0x2bc0aea9), TOBN(0x4f668fd5, 0xde329431), + TOBN(0x03a32ab1, 0x38e4b87e), TOBN(0xe1374517, 0x73d0ef0b), + TOBN(0x1a46f7e6, 0x853ac983), TOBN(0xc3bdf42e, 0x68e78a57), + TOBN(0xacf20785, 0x2ea96dd1), TOBN(0xa10649b9, 0xf1638460), + TOBN(0xf2369f0b, 0x879fbbed), TOBN(0x0ff0ae86, 0xda9d1869), + TOBN(0x5251d759, 0x56766f45), TOBN(0x4984d8c0, 0x2be8d0fc), + TOBN(0x7ecc95a6, 0xd21008f0), TOBN(0x29bd54a0, 0x3a1a1c49), + TOBN(0xab9828c5, 0xd26c50f3), TOBN(0x32c0087c, 0x51d0d251), + TOBN(0x9bac3ce6, 0x0c1cdb26), TOBN(0xcd94d947, 0x557ca205), + TOBN(0x1b1bd598, 0x9db1fdcd), TOBN(0x0eda0108, 0xa3d8b149), + TOBN(0x95066610, 0x56152fcc), TOBN(0xc2f037e6, 0xe7192b33), + TOBN(0xdeffb41a, 0xc92e05a4), TOBN(0x1105f6c2, 0xc2f6c62e), + TOBN(0x68e73500, 0x8733913c), TOBN(0xcce86163, 0x3f3adc40), + TOBN(0xf407a942, 0x38a278e9), TOBN(0xd13c1b9d, 0x2ab21292), + TOBN(0x93ed7ec7, 0x1c74cf5c), TOBN(0x8887dc48, 0xf1a4c1b4), + TOBN(0x3830ff30, 0x4b3a11f1), TOBN(0x358c5a3c, 0x58937cb6), + TOBN(0x027dc404, 0x89022829), TOBN(0x40e93977, 0x3b798f79), + TOBN(0x90ad3337, 0x38be6ead), TOBN(0x9c23f6bc, 0xf34c0a5d), + TOBN(0xd1711a35, 0xfbffd8bb), TOBN(0x60fcfb49, 0x1949d3dd), + TOBN(0x09c8ef4b, 0x7825d93a), TOBN(0x24233cff, 0xa0a8c968), + TOBN(0x67ade46c, 0xe6d982af), TOBN(0xebb6bf3e, 0xe7544d7c), + TOBN(0xd6b9ba76, 0x3d8bd087), TOBN(0x46fe382d, 0x4dc61280), + TOBN(0xbd39a7e8, 0xb5bdbd75), TOBN(0xab381331, 0xb8f228fe), + TOBN(0x0709a77c, 0xce1c4300), TOBN(0x6a247e56, 0xf337ceac), + TOBN(0x8f34f21b, 0x636288be), TOBN(0x9dfdca74, 0xc8a7c305), + TOBN(0x6decfd1b, 0xea919e04), TOBN(0xcdf2688d, 0x8e1991f8), + TOBN(0xe607df44, 0xd0f8a67e), TOBN(0xd985df4b, 0x0b58d010), + TOBN(0x57f834c5, 0x0c24f8f4), TOBN(0xe976ef56, 0xa0bf01ae), + TOBN(0x536395ac, 0xa1c32373), TOBN(0x351027aa, 0x734c0a13), + TOBN(0xd2f1b5d6, 0x5e6bd5bc), TOBN(0x2b539e24, 0x223debed), + TOBN(0xd4994cec, 0x0eaa1d71), TOBN(0x2a83381d, 0x661dcf65), + TOBN(0x5f1aed2f, 0x7b54c740), TOBN(0x0bea3fa5, 0xd6dda5ee), + TOBN(0x9d4fb684, 0x36cc6134), TOBN(0x8eb9bbf3, 0xc0a443dd), + TOBN(0xfc500e2e, 0x383b7d2a), TOBN(0x7aad621c, 0x5b775257), + TOBN(0x69284d74, 0x0a8f7cc0), TOBN(0xe820c2ce, 0x07562d65), + TOBN(0xbf9531b9, 0x499758ee), TOBN(0x73e95ca5, 0x6ee0cc2d), + TOBN(0xf61790ab, 0xfbaf50a5), TOBN(0xdf55e76b, 0x684e0750), + TOBN(0xec516da7, 0xf176b005), TOBN(0x575553bb, 0x7a2dddc7), + TOBN(0x37c87ca3, 0x553afa73), TOBN(0x315f3ffc, 0x4d55c251), + TOBN(0xe846442a, 0xaf3e5d35), TOBN(0x61b91149, 0x6495ff28), + TOBN(0x23cc95d3, 0xfa326dc3), TOBN(0x1df4da1f, 0x18fc2cea), + TOBN(0x24bf9adc, 0xd0a37d59), TOBN(0xb6710053, 0x320d6e1e), + TOBN(0x96f9667e, 0x618344d1), TOBN(0xcc7ce042, 0xa06445af), + TOBN(0xa02d8514, 0xd68dbc3a), TOBN(0x4ea109e4, 0x280b5a5b), + TOBN(0x5741a7ac, 0xb40961bf), TOBN(0x4ada5937, 0x6aa56bfa), + TOBN(0x7feb9145, 0x02b765d1), TOBN(0x561e97be, 0xe6ad1582), + TOBN(0xbbc4a5b6, 0xda3982f5), TOBN(0x0c2659ed, 0xb546f468), + TOBN(0xb8e7e6aa, 0x59612d20), TOBN(0xd83dfe20, 0xac19e8e0), + TOBN(0x8530c45f, 0xb835398c), TOBN(0x6106a8bf, 0xb38a41c2), + TOBN(0x21e8f9a6, 0x35f5dcdb), TOBN(0x39707137, 0xcae498ed), + TOBN(0x70c23834, 0xd8249f00), TOBN(0x9f14b58f, 0xab2537a0), + TOBN(0xd043c365, 0x5f61c0c2), TOBN(0xdc5926d6, 0x09a194a7), + TOBN(0xddec0339, 0x8e77738a), TOBN(0xd07a63ef, 0xfba46426), + TOBN(0x2e58e79c, 0xee7f6e86), TOBN(0xe59b0459, 0xff32d241), + TOBN(0xc5ec84e5, 0x20fa0338), TOBN(0x97939ac8, 0xeaff5ace), + TOBN(0x0310a4e3, 0xb4a38313), TOBN(0x9115fba2, 0x8f9d9885), + TOBN(0x8dd710c2, 0x5fadf8c3), TOBN(0x66be38a2, 0xce19c0e2), + TOBN(0xd42a279c, 0x4cfe5022), TOBN(0x597bb530, 0x0e24e1b8), + TOBN(0x3cde86b7, 0xc153ca7f), TOBN(0xa8d30fb3, 0x707d63bd), + TOBN(0xac905f92, 0xbd60d21e), TOBN(0x98e7ffb6, 0x7b9a54ab), + TOBN(0xd7147df8, 0xe9726a30), TOBN(0xb5e216ff, 0xafce3533), + TOBN(0xb550b799, 0x2ff1ec40), TOBN(0x6b613b87, 0xa1e953fd), + TOBN(0x87b88dba, 0x792d5610), TOBN(0x2ee1270a, 0xa190fbe1), + TOBN(0x02f4e2dc, 0x2ef581da), TOBN(0x016530e4, 0xeff82a95), + TOBN(0xcbb93dfd, 0x8fd6ee89), TOBN(0x16d3d986, 0x46848fff), + TOBN(0x600eff24, 0x1da47adf), TOBN(0x1b9754a0, 0x0ad47a71), + TOBN(0x8f9266df, 0x70c33b98), TOBN(0xaadc87ae, 0xdf34186e), + TOBN(0x0d2ce8e1, 0x4ad24132), TOBN(0x8a47cbfc, 0x19946eba), + TOBN(0x47feeb66, 0x62b5f3af), TOBN(0xcefab561, 0x0abb3734), + TOBN(0x449de60e, 0x19f35cb1), TOBN(0x39f8db14, 0x157f0eb9), + TOBN(0xffaecc5b, 0x3c61bfd6), TOBN(0xa5a4d41d, 0x41216703), + TOBN(0x7f8fabed, 0x224e1cc2), TOBN(0x0d5a8186, 0x871ad953), + TOBN(0xf10774f7, 0xd22da9a9), TOBN(0x45b8a678, 0xcc8a9b0d), + TOBN(0xd9c2e722, 0xbdc32cff), TOBN(0xbf71b5f5, 0x337202a5), + TOBN(0x95c57f2f, 0x69fc4db9), TOBN(0xb6dad34c, 0x765d01e1), + TOBN(0x7e0bd13f, 0xcb904635), TOBN(0x61751253, 0x763a588c), + TOBN(0xd85c2997, 0x81af2c2d), TOBN(0xc0f7d9c4, 0x81b9d7da), + TOBN(0x838a34ae, 0x08533e8d), TOBN(0x15c4cb08, 0x311d8311), + TOBN(0x97f83285, 0x8e121e14), TOBN(0xeea7dc1e, 0x85000a5f), + TOBN(0x0c6059b6, 0x5d256274), TOBN(0xec9beace, 0xb95075c0), + TOBN(0x173daad7, 0x1df97828), TOBN(0xbf851cb5, 0xa8937877), + TOBN(0xb083c594, 0x01646f3c), TOBN(0x3bad30cf, 0x50c6d352), + TOBN(0xfeb2b202, 0x496bbcea), TOBN(0x3cf9fd4f, 0x18a1e8ba), + TOBN(0xd26de7ff, 0x1c066029), TOBN(0x39c81e9e, 0x4e9ed4f8), + TOBN(0xd8be0cb9, 0x7b390d35), TOBN(0x01df2bbd, 0x964aab27), + TOBN(0x3e8c1a65, 0xc3ef64f8), TOBN(0x567291d1, 0x716ed1dd), + TOBN(0x95499c6c, 0x5f5406d3), TOBN(0x71fdda39, 0x5ba8e23f), + TOBN(0xcfeb320e, 0xd5096ece), TOBN(0xbe7ba92b, 0xca66dd16), + TOBN(0x4608d36b, 0xc6fb5a7d), TOBN(0xe3eea15a, 0x6d2dd0e0), + TOBN(0x75b0a3eb, 0x8f97a36a), TOBN(0xf59814cc, 0x1c83de1e), + TOBN(0x56c9c5b0, 0x1c33c23f), TOBN(0xa96c1da4, 0x6faa4136), + TOBN(0x46bf2074, 0xde316551), TOBN(0x3b866e7b, 0x1f756c8f), + TOBN(0x727727d8, 0x1495ed6b), TOBN(0xb2394243, 0xb682dce7), + TOBN(0x8ab8454e, 0x758610f3), TOBN(0xc243ce84, 0x857d72a4), + TOBN(0x7b320d71, 0xdbbf370f), TOBN(0xff9afa37, 0x78e0f7ca), + TOBN(0x0119d1e0, 0xea7b523f), TOBN(0xb997f8cb, 0x058c7d42), + TOBN(0x285bcd2a, 0x37bbb184), TOBN(0x51dcec49, 0xa45d1fa6), + TOBN(0x6ade3b64, 0xe29634cb), TOBN(0x080c94a7, 0x26b86ef1), + TOBN(0xba583db1, 0x2283fbe3), TOBN(0x902bddc8, 0x5a9315ed), + TOBN(0x07c1ccb3, 0x86964bec), TOBN(0x78f4eacf, 0xb6258301), + TOBN(0x4bdf3a49, 0x56f90823), TOBN(0xba0f5080, 0x741d777b), + TOBN(0x091d71c3, 0xf38bf760), TOBN(0x9633d50f, 0x9b625b02), + TOBN(0x03ecb743, 0xb8c9de61), TOBN(0xb4751254, 0x5de74720), + TOBN(0x9f9defc9, 0x74ce1cb2), TOBN(0x774a4f6a, 0x00bd32ef), + TOBN(0xaca385f7, 0x73848f22), TOBN(0x53dad716, 0xf3f8558e), + TOBN(0xab7b34b0, 0x93c471f9), TOBN(0xf530e069, 0x19644bc7), + TOBN(0x3d9fb1ff, 0xdd59d31a), TOBN(0x4382e0df, 0x08daa795), + TOBN(0x165c6f4b, 0xd5cc88d7), TOBN(0xeaa392d5, 0x4a18c900), + TOBN(0x94203c67, 0x648024ee), TOBN(0x188763f2, 0x8c2fabcd), + TOBN(0xa80f87ac, 0xbbaec835), TOBN(0x632c96e0, 0xf29d8d54), + TOBN(0x29b0a60e, 0x4c00a95e), TOBN(0x2ef17f40, 0xe011e9fa), + TOBN(0xf6c0e1d1, 0x15b77223), TOBN(0xaaec2c62, 0x14b04e32), + TOBN(0xd35688d8, 0x3d84e58c), TOBN(0x2af5094c, 0x958571db), + TOBN(0x4fff7e19, 0x760682a6), TOBN(0x4cb27077, 0xe39a407c), + TOBN(0x0f59c547, 0x4ff0e321), TOBN(0x169f34a6, 0x1b34c8ff), + TOBN(0x2bff1096, 0x52bc1ba7), TOBN(0xa25423b7, 0x83583544), + TOBN(0x5d55d5d5, 0x0ac8b782), TOBN(0xff6622ec, 0x2db3c892), + TOBN(0x48fce741, 0x6b8bb642), TOBN(0x31d6998c, 0x69d7e3dc), + TOBN(0xdbaf8004, 0xcadcaed0), TOBN(0x801b0142, 0xd81d053c), + TOBN(0x94b189fc, 0x59630ec6), TOBN(0x120e9934, 0xaf762c8e), + TOBN(0x53a29aa4, 0xfdc6a404), TOBN(0x19d8e01e, 0xa1909948), + TOBN(0x3cfcabf1, 0xd7e89681), TOBN(0x3321a50d, 0x4e132d37), + TOBN(0xd0496863, 0xe9a86111), TOBN(0x8c0cde61, 0x06a3bc65), + TOBN(0xaf866c49, 0xfc9f8eef), TOBN(0x2066350e, 0xff7f5141), + TOBN(0x4f8a4689, 0xe56ddfbd), TOBN(0xea1b0c07, 0xfe32983a), + TOBN(0x2b317462, 0x873cb8cb), TOBN(0x658deddc, 0x2d93229f), + TOBN(0x65efaf4d, 0x0f64ef58), TOBN(0xfe43287d, 0x730cc7a8), + TOBN(0xaebc0c72, 0x3d047d70), TOBN(0x92efa539, 0xd92d26c9), + TOBN(0x06e78457, 0x94b56526), TOBN(0x415cb80f, 0x0961002d), + TOBN(0x89e5c565, 0x76dcb10f), TOBN(0x8bbb6982, 0xff9259fe), + TOBN(0x4fe8795b, 0x9abc2668), TOBN(0xb5d4f534, 0x1e678fb1), + TOBN(0x6601f3be, 0x7b7da2b9), TOBN(0x98da59e2, 0xa13d6805), + TOBN(0x190d8ea6, 0x01799a52), TOBN(0xa20cec41, 0xb86d2952), + TOBN(0x3062ffb2, 0x7fff2a7c), TOBN(0x741b32e5, 0x79f19d37), + TOBN(0xf80d8181, 0x4eb57d47), TOBN(0x7a2d0ed4, 0x16aef06b), + TOBN(0x09735fb0, 0x1cecb588), TOBN(0x1641caaa, 0xc6061f5b)} + , + {TOBN(0x7f99824f, 0x20151427), TOBN(0x206828b6, 0x92430206), + TOBN(0xaa9097d7, 0xe1112357), TOBN(0xacf9a2f2, 0x09e414ec), + TOBN(0xdbdac9da, 0x27915356), TOBN(0x7e0734b7, 0x001efee3), + TOBN(0x54fab5bb, 0xd2b288e2), TOBN(0x4c630fc4, 0xf62dd09c), + TOBN(0x8537107a, 0x1ac2703b), TOBN(0xb49258d8, 0x6bc857b5), + TOBN(0x57df14de, 0xbcdaccd1), TOBN(0x24ab68d7, 0xc4ae8529), + TOBN(0x7ed8b5d4, 0x734e59d0), TOBN(0x5f8740c8, 0xc495cc80), + TOBN(0x84aedd5a, 0x291db9b3), TOBN(0x80b360f8, 0x4fb995be), + TOBN(0xae915f5d, 0x5fa067d1), TOBN(0x4134b57f, 0x9668960c), + TOBN(0xbd3656d6, 0xa48edaac), TOBN(0xdac1e3e4, 0xfc1d7436), + TOBN(0x674ff869, 0xd81fbb26), TOBN(0x449ed3ec, 0xb26c33d4), + TOBN(0x85138705, 0xd94203e8), TOBN(0xccde538b, 0xbeeb6f4a), + TOBN(0x55d5c68d, 0xa61a76fa), TOBN(0x598b441d, 0xca1554dc), + TOBN(0xd39923b9, 0x773b279c), TOBN(0x33331d3c, 0x36bf9efc), + TOBN(0x2d4c848e, 0x298de399), TOBN(0xcfdb8e77, 0xa1a27f56), + TOBN(0x94c855ea, 0x57b8ab70), TOBN(0xdcdb9dae, 0x6f7879ba), + TOBN(0x7bdff8c2, 0x019f2a59), TOBN(0xb3ce5bb3, 0xcb4fbc74), + TOBN(0xea907f68, 0x8a9173dd), TOBN(0x6cd3d0d3, 0x95a75439), + TOBN(0x92ecc4d6, 0xefed021c), TOBN(0x09a9f9b0, 0x6a77339a), + TOBN(0x87ca6b15, 0x7188c64a), TOBN(0x10c29968, 0x44899158), + TOBN(0x5859a229, 0xed6e82ef), TOBN(0x16f338e3, 0x65ebaf4e), + TOBN(0x0cd31387, 0x5ead67ae), TOBN(0x1c73d228, 0x54ef0bb4), + TOBN(0x4cb55131, 0x74a5c8c7), TOBN(0x01cd2970, 0x7f69ad6a), + TOBN(0xa04d00dd, 0xe966f87e), TOBN(0xd96fe447, 0x0b7b0321), + TOBN(0x342ac06e, 0x88fbd381), TOBN(0x02cd4a84, 0x5c35a493), + TOBN(0xe8fa89de, 0x54f1bbcd), TOBN(0x341d6367, 0x2575ed4c), + TOBN(0xebe357fb, 0xd238202b), TOBN(0x600b4d1a, 0xa984ead9), + TOBN(0xc35c9f44, 0x52436ea0), TOBN(0x96fe0a39, 0xa370751b), + TOBN(0x4c4f0736, 0x7f636a38), TOBN(0x9f943fb7, 0x0e76d5cb), + TOBN(0xb03510ba, 0xa8b68b8b), TOBN(0xc246780a, 0x9ed07a1f), + TOBN(0x3c051415, 0x6d549fc2), TOBN(0xc2953f31, 0x607781ca), + TOBN(0x955e2c69, 0xd8d95413), TOBN(0xb300fadc, 0x7bd282e3), + TOBN(0x81fe7b50, 0x87e9189f), TOBN(0xdb17375c, 0xf42dda27), + TOBN(0x22f7d896, 0xcf0a5904), TOBN(0xa0e57c5a, 0xebe348e6), + TOBN(0xa61011d3, 0xf40e3c80), TOBN(0xb1189321, 0x8db705c5), + TOBN(0x4ed9309e, 0x50fedec3), TOBN(0xdcf14a10, 0x4d6d5c1d), + TOBN(0x056c265b, 0x55691342), TOBN(0xe8e08504, 0x91049dc7), + TOBN(0x131329f5, 0xc9bae20a), TOBN(0x96c8b3e8, 0xd9dccdb4), + TOBN(0x8c5ff838, 0xfb4ee6b4), TOBN(0xfc5a9aeb, 0x41e8ccf0), + TOBN(0x7417b764, 0xfae050c6), TOBN(0x0953c3d7, 0x00452080), + TOBN(0x21372682, 0x38dfe7e8), TOBN(0xea417e15, 0x2bb79d4b), + TOBN(0x59641f1c, 0x76e7cf2d), TOBN(0x271e3059, 0xea0bcfcc), + TOBN(0x624c7dfd, 0x7253ecbd), TOBN(0x2f552e25, 0x4fca6186), + TOBN(0xcbf84ecd, 0x4d866e9c), TOBN(0x73967709, 0xf68d4610), + TOBN(0xa14b1163, 0xc27901b4), TOBN(0xfd9236e0, 0x899b8bf3), + TOBN(0x42b091ec, 0xcbc6da0a), TOBN(0xbb1dac6f, 0x5ad1d297), + TOBN(0x80e61d53, 0xa91cf76e), TOBN(0x4110a412, 0xd31f1ee7), + TOBN(0x2d87c3ba, 0x13efcf77), TOBN(0x1f374bb4, 0xdf450d76), + TOBN(0x5e78e2f2, 0x0d188dab), TOBN(0xe3968ed0, 0xf4b885ef), + TOBN(0x46c0568e, 0x7314570f), TOBN(0x31616338, 0x01170521), + TOBN(0x18e1e7e2, 0x4f0c8afe), TOBN(0x4caa75ff, 0xdeea78da), + TOBN(0x82db67f2, 0x7c5d8a51), TOBN(0x36a44d86, 0x6f505370), + TOBN(0xd72c5bda, 0x0333974f), TOBN(0x5db516ae, 0x27a70146), + TOBN(0x34705281, 0x210ef921), TOBN(0xbff17a8f, 0x0c9c38e5), + TOBN(0x78f4814e, 0x12476da1), TOBN(0xc1e16613, 0x33c16980), + TOBN(0x9e5b386f, 0x424d4bca), TOBN(0x4c274e87, 0xc85740de), + TOBN(0xb6a9b88d, 0x6c2f5226), TOBN(0x14d1b944, 0x550d7ca8), + TOBN(0x580c85fc, 0x1fc41709), TOBN(0xc1da368b, 0x54c6d519), + TOBN(0x2b0785ce, 0xd5113cf7), TOBN(0x0670f633, 0x5a34708f), + TOBN(0x46e23767, 0x15cc3f88), TOBN(0x1b480cfa, 0x50c72c8f), + TOBN(0x20288602, 0x4147519a), TOBN(0xd0981eac, 0x26b372f0), + TOBN(0xa9d4a7ca, 0xa785ebc8), TOBN(0xd953c50d, 0xdbdf58e9), + TOBN(0x9d6361cc, 0xfd590f8f), TOBN(0x72e9626b, 0x44e6c917), + TOBN(0x7fd96110, 0x22eb64cf), TOBN(0x863ebb7e, 0x9eb288f3), + TOBN(0x6e6ab761, 0x6aca8ee7), TOBN(0x97d10b39, 0xd7b40358), + TOBN(0x1687d377, 0x1e5feb0d), TOBN(0xc83e50e4, 0x8265a27a), + TOBN(0x8f75a9fe, 0xc954b313), TOBN(0xcc2e8f47, 0x310d1f61), + TOBN(0xf5ba81c5, 0x6557d0e0), TOBN(0x25f9680c, 0x3eaf6207), + TOBN(0xf95c6609, 0x4354080b), TOBN(0x5225bfa5, 0x7bf2fe1c), + TOBN(0xc5c004e2, 0x5c7d98fa), TOBN(0x3561bf1c, 0x019aaf60), + TOBN(0x5e6f9f17, 0xba151474), TOBN(0xdec2f934, 0xb04f6eca), + TOBN(0x64e368a1, 0x269acb1e), TOBN(0x1332d9e4, 0x0cdda493), + TOBN(0x60d6cf69, 0xdf23de05), TOBN(0x66d17da2, 0x009339a0), + TOBN(0x9fcac985, 0x0a693923), TOBN(0xbcf057fc, 0xed7c6a6d), + TOBN(0xc3c5c8c5, 0xf0b5662c), TOBN(0x25318dd8, 0xdcba4f24), + TOBN(0x60e8cb75, 0x082b69ff), TOBN(0x7c23b3ee, 0x1e728c01), + TOBN(0x15e10a0a, 0x097e4403), TOBN(0xcb3d0a86, 0x19854665), + TOBN(0x88d8e211, 0xd67d4826), TOBN(0xb39af66e, 0x0b9d2839), + TOBN(0xa5f94588, 0xbd475ca8), TOBN(0xe06b7966, 0xc077b80b), + TOBN(0xfedb1485, 0xda27c26c), TOBN(0xd290d33a, 0xfe0fd5e0), + TOBN(0xa40bcc47, 0xf34fb0fa), TOBN(0xb4760cc8, 0x1fb1ab09), + TOBN(0x8fca0993, 0xa273bfe3), TOBN(0x13e4fe07, 0xf70b213c), + TOBN(0x3bcdb992, 0xfdb05163), TOBN(0x8c484b11, 0x0c2b19b6), + TOBN(0x1acb815f, 0xaaf2e3e2), TOBN(0xc6905935, 0xb89ff1b4), + TOBN(0xb2ad6f9d, 0x586e74e1), TOBN(0x488883ad, 0x67b80484), + TOBN(0x758aa2c7, 0x369c3ddb), TOBN(0x8ab74e69, 0x9f9afd31), + TOBN(0x10fc2d28, 0x5e21beb1), TOBN(0x3484518a, 0x318c42f9), + TOBN(0x377427dc, 0x53cf40c3), TOBN(0x9de0781a, 0x391bc1d9), + TOBN(0x8faee858, 0x693807e1), TOBN(0xa3865327, 0x4e81ccc7), + TOBN(0x02c30ff2, 0x6f835b84), TOBN(0xb604437b, 0x0d3d38d4), + TOBN(0xb3fc8a98, 0x5ca1823d), TOBN(0xb82f7ec9, 0x03be0324), + TOBN(0xee36d761, 0xcf684a33), TOBN(0x5a01df0e, 0x9f29bf7d), + TOBN(0x686202f3, 0x1306583d), TOBN(0x05b10da0, 0x437c622e), + TOBN(0xbf9aaa0f, 0x076a7bc8), TOBN(0x25e94efb, 0x8f8f4e43), + TOBN(0x8a35c9b7, 0xfa3dc26d), TOBN(0xe0e5fb93, 0x96ff03c5), + TOBN(0xa77e3843, 0xebc394ce), TOBN(0xcede6595, 0x8361de60), + TOBN(0xd27c22f6, 0xa1993545), TOBN(0xab01cc36, 0x24d671ba), + TOBN(0x63fa2877, 0xa169c28e), TOBN(0x925ef904, 0x2eb08376), + TOBN(0x3b2fa3cf, 0x53aa0b32), TOBN(0xb27beb5b, 0x71c49d7a), + TOBN(0xb60e1834, 0xd105e27f), TOBN(0xd6089788, 0x4f68570d), + TOBN(0x23094ce0, 0xd6fbc2ac), TOBN(0x738037a1, 0x815ff551), + TOBN(0xda73b1bb, 0x6bef119c), TOBN(0xdcf6c430, 0xeef506ba), + TOBN(0x00e4fe7b, 0xe3ef104a), TOBN(0xebdd9a2c, 0x0a065628), + TOBN(0x853a81c3, 0x8792043e), TOBN(0x22ad6ece, 0xb3b59108), + TOBN(0x9fb813c0, 0x39cd297d), TOBN(0x8ec7e16e, 0x05bda5d9), + TOBN(0x2834797c, 0x0d104b96), TOBN(0xcc11a2e7, 0x7c511510), + TOBN(0x96ca5a53, 0x96ee6380), TOBN(0x054c8655, 0xcea38742), + TOBN(0xb5946852, 0xd54dfa7d), TOBN(0x97c422e7, 0x1f4ab207), + TOBN(0xbf907509, 0x0c22b540), TOBN(0x2cde42aa, 0xb7c267d4), + TOBN(0xba18f9ed, 0x5ab0d693), TOBN(0x3ba62aa6, 0x6e4660d9), + TOBN(0xb24bf97b, 0xab9ea96a), TOBN(0x5d039642, 0xe3b60e32), + TOBN(0x4e6a4506, 0x7c4d9bd5), TOBN(0x666c5b9e, 0x7ed4a6a4), + TOBN(0xfa3fdcd9, 0x8edbd7cc), TOBN(0x4660bb87, 0xc6ccd753), + TOBN(0x9ae90820, 0x21e6b64f), TOBN(0x8a56a713, 0xb36bfb3f), + TOBN(0xabfce096, 0x5726d47f), TOBN(0x9eed01b2, 0x0b1a9a7f), + TOBN(0x30e9cad4, 0x4eb74a37), TOBN(0x7b2524cc, 0x53e9666d), + TOBN(0x6a29683b, 0x8f4b002f), TOBN(0xc2200d7a, 0x41f4fc20), + TOBN(0xcf3af47a, 0x3a338acc), TOBN(0x6539a4fb, 0xe7128975), + TOBN(0xcec31c14, 0xc33c7fcf), TOBN(0x7eb6799b, 0xc7be322b), + TOBN(0x119ef4e9, 0x6646f623), TOBN(0x7b7a26a5, 0x54d7299b), + TOBN(0xcb37f08d, 0x403f46f2), TOBN(0x94b8fc43, 0x1a0ec0c7), + TOBN(0xbb8514e3, 0xc332142f), TOBN(0xf3ed2c33, 0xe80d2a7a), + TOBN(0x8d2080af, 0xb639126c), TOBN(0xf7b6be60, 0xe3553ade), + TOBN(0x3950aa9f, 0x1c7e2b09), TOBN(0x847ff958, 0x6410f02b), + TOBN(0x877b7cf5, 0x678a31b0), TOBN(0xd50301ae, 0x3998b620), + TOBN(0x734257c5, 0xc00fb396), TOBN(0xf9fb18a0, 0x04e672a6), + TOBN(0xff8bd8eb, 0xe8758851), TOBN(0x1e64e4c6, 0x5d99ba44), + TOBN(0x4b8eaedf, 0x7dfd93b7), TOBN(0xba2f2a98, 0x04e76b8c), + TOBN(0x7d790cba, 0xe8053433), TOBN(0xc8e725a0, 0x3d2c9585), + TOBN(0x58c5c476, 0xcdd8f5ed), TOBN(0xd106b952, 0xefa9fe1d), + TOBN(0x3c5c775b, 0x0eff13a9), TOBN(0x242442ba, 0xe057b930), + TOBN(0xe9f458d4, 0xc9b70cbd), TOBN(0x69b71448, 0xa3cdb89a), + TOBN(0x41ee46f6, 0x0e2ed742), TOBN(0x573f1045, 0x40067493), + TOBN(0xb1e154ff, 0x9d54c304), TOBN(0x2ad0436a, 0x8d3a7502), + TOBN(0xee4aaa2d, 0x431a8121), TOBN(0xcd38b3ab, 0x886f11ed), + TOBN(0x57d49ea6, 0x034a0eb7), TOBN(0xd2b773bd, 0xf7e85e58), + TOBN(0x4a559ac4, 0x9b5c1f14), TOBN(0xc444be1a, 0x3e54df2b), + TOBN(0x13aad704, 0xeda41891), TOBN(0xcd927bec, 0x5eb5c788), + TOBN(0xeb3c8516, 0xe48c8a34), TOBN(0x1b7ac812, 0x4b546669), + TOBN(0x1815f896, 0x594df8ec), TOBN(0x87c6a79c, 0x79227865), + TOBN(0xae02a2f0, 0x9b56ddbd), TOBN(0x1339b5ac, 0x8a2f1cf3), + TOBN(0xf2b569c7, 0x839dff0d), TOBN(0xb0b9e864, 0xfee9a43d), + TOBN(0x4ff8ca41, 0x77bb064e), TOBN(0x145a2812, 0xfd249f63), + TOBN(0x3ab7beac, 0xf86f689a), TOBN(0x9bafec27, 0x01d35f5e), + TOBN(0x28054c65, 0x4265aa91), TOBN(0xa4b18304, 0x035efe42), + TOBN(0x6887b0e6, 0x9639dec7), TOBN(0xf4b8f6ad, 0x3d52aea5), + TOBN(0xfb9293cc, 0x971a8a13), TOBN(0x3f159e5d, 0x4c934d07), + TOBN(0x2c50e9b1, 0x09acbc29), TOBN(0x08eb65e6, 0x7154d129), + TOBN(0x4feff589, 0x30b75c3e), TOBN(0x0bb82fe2, 0x94491c93), + TOBN(0xd8ac377a, 0x89af62bb), TOBN(0xd7b51490, 0x9685e49f), + TOBN(0xabca9a7b, 0x04497f19), TOBN(0x1b35ed0a, 0x1a7ad13f), + TOBN(0x6b601e21, 0x3ec86ed6), TOBN(0xda91fcb9, 0xce0c76f1), + TOBN(0x9e28507b, 0xd7ab27e1), TOBN(0x7c19a555, 0x63945b7b), + TOBN(0x6b43f0a1, 0xaafc9827), TOBN(0x443b4fbd, 0x3aa55b91), + TOBN(0x962b2e65, 0x6962c88f), TOBN(0x139da8d4, 0xce0db0ca), + TOBN(0xb93f05dd, 0x1b8d6c4f), TOBN(0x779cdff7, 0x180b9824), + TOBN(0xbba23fdd, 0xae57c7b7), TOBN(0x345342f2, 0x1b932522), + TOBN(0xfd9c80fe, 0x556d4aa3), TOBN(0xa03907ba, 0x6525bb61), + TOBN(0x38b010e1, 0xff218933), TOBN(0xc066b654, 0xaa52117b), + TOBN(0x8e141920, 0x94f2e6ea), TOBN(0x66a27dca, 0x0d32f2b2), + TOBN(0x69c7f993, 0x048b3717), TOBN(0xbf5a989a, 0xb178ae1c), + TOBN(0x49fa9058, 0x564f1d6b), TOBN(0x27ec6e15, 0xd31fde4e), + TOBN(0x4cce0373, 0x7276e7fc), TOBN(0x64086d79, 0x89d6bf02), + TOBN(0x5a72f046, 0x4ccdd979), TOBN(0x909c3566, 0x47775631), + TOBN(0x1c07bc6b, 0x75dd7125), TOBN(0xb4c6bc97, 0x87a0428d), + TOBN(0x507ece52, 0xfdeb6b9d), TOBN(0xfca56512, 0xb2c95432), + TOBN(0x15d97181, 0xd0e8bd06), TOBN(0x384dd317, 0xc6bb46ea), + TOBN(0x5441ea20, 0x3952b624), TOBN(0xbcf70dee, 0x4e7dc2fb), + TOBN(0x372b016e, 0x6628e8c3), TOBN(0x07a0d667, 0xb60a7522), + TOBN(0xcf05751b, 0x0a344ee2), TOBN(0x0ec09a48, 0x118bdeec), + TOBN(0x6e4b3d4e, 0xd83dce46), TOBN(0x43a6316d, 0x99d2fc6e), + TOBN(0xa99d8989, 0x56cf044c), TOBN(0x7c7f4454, 0xae3e5fb7), + TOBN(0xb2e6b121, 0xfbabbe92), TOBN(0x281850fb, 0xe1330076), + TOBN(0x093581ec, 0x97890015), TOBN(0x69b1dded, 0x75ff77f5), + TOBN(0x7cf0b18f, 0xab105105), TOBN(0x953ced31, 0xa89ccfef), + TOBN(0x3151f85f, 0xeb914009), TOBN(0x3c9f1b87, 0x88ed48ad), + TOBN(0xc9aba1a1, 0x4a7eadcb), TOBN(0x928e7501, 0x522e71cf), + TOBN(0xeaede727, 0x3a2e4f83), TOBN(0x467e10d1, 0x1ce3bbd3), + TOBN(0xf3442ac3, 0xb955dcf0), TOBN(0xba96307d, 0xd3d5e527), + TOBN(0xf763a10e, 0xfd77f474), TOBN(0x5d744bd0, 0x6a6e1ff0), + TOBN(0xd287282a, 0xa777899e), TOBN(0xe20eda8f, 0xd03f3cde), + TOBN(0x6a7e75bb, 0x50b07d31), TOBN(0x0b7e2a94, 0x6f379de4), + TOBN(0x31cb64ad, 0x19f593cf), TOBN(0x7b1a9e4f, 0x1e76ef1d), + TOBN(0xe18c9c9d, 0xb62d609c), TOBN(0x439bad6d, 0xe779a650), + TOBN(0x219d9066, 0xe032f144), TOBN(0x1db632b8, 0xe8b2ec6a), + TOBN(0xff0d0fd4, 0xfda12f78), TOBN(0x56fb4c2d, 0x2a25d265), + TOBN(0x5f4e2ee1, 0x255a03f1), TOBN(0x61cd6af2, 0xe96af176), + TOBN(0xe0317ba8, 0xd068bc97), TOBN(0x927d6bab, 0x264b988e), + TOBN(0xa18f07e0, 0xe90fb21e), TOBN(0x00fd2b80, 0xbba7fca1), + TOBN(0x20387f27, 0x95cd67b5), TOBN(0x5b89a4e7, 0xd39707f7), + TOBN(0x8f83ad3f, 0x894407ce), TOBN(0xa0025b94, 0x6c226132), + TOBN(0xc79563c7, 0xf906c13b), TOBN(0x5f548f31, 0x4e7bb025), + TOBN(0x2b4c6b8f, 0xeac6d113), TOBN(0xa67e3f9c, 0x0e813c76), + TOBN(0x3982717c, 0x3fe1f4b9), TOBN(0x58865819, 0x26d8050e), + TOBN(0x99f3640c, 0xf7f06f20), TOBN(0xdc610216, 0x2a66ebc2), + TOBN(0x52f2c175, 0x767a1e08), TOBN(0x05660e1a, 0x5999871b), + TOBN(0x6b0f1762, 0x6d3c4693), TOBN(0xf0e7d627, 0x37ed7bea), + TOBN(0xc51758c7, 0xb75b226d), TOBN(0x40a88628, 0x1f91613b), + TOBN(0x889dbaa7, 0xbbb38ce0), TOBN(0xe0404b65, 0xbddcad81), + TOBN(0xfebccd3a, 0x8bc9671f), TOBN(0xfbf9a357, 0xee1f5375), + TOBN(0x5dc169b0, 0x28f33398), TOBN(0xb07ec11d, 0x72e90f65), + TOBN(0xae7f3b4a, 0xfaab1eb1), TOBN(0xd970195e, 0x5f17538a), + TOBN(0x52b05cbe, 0x0181e640), TOBN(0xf5debd62, 0x2643313d), + TOBN(0x76148154, 0x5df31f82), TOBN(0x23e03b33, 0x3a9e13c5), + TOBN(0xff758949, 0x4fde0c1f), TOBN(0xbf8a1abe, 0xe5b6ec20), + TOBN(0x702278fb, 0x87e1db6c), TOBN(0xc447ad7a, 0x35ed658f), + TOBN(0x48d4aa38, 0x03d0ccf2), TOBN(0x80acb338, 0x819a7c03), + TOBN(0x9bc7c89e, 0x6e17cecc), TOBN(0x46736b8b, 0x03be1d82), + TOBN(0xd65d7b60, 0xc0432f96), TOBN(0xddebe7a3, 0xdeb5442f), + TOBN(0x79a25307, 0x7dff69a2), TOBN(0x37a56d94, 0x02cf3122), + TOBN(0x8bab8aed, 0xf2350d0a), TOBN(0x13c3f276, 0x037b0d9a), + TOBN(0xc664957c, 0x44c65cae), TOBN(0x88b44089, 0xc2e71a88), + TOBN(0xdb88e5a3, 0x5cb02664), TOBN(0x5d4c0bf1, 0x8686c72e), + TOBN(0xea3d9b62, 0xa682d53e), TOBN(0x9b605ef4, 0x0b2ad431), + TOBN(0x71bac202, 0xc69645d0), TOBN(0xa115f03a, 0x6a1b66e7), + TOBN(0xfe2c563a, 0x158f4dc4), TOBN(0xf715b3a0, 0x4d12a78c), + TOBN(0x8f7f0a48, 0xd413213a), TOBN(0x2035806d, 0xc04becdb), + TOBN(0xecd34a99, 0x5d8587f5), TOBN(0x4d8c3079, 0x9f6d3a71), + TOBN(0x1b2a2a67, 0x8d95a8f6), TOBN(0xc58c9d7d, 0xf2110d0d), + TOBN(0xdeee81d5, 0xcf8fba3f), TOBN(0xa42be3c0, 0x0c7cdf68), + TOBN(0x2126f742, 0xd43b5eaa), TOBN(0x054a0766, 0xdfa59b85), + TOBN(0x9d0d5e36, 0x126bfd45), TOBN(0xa1f8fbd7, 0x384f8a8f), + TOBN(0x317680f5, 0xd563fccc), TOBN(0x48ca5055, 0xf280a928), + TOBN(0xe00b81b2, 0x27b578cf), TOBN(0x10aad918, 0x2994a514), + TOBN(0xd9e07b62, 0xb7bdc953), TOBN(0x9f0f6ff2, 0x5bc086dd), + TOBN(0x09d1ccff, 0x655eee77), TOBN(0x45475f79, 0x5bef7df1), + TOBN(0x3faa28fa, 0x86f702cc), TOBN(0x92e60905, 0x0f021f07), + TOBN(0xe9e62968, 0x7f8fa8c6), TOBN(0xbd71419a, 0xf036ea2c), + TOBN(0x171ee1cc, 0x6028da9a), TOBN(0x5352fe1a, 0xc251f573), + TOBN(0xf8ff236e, 0x3fa997f4), TOBN(0xd831b6c9, 0xa5749d5f), + TOBN(0x7c872e1d, 0xe350e2c2), TOBN(0xc56240d9, 0x1e0ce403), + TOBN(0xf9deb077, 0x6974f5cb), TOBN(0x7d50ba87, 0x961c3728), + TOBN(0xd6f89426, 0x5a3a2518), TOBN(0xcf817799, 0xc6303d43), + TOBN(0x510a0471, 0x619e5696), TOBN(0xab049ff6, 0x3a5e307b), + TOBN(0xe4cdf9b0, 0xfeb13ec7), TOBN(0xd5e97117, 0x9d8ff90c), + TOBN(0xf6f64d06, 0x9afa96af), TOBN(0x00d0bf5e, 0x9d2012a2), + TOBN(0xe63f301f, 0x358bcdc0), TOBN(0x07689e99, 0x0a9d47f8), + TOBN(0x1f689e2f, 0x4f43d43a), TOBN(0x4d542a16, 0x90920904), + TOBN(0xaea293d5, 0x9ca0a707), TOBN(0xd061fe45, 0x8ac68065), + TOBN(0x1033bf1b, 0x0090008c), TOBN(0x29749558, 0xc08a6db6), + TOBN(0x74b5fc59, 0xc1d5d034), TOBN(0xf712e9f6, 0x67e215e0), + TOBN(0xfd520cbd, 0x860200e6), TOBN(0x0229acb4, 0x3ea22588), + TOBN(0x9cd1e14c, 0xfff0c82e), TOBN(0x87684b62, 0x59c69e73), + TOBN(0xda85e61c, 0x96ccb989), TOBN(0x2d5dbb02, 0xa3d06493), + TOBN(0xf22ad33a, 0xe86b173c), TOBN(0xe8e41ea5, 0xa79ff0e3), + TOBN(0x01d2d725, 0xdd0d0c10), TOBN(0x31f39088, 0x032d28f9), + TOBN(0x7b3f71e1, 0x7829839e), TOBN(0x0cf691b4, 0x4502ae58), + TOBN(0xef658dbd, 0xbefc6115), TOBN(0xa5cd6ee5, 0xb3ab5314), + TOBN(0x206c8d7b, 0x5f1d2347), TOBN(0x794645ba, 0x4cc2253a), + TOBN(0xd517d8ff, 0x58389e08), TOBN(0x4fa20dee, 0x9f847288), + TOBN(0xeba072d8, 0xd797770a), TOBN(0x7360c91d, 0xbf429e26), + TOBN(0x7200a3b3, 0x80af8279), TOBN(0x6a1c9150, 0x82dadce3), + TOBN(0x0ee6d3a7, 0xc35d8794), TOBN(0x042e6558, 0x0356bae5), + TOBN(0x9f59698d, 0x643322fd), TOBN(0x9379ae15, 0x50a61967), + TOBN(0x64b9ae62, 0xfcc9981e), TOBN(0xaed3d631, 0x6d2934c6), + TOBN(0x2454b302, 0x5e4e65eb), TOBN(0xab09f647, 0xf9950428)} + , + {TOBN(0xb2083a12, 0x22248acc), TOBN(0x1f6ec0ef, 0x3264e366), + TOBN(0x5659b704, 0x5afdee28), TOBN(0x7a823a40, 0xe6430bb5), + TOBN(0x24592a04, 0xe1900a79), TOBN(0xcde09d4a, 0xc9ee6576), + TOBN(0x52b6463f, 0x4b5ea54a), TOBN(0x1efe9ed3, 0xd3ca65a7), + TOBN(0xe27a6dbe, 0x305406dd), TOBN(0x8eb7dc7f, 0xdd5d1957), + TOBN(0xf54a6876, 0x387d4d8f), TOBN(0x9c479409, 0xc7762de4), + TOBN(0xbe4d5b5d, 0x99b30778), TOBN(0x25380c56, 0x6e793682), + TOBN(0x602d37f3, 0xdac740e3), TOBN(0x140deabe, 0x1566e4ae), + TOBN(0x4481d067, 0xafd32acf), TOBN(0xd8f0fcca, 0xe1f71ccf), + TOBN(0xd208dd0c, 0xb596f2da), TOBN(0xd049d730, 0x9aad93f9), + TOBN(0xc79f263d, 0x42ab580e), TOBN(0x09411bb1, 0x23f707b4), + TOBN(0x8cfde1ff, 0x835e0eda), TOBN(0x72707490, 0x90f03402), + TOBN(0xeaee6126, 0xc49a861e), TOBN(0x024f3b65, 0xe14f0d06), + TOBN(0x51a3f1e8, 0xc69bfc17), TOBN(0xc3c3a8e9, 0xa7686381), + TOBN(0x3400752c, 0xb103d4c8), TOBN(0x02bc4613, 0x9218b36b), + TOBN(0xc67f75eb, 0x7651504a), TOBN(0xd6848b56, 0xd02aebfa), + TOBN(0xbd9802e6, 0xc30fa92b), TOBN(0x5a70d96d, 0x9a552784), + TOBN(0x9085c4ea, 0x3f83169b), TOBN(0xfa9423bb, 0x06908228), + TOBN(0x2ffebe12, 0xfe97a5b9), TOBN(0x85da6049, 0x71b99118), + TOBN(0x9cbc2f7f, 0x63178846), TOBN(0xfd96bc70, 0x9153218e), + TOBN(0x958381db, 0x1782269b), TOBN(0xae34bf79, 0x2597e550), + TOBN(0xbb5c6064, 0x5f385153), TOBN(0x6f0e96af, 0xe3088048), + TOBN(0xbf6a0215, 0x77884456), TOBN(0xb3b5688c, 0x69310ea7), + TOBN(0x17c94295, 0x04fad2de), TOBN(0xe020f0e5, 0x17896d4d), + TOBN(0x730ba0ab, 0x0976505f), TOBN(0x567f6813, 0x095e2ec5), + TOBN(0x47062010, 0x6331ab71), TOBN(0x72cfa977, 0x41d22b9f), + TOBN(0x33e55ead, 0x8a2373da), TOBN(0xa8d0d5f4, 0x7ba45a68), + TOBN(0xba1d8f9c, 0x03029d15), TOBN(0x8f34f1cc, 0xfc55b9f3), + TOBN(0xcca4428d, 0xbbe5a1a9), TOBN(0x8187fd5f, 0x3126bd67), + TOBN(0x0036973a, 0x48105826), TOBN(0xa39b6663, 0xb8bd61a0), + TOBN(0x6d42deef, 0x2d65a808), TOBN(0x4969044f, 0x94636b19), + TOBN(0xf611ee47, 0xdd5d564c), TOBN(0x7b2f3a49, 0xd2873077), + TOBN(0x94157d45, 0x300eb294), TOBN(0x2b2a656e, 0x169c1494), + TOBN(0xc000dd76, 0xd3a47aa9), TOBN(0xa2864e4f, 0xa6243ea4), + TOBN(0x82716c47, 0xdb89842e), TOBN(0x12dfd7d7, 0x61479fb7), + TOBN(0x3b9a2c56, 0xe0b2f6dc), TOBN(0x46be862a, 0xd7f85d67), + TOBN(0x03b0d8dd, 0x0f82b214), TOBN(0x460c34f9, 0xf103cbc6), + TOBN(0xf32e5c03, 0x18d79e19), TOBN(0x8b8888ba, 0xa84117f8), + TOBN(0x8f3c37dc, 0xc0722677), TOBN(0x10d21be9, 0x1c1c0f27), + TOBN(0xd47c8468, 0xe0f7a0c6), TOBN(0x9bf02213, 0xadecc0e0), + TOBN(0x0baa7d12, 0x42b48b99), TOBN(0x1bcb665d, 0x48424096), + TOBN(0x8b847cd6, 0xebfb5cfb), TOBN(0x87c2ae56, 0x9ad4d10d), + TOBN(0xf1cbb122, 0x0de36726), TOBN(0xe7043c68, 0x3fdfbd21), + TOBN(0x4bd0826a, 0x4e79d460), TOBN(0x11f5e598, 0x4bd1a2cb), + TOBN(0x97554160, 0xb7fe7b6e), TOBN(0x7d16189a, 0x400a3fb2), + TOBN(0xd73e9bea, 0xe328ca1e), TOBN(0x0dd04b97, 0xe793d8cc), + TOBN(0xa9c83c9b, 0x506db8cc), TOBN(0x5cd47aae, 0xcf38814c), + TOBN(0x26fc430d, 0xb64b45e6), TOBN(0x079b5499, 0xd818ea84), + TOBN(0xebb01102, 0xc1c24a3b), TOBN(0xca24e568, 0x1c161c1a), + TOBN(0x103eea69, 0x36f00a4a), TOBN(0x9ad76ee8, 0x76176c7b), + TOBN(0x97451fc2, 0x538e0ff7), TOBN(0x94f89809, 0x6604b3b0), + TOBN(0x6311436e, 0x3249cfd7), TOBN(0x27b4a7bd, 0x41224f69), + TOBN(0x03b5d21a, 0xe0ac2941), TOBN(0x279b0254, 0xc2d31937), + TOBN(0x3307c052, 0xcac992d0), TOBN(0x6aa7cb92, 0xefa8b1f3), + TOBN(0x5a182580, 0x0d37c7a5), TOBN(0x13380c37, 0x342d5422), + TOBN(0x92ac2d66, 0xd5d2ef92), TOBN(0x035a70c9, 0x030c63c6), + TOBN(0xc16025dd, 0x4ce4f152), TOBN(0x1f419a71, 0xf9df7c06), + TOBN(0x6d5b2214, 0x91e4bb14), TOBN(0xfc43c6cc, 0x839fb4ce), + TOBN(0x49f06591, 0x925d6b2d), TOBN(0x4b37d9d3, 0x62186598), + TOBN(0x8c54a971, 0xd01b1629), TOBN(0xe1a9c29f, 0x51d50e05), + TOBN(0x5109b785, 0x71ba1861), TOBN(0x48b22d5c, 0xd0c8f93d), + TOBN(0xe8fa84a7, 0x8633bb93), TOBN(0x53fba6ba, 0x5aebbd08), + TOBN(0x7ff27df3, 0xe5eea7d8), TOBN(0x521c8796, 0x68ca7158), + TOBN(0xb9d5133b, 0xce6f1a05), TOBN(0x2d50cd53, 0xfd0ebee4), + TOBN(0xc82115d6, 0xc5a3ef16), TOBN(0x993eff9d, 0xba079221), + TOBN(0xe4da2c5e, 0x4b5da81c), TOBN(0x9a89dbdb, 0x8033fd85), + TOBN(0x60819ebf, 0x2b892891), TOBN(0x53902b21, 0x5d14a4d5), + TOBN(0x6ac35051, 0xd7fda421), TOBN(0xcc6ab885, 0x61c83284), + TOBN(0x14eba133, 0xf74cff17), TOBN(0x240aaa03, 0xecb813f2), + TOBN(0xcfbb6540, 0x6f665bee), TOBN(0x084b1fe4, 0xa425ad73), + TOBN(0x009d5d16, 0xd081f6a6), TOBN(0x35304fe8, 0xeef82c90), + TOBN(0xf20346d5, 0xaa9eaa22), TOBN(0x0ada9f07, 0xac1c91e3), + TOBN(0xa6e21678, 0x968a6144), TOBN(0x54c1f77c, 0x07b31a1e), + TOBN(0xd6bb787e, 0x5781fbe1), TOBN(0x61bd2ee0, 0xe31f1c4a), + TOBN(0xf25aa1e9, 0x781105fc), TOBN(0x9cf2971f, 0x7b2f8e80), + TOBN(0x26d15412, 0xcdff919b), TOBN(0x01db4ebe, 0x34bc896e), + TOBN(0x7d9b3e23, 0xb40df1cf), TOBN(0x59337373, 0x94e971b4), + TOBN(0xbf57bd14, 0x669cf921), TOBN(0x865daedf, 0x0c1a1064), + TOBN(0x3eb70bd3, 0x83279125), TOBN(0xbc3d5b9f, 0x34ecdaab), + TOBN(0x91e3ed7e, 0x5f755caf), TOBN(0x49699f54, 0xd41e6f02), + TOBN(0x185770e1, 0xd4a7a15b), TOBN(0x08f3587a, 0xeaac87e7), + TOBN(0x352018db, 0x473133ea), TOBN(0x674ce719, 0x04fd30fc), + TOBN(0x7b8d9835, 0x088b3e0e), TOBN(0x7a0356a9, 0x5d0d47a1), + TOBN(0x9d9e7659, 0x6474a3c4), TOBN(0x61ea48a7, 0xff66966c), + TOBN(0x30417758, 0x0f3e4834), TOBN(0xfdbb21c2, 0x17a9afcb), + TOBN(0x756fa17f, 0x2f9a67b3), TOBN(0x2a6b2421, 0xa245c1a8), + TOBN(0x64be2794, 0x4af02291), TOBN(0xade465c6, 0x2a5804fe), + TOBN(0x8dffbd39, 0xa6f08fd7), TOBN(0xc4efa84c, 0xaa14403b), + TOBN(0xa1b91b2a, 0x442b0f5c), TOBN(0xb748e317, 0xcf997736), + TOBN(0x8d1b62bf, 0xcee90e16), TOBN(0x907ae271, 0x0b2078c0), + TOBN(0xdf31534b, 0x0c9bcddd), TOBN(0x043fb054, 0x39adce83), + TOBN(0x99031043, 0xd826846a), TOBN(0x61a9c0d6, 0xb144f393), + TOBN(0xdab48046, 0x47718427), TOBN(0xdf17ff9b, 0x6e830f8b), + TOBN(0x408d7ee8, 0xe49a1347), TOBN(0x6ac71e23, 0x91c1d4ae), + TOBN(0xc8cbb9fd, 0x1defd73c), TOBN(0x19840657, 0xbbbbfec5), + TOBN(0x39db1cb5, 0x9e7ef8ea), TOBN(0x78aa8296, 0x64105f30), + TOBN(0xa3d9b7f0, 0xa3738c29), TOBN(0x0a2f235a, 0xbc3250a3), + TOBN(0x55e506f6, 0x445e4caf), TOBN(0x0974f73d, 0x33475f7a), + TOBN(0xd37dbba3, 0x5ba2f5a8), TOBN(0x542c6e63, 0x6af40066), + TOBN(0x26d99b53, 0xc5d73e2c), TOBN(0x06060d7d, 0x6c3ca33e), + TOBN(0xcdbef1c2, 0x065fef4a), TOBN(0x77e60f7d, 0xfd5b92e3), + TOBN(0xd7c549f0, 0x26708350), TOBN(0x201b3ad0, 0x34f121bf), + TOBN(0x5fcac2a1, 0x0334fc14), TOBN(0x8a9a9e09, 0x344552f6), + TOBN(0x7dd8a1d3, 0x97653082), TOBN(0x5fc0738f, 0x79d4f289), + TOBN(0x787d244d, 0x17d2d8c3), TOBN(0xeffc6345, 0x70830684), + TOBN(0x5ddb96dd, 0xe4f73ae5), TOBN(0x8efb14b1, 0x172549a5), + TOBN(0x6eb73eee, 0x2245ae7a), TOBN(0xbca4061e, 0xea11f13e), + TOBN(0xb577421d, 0x30b01f5d), TOBN(0xaa688b24, 0x782e152c), + TOBN(0x67608e71, 0xbd3502ba), TOBN(0x4ef41f24, 0xb4de75a0), + TOBN(0xb08dde5e, 0xfd6125e5), TOBN(0xde484825, 0xa409543f), + TOBN(0x1f198d98, 0x65cc2295), TOBN(0x428a3771, 0x6e0edfa2), + TOBN(0x4f9697a2, 0xadf35fc7), TOBN(0x01a43c79, 0xf7cac3c7), + TOBN(0xb05d7059, 0x0fd3659a), TOBN(0x8927f30c, 0xbb7f2d9a), + TOBN(0x4023d1ac, 0x8cf984d3), TOBN(0x32125ed3, 0x02897a45), + TOBN(0xfb572dad, 0x3d414205), TOBN(0x73000ef2, 0xe3fa82a9), + TOBN(0x4c0868e9, 0xf10a5581), TOBN(0x5b61fc67, 0x6b0b3ca5), + TOBN(0xc1258d5b, 0x7cae440c), TOBN(0x21c08b41, 0x402b7531), + TOBN(0xf61a8955, 0xde932321), TOBN(0x3568faf8, 0x2d1408af), + TOBN(0x71b15e99, 0x9ecf965b), TOBN(0xf14ed248, 0xe917276f), + TOBN(0xc6f4caa1, 0x820cf9e2), TOBN(0x681b20b2, 0x18d83c7e), + TOBN(0x6cde738d, 0xc6c01120), TOBN(0x71db0813, 0xae70e0db), + TOBN(0x95fc0644, 0x74afe18c), TOBN(0x34619053, 0x129e2be7), + TOBN(0x80615cea, 0xdb2a3b15), TOBN(0x0a49a19e, 0xdb4c7073), + TOBN(0x0e1b84c8, 0x8fd2d367), TOBN(0xd74bf462, 0x033fb8aa), + TOBN(0x889f6d65, 0x533ef217), TOBN(0x7158c7e4, 0xc3ca2e87), + TOBN(0xfb670dfb, 0xdc2b4167), TOBN(0x75910a01, 0x844c257f), + TOBN(0xf336bf07, 0xcf88577d), TOBN(0x22245250, 0xe45e2ace), + TOBN(0x2ed92e8d, 0x7ca23d85), TOBN(0x29f8be4c, 0x2b812f58), + TOBN(0xdd9ebaa7, 0x076fe12b), TOBN(0x3f2400cb, 0xae1537f9), + TOBN(0x1aa93528, 0x17bdfb46), TOBN(0xc0f98430, 0x67883b41), + TOBN(0x5590ede1, 0x0170911d), TOBN(0x7562f5bb, 0x34d4b17f), + TOBN(0xe1fa1df2, 0x1826b8d2), TOBN(0xb40b796a, 0x6bd80d59), + TOBN(0xd65bf197, 0x3467ba92), TOBN(0x8c9b46db, 0xf70954b0), + TOBN(0x97c8a0f3, 0x0e78f15d), TOBN(0xa8f3a69a, 0x85a4c961), + TOBN(0x4242660f, 0x61e4ce9b), TOBN(0xbf06aab3, 0x6ea6790c), + TOBN(0xc6706f8e, 0xec986416), TOBN(0x9e56dec1, 0x9a9fc225), + TOBN(0x527c46f4, 0x9a9898d9), TOBN(0xd799e77b, 0x5633cdef), + TOBN(0x24eacc16, 0x7d9e4297), TOBN(0xabb61cea, 0x6b1cb734), + TOBN(0xbee2e8a7, 0xf778443c), TOBN(0x3bb42bf1, 0x29de2fe6), + TOBN(0xcbed86a1, 0x3003bb6f), TOBN(0xd3918e6c, 0xd781cdf6), + TOBN(0x4bee3271, 0x9a5103f1), TOBN(0x5243efc6, 0xf50eac06), + TOBN(0xb8e122cb, 0x6adcc119), TOBN(0x1b7faa84, 0xc0b80a08), + TOBN(0x32c3d1bd, 0x6dfcd08c), TOBN(0x129dec4e, 0x0be427de), + TOBN(0x98ab679c, 0x1d263c83), TOBN(0xafc83cb7, 0xcef64eff), + TOBN(0x85eb6088, 0x2fa6be76), TOBN(0x892585fb, 0x1328cbfe), + TOBN(0xc154d3ed, 0xcf618dda), TOBN(0xc44f601b, 0x3abaf26e), + TOBN(0x7bf57d0b, 0x2be1fdfd), TOBN(0xa833bd2d, 0x21137fee), + TOBN(0x9353af36, 0x2db591a8), TOBN(0xc76f26dc, 0x5562a056), + TOBN(0x1d87e47d, 0x3fdf5a51), TOBN(0x7afb5f93, 0x55c9cab0), + TOBN(0x91bbf58f, 0x89e0586e), TOBN(0x7c72c018, 0x0d843709), + TOBN(0xa9a5aafb, 0x99b5c3dc), TOBN(0xa48a0f1d, 0x3844aeb0), + TOBN(0x7178b7dd, 0xb667e482), TOBN(0x453985e9, 0x6e23a59a), + TOBN(0x4a54c860, 0x01b25dd8), TOBN(0x0dd37f48, 0xfb897c8a), + TOBN(0x5f8aa610, 0x0ea90cd9), TOBN(0xc8892c68, 0x16d5830d), + TOBN(0xeb4befc0, 0xef514ca5), TOBN(0x478eb679, 0xe72c9ee6), + TOBN(0x9bca20da, 0xdbc40d5f), TOBN(0xf015de21, 0xdde4f64a), + TOBN(0xaa6a4de0, 0xeaf4b8a5), TOBN(0x68cfd9ca, 0x4bc60e32), + TOBN(0x668a4b01, 0x7fd15e70), TOBN(0xd9f0694a, 0xf27dc09d), + TOBN(0xf6c3cad5, 0xba708bcd), TOBN(0x5cd2ba69, 0x5bb95c2a), + TOBN(0xaa28c1d3, 0x33c0a58f), TOBN(0x23e274e3, 0xabc77870), + TOBN(0x44c3692d, 0xdfd20a4a), TOBN(0x091c5fd3, 0x81a66653), + TOBN(0x6c0bb691, 0x09a0757d), TOBN(0x9072e8b9, 0x667343ea), + TOBN(0x31d40eb0, 0x80848bec), TOBN(0x95bd480a, 0x79fd36cc), + TOBN(0x01a77c61, 0x65ed43f5), TOBN(0xafccd127, 0x2e0d40bf), + TOBN(0xeccfc82d, 0x1cc1884b), TOBN(0xc85ac201, 0x5d4753b4), + TOBN(0xc7a6caac, 0x658e099f), TOBN(0xcf46369e, 0x04b27390), + TOBN(0xe2e7d049, 0x506467ea), TOBN(0x481b63a2, 0x37cdeccc), + TOBN(0x4029abd8, 0xed80143a), TOBN(0x28bfe3c7, 0xbcb00b88), + TOBN(0x3bec1009, 0x0643d84a), TOBN(0x885f3668, 0xabd11041), + TOBN(0xdb02432c, 0xf83a34d6), TOBN(0x32f7b360, 0x719ceebe), + TOBN(0xf06c7837, 0xdad1fe7a), TOBN(0x60a157a9, 0x5441a0b0), + TOBN(0x704970e9, 0xe2d47550), TOBN(0xcd2bd553, 0x271b9020), + TOBN(0xff57f82f, 0x33e24a0b), TOBN(0x9cbee23f, 0xf2565079), + TOBN(0x16353427, 0xeb5f5825), TOBN(0x276feec4, 0xe948d662), + TOBN(0xd1b62bc6, 0xda10032b), TOBN(0x718351dd, 0xf0e72a53), + TOBN(0x93452076, 0x2420e7ba), TOBN(0x96368fff, 0x3a00118d), + TOBN(0x00ce2d26, 0x150a49e4), TOBN(0x0c28b636, 0x3f04706b), + TOBN(0xbad65a46, 0x58b196d0), TOBN(0x6c8455fc, 0xec9f8b7c), + TOBN(0xe90c895f, 0x2d71867e), TOBN(0x5c0be31b, 0xedf9f38c), + TOBN(0x2a37a15e, 0xd8f6ec04), TOBN(0x239639e7, 0x8cd85251), + TOBN(0xd8975315, 0x9c7c4c6b), TOBN(0x603aa3c0, 0xd7409af7), + TOBN(0xb8d53d0c, 0x007132fb), TOBN(0x68d12af7, 0xa6849238), + TOBN(0xbe0607e7, 0xbf5d9279), TOBN(0x9aa50055, 0xaada74ce), + TOBN(0xe81079cb, 0xba7e8ccb), TOBN(0x610c71d1, 0xa5f4ff5e), + TOBN(0x9e2ee1a7, 0x5aa07093), TOBN(0xca84004b, 0xa75da47c), + TOBN(0x074d3951, 0x3de75401), TOBN(0xf938f756, 0xbb311592), + TOBN(0x96197618, 0x00a43421), TOBN(0x39a25362, 0x07bc78c8), + TOBN(0x278f710a, 0x0a171276), TOBN(0xb28446ea, 0x8d1a8f08), + TOBN(0x184781bf, 0xe3b6a661), TOBN(0x7751cb1d, 0xe6d279f7), + TOBN(0xf8ff95d6, 0xc59eb662), TOBN(0x186d90b7, 0x58d3dea7), + TOBN(0x0e4bb6c1, 0xdfb4f754), TOBN(0x5c5cf56b, 0x2b2801dc), + TOBN(0xc561e452, 0x1f54564d), TOBN(0xb4fb8c60, 0xf0dd7f13), + TOBN(0xf8849630, 0x33ff98c7), TOBN(0x9619fffa, 0xcf17769c), + TOBN(0xf8090bf6, 0x1bfdd80a), TOBN(0x14d9a149, 0x422cfe63), + TOBN(0xb354c360, 0x6f6df9ea), TOBN(0xdbcf770d, 0x218f17ea), + TOBN(0x207db7c8, 0x79eb3480), TOBN(0x213dbda8, 0x559b6a26), + TOBN(0xac4c200b, 0x29fc81b3), TOBN(0xebc3e09f, 0x171d87c1), + TOBN(0x91799530, 0x1481aa9e), TOBN(0x051b92e1, 0x92e114fa), + TOBN(0xdf8f92e9, 0xecb5537f), TOBN(0x44b1b2cc, 0x290c7483), + TOBN(0xa711455a, 0x2adeb016), TOBN(0x964b6856, 0x81a10c2c), + TOBN(0x4f159d99, 0xcec03623), TOBN(0x05532225, 0xef3271ea), + TOBN(0xb231bea3, 0xc5ee4849), TOBN(0x57a54f50, 0x7094f103), + TOBN(0x3e2d421d, 0x9598b352), TOBN(0xe865a49c, 0x67412ab4), + TOBN(0xd2998a25, 0x1cc3a912), TOBN(0x5d092808, 0x0c74d65d), + TOBN(0x73f45908, 0x4088567a), TOBN(0xeb6b280e, 0x1f214a61), + TOBN(0x8c9adc34, 0xcaf0c13d), TOBN(0x39d12938, 0xf561fb80), + TOBN(0xb2dc3a5e, 0xbc6edfb4), TOBN(0x7485b1b1, 0xfe4d210e), + TOBN(0x062e0400, 0xe186ae72), TOBN(0x91e32d5c, 0x6eeb3b88), + TOBN(0x6df574d7, 0x4be59224), TOBN(0xebc88ccc, 0x716d55f3), + TOBN(0x26c2e6d0, 0xcad6ed33), TOBN(0xc6e21e7d, 0x0d3e8b10), + TOBN(0x2cc5840e, 0x5bcc36bb), TOBN(0x9292445e, 0x7da74f69), + TOBN(0x8be8d321, 0x4e5193a8), TOBN(0x3ec23629, 0x8df06413), + TOBN(0xc7e9ae85, 0xb134defa), TOBN(0x6073b1d0, 0x1bb2d475), + TOBN(0xb9ad615e, 0x2863c00d), TOBN(0x9e29493d, 0x525f4ac4), + TOBN(0xc32b1dea, 0x4e9acf4f), TOBN(0x3e1f01c8, 0xa50db88d), + TOBN(0xb05d70ea, 0x04da916c), TOBN(0x714b0d0a, 0xd865803e), + TOBN(0x4bd493fc, 0x9920cb5e), TOBN(0x5b44b1f7, 0x92c7a3ac), + TOBN(0xa2a77293, 0xbcec9235), TOBN(0x5ee06e87, 0xcd378553), + TOBN(0xceff8173, 0xda621607), TOBN(0x2bb03e4c, 0x99f5d290), + TOBN(0x2945106a, 0xa6f734ac), TOBN(0xb5056604, 0xd25c4732), + TOBN(0x5945920c, 0xe079afee), TOBN(0x686e17a0, 0x6789831f), + TOBN(0x5966bee8, 0xb74a5ae5), TOBN(0x38a673a2, 0x1e258d46), + TOBN(0xbd1cc1f2, 0x83141c95), TOBN(0x3b2ecf4f, 0x0e96e486), + TOBN(0xcd3aa896, 0x74e5fc78), TOBN(0x415ec10c, 0x2482fa7a), + TOBN(0x15234419, 0x80503380), TOBN(0x513d917a, 0xd314b392), + TOBN(0xb0b52f4e, 0x63caecae), TOBN(0x07bf22ad, 0x2dc7780b), + TOBN(0xe761e8a1, 0xe4306839), TOBN(0x1b3be962, 0x5dd7feaa), + TOBN(0x4fe728de, 0x74c778f1), TOBN(0xf1fa0bda, 0x5e0070f6), + TOBN(0x85205a31, 0x6ec3f510), TOBN(0x2c7e4a14, 0xd2980475), + TOBN(0xde3c19c0, 0x6f30ebfd), TOBN(0xdb1c1f38, 0xd4b7e644), + TOBN(0xfe291a75, 0x5dce364a), TOBN(0xb7b22a3c, 0x058f5be3), + TOBN(0x2cd2c302, 0x37fea38c), TOBN(0x2930967a, 0x2e17be17), + TOBN(0x87f009de, 0x0c061c65), TOBN(0xcb014aac, 0xedc6ed44), + TOBN(0x49bd1cb4, 0x3bafb1eb), TOBN(0x81bd8b5c, 0x282d3688), + TOBN(0x1cdab87e, 0xf01a17af), TOBN(0x21f37ac4, 0xe710063b), + TOBN(0x5a6c5676, 0x42fc8193), TOBN(0xf4753e70, 0x56a6015c), + TOBN(0x020f795e, 0xa15b0a44), TOBN(0x8f37c8d7, 0x8958a958), + TOBN(0x63b7e89b, 0xa4b675b5), TOBN(0xb4fb0c0c, 0x0fc31aea), + TOBN(0xed95e639, 0xa7ff1f2e), TOBN(0x9880f5a3, 0x619614fb), + TOBN(0xdeb6ff02, 0x947151ab), TOBN(0x5bc5118c, 0xa868dcdb), + TOBN(0xd8da2055, 0x4c20cea5), TOBN(0xcac2776e, 0x14c4d69a), + TOBN(0xcccb22c1, 0x622d599b), TOBN(0xa4ddb653, 0x68a9bb50), + TOBN(0x2c4ff151, 0x1b4941b4), TOBN(0xe1ff19b4, 0x6efba588), + TOBN(0x35034363, 0xc48345e0), TOBN(0x45542e3d, 0x1e29dfc4), + TOBN(0xf197cb91, 0x349f7aed), TOBN(0x3b2b5a00, 0x8fca8420), + TOBN(0x7c175ee8, 0x23aaf6d8), TOBN(0x54dcf421, 0x35af32b6), + TOBN(0x0ba14307, 0x27d6561e), TOBN(0x879d5ee4, 0xd175b1e2), + TOBN(0xc7c43673, 0x99807db5), TOBN(0x77a54455, 0x9cd55bcd), + TOBN(0xe6c2ff13, 0x0105c072), TOBN(0x18f7a99f, 0x8dda7da4), + TOBN(0x4c301820, 0x0e2d35c1), TOBN(0x06a53ca0, 0xd9cc6c82), + TOBN(0xaa21cc1e, 0xf1aa1d9e), TOBN(0x32414334, 0x4a75b1e8), + TOBN(0x2a6d1328, 0x0ebe9fdc), TOBN(0x16bd173f, 0x98a4755a), + TOBN(0xfbb9b245, 0x2133ffd9), TOBN(0x39a8b2f1, 0x830f1a20), + TOBN(0x484bc97d, 0xd5a1f52a), TOBN(0xd6aebf56, 0xa40eddf8), + TOBN(0x32257acb, 0x76ccdac6), TOBN(0xaf4d36ec, 0x1586ff27), + TOBN(0x8eaa8863, 0xf8de7dd1), TOBN(0x0045d5cf, 0x88647c16)} + , + {TOBN(0xa6f3d574, 0xc005979d), TOBN(0xc2072b42, 0x6a40e350), + TOBN(0xfca5c156, 0x8de2ecf9), TOBN(0xa8c8bf5b, 0xa515344e), + TOBN(0x97aee555, 0x114df14a), TOBN(0xd4374a4d, 0xfdc5ec6b), + TOBN(0x754cc28f, 0x2ca85418), TOBN(0x71cb9e27, 0xd3c41f78), + TOBN(0x89105079, 0x03605c39), TOBN(0xf0843d9e, 0xa142c96c), + TOBN(0xf3744934, 0x16923684), TOBN(0x732caa2f, 0xfa0a2893), + TOBN(0xb2e8c270, 0x61160170), TOBN(0xc32788cc, 0x437fbaa3), + TOBN(0x39cd818e, 0xa6eda3ac), TOBN(0xe2e94239, 0x9e2b2e07), + TOBN(0x6967d39b, 0x0260e52a), TOBN(0xd42585cc, 0x90653325), + TOBN(0x0d9bd605, 0x21ca7954), TOBN(0x4fa20877, 0x81ed57b3), + TOBN(0x60c1eff8, 0xe34a0bbe), TOBN(0x56b0040c, 0x84f6ef64), + TOBN(0x28be2b24, 0xb1af8483), TOBN(0xb2278163, 0xf5531614), + TOBN(0x8df27545, 0x5922ac1c), TOBN(0xa7b3ef5c, 0xa52b3f63), + TOBN(0x8e77b214, 0x71de57c4), TOBN(0x31682c10, 0x834c008b), + TOBN(0xc76824f0, 0x4bd55d31), TOBN(0xb6d1c086, 0x17b61c71), + TOBN(0x31db0903, 0xc2a5089d), TOBN(0x9c092172, 0x184e5d3f), + TOBN(0xdd7ced5b, 0xc00cc638), TOBN(0x1a2015eb, 0x61278fc2), + TOBN(0x2e8e5288, 0x6a37f8d6), TOBN(0xc457786f, 0xe79933ad), + TOBN(0xb3fe4cce, 0x2c51211a), TOBN(0xad9b10b2, 0x24c20498), + TOBN(0x90d87a4f, 0xd28db5e5), TOBN(0x698cd105, 0x3aca2fc3), + TOBN(0x4f112d07, 0xe91b536d), TOBN(0xceb982f2, 0x9eba09d6), + TOBN(0x3c157b2c, 0x197c396f), TOBN(0xe23c2d41, 0x7b66eb24), + TOBN(0x480c57d9, 0x3f330d37), TOBN(0xb3a4c8a1, 0x79108deb), + TOBN(0x702388de, 0xcb199ce5), TOBN(0x0b019211, 0xb944a8d4), + TOBN(0x24f2a692, 0x840bb336), TOBN(0x7c353bdc, 0xa669fa7b), + TOBN(0xda20d6fc, 0xdec9c300), TOBN(0x625fbe2f, 0xa13a4f17), + TOBN(0xa2b1b61a, 0xdbc17328), TOBN(0x008965bf, 0xa9515621), + TOBN(0x49690939, 0xc620ff46), TOBN(0x182dd27d, 0x8717e91c), + TOBN(0x5ace5035, 0xea6c3997), TOBN(0x54259aaa, 0xc2610bef), + TOBN(0xef18bb3f, 0x3c80dd39), TOBN(0x6910b95b, 0x5fc3fa39), + TOBN(0xfce2f510, 0x43e09aee), TOBN(0xced56c9f, 0xa7675665), + TOBN(0x10e265ac, 0xd872db61), TOBN(0x6982812e, 0xae9fce69), + TOBN(0x29be11c6, 0xce800998), TOBN(0x72bb1752, 0xb90360d9), + TOBN(0x2c193197, 0x5a4ad590), TOBN(0x2ba2f548, 0x9fc1dbc0), + TOBN(0x7fe4eebb, 0xe490ebe0), TOBN(0x12a0a4cd, 0x7fae11c0), + TOBN(0x7197cf81, 0xe903ba37), TOBN(0xcf7d4aa8, 0xde1c6dd8), + TOBN(0x92af6bf4, 0x3fd5684c), TOBN(0x2b26eecf, 0x80360aa1), + TOBN(0xbd960f30, 0x00546a82), TOBN(0x407b3c43, 0xf59ad8fe), + TOBN(0x86cae5fe, 0x249c82ba), TOBN(0x9e0faec7, 0x2463744c), + TOBN(0x87f551e8, 0x94916272), TOBN(0x033f9344, 0x6ceb0615), + TOBN(0x1e5eb0d1, 0x8be82e84), TOBN(0x89967f0e, 0x7a582fef), + TOBN(0xbcf687d5, 0xa6e921fa), TOBN(0xdfee4cf3, 0xd37a09ba), + TOBN(0x94f06965, 0xb493c465), TOBN(0x638b9a1c, 0x7635c030), + TOBN(0x76667864, 0x66f05e9f), TOBN(0xccaf6808, 0xc04da725), + TOBN(0xca2eb690, 0x768fccfc), TOBN(0xf402d37d, 0xb835b362), + TOBN(0x0efac0d0, 0xe2fdfcce), TOBN(0xefc9cdef, 0xb638d990), + TOBN(0x2af12b72, 0xd1669a8b), TOBN(0x33c536bc, 0x5774ccbd), + TOBN(0x30b21909, 0xfb34870e), TOBN(0xc38fa2f7, 0x7df25aca), + TOBN(0x74c5f02b, 0xbf81f3f5), TOBN(0x0525a5ae, 0xaf7e4581), + TOBN(0x88d2aaba, 0x433c54ae), TOBN(0xed9775db, 0x806a56c5), + TOBN(0xd320738a, 0xc0edb37d), TOBN(0x25fdb6ee, 0x66cc1f51), + TOBN(0xac661d17, 0x10600d76), TOBN(0x931ec1f3, 0xbdd1ed76), + TOBN(0x65c11d62, 0x19ee43f1), TOBN(0x5cd57c3e, 0x60829d97), + TOBN(0xd26c91a3, 0x984be6e8), TOBN(0xf08d9309, 0x8b0c53bd), + TOBN(0x94bc9e5b, 0xc016e4ea), TOBN(0xd3916839, 0x11d43d2b), + TOBN(0x886c5ad7, 0x73701155), TOBN(0xe0377626, 0x20b00715), + TOBN(0x7f01c9ec, 0xaa80ba59), TOBN(0x3083411a, 0x68538e51), + TOBN(0x970370f1, 0xe88128af), TOBN(0x625cc3db, 0x91dec14b), + TOBN(0xfef9666c, 0x01ac3107), TOBN(0xb2a8d577, 0xd5057ac3), + TOBN(0xb0f26299, 0x92be5df7), TOBN(0xf579c8e5, 0x00353924), + TOBN(0xb8fa3d93, 0x1341ed7a), TOBN(0x4223272c, 0xa7b59d49), + TOBN(0x3dcb1947, 0x83b8c4a4), TOBN(0x4e413c01, 0xed1302e4), + TOBN(0x6d999127, 0xe17e44ce), TOBN(0xee86bf75, 0x33b3adfb), + TOBN(0xf6902fe6, 0x25aa96ca), TOBN(0xb73540e4, 0xe5aae47d), + TOBN(0x32801d7b, 0x1b4a158c), TOBN(0xe571c99e, 0x27e2a369), + TOBN(0x40cb76c0, 0x10d9f197), TOBN(0xc308c289, 0x3167c0ae), + TOBN(0xa6ef9dd3, 0xeb7958f2), TOBN(0xa7226dfc, 0x300879b1), + TOBN(0x6cd0b362, 0x7edf0636), TOBN(0x4efbce6c, 0x7bc37eed), + TOBN(0x75f92a05, 0x8d699021), TOBN(0x586d4c79, 0x772566e3), + TOBN(0x378ca5f1, 0x761ad23a), TOBN(0x650d86fc, 0x1465a8ac), + TOBN(0x7a4ed457, 0x842ba251), TOBN(0x6b65e3e6, 0x42234933), + TOBN(0xaf1543b7, 0x31aad657), TOBN(0xa4cefe98, 0xcbfec369), + TOBN(0xb587da90, 0x9f47befb), TOBN(0x6562e9fb, 0x41312d13), + TOBN(0xa691ea59, 0xeff1cefe), TOBN(0xcc30477a, 0x05fc4cf6), + TOBN(0xa1632461, 0x0b0ffd3d), TOBN(0xa1f16f3b, 0x5b355956), + TOBN(0x5b148d53, 0x4224ec24), TOBN(0xdc834e7b, 0xf977012a), + TOBN(0x7bfc5e75, 0xb2c69dbc), TOBN(0x3aa77a29, 0x03c3da6c), + TOBN(0xde0df03c, 0xca910271), TOBN(0xcbd5ca4a, 0x7806dc55), + TOBN(0xe1ca5807, 0x6db476cb), TOBN(0xfde15d62, 0x5f37a31e), + TOBN(0xf49af520, 0xf41af416), TOBN(0x96c5c5b1, 0x7d342db5), + TOBN(0x155c43b7, 0xeb4ceb9b), TOBN(0x2e993010, 0x4e77371a), + TOBN(0x1d2987da, 0x675d43af), TOBN(0xef2bc1c0, 0x8599fd72), + TOBN(0x96894b7b, 0x9342f6b2), TOBN(0x201eadf2, 0x7c8e71f0), + TOBN(0xf3479d9f, 0x4a1f3efc), TOBN(0xe0f8a742, 0x702a9704), + TOBN(0xeafd44b6, 0xb3eba40c), TOBN(0xf9739f29, 0xc1c1e0d0), + TOBN(0x0091471a, 0x619d505e), TOBN(0xc15f9c96, 0x9d7c263e), + TOBN(0x5be47285, 0x83afbe33), TOBN(0xa3b6d6af, 0x04f1e092), + TOBN(0xe76526b9, 0x751a9d11), TOBN(0x2ec5b26d, 0x9a4ae4d2), + TOBN(0xeb66f4d9, 0x02f6fb8d), TOBN(0x4063c561, 0x96912164), + TOBN(0xeb7050c1, 0x80ef3000), TOBN(0x288d1c33, 0xeaa5b3f0), + TOBN(0xe87c68d6, 0x07806fd8), TOBN(0xb2f7f9d5, 0x4bbbf50f), + TOBN(0x25972f3a, 0xac8d6627), TOBN(0xf8547774, 0x10e8c13b), + TOBN(0xcc50ef6c, 0x872b4a60), TOBN(0xab2a34a4, 0x4613521b), + TOBN(0x39c5c190, 0x983e15d1), TOBN(0x61dde5df, 0x59905512), + TOBN(0xe417f621, 0x9f2275f3), TOBN(0x0750c8b6, 0x451d894b), + TOBN(0x75b04ab9, 0x78b0bdaa), TOBN(0x3bfd9fd4, 0x458589bd), + TOBN(0xf1013e30, 0xee9120b6), TOBN(0x2b51af93, 0x23a4743e), + TOBN(0xea96ffae, 0x48d14d9e), TOBN(0x71dc0dbe, 0x698a1d32), + TOBN(0x914962d2, 0x0180cca4), TOBN(0x1ae60677, 0xc3568963), + TOBN(0x8cf227b1, 0x437bc444), TOBN(0xc650c83b, 0xc9962c7a), + TOBN(0x23c2c7dd, 0xfe7ccfc4), TOBN(0xf925c89d, 0x1b929d48), + TOBN(0x4460f74b, 0x06783c33), TOBN(0xac2c8d49, 0xa590475a), + TOBN(0xfb40b407, 0xb807bba0), TOBN(0x9d1e362d, 0x69ff8f3a), + TOBN(0xa33e9681, 0xcbef64a4), TOBN(0x67ece5fa, 0x332fb4b2), + TOBN(0x6900a99b, 0x739f10e3), TOBN(0xc3341ca9, 0xff525925), + TOBN(0xee18a626, 0xa9e2d041), TOBN(0xa5a83685, 0x29580ddd), + TOBN(0xf3470c81, 0x9d7de3cd), TOBN(0xedf02586, 0x2062cf9c), + TOBN(0xf43522fa, 0xc010edb0), TOBN(0x30314135, 0x13a4b1ae), + TOBN(0xc792e02a, 0xdb22b94b), TOBN(0x993d8ae9, 0xa1eaa45b), + TOBN(0x8aad6cd3, 0xcd1e1c63), TOBN(0x89529ca7, 0xc5ce688a), + TOBN(0x2ccee3aa, 0xe572a253), TOBN(0xe02b6438, 0x02a21efb), + TOBN(0xa7091b6e, 0xc9430358), TOBN(0x06d1b1fa, 0x9d7db504), + TOBN(0x58846d32, 0xc4744733), TOBN(0x40517c71, 0x379f9e34), + TOBN(0x2f65655f, 0x130ef6ca), TOBN(0x526e4488, 0xf1f3503f), + TOBN(0x8467bd17, 0x7ee4a976), TOBN(0x1d9dc913, 0x921363d1), + TOBN(0xd8d24c33, 0xb069e041), TOBN(0x5eb5da0a, 0x2cdf7f51), + TOBN(0x1c0f3cb1, 0x197b994f), TOBN(0x3c95a6c5, 0x2843eae9), + TOBN(0x7766ffc9, 0xa6097ea5), TOBN(0x7bea4093, 0xd723b867), + TOBN(0xb48e1f73, 0x4db378f9), TOBN(0x70025b00, 0xe37b77ac), + TOBN(0x943dc8e7, 0xaf24ad46), TOBN(0xb98a15ac, 0x16d00a85), + TOBN(0x3adc38ba, 0x2743b004), TOBN(0xb1c7f4f7, 0x334415ee), + TOBN(0xea43df8f, 0x1e62d05a), TOBN(0x32618905, 0x9d76a3b6), + TOBN(0x2fbd0bb5, 0xa23a0f46), TOBN(0x5bc971db, 0x6a01918c), + TOBN(0x7801d94a, 0xb4743f94), TOBN(0xb94df65e, 0x676ae22b), + TOBN(0xaafcbfab, 0xaf95894c), TOBN(0x7b9bdc07, 0x276b2241), + TOBN(0xeaf98362, 0x5bdda48b), TOBN(0x5977faf2, 0xa3fcb4df), + TOBN(0xbed042ef, 0x052c4b5b), TOBN(0x9fe87f71, 0x067591f0), + TOBN(0xc89c73ca, 0x22f24ec7), TOBN(0x7d37fa9e, 0xe64a9f1b), + TOBN(0x2710841a, 0x15562627), TOBN(0x2c01a613, 0xc243b034), + TOBN(0x1d135c56, 0x2bc68609), TOBN(0xc2ca1715, 0x8b03f1f6), + TOBN(0xc9966c2d, 0x3eb81d82), TOBN(0xc02abf4a, 0x8f6df13e), + TOBN(0x77b34bd7, 0x8f72b43b), TOBN(0xaff6218f, 0x360c82b0), + TOBN(0x0aa5726c, 0x8d55b9d2), TOBN(0xdc0adbe9, 0x99e9bffb), + TOBN(0x9097549c, 0xefb9e72a), TOBN(0x16755712, 0x9dfb3111), + TOBN(0xdd8bf984, 0xf26847f9), TOBN(0xbcb8e387, 0xdfb30cb7), + TOBN(0xc1fd32a7, 0x5171ef9c), TOBN(0x977f3fc7, 0x389b363f), + TOBN(0x116eaf2b, 0xf4babda0), TOBN(0xfeab68bd, 0xf7113c8e), + TOBN(0xd1e3f064, 0xb7def526), TOBN(0x1ac30885, 0xe0b3fa02), + TOBN(0x1c5a6e7b, 0x40142d9d), TOBN(0x839b5603, 0x30921c0b), + TOBN(0x48f301fa, 0x36a116a3), TOBN(0x380e1107, 0xcfd9ee6d), + TOBN(0x7945ead8, 0x58854be1), TOBN(0x4111c12e, 0xcbd4d49d), + TOBN(0xece3b1ec, 0x3a29c2ef), TOBN(0x6356d404, 0x8d3616f5), + TOBN(0x9f0d6a8f, 0x594d320e), TOBN(0x0989316d, 0xf651ccd2), + TOBN(0x6c32117a, 0x0f8fdde4), TOBN(0x9abe5cc5, 0xa26a9bbc), + TOBN(0xcff560fb, 0x9723f671), TOBN(0x21b2a12d, 0x7f3d593c), + TOBN(0xe4cb18da, 0x24ba0696), TOBN(0x186e2220, 0xc3543384), + TOBN(0x722f64e0, 0x88312c29), TOBN(0x94282a99, 0x17dc7752), + TOBN(0x62467bbf, 0x5a85ee89), TOBN(0xf435c650, 0xf10076a0), + TOBN(0xc9ff1539, 0x43b3a50b), TOBN(0x7132130c, 0x1a53efbc), + TOBN(0x31bfe063, 0xf7b0c5b7), TOBN(0xb0179a7d, 0x4ea994cc), + TOBN(0x12d064b3, 0xc85f455b), TOBN(0x47259328, 0x8f6e0062), + TOBN(0xf64e590b, 0xb875d6d9), TOBN(0x22dd6225, 0xad92bcc7), + TOBN(0xb658038e, 0xb9c3bd6d), TOBN(0x00cdb0d6, 0xfbba27c8), + TOBN(0x0c681337, 0x1062c45d), TOBN(0xd8515b8c, 0x2d33407d), + TOBN(0xcb8f699e, 0x8cbb5ecf), TOBN(0x8c4347f8, 0xc608d7d8), + TOBN(0x2c11850a, 0xbb3e00db), TOBN(0x20a8dafd, 0xecb49d19), + TOBN(0xbd781480, 0x45ee2f40), TOBN(0x75e354af, 0x416b60cf), + TOBN(0xde0b58a1, 0x8d49a8c4), TOBN(0xe40e94e2, 0xfa359536), + TOBN(0xbd4fa59f, 0x62accd76), TOBN(0x05cf466a, 0x8c762837), + TOBN(0xb5abda99, 0x448c277b), TOBN(0x5a9e01bf, 0x48b13740), + TOBN(0x9d457798, 0x326aad8d), TOBN(0xbdef4954, 0xc396f7e7), + TOBN(0x6fb274a2, 0xc253e292), TOBN(0x2800bf0a, 0x1cfe53e7), + TOBN(0x22426d31, 0x44438fd4), TOBN(0xef233923, 0x5e259f9a), + TOBN(0x4188503c, 0x03f66264), TOBN(0x9e5e7f13, 0x7f9fdfab), + TOBN(0x565eb76c, 0x5fcc1aba), TOBN(0xea632548, 0x59b5bff8), + TOBN(0x5587c087, 0xaab6d3fa), TOBN(0x92b639ea, 0x6ce39c1b), + TOBN(0x0706e782, 0x953b135c), TOBN(0x7308912e, 0x425268ef), + TOBN(0x599e92c7, 0x090e7469), TOBN(0x83b90f52, 0x9bc35e75), + TOBN(0x4750b3d0, 0x244975b3), TOBN(0xf3a44358, 0x11965d72), + TOBN(0x179c6774, 0x9c8dc751), TOBN(0xff18cdfe, 0xd23d9ff0), + TOBN(0xc4013833, 0x2028e247), TOBN(0x96e280e2, 0xf3bfbc79), + TOBN(0xf60417bd, 0xd0880a84), TOBN(0x263c9f3d, 0x2a568151), + TOBN(0x36be15b3, 0x2d2ce811), TOBN(0x846dc0c2, 0xf8291d21), + TOBN(0x5cfa0ecb, 0x789fcfdb), TOBN(0x45a0beed, 0xd7535b9a), + TOBN(0xec8e9f07, 0x96d69af1), TOBN(0x31a7c5b8, 0x599ab6dc), + TOBN(0xd36d45ef, 0xf9e2e09f), TOBN(0x3cf49ef1, 0xdcee954b), + TOBN(0x6be34cf3, 0x086cff9b), TOBN(0x88dbd491, 0x39a3360f), + TOBN(0x1e96b8cc, 0x0dbfbd1d), TOBN(0xc1e5f7bf, 0xcb7e2552), + TOBN(0x0547b214, 0x28819d98), TOBN(0xc770dd9c, 0x7aea9dcb), + TOBN(0xaef0d4c7, 0x041d68c8), TOBN(0xcc2b9818, 0x13cb9ba8), + TOBN(0x7fc7bc76, 0xfe86c607), TOBN(0x6b7b9337, 0x502a9a95), + TOBN(0x1948dc27, 0xd14dab63), TOBN(0x249dd198, 0xdae047be), + TOBN(0xe8356584, 0xa981a202), TOBN(0x3531dd18, 0x3a893387), + TOBN(0x1be11f90, 0xc85c7209), TOBN(0x93d2fe1e, 0xe2a52b5a), + TOBN(0x8225bfe2, 0xec6d6b97), TOBN(0x9cf6d6f4, 0xbd0aa5de), + TOBN(0x911459cb, 0x54779f5f), TOBN(0x5649cddb, 0x86aeb1f3), + TOBN(0x32133579, 0x3f26ce5a), TOBN(0xc289a102, 0x550f431e), + TOBN(0x559dcfda, 0x73b84c6f), TOBN(0x84973819, 0xee3ac4d7), + TOBN(0xb51e55e6, 0xf2606a82), TOBN(0xe25f7061, 0x90f2fb57), + TOBN(0xacef6c2a, 0xb1a4e37c), TOBN(0x864e359d, 0x5dcf2706), + TOBN(0x479e6b18, 0x7ce57316), TOBN(0x2cab2500, 0x3a96b23d), + TOBN(0xed489862, 0x8ef16df7), TOBN(0x2056538c, 0xef3758b5), + TOBN(0xa7df865e, 0xf15d3101), TOBN(0x80c5533a, 0x61b553d7), + TOBN(0x366e1997, 0x4ed14294), TOBN(0x6620741f, 0xb3c0bcd6), + TOBN(0x21d1d9c4, 0xedc45418), TOBN(0x005b859e, 0xc1cc4a9d), + TOBN(0xdf01f630, 0xa1c462f0), TOBN(0x15d06cf3, 0xf26820c7), + TOBN(0x9f7f24ee, 0x3484be47), TOBN(0x2ff33e96, 0x4a0c902f), + TOBN(0x00bdf457, 0x5a0bc453), TOBN(0x2378dfaf, 0x1aa238db), + TOBN(0x272420ec, 0x856720f2), TOBN(0x2ad9d95b, 0x96797291), + TOBN(0xd1242cc6, 0x768a1558), TOBN(0x2e287f8b, 0x5cc86aa8), + TOBN(0x796873d0, 0x990cecaa), TOBN(0xade55f81, 0x675d4080), + TOBN(0x2645eea3, 0x21f0cd84), TOBN(0x7a1efa0f, 0xb4e17d02), + TOBN(0xf6858420, 0x037cc061), TOBN(0x682e05f0, 0xd5d43e12), + TOBN(0x59c36994, 0x27218710), TOBN(0x85cbba4d, 0x3f7cd2fc), + TOBN(0x726f9729, 0x7a3cd22a), TOBN(0x9f8cd5dc, 0x4a628397), + TOBN(0x17b93ab9, 0xc23165ed), TOBN(0xff5f5dbf, 0x122823d4), + TOBN(0xc1e4e4b5, 0x654a446d), TOBN(0xd1a9496f, 0x677257ba), + TOBN(0x6387ba94, 0xde766a56), TOBN(0x23608bc8, 0x521ec74a), + TOBN(0x16a522d7, 0x6688c4d4), TOBN(0x9d6b4282, 0x07373abd), + TOBN(0xa62f07ac, 0xb42efaa3), TOBN(0xf73e00f7, 0xe3b90180), + TOBN(0x36175fec, 0x49421c3e), TOBN(0xc4e44f9b, 0x3dcf2678), + TOBN(0x76df436b, 0x7220f09f), TOBN(0x172755fb, 0x3aa8b6cf), + TOBN(0xbab89d57, 0x446139cc), TOBN(0x0a0a6e02, 0x5fe0208f), + TOBN(0xcdbb63e2, 0x11e5d399), TOBN(0x33ecaa12, 0xa8977f0b), + TOBN(0x59598b21, 0xf7c42664), TOBN(0xb3e91b32, 0xab65d08a), + TOBN(0x035822ee, 0xf4502526), TOBN(0x1dcf0176, 0x720a82a9), + TOBN(0x50f8598f, 0x3d589e02), TOBN(0xdf0478ff, 0xb1d63d2c), + TOBN(0x8b8068bd, 0x1571cd07), TOBN(0x30c3aa4f, 0xd79670cd), + TOBN(0x25e8fd4b, 0x941ade7f), TOBN(0x3d1debdc, 0x32790011), + TOBN(0x65b6dcbd, 0x3a3f9ff0), TOBN(0x282736a4, 0x793de69c), + TOBN(0xef69a0c3, 0xd41d3bd3), TOBN(0xb533b8c9, 0x07a26bde), + TOBN(0xe2801d97, 0xdb2edf9f), TOBN(0xdc4a8269, 0xe1877af0), + TOBN(0x6c1c5851, 0x3d590dbe), TOBN(0x84632f6b, 0xee4e9357), + TOBN(0xd36d36b7, 0x79b33374), TOBN(0xb46833e3, 0x9bbca2e6), + TOBN(0x37893913, 0xf7fc0586), TOBN(0x385315f7, 0x66bf4719), + TOBN(0x72c56293, 0xb31855dc), TOBN(0xd1416d4e, 0x849061fe), + TOBN(0xbeb3ab78, 0x51047213), TOBN(0x447f6e61, 0xf040c996), + TOBN(0xd06d310d, 0x638b1d0c), TOBN(0xe28a413f, 0xbad1522e), + TOBN(0x685a76cb, 0x82003f86), TOBN(0x610d07f7, 0x0bcdbca3), + TOBN(0x6ff66021, 0x9ca4c455), TOBN(0x7df39b87, 0xcea10eec), + TOBN(0xb9255f96, 0xe22db218), TOBN(0x8cc6d9eb, 0x08a34c44), + TOBN(0xcd4ffb86, 0x859f9276), TOBN(0x8fa15eb2, 0x50d07335), + TOBN(0xdf553845, 0xcf2c24b5), TOBN(0x89f66a9f, 0x52f9c3ba), + TOBN(0x8f22b5b9, 0xe4a7ceb3), TOBN(0xaffef809, 0x0e134686), + TOBN(0x3e53e1c6, 0x8eb8fac2), TOBN(0x93c1e4eb, 0x28aec98e), + TOBN(0xb6b91ec5, 0x32a43bcb), TOBN(0x2dbfa947, 0xb2d74a51), + TOBN(0xe065d190, 0xca84bad7), TOBN(0xfb13919f, 0xad58e65c), + TOBN(0x3c41718b, 0xf1cb6e31), TOBN(0x688969f0, 0x06d05c3f), + TOBN(0xd4f94ce7, 0x21264d45), TOBN(0xfdfb65e9, 0x7367532b), + TOBN(0x5b1be8b1, 0x0945a39d), TOBN(0x229f789c, 0x2b8baf3b), + TOBN(0xd8f41f3e, 0x6f49f15d), TOBN(0x678ce828, 0x907f0792), + TOBN(0xc69ace82, 0xfca6e867), TOBN(0x106451ae, 0xd01dcc89), + TOBN(0x1bb4f7f0, 0x19fc32d2), TOBN(0x64633dfc, 0xb00c52d2), + TOBN(0x8f13549a, 0xad9ea445), TOBN(0x99a3bf50, 0xfb323705), + TOBN(0x0c9625a2, 0x534d4dbc), TOBN(0x45b8f1d1, 0xc2a2fea3), + TOBN(0x76ec21a1, 0xa530fc1a), TOBN(0x4bac9c2a, 0x9e5bd734), + TOBN(0x5996d76a, 0x7b4e3587), TOBN(0x0045cdee, 0x1182d9e3), + TOBN(0x1aee24b9, 0x1207f13d), TOBN(0x66452e97, 0x97345a41), + TOBN(0x16e5b054, 0x9f950cd0), TOBN(0x9cc72fb1, 0xd7fdd075), + TOBN(0x6edd61e7, 0x66249663), TOBN(0xde4caa4d, 0xf043cccb), + TOBN(0x11b1f57a, 0x55c7ac17), TOBN(0x779cbd44, 0x1a85e24d), + TOBN(0x78030f86, 0xe46081e7), TOBN(0xfd4a6032, 0x8e20f643), + TOBN(0xcc7a6488, 0x0a750c0f), TOBN(0x39bacfe3, 0x4e548e83), + TOBN(0x3d418c76, 0x0c110f05), TOBN(0x3e4daa4c, 0xb1f11588), + TOBN(0x2733e7b5, 0x5ffc69ff), TOBN(0x46f147bc, 0x92053127), + TOBN(0x885b2434, 0xd722df94), TOBN(0x6a444f65, 0xe6fc6b7c)} + , + {TOBN(0x7a1a465a, 0xc3f16ea8), TOBN(0x115a461d, 0xb2f1d11c), + TOBN(0x4767dd95, 0x6c68a172), TOBN(0x3392f2eb, 0xd13a4698), + TOBN(0xc7a99ccd, 0xe526cdc7), TOBN(0x8e537fdc, 0x22292b81), + TOBN(0x76d8cf69, 0xa6d39198), TOBN(0xffc5ff43, 0x2446852d), + TOBN(0x97b14f7e, 0xa90567e6), TOBN(0x513257b7, 0xb6ae5cb7), + TOBN(0x85454a3c, 0x9f10903d), TOBN(0xd8d2c9ad, 0x69bc3724), + TOBN(0x38da9324, 0x6b29cb44), TOBN(0xb540a21d, 0x77c8cbac), + TOBN(0x9bbfe435, 0x01918e42), TOBN(0xfffa707a, 0x56c3614e), + TOBN(0x0ce4e3f1, 0xd4e353b7), TOBN(0x062d8a14, 0xef46b0a0), + TOBN(0x6408d5ab, 0x574b73fd), TOBN(0xbc41d1c9, 0xd3273ffd), + TOBN(0x3538e1e7, 0x6be77800), TOBN(0x71fe8b37, 0xc5655031), + TOBN(0x1cd91621, 0x6b9b331a), TOBN(0xad825d0b, 0xbb388f73), + TOBN(0x56c2e05b, 0x1cb76219), TOBN(0x0ec0bf91, 0x71567e7e), + TOBN(0xe7076f86, 0x61c4c910), TOBN(0xd67b085b, 0xbabc04d9), + TOBN(0x9fb90459, 0x5e93a96a), TOBN(0x7526c1ea, 0xfbdc249a), + TOBN(0x0d44d367, 0xecdd0bb7), TOBN(0x95399917, 0x9dc0d695), + TOBN(0x61360ee9, 0x9e240d18), TOBN(0x057cdcac, 0xb4b94466), + TOBN(0xe7667cd1, 0x2fe5325c), TOBN(0x1fa297b5, 0x21974e3b), + TOBN(0xfa4081e7, 0xdb083d76), TOBN(0x31993be6, 0xf206bd15), + TOBN(0x8949269b, 0x14c19f8c), TOBN(0x21468d72, 0xa9d92357), + TOBN(0x2ccbc583, 0xa4c506ec), TOBN(0x957ed188, 0xd1acfe97), + TOBN(0x8baed833, 0x12f1aea2), TOBN(0xef2a6cb4, 0x8325362d), + TOBN(0x130dde42, 0x8e195c43), TOBN(0xc842025a, 0x0e6050c6), + TOBN(0x2da972a7, 0x08686a5d), TOBN(0xb52999a1, 0xe508b4a8), + TOBN(0xd9f090b9, 0x10a5a8bd), TOBN(0xca91d249, 0x096864da), + TOBN(0x8e6a93be, 0x3f67dbc1), TOBN(0xacae6fba, 0xf5f4764c), + TOBN(0x1563c6e0, 0xd21411a0), TOBN(0x28fa787f, 0xda0a4ad8), + TOBN(0xd524491c, 0x908c8030), TOBN(0x1257ba0e, 0x4c795f07), + TOBN(0x83f49167, 0xceca9754), TOBN(0x426d2cf6, 0x4b7939a0), + TOBN(0x2555e355, 0x723fd0bf), TOBN(0xa96e6d06, 0xc4f144e2), + TOBN(0x4768a8dd, 0x87880e61), TOBN(0x15543815, 0xe508e4d5), + TOBN(0x09d7e772, 0xb1b65e15), TOBN(0x63439dd6, 0xac302fa0), + TOBN(0xb93f802f, 0xc14e35c2), TOBN(0x71735b7c, 0x4341333c), + TOBN(0x03a25104, 0x16d4f362), TOBN(0x3f4d069b, 0xbf433c8e), + TOBN(0x0d83ae01, 0xf78f5a7c), TOBN(0x50a8ffbe, 0x7c4eed07), + TOBN(0xc74f8906, 0x76e10f83), TOBN(0x7d080966, 0x9ddaf8e1), + TOBN(0xb11df8e1, 0x698e04cc), TOBN(0x877be203, 0x169005c8), + TOBN(0x32749e8c, 0x4f3c6179), TOBN(0x2dbc9d0a, 0x7853fc05), + TOBN(0x187d4f93, 0x9454d937), TOBN(0xe682ce9d, 0xb4800e1b), + TOBN(0xa9129ad8, 0x165e68e8), TOBN(0x0fe29735, 0xbe7f785b), + TOBN(0x5303f40c, 0x5b9e02b7), TOBN(0xa37c9692, 0x35ee04e8), + TOBN(0x5f46cc20, 0x34d6632b), TOBN(0x55ef72b2, 0x96ac545b), + TOBN(0xabec5c1f, 0x7b91b062), TOBN(0x0a79e1c7, 0xbb33e821), + TOBN(0xbb04b428, 0x3a9f4117), TOBN(0x0de1f28f, 0xfd2a475a), + TOBN(0x31019ccf, 0x3a4434b4), TOBN(0xa3458111, 0x1a7954dc), + TOBN(0xa9dac80d, 0xe34972a7), TOBN(0xb043d054, 0x74f6b8dd), + TOBN(0x021c319e, 0x11137b1a), TOBN(0x00a754ce, 0xed5cc03f), + TOBN(0x0aa2c794, 0xcbea5ad4), TOBN(0x093e67f4, 0x70c015b6), + TOBN(0x72cdfee9, 0xc97e3f6b), TOBN(0xc10bcab4, 0xb6da7461), + TOBN(0x3b02d2fc, 0xb59806b9), TOBN(0x85185e89, 0xa1de6f47), + TOBN(0x39e6931f, 0x0eb6c4d4), TOBN(0x4d4440bd, 0xd4fa5b04), + TOBN(0x5418786e, 0x34be7eb8), TOBN(0x6380e521, 0x9d7259bc), + TOBN(0x20ac0351, 0xd598d710), TOBN(0x272c4166, 0xcb3a4da4), + TOBN(0xdb82fe1a, 0xca71de1f), TOBN(0x746e79f2, 0xd8f54b0f), + TOBN(0x6e7fc736, 0x4b573e9b), TOBN(0x75d03f46, 0xfd4b5040), + TOBN(0x5c1cc36d, 0x0b98d87b), TOBN(0x513ba3f1, 0x1f472da1), + TOBN(0x79d0af26, 0xabb177dd), TOBN(0xf82ab568, 0x7891d564), + TOBN(0x2b6768a9, 0x72232173), TOBN(0xefbb3bb0, 0x8c1f6619), + TOBN(0xb29c11db, 0xa6d18358), TOBN(0x519e2797, 0xb0916d3a), + TOBN(0xd4dc18f0, 0x9188e290), TOBN(0x648e86e3, 0x98b0ca7f), + TOBN(0x859d3145, 0x983c38b5), TOBN(0xb14f176c, 0x637abc8b), + TOBN(0x2793fb9d, 0xcaff7be6), TOBN(0xebe5a55f, 0x35a66a5a), + TOBN(0x7cec1dcd, 0x9f87dc59), TOBN(0x7c595cd3, 0xfbdbf560), + TOBN(0x5b543b22, 0x26eb3257), TOBN(0x69080646, 0xc4c935fd), + TOBN(0x7f2e4403, 0x81e9ede3), TOBN(0x243c3894, 0xcaf6df0a), + TOBN(0x7c605bb1, 0x1c073b11), TOBN(0xcd06a541, 0xba6a4a62), + TOBN(0x29168949, 0x49d4e2e5), TOBN(0x33649d07, 0x4af66880), + TOBN(0xbfc0c885, 0xe9a85035), TOBN(0xb4e52113, 0xfc410f4b), + TOBN(0xdca3b706, 0x78a6513b), TOBN(0x92ea4a2a, 0x9edb1943), + TOBN(0x02642216, 0xdb6e2dd8), TOBN(0x9b45d0b4, 0x9fd57894), + TOBN(0x114e70db, 0xc69d11ae), TOBN(0x1477dd19, 0x4c57595f), + TOBN(0xbc2208b4, 0xec77c272), TOBN(0x95c5b4d7, 0xdb68f59c), + TOBN(0xb8c4fc63, 0x42e532b7), TOBN(0x386ba422, 0x9ae35290), + TOBN(0xfb5dda42, 0xd201ecbc), TOBN(0x2353dc8b, 0xa0e38fd6), + TOBN(0x9a0b85ea, 0x68f7e978), TOBN(0x96ec5682, 0x2ad6d11f), + TOBN(0x5e279d6c, 0xe5f6886d), TOBN(0xd3fe03cd, 0x3cb1914d), + TOBN(0xfe541fa4, 0x7ea67c77), TOBN(0x952bd2af, 0xe3ea810c), + TOBN(0x791fef56, 0x8d01d374), TOBN(0xa3a1c621, 0x0f11336e), + TOBN(0x5ad0d5a9, 0xc7ec6d79), TOBN(0xff7038af, 0x3225c342), + TOBN(0x003c6689, 0xbc69601b), TOBN(0x25059bc7, 0x45e8747d), + TOBN(0xfa4965b2, 0xf2086fbf), TOBN(0xf6840ea6, 0x86916078), + TOBN(0xd7ac7620, 0x70081d6c), TOBN(0xe600da31, 0xb5328645), + TOBN(0x01916f63, 0x529b8a80), TOBN(0xe80e4858, 0x2d7d6f3e), + TOBN(0x29eb0fe8, 0xd664ca7c), TOBN(0xf017637b, 0xe7b43b0c), + TOBN(0x9a75c806, 0x76cb2566), TOBN(0x8f76acb1, 0xb24892d9), + TOBN(0x7ae7b9cc, 0x1f08fe45), TOBN(0x19ef7329, 0x6a4907d8), + TOBN(0x2db4ab71, 0x5f228bf0), TOBN(0xf3cdea39, 0x817032d7), + TOBN(0x0b1f482e, 0xdcabe3c0), TOBN(0x3baf76b4, 0xbb86325c), + TOBN(0xd49065e0, 0x10089465), TOBN(0x3bab5d29, 0x8e77c596), + TOBN(0x7636c3a6, 0x193dbd95), TOBN(0xdef5d294, 0xb246e499), + TOBN(0xb22c58b9, 0x286b2475), TOBN(0xa0b93939, 0xcd80862b), + TOBN(0x3002c83a, 0xf0992388), TOBN(0x6de01f9b, 0xeacbe14c), + TOBN(0x6aac688e, 0xadd70482), TOBN(0x708de92a, 0x7b4a4e8a), + TOBN(0x75b6dd73, 0x758a6eef), TOBN(0xea4bf352, 0x725b3c43), + TOBN(0x10041f2c, 0x87912868), TOBN(0xb1b1be95, 0xef09297a), + TOBN(0x19ae23c5, 0xa9f3860a), TOBN(0xc4f0f839, 0x515dcf4b), + TOBN(0x3c7ecca3, 0x97f6306a), TOBN(0x744c44ae, 0x68a3a4b0), + TOBN(0x69cd13a0, 0xb3a1d8a2), TOBN(0x7cad0a1e, 0x5256b578), + TOBN(0xea653fcd, 0x33791d9e), TOBN(0x9cc2a05d, 0x74b2e05f), + TOBN(0x73b391dc, 0xfd7affa2), TOBN(0xddb7091e, 0xb6b05442), + TOBN(0xc71e27bf, 0x8538a5c6), TOBN(0x195c63dd, 0x89abff17), + TOBN(0xfd315285, 0x1b71e3da), TOBN(0x9cbdfda7, 0xfa680fa0), + TOBN(0x9db876ca, 0x849d7eab), TOBN(0xebe2764b, 0x3c273271), + TOBN(0x663357e3, 0xf208dcea), TOBN(0x8c5bd833, 0x565b1b70), + TOBN(0xccc3b4f5, 0x9837fc0d), TOBN(0x9b641ba8, 0xa79cf00f), + TOBN(0x7428243d, 0xdfdf3990), TOBN(0x83a594c4, 0x020786b1), + TOBN(0xb712451a, 0x526c4502), TOBN(0x9d39438e, 0x6adb3f93), + TOBN(0xfdb261e3, 0xe9ff0ccd), TOBN(0x80344e3c, 0xe07af4c3), + TOBN(0x75900d7c, 0x2fa4f126), TOBN(0x08a3b865, 0x5c99a232), + TOBN(0x2478b6bf, 0xdb25e0c3), TOBN(0x482cc2c2, 0x71db2edf), + TOBN(0x37df7e64, 0x5f321bb8), TOBN(0x8a93821b, 0x9a8005b4), + TOBN(0x3fa2f10c, 0xcc8c1958), TOBN(0x0d332218, 0x2c269d0a), + TOBN(0x20ab8119, 0xe246b0e6), TOBN(0xb39781e4, 0xd349fd17), + TOBN(0xd293231e, 0xb31aa100), TOBN(0x4b779c97, 0xbb032168), + TOBN(0x4b3f19e1, 0xc8470500), TOBN(0x45b7efe9, 0x0c4c869d), + TOBN(0xdb84f38a, 0xa1a6bbcc), TOBN(0x3b59cb15, 0xb2fddbc1), + TOBN(0xba5514df, 0x3fd165e8), TOBN(0x499fd6a9, 0x061f8811), + TOBN(0x72cd1fe0, 0xbfef9f00), TOBN(0x120a4bb9, 0x79ad7e8a), + TOBN(0xf2ffd095, 0x5f4a5ac5), TOBN(0xcfd174f1, 0x95a7a2f0), + TOBN(0xd42301ba, 0x9d17baf1), TOBN(0xd2fa487a, 0x77f22089), + TOBN(0x9cb09efe, 0xb1dc77e1), TOBN(0xe9566939, 0x21c99682), + TOBN(0x8c546901, 0x6c6067bb), TOBN(0xfd378574, 0x61c24456), + TOBN(0x2b6a6cbe, 0x81796b33), TOBN(0x62d550f6, 0x58e87f8b), + TOBN(0x1b763e1c, 0x7f1b01b4), TOBN(0x4b93cfea, 0x1b1b5e12), + TOBN(0xb9345238, 0x1d531696), TOBN(0x57201c00, 0x88cdde69), + TOBN(0xdde92251, 0x9a86afc7), TOBN(0xe3043895, 0xbd35cea8), + TOBN(0x7608c1e1, 0x8555970d), TOBN(0x8267dfa9, 0x2535935e), + TOBN(0xd4c60a57, 0x322ea38b), TOBN(0xe0bf7977, 0x804ef8b5), + TOBN(0x1a0dab28, 0xc06fece4), TOBN(0xd405991e, 0x94e7b49d), + TOBN(0xc542b6d2, 0x706dab28), TOBN(0xcb228da3, 0xa91618fb), + TOBN(0x224e4164, 0x107d1cea), TOBN(0xeb9fdab3, 0xd0f5d8f1), + TOBN(0xc02ba386, 0x0d6e41cd), TOBN(0x676a72c5, 0x9b1f7146), + TOBN(0xffd6dd98, 0x4d6cb00b), TOBN(0xcef9c5ca, 0xde2e8d7c), + TOBN(0xa1bbf5d7, 0x641c7936), TOBN(0x1b95b230, 0xee8f772e), + TOBN(0xf765a92e, 0xe8ac25b1), TOBN(0xceb04cfc, 0x3a18b7c6), + TOBN(0x27944cef, 0x0acc8966), TOBN(0xcbb3c957, 0x434c1004), + TOBN(0x9c9971a1, 0xa43ff93c), TOBN(0x5bc2db17, 0xa1e358a9), + TOBN(0x45b4862e, 0xa8d9bc82), TOBN(0x70ebfbfb, 0x2201e052), + TOBN(0xafdf64c7, 0x92871591), TOBN(0xea5bcae6, 0xb42d0219), + TOBN(0xde536c55, 0x2ad8f03c), TOBN(0xcd6c3f4d, 0xa76aa33c), + TOBN(0xbeb5f623, 0x0bca6de3), TOBN(0xdd20dd99, 0xb1e706fd), + TOBN(0x90b3ff9d, 0xac9059d4), TOBN(0x2d7b2902, 0x7ccccc4e), + TOBN(0x8a090a59, 0xce98840f), TOBN(0xa5d947e0, 0x8410680a), + TOBN(0x49ae346a, 0x923379a5), TOBN(0x7dbc84f9, 0xb28a3156), + TOBN(0xfd40d916, 0x54a1aff2), TOBN(0xabf318ba, 0x3a78fb9b), + TOBN(0x50152ed8, 0x3029f95e), TOBN(0x9fc1dd77, 0xc58ad7fa), + TOBN(0x5fa57915, 0x13595c17), TOBN(0xb9504668, 0x8f62b3a9), + TOBN(0x907b5b24, 0xff3055b0), TOBN(0x2e995e35, 0x9a84f125), + TOBN(0x87dacf69, 0x7e9bbcfb), TOBN(0x95d0c1d6, 0xe86d96e3), + TOBN(0x65726e3c, 0x2d95a75c), TOBN(0x2c3c9001, 0xacd27f21), + TOBN(0x1deab561, 0x6c973f57), TOBN(0x108b7e2c, 0xa5221643), + TOBN(0x5fee9859, 0xc4ef79d4), TOBN(0xbd62b88a, 0x40d4b8c6), + TOBN(0xb4dd29c4, 0x197c75d6), TOBN(0x266a6df2, 0xb7076feb), + TOBN(0x9512d0ea, 0x4bf2df11), TOBN(0x1320c24f, 0x6b0cc9ec), + TOBN(0x6bb1e0e1, 0x01a59596), TOBN(0x8317c5bb, 0xeff9aaac), + TOBN(0x65bb405e, 0x385aa6c9), TOBN(0x613439c1, 0x8f07988f), + TOBN(0xd730049f, 0x16a66e91), TOBN(0xe97f2820, 0xfa1b0e0d), + TOBN(0x4131e003, 0x304c28ea), TOBN(0x820ab732, 0x526bac62), + TOBN(0xb2ac9ef9, 0x28714423), TOBN(0x54ecfffa, 0xadb10cb2), + TOBN(0x8781476e, 0xf886a4cc), TOBN(0x4b2c87b5, 0xdb2f8d49), + TOBN(0xe857cd20, 0x0a44295d), TOBN(0x707d7d21, 0x58c6b044), + TOBN(0xae8521f9, 0xf596757c), TOBN(0x87448f03, 0x67b2b714), + TOBN(0x13a9bc45, 0x5ebcd58d), TOBN(0x79bcced9, 0x9122d3c1), + TOBN(0x3c644247, 0x9e076642), TOBN(0x0cf22778, 0x2df4767d), + TOBN(0x5e61aee4, 0x71d444b6), TOBN(0x211236bf, 0xc5084a1d), + TOBN(0x7e15bc9a, 0x4fd3eaf6), TOBN(0x68df2c34, 0xab622bf5), + TOBN(0x9e674f0f, 0x59bf4f36), TOBN(0xf883669b, 0xd7f34d73), + TOBN(0xc48ac1b8, 0x31497b1d), TOBN(0x323b925d, 0x5106703b), + TOBN(0x22156f42, 0x74082008), TOBN(0xeffc521a, 0xc8482bcb), + TOBN(0x5c6831bf, 0x12173479), TOBN(0xcaa2528f, 0xc4739490), + TOBN(0x84d2102a, 0x8f1b3c4d), TOBN(0xcf64dfc1, 0x2d9bec0d), + TOBN(0x433febad, 0x78a546ef), TOBN(0x1f621ec3, 0x7b73cef1), + TOBN(0x6aecd627, 0x37338615), TOBN(0x162082ab, 0x01d8edf6), + TOBN(0x833a8119, 0x19e86b66), TOBN(0x6023a251, 0xd299b5db), + TOBN(0xf5bb0c3a, 0xbbf04b89), TOBN(0x6735eb69, 0xae749a44), + TOBN(0xd0e058c5, 0x4713de3b), TOBN(0xfdf2593e, 0x2c3d4ccd), + TOBN(0x1b8f414e, 0xfdd23667), TOBN(0xdd52aaca, 0xfa2015ee), + TOBN(0x3e31b517, 0xbd9625ff), TOBN(0x5ec9322d, 0x8db5918c), + TOBN(0xbc73ac85, 0xa96f5294), TOBN(0x82aa5bf3, 0x61a0666a), + TOBN(0x49755810, 0xbf08ac42), TOBN(0xd21cdfd5, 0x891cedfc), + TOBN(0x918cb57b, 0x67f8be10), TOBN(0x365d1a7c, 0x56ffa726), + TOBN(0x2435c504, 0x6532de93), TOBN(0xc0fc5e10, 0x2674cd02), + TOBN(0x6e51fcf8, 0x9cbbb142), TOBN(0x1d436e5a, 0xafc50692), + TOBN(0x766bffff, 0x3fbcae22), TOBN(0x3148c2fd, 0xfd55d3b8), + TOBN(0x52c7fdc9, 0x233222fa), TOBN(0x89ff1092, 0xe419fb6b), + TOBN(0x3cd6db99, 0x25254977), TOBN(0x2e85a161, 0x1cf12ca7), + TOBN(0xadd2547c, 0xdc810bc9), TOBN(0xea3f458f, 0x9d257c22), + TOBN(0x642c1fbe, 0x27d6b19b), TOBN(0xed07e6b5, 0x140481a6), + TOBN(0x6ada1d42, 0x86d2e0f8), TOBN(0xe5920122, 0x0e8a9fd5), + TOBN(0x02c936af, 0x708c1b49), TOBN(0x60f30fee, 0x2b4bfaff), + TOBN(0x6637ad06, 0x858e6a61), TOBN(0xce4c7767, 0x3fd374d0), + TOBN(0x39d54b2d, 0x7188defb), TOBN(0xa8c9d250, 0xf56a6b66), + TOBN(0x58fc0f5e, 0xb24fe1dc), TOBN(0x9eaf9dee, 0x6b73f24c), + TOBN(0xa90d588b, 0x33650705), TOBN(0xde5b62c5, 0xaf2ec729), + TOBN(0x5c72cfae, 0xd3c2b36e), TOBN(0x868c19d5, 0x034435da), + TOBN(0x88605f93, 0xe17ee145), TOBN(0xaa60c4ee, 0x77a5d5b1), + TOBN(0xbcf5bfd2, 0x3b60c472), TOBN(0xaf4ef13c, 0xeb1d3049), + TOBN(0x373f44fc, 0xe13895c9), TOBN(0xf29b382f, 0x0cbc9822), + TOBN(0x1bfcb853, 0x73efaef6), TOBN(0xcf56ac9c, 0xa8c96f40), + TOBN(0xd7adf109, 0x7a191e24), TOBN(0x98035f44, 0xbf8a8dc2), + TOBN(0xf40a71b9, 0x1e750c84), TOBN(0xc57f7b0c, 0x5dc6c469), + TOBN(0x49a0e79c, 0x6fbc19c1), TOBN(0x6b0f5889, 0xa48ebdb8), + TOBN(0x5d3fd084, 0xa07c4e9f), TOBN(0xc3830111, 0xab27de14), + TOBN(0x0e4929fe, 0x33e08dcc), TOBN(0xf4a5ad24, 0x40bb73a3), + TOBN(0xde86c2bf, 0x490f97ca), TOBN(0x288f09c6, 0x67a1ce18), + TOBN(0x364bb886, 0x1844478d), TOBN(0x7840fa42, 0xceedb040), + TOBN(0x1269fdd2, 0x5a631b37), TOBN(0x94761f1e, 0xa47c8b7d), + TOBN(0xfc0c2e17, 0x481c6266), TOBN(0x85e16ea2, 0x3daa5fa7), + TOBN(0xccd86033, 0x92491048), TOBN(0x0c2f6963, 0xf4d402d7), + TOBN(0x6336f7df, 0xdf6a865c), TOBN(0x0a2a463c, 0xb5c02a87), + TOBN(0xb0e29be7, 0xbf2f12ee), TOBN(0xf0a22002, 0x66bad988), + TOBN(0x27f87e03, 0x9123c1d7), TOBN(0x21669c55, 0x328a8c98), + TOBN(0x186b9803, 0x92f14529), TOBN(0xd3d056cc, 0x63954df3), + TOBN(0x2f03fd58, 0x175a46f6), TOBN(0x63e34ebe, 0x11558558), + TOBN(0xe13fedee, 0x5b80cfa5), TOBN(0xe872a120, 0xd401dbd1), + TOBN(0x52657616, 0xe8a9d667), TOBN(0xbc8da4b6, 0xe08d6693), + TOBN(0x370fb9bb, 0x1b703e75), TOBN(0x6773b186, 0xd4338363), + TOBN(0x18dad378, 0xecef7bff), TOBN(0xaac787ed, 0x995677da), + TOBN(0x4801ea8b, 0x0437164b), TOBN(0xf430ad20, 0x73fe795e), + TOBN(0xb164154d, 0x8ee5eb73), TOBN(0x0884ecd8, 0x108f7c0e), + TOBN(0x0e6ec096, 0x5f520698), TOBN(0x640631fe, 0x44f7b8d9), + TOBN(0x92fd34fc, 0xa35a68b9), TOBN(0x9c5a4b66, 0x4d40cf4e), + TOBN(0x949454bf, 0x80b6783d), TOBN(0x80e701fe, 0x3a320a10), + TOBN(0x8d1a564a, 0x1a0a39b2), TOBN(0x1436d53d, 0x320587db), + TOBN(0xf5096e6d, 0x6556c362), TOBN(0xbc23a3c0, 0xe2455d7e), + TOBN(0x3a7aee54, 0x807230f9), TOBN(0x9ba1cfa6, 0x22ae82fd), + TOBN(0x833a057a, 0x99c5d706), TOBN(0x8be85f4b, 0x842315c9), + TOBN(0xd083179a, 0x66a72f12), TOBN(0x2fc77d5d, 0xcdcc73cd), + TOBN(0x22b88a80, 0x5616ee30), TOBN(0xfb09548f, 0xe7ab1083), + TOBN(0x8ad6ab0d, 0x511270cd), TOBN(0x61f6c57a, 0x6924d9ab), + TOBN(0xa0f7bf72, 0x90aecb08), TOBN(0x849f87c9, 0x0df784a4), + TOBN(0x27c79c15, 0xcfaf1d03), TOBN(0xbbf9f675, 0xc463face), + TOBN(0x91502c65, 0x765ba543), TOBN(0x18ce3cac, 0x42ea60dd), + TOBN(0xe5cee6ac, 0x6e43ecb3), TOBN(0x63e4e910, 0x68f2aeeb), + TOBN(0x26234fa3, 0xc85932ee), TOBN(0x96883e8b, 0x4c90c44d), + TOBN(0x29b9e738, 0xa18a50f6), TOBN(0xbfc62b2a, 0x3f0420df), + TOBN(0xd22a7d90, 0x6d3e1fa9), TOBN(0x17115618, 0xfe05b8a3), + TOBN(0x2a0c9926, 0xbb2b9c01), TOBN(0xc739fcc6, 0xe07e76a2), + TOBN(0x540e9157, 0x165e439a), TOBN(0x06353a62, 0x6a9063d8), + TOBN(0x84d95594, 0x61e927a3), TOBN(0x013b9b26, 0xe2e0be7f), + TOBN(0x4feaec3b, 0x973497f1), TOBN(0x15c0f94e, 0x093ebc2d), + TOBN(0x6af5f227, 0x33af0583), TOBN(0x0c2af206, 0xc61f3340), + TOBN(0xd25dbdf1, 0x4457397c), TOBN(0x2e8ed017, 0xcabcbae0), + TOBN(0xe3010938, 0xc2815306), TOBN(0xbaa99337, 0xe8c6cd68), + TOBN(0x08513182, 0x3b0ec7de), TOBN(0x1e1b822b, 0x58df05df), + TOBN(0x5c14842f, 0xa5c3b683), TOBN(0x98fe977e, 0x3eba34ce), + TOBN(0xfd2316c2, 0x0d5e8873), TOBN(0xe48d839a, 0xbd0d427d), + TOBN(0x495b2218, 0x623fc961), TOBN(0x24ee56e7, 0xb46fba5e), + TOBN(0x9184a55b, 0x91e4de58), TOBN(0xa7488ca5, 0xdfdea288), + TOBN(0xa723862e, 0xa8dcc943), TOBN(0x92d762b2, 0x849dc0fc), + TOBN(0x3c444a12, 0x091ff4a9), TOBN(0x581113fa, 0x0cada274), + TOBN(0xb9de0a45, 0x30d8eae2), TOBN(0x5e0fcd85, 0xdf6b41ea), + TOBN(0x6233ea68, 0xc094dbb5), TOBN(0xb77d062e, 0xd968d410), + TOBN(0x3e719bbc, 0x58b3002d), TOBN(0x68e7dd3d, 0x3dc49d58), + TOBN(0x8d825740, 0x013a5e58), TOBN(0x21311747, 0x3c9e3c1b), + TOBN(0x0cb0a2a7, 0x7c99b6ab), TOBN(0x5c48a3b3, 0xc2f888f2)} + , + {TOBN(0xc7913e91, 0x991724f3), TOBN(0x5eda799c, 0x39cbd686), + TOBN(0xddb595c7, 0x63d4fc1e), TOBN(0x6b63b80b, 0xac4fed54), + TOBN(0x6ea0fc69, 0x7e5fb516), TOBN(0x737708ba, 0xd0f1c964), + TOBN(0x9628745f, 0x11a92ca5), TOBN(0x61f37958, 0x9a86967a), + TOBN(0x9af39b2c, 0xaa665072), TOBN(0x78322fa4, 0xefd324ef), + TOBN(0x3d153394, 0xc327bd31), TOBN(0x81d5f271, 0x3129dab0), + TOBN(0xc72e0c42, 0xf48027f5), TOBN(0xaa40cdbc, 0x8536e717), + TOBN(0xf45a657a, 0x2d369d0f), TOBN(0xb03bbfc4, 0xea7f74e6), + TOBN(0x46a8c418, 0x0d738ded), TOBN(0x6f1a5bb0, 0xe0de5729), + TOBN(0xf10230b9, 0x8ba81675), TOBN(0x32c6f30c, 0x112b33d4), + TOBN(0x7559129d, 0xd8fffb62), TOBN(0x6a281b47, 0xb459bf05), + TOBN(0x77c1bd3a, 0xfa3b6776), TOBN(0x0709b380, 0x7829973a), + TOBN(0x8c26b232, 0xa3326505), TOBN(0x38d69272, 0xee1d41bf), + TOBN(0x0459453e, 0xffe32afa), TOBN(0xce8143ad, 0x7cb3ea87), + TOBN(0x932ec1fa, 0x7e6ab666), TOBN(0x6cd2d230, 0x22286264), + TOBN(0x459a46fe, 0x6736f8ed), TOBN(0x50bf0d00, 0x9eca85bb), + TOBN(0x0b825852, 0x877a21ec), TOBN(0x300414a7, 0x0f537a94), + TOBN(0x3f1cba40, 0x21a9a6a2), TOBN(0x50824eee, 0x76943c00), + TOBN(0xa0dbfcec, 0xf83cba5d), TOBN(0xf9538148, 0x93b4f3c0), + TOBN(0x61744162, 0x48f24dd7), TOBN(0x5322d64d, 0xe4fb09dd), + TOBN(0x57447384, 0x3d9325f3), TOBN(0xa9bef2d0, 0xf371cb84), + TOBN(0x77d2188b, 0xa61e36c5), TOBN(0xbbd6a7d7, 0xc602df72), + TOBN(0xba3aa902, 0x8f61bc0b), TOBN(0xf49085ed, 0x6ed0b6a1), + TOBN(0x8bc625d6, 0xae6e8298), TOBN(0x832b0b1d, 0xa2e9c01d), + TOBN(0xa337c447, 0xf1f0ced1), TOBN(0x800cc793, 0x9492dd2b), + TOBN(0x4b93151d, 0xbea08efa), TOBN(0x820cf3f8, 0xde0a741e), + TOBN(0xff1982dc, 0x1c0f7d13), TOBN(0xef921960, 0x84dde6ca), + TOBN(0x1ad7d972, 0x45f96ee3), TOBN(0x319c8dbe, 0x29dea0c7), + TOBN(0xd3ea3871, 0x7b82b99b), TOBN(0x75922d4d, 0x470eb624), + TOBN(0x8f66ec54, 0x3b95d466), TOBN(0x66e673cc, 0xbee1e346), + TOBN(0x6afe67c4, 0xb5f2b89a), TOBN(0x3de9c1e6, 0x290e5cd3), + TOBN(0x8c278bb6, 0x310a2ada), TOBN(0x420fa384, 0x0bdb323b), + TOBN(0x0ae1d63b, 0x0eb919b0), TOBN(0xd74ee51d, 0xa74b9620), + TOBN(0x395458d0, 0xa674290c), TOBN(0x324c930f, 0x4620a510), + TOBN(0x2d1f4d19, 0xfbac27d4), TOBN(0x4086e8ca, 0x9bedeeac), + TOBN(0x0cdd211b, 0x9b679ab8), TOBN(0x5970167d, 0x7090fec4), + TOBN(0x3420f2c9, 0xfaf1fc63), TOBN(0x616d333a, 0x328c8bb4), + TOBN(0x7d65364c, 0x57f1fe4a), TOBN(0x9343e877, 0x55e5c73a), + TOBN(0x5795176b, 0xe970e78c), TOBN(0xa36ccebf, 0x60533627), + TOBN(0xfc7c7380, 0x09cdfc1b), TOBN(0xb39a2afe, 0xb3fec326), + TOBN(0xb7ff1ba1, 0x6224408a), TOBN(0xcc856e92, 0x247cfc5e), + TOBN(0x01f102e7, 0xc18bc493), TOBN(0x4613ab74, 0x2091c727), + TOBN(0xaa25e89c, 0xc420bf2b), TOBN(0x00a53176, 0x90337ec2), + TOBN(0xd2be9f43, 0x7d025fc7), TOBN(0x3316fb85, 0x6e6fe3dc), + TOBN(0x27520af5, 0x9ac50814), TOBN(0xfdf95e78, 0x9a8e4223), + TOBN(0xb7e7df2a, 0x56bec5a0), TOBN(0xf7022f7d, 0xdf159e5d), + TOBN(0x93eeeab1, 0xcac1fe8f), TOBN(0x8040188c, 0x37451168), + TOBN(0x7ee8aa8a, 0xd967dce6), TOBN(0xfa0e79e7, 0x3abc9299), + TOBN(0x67332cfc, 0x2064cfd1), TOBN(0x339c31de, 0xb0651934), + TOBN(0x719b28d5, 0x2a3bcbea), TOBN(0xee74c82b, 0x9d6ae5c6), + TOBN(0x0927d05e, 0xbaf28ee6), TOBN(0x82cecf2c, 0x9d719028), + TOBN(0x0b0d353e, 0xddb30289), TOBN(0xfe4bb977, 0xfddb2e29), + TOBN(0xbb5bb990, 0x640bfd9e), TOBN(0xd226e277, 0x82f62108), + TOBN(0x4bf00985, 0x02ffdd56), TOBN(0x7756758a, 0x2ca1b1b5), + TOBN(0xc32b62a3, 0x5285fe91), TOBN(0xedbc546a, 0x8c9cd140), + TOBN(0x1e47a013, 0xaf5cb008), TOBN(0xbca7e720, 0x073ce8f2), + TOBN(0xe10b2ab8, 0x17a91cae), TOBN(0xb89aab65, 0x08e27f63), + TOBN(0x7b3074a7, 0xdba3ddf9), TOBN(0x1c20ce09, 0x330c2972), + TOBN(0x6b9917b4, 0x5fcf7e33), TOBN(0xe6793743, 0x945ceb42), + TOBN(0x18fc2215, 0x5c633d19), TOBN(0xad1adb3c, 0xc7485474), + TOBN(0x646f9679, 0x6424c49b), TOBN(0xf888dfe8, 0x67c241c9), + TOBN(0xe12d4b93, 0x24f68b49), TOBN(0x9a6b62d8, 0xa571df20), + TOBN(0x81b4b26d, 0x179483cb), TOBN(0x666f9632, 0x9511fae2), + TOBN(0xd281b3e4, 0xd53aa51f), TOBN(0x7f96a765, 0x7f3dbd16), + TOBN(0xa7f8b5bf, 0x074a30ce), TOBN(0xd7f52107, 0x005a32e6), + TOBN(0x6f9e0907, 0x50237ed4), TOBN(0x2f21da47, 0x8096fa2b), + TOBN(0xf3e19cb4, 0xeec863a0), TOBN(0xd18f77fd, 0x9527620a), + TOBN(0x9505c81c, 0x407c1cf8), TOBN(0x9998db4e, 0x1b6ec284), + TOBN(0x7e3389e5, 0xc247d44d), TOBN(0x12507141, 0x3f4f3d80), + TOBN(0xd4ba0110, 0x4a78a6c7), TOBN(0x312874a0, 0x767720be), + TOBN(0xded059a6, 0x75944370), TOBN(0xd6123d90, 0x3b2c0bdd), + TOBN(0xa56b717b, 0x51c108e3), TOBN(0x9bb7940e, 0x070623e9), + TOBN(0x794e2d59, 0x84ac066c), TOBN(0xf5954a92, 0xe68c69a0), + TOBN(0x28c52458, 0x4fd99dcc), TOBN(0x60e639fc, 0xb1012517), + TOBN(0xc2e60125, 0x7de79248), TOBN(0xe9ef6404, 0xf12fc6d7), + TOBN(0x4c4f2808, 0x2a3b5d32), TOBN(0x865ad32e, 0xc768eb8a), + TOBN(0xac02331b, 0x13fb70b6), TOBN(0x037b44c1, 0x95599b27), + TOBN(0x1a860fc4, 0x60bd082c), TOBN(0xa2e25745, 0xc980cd01), + TOBN(0xee3387a8, 0x1da0263e), TOBN(0x931bfb95, 0x2d10f3d6), + TOBN(0x5b687270, 0xa1f24a32), TOBN(0xf140e65d, 0xca494b86), + TOBN(0x4f4ddf91, 0xb2f1ac7a), TOBN(0xf99eaabb, 0x760fee27), + TOBN(0x57f4008a, 0x49c228e5), TOBN(0x090be440, 0x1cf713bb), + TOBN(0xac91fbe4, 0x5004f022), TOBN(0xd838c2c2, 0x569e1af6), + TOBN(0xd6c7d20b, 0x0f1daaa5), TOBN(0xaa063ac1, 0x1bbb02c0), + TOBN(0x0938a422, 0x59558a78), TOBN(0x5343c669, 0x8435da2f), + TOBN(0x96f67b18, 0x034410dc), TOBN(0x7cc1e424, 0x84510804), + TOBN(0x86a1543f, 0x16dfbb7d), TOBN(0x921fa942, 0x5b5bd592), + TOBN(0x9dcccb6e, 0xb33dd03c), TOBN(0x8581ddd9, 0xb843f51e), + TOBN(0x54935fcb, 0x81d73c9e), TOBN(0x6d07e979, 0x0a5e97ab), + TOBN(0x4dc7b30a, 0xcf3a6bab), TOBN(0x147ab1f3, 0x170bee11), + TOBN(0x0aaf8e3d, 0x9fafdee4), TOBN(0xfab3dbcb, 0x538a8b95), + TOBN(0x405df4b3, 0x6ef13871), TOBN(0xf1f4e9cb, 0x088d5a49), + TOBN(0x9bcd24d3, 0x66b33f1d), TOBN(0x3b97b820, 0x5ce445c0), + TOBN(0xe2926549, 0xba93ff61), TOBN(0xd9c341ce, 0x4dafe616), + TOBN(0xfb30a76e, 0x16efb6f3), TOBN(0xdf24b8ca, 0x605b953c), + TOBN(0x8bd52afe, 0xc2fffb9f), TOBN(0xbbac5ff7, 0xe19d0b96), + TOBN(0x43c01b87, 0x459afccd), TOBN(0x6bd45143, 0xb7432652), + TOBN(0x84734530, 0x55b5d78e), TOBN(0x81088fdb, 0x1554ba7d), + TOBN(0xada0a52c, 0x1e269375), TOBN(0xf9f037c4, 0x2dc5ec10), + TOBN(0xc0660607, 0x94bfbc11), TOBN(0xc0a630bb, 0xc9c40d2f), + TOBN(0x5efc797e, 0xab64c31e), TOBN(0xffdb1dab, 0x74507144), + TOBN(0xf6124287, 0x1ca6790c), TOBN(0xe9609d81, 0xe69bf1bf), + TOBN(0xdb898595, 0x00d24fc9), TOBN(0x9c750333, 0xe51fb417), + TOBN(0x51830a91, 0xfef7bbde), TOBN(0x0ce67dc8, 0x945f585c), + TOBN(0x9a730ed4, 0x4763eb50), TOBN(0x24a0e221, 0xc1ab0d66), + TOBN(0x643b6393, 0x648748f3), TOBN(0x1982daa1, 0x6d3c6291), + TOBN(0x6f00a9f7, 0x8bbc5549), TOBN(0x7a1783e1, 0x7f36384e), + TOBN(0xe8346323, 0xde977f50), TOBN(0x91ab688d, 0xb245502a), + TOBN(0x331ab6b5, 0x6d0bdd66), TOBN(0x0a6ef32e, 0x64b71229), + TOBN(0x1028150e, 0xfe7c352f), TOBN(0x27e04350, 0xce7b39d3), + TOBN(0x2a3c8acd, 0xc1070c82), TOBN(0xfb2034d3, 0x80c9feef), + TOBN(0x2d729621, 0x709f3729), TOBN(0x8df290bf, 0x62cb4549), + TOBN(0x02f99f33, 0xfc2e4326), TOBN(0x3b30076d, 0x5eddf032), + TOBN(0xbb21f8cf, 0x0c652fb5), TOBN(0x314fb49e, 0xed91cf7b), + TOBN(0xa013eca5, 0x2f700750), TOBN(0x2b9e3c23, 0x712a4575), + TOBN(0xe5355557, 0xaf30fbb0), TOBN(0x1ada3516, 0x7c77e771), + TOBN(0x45f6ecb2, 0x7b135670), TOBN(0xe85d19df, 0x7cfc202e), + TOBN(0x0f1b50c7, 0x58d1be9f), TOBN(0x5ebf2c0a, 0xead2e344), + TOBN(0x1531fe4e, 0xabc199c9), TOBN(0xc7032592, 0x56bab0ae), + TOBN(0x16ab2e48, 0x6c1fec54), TOBN(0x0f87fda8, 0x04280188), + TOBN(0xdc9f46fc, 0x609e4a74), TOBN(0x2a44a143, 0xba667f91), + TOBN(0xbc3d8b95, 0xb4d83436), TOBN(0xa01e4bd0, 0xc7bd2958), + TOBN(0x7b182932, 0x73483c90), TOBN(0xa79c6aa1, 0xa7c7b598), + TOBN(0xbf3983c6, 0xeaaac07e), TOBN(0x8f18181e, 0x96e0d4e6), + TOBN(0x8553d37c, 0x051af62b), TOBN(0xe9a998eb, 0x0bf94496), + TOBN(0xe0844f9f, 0xb0d59aa1), TOBN(0x983fd558, 0xe6afb813), + TOBN(0x9670c0ca, 0x65d69804), TOBN(0x732b22de, 0x6ea5ff2d), + TOBN(0xd7640ba9, 0x5fd8623b), TOBN(0x9f619163, 0xa6351782), + TOBN(0x0bfc27ee, 0xacee5043), TOBN(0xae419e73, 0x2eb10f02), + TOBN(0x19c028d1, 0x8943fb05), TOBN(0x71f01cf7, 0xff13aa2a), + TOBN(0x7790737e, 0x8887a132), TOBN(0x67513309, 0x66318410), + TOBN(0x9819e8a3, 0x7ddb795e), TOBN(0xfecb8ef5, 0xdad100b2), + TOBN(0x59f74a22, 0x3021926a), TOBN(0xb7c28a49, 0x6f9b4c1c), + TOBN(0xed1a733f, 0x912ad0ab), TOBN(0x42a910af, 0x01a5659c), + TOBN(0x3842c6e0, 0x7bd68cab), TOBN(0x2b57fa38, 0x76d70ac8), + TOBN(0x8a6707a8, 0x3c53aaeb), TOBN(0x62c1c510, 0x65b4db18), + TOBN(0x8de2c1fb, 0xb2d09dc7), TOBN(0xc3dfed12, 0x266bd23b), + TOBN(0x927d039b, 0xd5b27db6), TOBN(0x2fb2f0f1, 0x103243da), + TOBN(0xf855a07b, 0x80be7399), TOBN(0xed9327ce, 0x1f9f27a8), + TOBN(0xa0bd99c7, 0x729bdef7), TOBN(0x2b67125e, 0x28250d88), + TOBN(0x784b26e8, 0x8670ced7), TOBN(0xe3dfe41f, 0xc31bd3b4), + TOBN(0x9e353a06, 0xbcc85cbc), TOBN(0x302e2909, 0x60178a9d), + TOBN(0x860abf11, 0xa6eac16e), TOBN(0x76447000, 0xaa2b3aac), + TOBN(0x46ff9d19, 0x850afdab), TOBN(0x35bdd6a5, 0xfdb2d4c1), + TOBN(0xe82594b0, 0x7e5c9ce9), TOBN(0x0f379e53, 0x20af346e), + TOBN(0x608b31e3, 0xbc65ad4a), TOBN(0x710c6b12, 0x267c4826), + TOBN(0x51c966f9, 0x71954cf1), TOBN(0xb1cec793, 0x0d0aa215), + TOBN(0x1f155989, 0x86bd23a8), TOBN(0xae2ff99c, 0xf9452e86), + TOBN(0xd8dd953c, 0x340ceaa2), TOBN(0x26355275, 0x2e2e9333), + TOBN(0x15d4e5f9, 0x8586f06d), TOBN(0xd6bf94a8, 0xf7cab546), + TOBN(0x33c59a0a, 0xb76a9af0), TOBN(0x52740ab3, 0xba095af7), + TOBN(0xc444de8a, 0x24389ca0), TOBN(0xcc6f9863, 0x706da0cb), + TOBN(0xb5a741a7, 0x6b2515cf), TOBN(0x71c41601, 0x9585c749), + TOBN(0x78350d4f, 0xe683de97), TOBN(0x31d61524, 0x63d0b5f5), + TOBN(0x7a0cc5e1, 0xfbce090b), TOBN(0xaac927ed, 0xfbcb2a5b), + TOBN(0xe920de49, 0x20d84c35), TOBN(0x8c06a0b6, 0x22b4de26), + TOBN(0xd34dd58b, 0xafe7ddf3), TOBN(0x55851fed, 0xc1e6e55b), + TOBN(0xd1395616, 0x960696e7), TOBN(0x940304b2, 0x5f22705f), + TOBN(0x6f43f861, 0xb0a2a860), TOBN(0xcf121282, 0x0e7cc981), + TOBN(0x12186212, 0x0ab64a96), TOBN(0x09215b9a, 0xb789383c), + TOBN(0x311eb305, 0x37387c09), TOBN(0xc5832fce, 0xf03ee760), + TOBN(0x30358f58, 0x32f7ea19), TOBN(0xe01d3c34, 0x91d53551), + TOBN(0x1ca5ee41, 0xda48ea80), TOBN(0x34e71e8e, 0xcf4fa4c1), + TOBN(0x312abd25, 0x7af1e1c7), TOBN(0xe3afcdeb, 0x2153f4a5), + TOBN(0x9d5c84d7, 0x00235e9a), TOBN(0x0308d3f4, 0x8c4c836f), + TOBN(0xc0a66b04, 0x89332de5), TOBN(0x610dd399, 0x89e566ef), + TOBN(0xf8eea460, 0xd1ac1635), TOBN(0x84cbb3fb, 0x20a2c0df), + TOBN(0x40afb488, 0xe74a48c5), TOBN(0x29738198, 0xd326b150), + TOBN(0x2a17747f, 0xa6d74081), TOBN(0x60ea4c05, 0x55a26214), + TOBN(0x53514bb4, 0x1f88c5fe), TOBN(0xedd64567, 0x7e83426c), + TOBN(0xd5d6cbec, 0x96460b25), TOBN(0xa12fd0ce, 0x68dc115e), + TOBN(0xc5bc3ed2, 0x697840ea), TOBN(0x969876a8, 0xa6331e31), + TOBN(0x60c36217, 0x472ff580), TOBN(0xf4229705, 0x4ad41393), + TOBN(0x4bd99ef0, 0xa03b8b92), TOBN(0x501c7317, 0xc144f4f6), + TOBN(0x159009b3, 0x18464945), TOBN(0x6d5e594c, 0x74c5c6be), + TOBN(0x2d587011, 0x321a3660), TOBN(0xd1e184b1, 0x3898d022), + TOBN(0x5ba04752, 0x4c6a7e04), TOBN(0x47fa1e2b, 0x45550b65), + TOBN(0x9419daf0, 0x48c0a9a5), TOBN(0x66362953, 0x7c243236), + TOBN(0xcd0744b1, 0x5cb12a88), TOBN(0x561b6f9a, 0x2b646188), + TOBN(0x599415a5, 0x66c2c0c0), TOBN(0xbe3f0859, 0x0f83f09a), + TOBN(0x9141c5be, 0xb92041b8), TOBN(0x01ae38c7, 0x26477d0d), + TOBN(0xca8b71f3, 0xd12c7a94), TOBN(0xfab5b31f, 0x765c70db), + TOBN(0x76ae7492, 0x487443e9), TOBN(0x8595a310, 0x990d1349), + TOBN(0xf8dbeda8, 0x7d460a37), TOBN(0x7f7ad082, 0x1e45a38f), + TOBN(0xed1d4db6, 0x1059705a), TOBN(0xa3dd492a, 0xe6b9c697), + TOBN(0x4b92ee3a, 0x6eb38bd5), TOBN(0xbab2609d, 0x67cc0bb7), + TOBN(0x7fc4fe89, 0x6e70ee82), TOBN(0xeff2c56e, 0x13e6b7e3), + TOBN(0x9b18959e, 0x34d26fca), TOBN(0x2517ab66, 0x889d6b45), + TOBN(0xf167b4e0, 0xbdefdd4f), TOBN(0x69958465, 0xf366e401), + TOBN(0x5aa368ab, 0xa73bbec0), TOBN(0x12148709, 0x7b240c21), + TOBN(0x378c3233, 0x18969006), TOBN(0xcb4d73ce, 0xe1fe53d1), + TOBN(0x5f50a80e, 0x130c4361), TOBN(0xd67f5951, 0x7ef5212b), + TOBN(0xf145e21e, 0x9e70c72e), TOBN(0xb2e52e29, 0x5566d2fb), + TOBN(0x44eaba4a, 0x032397f5), TOBN(0x5e56937b, 0x7e31a7de), + TOBN(0x68dcf517, 0x456c61e1), TOBN(0xbc2e954a, 0xa8b0a388), + TOBN(0xe3552fa7, 0x60a8b755), TOBN(0x03442dae, 0x73ad0cde), + TOBN(0x37ffe747, 0xceb26210), TOBN(0x983545e8, 0x787baef9), + TOBN(0x8b8c8535, 0x86a3de31), TOBN(0xc621dbcb, 0xfacd46db), + TOBN(0x82e442e9, 0x59266fbb), TOBN(0xa3514c37, 0x339d471c), + TOBN(0x3a11b771, 0x62cdad96), TOBN(0xf0cb3b3c, 0xecf9bdf0), + TOBN(0x3fcbdbce, 0x478e2135), TOBN(0x7547b5cf, 0xbda35342), + TOBN(0xa97e81f1, 0x8a677af6), TOBN(0xc8c2bf83, 0x28817987), + TOBN(0xdf07eaaf, 0x45580985), TOBN(0xc68d1f05, 0xc93b45cb), + TOBN(0x106aa2fe, 0xc77b4cac), TOBN(0x4c1d8afc, 0x04a7ae86), + TOBN(0xdb41c3fd, 0x9eb45ab2), TOBN(0x5b234b5b, 0xd4b22e74), + TOBN(0xda253dec, 0xf215958a), TOBN(0x67e0606e, 0xa04edfa0), + TOBN(0xabbbf070, 0xef751b11), TOBN(0xf352f175, 0xf6f06dce), + TOBN(0xdfc4b6af, 0x6839f6b4), TOBN(0x53ddf9a8, 0x9959848e), + TOBN(0xda49c379, 0xc21520b0), TOBN(0x90864ff0, 0xdbd5d1b6), + TOBN(0x2f055d23, 0x5f49c7f7), TOBN(0xe51e4e6a, 0xa796b2d8), + TOBN(0xc361a67f, 0x5c9dc340), TOBN(0x5ad53c37, 0xbca7c620), + TOBN(0xda1d6588, 0x32c756d0), TOBN(0xad60d911, 0x8bb67e13), + TOBN(0xd6c47bdf, 0x0eeec8c6), TOBN(0x4a27fec1, 0x078a1821), + TOBN(0x081f7415, 0xc3099524), TOBN(0x8effdf0b, 0x82cd8060), + TOBN(0xdb70ec1c, 0x65842df8), TOBN(0x8821b358, 0xd319a901), + TOBN(0x72ee56ee, 0xde42b529), TOBN(0x5bb39592, 0x236e4286), + TOBN(0xd1183316, 0xfd6f7140), TOBN(0xf9fadb5b, 0xbd8e81f7), + TOBN(0x701d5e0c, 0x5a02d962), TOBN(0xfdee4dbf, 0x1b601324), + TOBN(0xbed17407, 0x35d7620e), TOBN(0x04e3c2c3, 0xf48c0012), + TOBN(0x9ee29da7, 0x3455449a), TOBN(0x562cdef4, 0x91a836c4), + TOBN(0x8f682a5f, 0x47701097), TOBN(0x617125d8, 0xff88d0c2), + TOBN(0x948fda24, 0x57bb86dd), TOBN(0x348abb8f, 0x289f7286), + TOBN(0xeb10eab5, 0x99d94bbd), TOBN(0xd51ba28e, 0x4684d160), + TOBN(0xabe0e51c, 0x30c8f41a), TOBN(0x66588b45, 0x13254f4a), + TOBN(0x147ebf01, 0xfad097a5), TOBN(0x49883ea8, 0x610e815d), + TOBN(0xe44d60ba, 0x8a11de56), TOBN(0xa970de6e, 0x827a7a6d), + TOBN(0x2be41424, 0x5e17fc19), TOBN(0xd833c657, 0x01214057), + TOBN(0x1375813b, 0x363e723f), TOBN(0x6820bb88, 0xe6a52e9b), + TOBN(0x7e7f6970, 0xd875d56a), TOBN(0xd6a0a9ac, 0x51fbf6bf), + TOBN(0x54ba8790, 0xa3083c12), TOBN(0xebaeb23d, 0x6ae7eb64), + TOBN(0xa8685c3a, 0xb99a907a), TOBN(0xf1e74550, 0x026bf40b), + TOBN(0x7b73a027, 0xc802cd9e), TOBN(0x9a8a927c, 0x4fef4635), + TOBN(0xe1b6f60c, 0x08191224), TOBN(0xc4126ebb, 0xde4ec091), + TOBN(0xe1dff4dc, 0x4ae38d84), TOBN(0xde3f57db, 0x4f2ef985), + TOBN(0x34964337, 0xd446a1dd), TOBN(0x7bf217a0, 0x859e77f6), + TOBN(0x8ff10527, 0x8e1d13f5), TOBN(0xa304ef03, 0x74eeae27), + TOBN(0xfc6f5e47, 0xd19dfa5a), TOBN(0xdb007de3, 0x7fad982b), + TOBN(0x28205ad1, 0x613715f5), TOBN(0x251e6729, 0x7889529e), + TOBN(0x72705184, 0x1ae98e78), TOBN(0xf818537d, 0x271cac32), + TOBN(0xc8a15b7e, 0xb7f410f5), TOBN(0xc474356f, 0x81f62393), + TOBN(0x92dbdc5a, 0xc242316b), TOBN(0xabe060ac, 0xdbf4aff5), + TOBN(0x6e8c38fe, 0x909a8ec6), TOBN(0x43e514e5, 0x6116cb94), + TOBN(0x2078fa38, 0x07d784f9), TOBN(0x1161a880, 0xf4b5b357), + TOBN(0x5283ce79, 0x13adea3d), TOBN(0x0756c3e6, 0xcc6a910b), + TOBN(0x60bcfe01, 0xaaa79697), TOBN(0x04a73b29, 0x56391db1), + TOBN(0xdd8dad47, 0x189b45a0), TOBN(0xbfac0dd0, 0x48d5b8d9), + TOBN(0x34ab3af5, 0x7d3d2ec2), TOBN(0x6fa2fc2d, 0x207bd3af), + TOBN(0x9ff40092, 0x66550ded), TOBN(0x719b3e87, 0x1fd5b913), + TOBN(0xa573a496, 0x6d17fbc7), TOBN(0x0cd1a70a, 0x73d2b24e), + TOBN(0x34e2c5ca, 0xb2676937), TOBN(0xe7050b06, 0xbf669f21), + TOBN(0xfbe948b6, 0x1ede9046), TOBN(0xa0530051, 0x97662659), + TOBN(0x58cbd4ed, 0xf10124c5), TOBN(0xde2646e4, 0xdd6c06c8), + TOBN(0x332f8108, 0x8cad38c0), TOBN(0x471b7e90, 0x6bd68ae2), + TOBN(0x56ac3fb2, 0x0d8e27a3), TOBN(0xb54660db, 0x136b4b0d), + TOBN(0x123a1e11, 0xa6fd8de4), TOBN(0x44dbffea, 0xa37799ef), + TOBN(0x4540b977, 0xce6ac17c), TOBN(0x495173a8, 0xaf60acef)} + , + {TOBN(0x9ebb284d, 0x391c2a82), TOBN(0xbcdd4863, 0x158308e8), + TOBN(0x006f16ec, 0x83f1edca), TOBN(0xa13e2c37, 0x695dc6c8), + TOBN(0x2ab756f0, 0x4a057a87), TOBN(0xa8765500, 0xa6b48f98), + TOBN(0x4252face, 0x68651c44), TOBN(0xa52b540b, 0xe1765e02), + TOBN(0x4f922fc5, 0x16a0d2bb), TOBN(0x0d5cc16c, 0x1a623499), + TOBN(0x9241cf3a, 0x57c62c8b), TOBN(0x2f5e6961, 0xfd1b667f), + TOBN(0x5c15c70b, 0xf5a01797), TOBN(0x3d20b44d, 0x60956192), + TOBN(0x04911b37, 0x071fdb52), TOBN(0xf648f916, 0x8d6f0f7b), + TOBN(0x6dc1acaf, 0xe60b7cf7), TOBN(0x25860a50, 0x84a9d869), + TOBN(0x56fc6f09, 0xe7ba8ac4), TOBN(0x828c5bd0, 0x6148d29e), + TOBN(0xac6b435e, 0xdc55ae5f), TOBN(0xa527f56c, 0xc0117411), + TOBN(0x94d5045e, 0xfd24342c), TOBN(0x2c4c0a35, 0x70b67c0d), + TOBN(0x027cc8b8, 0xfac61d9a), TOBN(0x7d25e062, 0xe3c6fe8a), + TOBN(0xe08805bf, 0xe5bff503), TOBN(0x13271e6c, 0x6ff632f7), + TOBN(0x55dca6c0, 0x232f76a5), TOBN(0x8957c32d, 0x701ef426), + TOBN(0xee728bcb, 0xa10a5178), TOBN(0x5ea60411, 0xb62c5173), + TOBN(0xfc4e964e, 0xd0b8892b), TOBN(0x9ea17683, 0x9301bb74), + TOBN(0x6265c5ae, 0xfcc48626), TOBN(0xe60cf82e, 0xbb3e9102), + TOBN(0x57adf797, 0xd4df5531), TOBN(0x235b59a1, 0x8deeefe2), + TOBN(0x60adcf58, 0x3f306eb1), TOBN(0x105c2753, 0x3d09492d), + TOBN(0x4090914b, 0xb5def996), TOBN(0x1cb69c83, 0x233dd1e7), + TOBN(0xc1e9c1d3, 0x9b3d5e76), TOBN(0x1f3338ed, 0xfccf6012), + TOBN(0xb1e95d0d, 0x2f5378a8), TOBN(0xacf4c2c7, 0x2f00cd21), + TOBN(0x6e984240, 0xeb5fe290), TOBN(0xd66c038d, 0x248088ae), + TOBN(0x804d264a, 0xf94d70cf), TOBN(0xbdb802ef, 0x7314bf7e), + TOBN(0x8fb54de2, 0x4333ed02), TOBN(0x740461e0, 0x285635d9), + TOBN(0x4113b2c8, 0x365e9383), TOBN(0xea762c83, 0x3fdef652), + TOBN(0x4eec6e2e, 0x47b956c1), TOBN(0xa3d814be, 0x65620fa4), + TOBN(0x9ad5462b, 0xb4d8bc50), TOBN(0x181c0b16, 0xa9195770), + TOBN(0xebd4fe1c, 0x78412a68), TOBN(0xae0341bc, 0xc0dff48c), + TOBN(0xb6bc45cf, 0x7003e866), TOBN(0xf11a6dea, 0x8a24a41b), + TOBN(0x5407151a, 0xd04c24c2), TOBN(0x62c9d27d, 0xda5b7b68), + TOBN(0x2e964235, 0x88cceff6), TOBN(0x8594c54f, 0x8b07ed69), + TOBN(0x1578e73c, 0xc84d0d0d), TOBN(0x7b4e1055, 0xff532868), + TOBN(0xa348c0d5, 0xb5ec995a), TOBN(0xbf4b9d55, 0x14289a54), + TOBN(0x9ba155a6, 0x58fbd777), TOBN(0x186ed7a8, 0x1a84491d), + TOBN(0xd4992b30, 0x614c0900), TOBN(0xda98d121, 0xbd00c24b), + TOBN(0x7f534dc8, 0x7ec4bfa1), TOBN(0x4a5ff674, 0x37dc34bc), + TOBN(0x68c196b8, 0x1d7ea1d7), TOBN(0x38cf2893, 0x80a6d208), + TOBN(0xfd56cd09, 0xe3cbbd6e), TOBN(0xec72e27e, 0x4205a5b6), + TOBN(0x15ea68f5, 0xa44f77f7), TOBN(0x7aa5f9fd, 0xb43c52bc), + TOBN(0x86ff676f, 0x94f0e609), TOBN(0xa4cde963, 0x2e2d432b), + TOBN(0x8cafa0c0, 0xeee470af), TOBN(0x84137d0e, 0x8a3f5ec8), + TOBN(0xebb40411, 0xfaa31231), TOBN(0xa239c13f, 0x6f7f7ccf), + TOBN(0x32865719, 0xa8afd30b), TOBN(0x86798328, 0x8a826dce), + TOBN(0xdf04e891, 0xc4a8fbe0), TOBN(0xbb6b6e1b, 0xebf56ad3), + TOBN(0x0a695b11, 0x471f1ff0), TOBN(0xd76c3389, 0xbe15baf0), + TOBN(0x018edb95, 0xbe96c43e), TOBN(0xf2beaaf4, 0x90794158), + TOBN(0x152db09e, 0xc3076a27), TOBN(0x5e82908e, 0xe416545d), + TOBN(0xa2c41272, 0x356d6f2e), TOBN(0xdc9c9642, 0x31fd74e1), + TOBN(0x66ceb88d, 0x519bf615), TOBN(0xe29ecd76, 0x05a2274e), + TOBN(0x3a0473c4, 0xbf5e2fa0), TOBN(0x6b6eb671, 0x64284e67), + TOBN(0xe8b97932, 0xb88756dd), TOBN(0xed4e8652, 0xf17e3e61), + TOBN(0xc2dd1499, 0x3ee1c4a4), TOBN(0xc0aaee17, 0x597f8c0e), + TOBN(0x15c4edb9, 0x6c168af3), TOBN(0x6563c7bf, 0xb39ae875), + TOBN(0xadfadb6f, 0x20adb436), TOBN(0xad55e8c9, 0x9a042ac0), + TOBN(0x975a1ed8, 0xb76da1f5), TOBN(0x10dfa466, 0xa58acb94), + TOBN(0x8dd7f7e3, 0xac060282), TOBN(0x6813e66a, 0x572a051e), + TOBN(0xb4ccae1e, 0x350cb901), TOBN(0xb653d656, 0x50cb7822), + TOBN(0x42484710, 0xdfab3b87), TOBN(0xcd7ee537, 0x9b670fd0), + TOBN(0x0a50b12e, 0x523b8bf6), TOBN(0x8009eb5b, 0x8f910c1b), + TOBN(0xf535af82, 0x4a167588), TOBN(0x0f835f9c, 0xfb2a2abd), + TOBN(0xf59b2931, 0x2afceb62), TOBN(0xc797df2a, 0x169d383f), + TOBN(0xeb3f5fb0, 0x66ac02b0), TOBN(0x029d4c6f, 0xdaa2d0ca), + TOBN(0xd4059bc1, 0xafab4bc5), TOBN(0x833f5c6f, 0x56783247), + TOBN(0xb5346630, 0x8d2d3605), TOBN(0x83387891, 0xd34d8433), + TOBN(0xd973b30f, 0xadd9419a), TOBN(0xbcca1099, 0xafe3fce8), + TOBN(0x08178315, 0x0809aac6), TOBN(0x01b7f21a, 0x540f0f11), + TOBN(0x65c29219, 0x909523c8), TOBN(0xa62f648f, 0xa3a1c741), + TOBN(0x88598d4f, 0x60c9e55a), TOBN(0xbce9141b, 0x0e4f347a), + TOBN(0x9af97d84, 0x35f9b988), TOBN(0x0210da62, 0x320475b6), + TOBN(0x3c076e22, 0x9191476c), TOBN(0x7520dbd9, 0x44fc7834), + TOBN(0x6a6b2cfe, 0xc1ab1bbd), TOBN(0xef8a65be, 0xdc650938), + TOBN(0x72855540, 0x805d7bc4), TOBN(0xda389396, 0xed11fdfd), + TOBN(0xa9d5bd36, 0x74660876), TOBN(0x11d67c54, 0xb45dff35), + TOBN(0x6af7d148, 0xa4f5da94), TOBN(0xbb8d4c3f, 0xc0bbeb31), + TOBN(0x87a7ebd1, 0xe0a1b12a), TOBN(0x1e4ef88d, 0x770ba95f), + TOBN(0x8c33345c, 0xdc2ae9cb), TOBN(0xcecf1276, 0x01cc8403), + TOBN(0x687c012e, 0x1b39b80f), TOBN(0xfd90d0ad, 0x35c33ba4), + TOBN(0xa3ef5a67, 0x5c9661c2), TOBN(0x368fc88e, 0xe017429e), + TOBN(0xd30c6761, 0x196a2fa2), TOBN(0x931b9817, 0xbd5b312e), + TOBN(0xba01000c, 0x72f54a31), TOBN(0xa203d2c8, 0x66eaa541), + TOBN(0xf2abdee0, 0x98939db3), TOBN(0xe37d6c2c, 0x3e606c02), + TOBN(0xf2921574, 0x521ff643), TOBN(0x2781b3c4, 0xd7e2fca3), + TOBN(0x664300b0, 0x7850ec06), TOBN(0xac5a38b9, 0x7d3a10cf), + TOBN(0x9233188d, 0xe34ab39d), TOBN(0xe77057e4, 0x5072cbb9), + TOBN(0xbcf0c042, 0xb59e78df), TOBN(0x4cfc91e8, 0x1d97de52), + TOBN(0x4661a26c, 0x3ee0ca4a), TOBN(0x5620a4c1, 0xfb8507bc), + TOBN(0x4b44d4aa, 0x049f842c), TOBN(0xceabc5d5, 0x1540e82b), + TOBN(0x306710fd, 0x15c6f156), TOBN(0xbe5ae52b, 0x63db1d72), + TOBN(0x06f1e7e6, 0x334957f1), TOBN(0x57e388f0, 0x31144a70), + TOBN(0xfb69bb2f, 0xdf96447b), TOBN(0x0f78ebd3, 0x73e38a12), + TOBN(0xb8222605, 0x2b7ce542), TOBN(0xe6d4ce99, 0x7472bde1), + TOBN(0x53e16ebe, 0x09d2f4da), TOBN(0x180ff42e, 0x53b92b2e), + TOBN(0xc59bcc02, 0x2c34a1c6), TOBN(0x3803d6f9, 0x422c46c2), + TOBN(0x18aff74f, 0x5c14a8a2), TOBN(0x55aebf80, 0x10a08b28), + TOBN(0x66097d58, 0x7135593f), TOBN(0x32e6eff7, 0x2be570cd), + TOBN(0x584e6a10, 0x2a8c860d), TOBN(0xcd185890, 0xa2eb4163), + TOBN(0x7ceae99d, 0x6d97e134), TOBN(0xd42c6b70, 0xdd8447ce), + TOBN(0x59ddbb4a, 0xb8c50273), TOBN(0x03c612df, 0x3cf34e1e), + TOBN(0x84b9ca15, 0x04b6c5a0), TOBN(0x35216f39, 0x18f0e3a3), + TOBN(0x3ec2d2bc, 0xbd986c00), TOBN(0x8bf546d9, 0xd19228fe), + TOBN(0xd1c655a4, 0x4cd623c3), TOBN(0x366ce718, 0x502b8e5a), + TOBN(0x2cfc84b4, 0xeea0bfe7), TOBN(0xe01d5cee, 0xcf443e8e), + TOBN(0x8ec045d9, 0x036520f8), TOBN(0xdfb3c3d1, 0x92d40e98), + TOBN(0x0bac4cce, 0xcc559a04), TOBN(0x35eccae5, 0x240ea6b1), + TOBN(0x180b32db, 0xf8a5a0ac), TOBN(0x547972a5, 0xeb699700), + TOBN(0xa3765801, 0xca26bca0), TOBN(0x57e09d0e, 0xa647f25a), + TOBN(0xb956970e, 0x2fdd23cc), TOBN(0xb80288bc, 0x5682e971), + TOBN(0xe6e6d91e, 0x9ae86ebc), TOBN(0x0564c83f, 0x8c9f1939), + TOBN(0x551932a2, 0x39560368), TOBN(0xe893752b, 0x049c28e2), + TOBN(0x0b03cee5, 0xa6a158c3), TOBN(0xe12d656b, 0x04964263), + TOBN(0x4b47554e, 0x63e3bc1d), TOBN(0xc719b6a2, 0x45044ff7), + TOBN(0x4f24d30a, 0xe48daa07), TOBN(0xa3f37556, 0xc8c1edc3), + TOBN(0x9a47bf76, 0x0700d360), TOBN(0xbb1a1824, 0x822ae4e2), + TOBN(0x22e275a3, 0x89f1fb4c), TOBN(0x72b1aa23, 0x9968c5f5), + TOBN(0xa75feaca, 0xbe063f64), TOBN(0x9b392f43, 0xbce47a09), + TOBN(0xd4241509, 0x1ad07aca), TOBN(0x4b0c591b, 0x8d26cd0f), + TOBN(0x2d42ddfd, 0x92f1169a), TOBN(0x63aeb1ac, 0x4cbf2392), + TOBN(0x1de9e877, 0x0691a2af), TOBN(0xebe79af7, 0xd98021da), + TOBN(0xcfdf2a4e, 0x40e50acf), TOBN(0xf0a98ad7, 0xaf01d665), + TOBN(0xefb640bf, 0x1831be1f), TOBN(0x6fe8bd2f, 0x80e9ada0), + TOBN(0x94c103a1, 0x6cafbc91), TOBN(0x170f8759, 0x8308e08c), + TOBN(0x5de2d2ab, 0x9780ff4f), TOBN(0x666466bc, 0x45b201f2), + TOBN(0x58af2010, 0xf5b343bc), TOBN(0x0f2e400a, 0xf2f142fe), + TOBN(0x3483bfde, 0xa85f4bdf), TOBN(0xf0b1d093, 0x03bfeaa9), + TOBN(0x2ea01b95, 0xc7081603), TOBN(0xe943e4c9, 0x3dba1097), + TOBN(0x47be92ad, 0xb438f3a6), TOBN(0x00bb7742, 0xe5bf6636), + TOBN(0x136b7083, 0x824297b4), TOBN(0x9d0e5580, 0x5584455f), + TOBN(0xab48cedc, 0xf1c7d69e), TOBN(0x53a9e481, 0x2a256e76), + TOBN(0x0402b0e0, 0x65eb2413), TOBN(0xdadbbb84, 0x8fc407a7), + TOBN(0xa65cd5a4, 0x8d7f5492), TOBN(0x21d44293, 0x74bae294), + TOBN(0x66917ce6, 0x3b5f1cc4), TOBN(0x37ae52ea, 0xce872e62), + TOBN(0xbb087b72, 0x2905f244), TOBN(0x12077086, 0x1e6af74f), + TOBN(0x4b644e49, 0x1058edea), TOBN(0x827510e3, 0xb638ca1d), + TOBN(0x8cf2b704, 0x6038591c), TOBN(0xffc8b47a, 0xfe635063), + TOBN(0x3ae220e6, 0x1b4d5e63), TOBN(0xbd864742, 0x9d961b4b), + TOBN(0x610c107e, 0x9bd16bed), TOBN(0x4270352a, 0x1127147b), + TOBN(0x7d17ffe6, 0x64cfc50e), TOBN(0x50dee01a, 0x1e36cb42), + TOBN(0x068a7622, 0x35dc5f9a), TOBN(0x9a08d536, 0xdf53f62c), + TOBN(0x4ed71457, 0x6be5f7de), TOBN(0xd93006f8, 0xc2263c9e), + TOBN(0xe073694c, 0xcacacb36), TOBN(0x2ff7a5b4, 0x3ae118ab), + TOBN(0x3cce53f1, 0xcd871236), TOBN(0xf156a39d, 0xc2aa6d52), + TOBN(0x9cc5f271, 0xb198d76d), TOBN(0xbc615b6f, 0x81383d39), + TOBN(0xa54538e8, 0xde3eee6b), TOBN(0x58c77538, 0xab910d91), + TOBN(0x31e5bdbc, 0x58d278bd), TOBN(0x3cde4adf, 0xb963acae), + TOBN(0xb1881fd2, 0x5302169c), TOBN(0x8ca60fa0, 0xa989ed8b), + TOBN(0xa1999458, 0xff96a0ee), TOBN(0xc1141f03, 0xac6c283d), + TOBN(0x7677408d, 0x6dfafed3), TOBN(0x33a01653, 0x39661588), + TOBN(0x3c9c15ec, 0x0b726fa0), TOBN(0x090cfd93, 0x6c9b56da), + TOBN(0xe34f4bae, 0xa3c40af5), TOBN(0x3469eadb, 0xd21129f1), + TOBN(0xcc51674a, 0x1e207ce8), TOBN(0x1e293b24, 0xc83b1ef9), + TOBN(0x17173d13, 0x1e6c0bb4), TOBN(0x19004695, 0x90776d35), + TOBN(0xe7980e34, 0x6de6f922), TOBN(0x873554cb, 0xf4dd9a22), + TOBN(0x0316c627, 0xcbf18a51), TOBN(0x4d93651b, 0x3032c081), + TOBN(0x207f2771, 0x3946834d), TOBN(0x2c08d7b4, 0x30cdbf80), + TOBN(0x137a4fb4, 0x86df2a61), TOBN(0xa1ed9c07, 0xecf7b4a2), + TOBN(0xb2e460e2, 0x7bd042ff), TOBN(0xb7f5e2fa, 0x5f62f5ec), + TOBN(0x7aa6ec6b, 0xcc2423b7), TOBN(0x75ce0a7f, 0xba63eea7), + TOBN(0x67a45fb1, 0xf250a6e1), TOBN(0x93bc919c, 0xe53cdc9f), + TOBN(0x9271f56f, 0x871942df), TOBN(0x2372ff6f, 0x7859ad66), + TOBN(0x5f4c2b96, 0x33cb1a78), TOBN(0xe3e29101, 0x5838aa83), + TOBN(0xa7ed1611, 0xe4e8110c), TOBN(0x2a2d70d5, 0x330198ce), + TOBN(0xbdf132e8, 0x6720efe0), TOBN(0xe61a8962, 0x66a471bf), + TOBN(0x796d3a85, 0x825808bd), TOBN(0x51dc3cb7, 0x3fd6e902), + TOBN(0x643c768a, 0x916219d1), TOBN(0x36cd7685, 0xa2ad7d32), + TOBN(0xe3db9d05, 0xb22922a4), TOBN(0x6494c87e, 0xdba29660), + TOBN(0xf0ac91df, 0xbcd2ebc7), TOBN(0x4deb57a0, 0x45107f8d), + TOBN(0x42271f59, 0xc3d12a73), TOBN(0x5f71687c, 0xa5c2c51d), + TOBN(0xcb1f50c6, 0x05797bcb), TOBN(0x29ed0ed9, 0xd6d34eb0), + TOBN(0xe5fe5b47, 0x4683c2eb), TOBN(0x4956eeb5, 0x97447c46), + TOBN(0x5b163a43, 0x71207167), TOBN(0x93fa2fed, 0x0248c5ef), + TOBN(0x67930af2, 0x31f63950), TOBN(0xa77797c1, 0x14caa2c9), + TOBN(0x526e80ee, 0x27ac7e62), TOBN(0xe1e6e626, 0x58b28aec), + TOBN(0x636178b0, 0xb3c9fef0), TOBN(0xaf7752e0, 0x6d5f90be), + TOBN(0x94ecaf18, 0xeece51cf), TOBN(0x2864d0ed, 0xca806e1f), + TOBN(0x6de2e383, 0x97c69134), TOBN(0x5a42c316, 0xeb291293), + TOBN(0xc7779219, 0x6a60bae0), TOBN(0xa24de346, 0x6b7599d1), + TOBN(0x49d374aa, 0xb75d4941), TOBN(0x98900586, 0x2d501ff0), + TOBN(0x9f16d40e, 0xeb7974cf), TOBN(0x1033860b, 0xcdd8c115), + TOBN(0xb6c69ac8, 0x2094cec3), TOBN(0x9976fb88, 0x403b770c), + TOBN(0x1dea026c, 0x4859590d), TOBN(0xb6acbb46, 0x8562d1fd), + TOBN(0x7cd6c461, 0x44569d85), TOBN(0xc3190a36, 0x97f0891d), + TOBN(0xc6f53195, 0x48d5a17d), TOBN(0x7d919966, 0xd749abc8), + TOBN(0x65104837, 0xdd1c8a20), TOBN(0x7e5410c8, 0x2f683419), + TOBN(0x958c3ca8, 0xbe94022e), TOBN(0x605c3197, 0x6145dac2), + TOBN(0x3fc07501, 0x01683d54), TOBN(0x1d7127c5, 0x595b1234), + TOBN(0x10b8f87c, 0x9481277f), TOBN(0x677db2a8, 0xe65a1adb), + TOBN(0xec2fccaa, 0xddce3345), TOBN(0x2a6811b7, 0x012a4350), + TOBN(0x96760ff1, 0xac598bdc), TOBN(0x054d652a, 0xd1bf4128), + TOBN(0x0a1151d4, 0x92a21005), TOBN(0xad7f3971, 0x33110fdf), + TOBN(0x8c95928c, 0x1960100f), TOBN(0x6c91c825, 0x7bf03362), + TOBN(0xc8c8b2a2, 0xce309f06), TOBN(0xfdb27b59, 0xca27204b), + TOBN(0xd223eaa5, 0x0848e32e), TOBN(0xb93e4b2e, 0xe7bfaf1e), + TOBN(0xc5308ae6, 0x44aa3ded), TOBN(0x317a666a, 0xc015d573), + TOBN(0xc888ce23, 0x1a979707), TOBN(0xf141c1e6, 0x0d5c4958), + TOBN(0xb53b7de5, 0x61906373), TOBN(0x858dbade, 0xeb999595), + TOBN(0x8cbb47b2, 0xa59e5c36), TOBN(0x660318b3, 0xdcf4e842), + TOBN(0xbd161ccd, 0x12ba4b7a), TOBN(0xf399daab, 0xf8c8282a), + TOBN(0x1587633a, 0xeeb2130d), TOBN(0xa465311a, 0xda38dd7d), + TOBN(0x5f75eec8, 0x64d3779b), TOBN(0x3c5d0476, 0xad64c171), + TOBN(0x87410371, 0x2a914428), TOBN(0x8096a891, 0x90e2fc29), + TOBN(0xd3d2ae9d, 0x23b3ebc2), TOBN(0x90bdd6db, 0xa580cfd6), + TOBN(0x52dbb7f3, 0xc5b01f6c), TOBN(0xe68eded4, 0xe102a2dc), + TOBN(0x17785b77, 0x99eb6df0), TOBN(0x26c3cc51, 0x7386b779), + TOBN(0x345ed988, 0x6417a48e), TOBN(0xe990b4e4, 0x07d6ef31), + TOBN(0x0f456b7e, 0x2586abba), TOBN(0x239ca6a5, 0x59c96e9a), + TOBN(0xe327459c, 0xe2eb4206), TOBN(0x3a4c3313, 0xa002b90a), + TOBN(0x2a114806, 0xf6a3f6fb), TOBN(0xad5cad2f, 0x85c251dd), + TOBN(0x92c1f613, 0xf5a784d3), TOBN(0xec7bfacf, 0x349766d5), + TOBN(0x04b3cd33, 0x3e23cb3b), TOBN(0x3979fe84, 0xc5a64b2d), + TOBN(0x192e2720, 0x7e589106), TOBN(0xa60c43d1, 0xa15b527f), + TOBN(0x2dae9082, 0xbe7cf3a6), TOBN(0xcc86ba92, 0xbc967274), + TOBN(0xf28a2ce8, 0xaea0a8a9), TOBN(0x404ca6d9, 0x6ee988b3), + TOBN(0xfd7e9c5d, 0x005921b8), TOBN(0xf56297f1, 0x44e79bf9), + TOBN(0xa163b460, 0x0d75ddc2), TOBN(0x30b23616, 0xa1f2be87), + TOBN(0x4b070d21, 0xbfe50e2b), TOBN(0x7ef8cfd0, 0xe1bfede1), + TOBN(0xadba0011, 0x2aac4ae0), TOBN(0x2a3e7d01, 0xb9ebd033), + TOBN(0x995277ec, 0xe38d9d1c), TOBN(0xb500249e, 0x9c5d2de3), + TOBN(0x8912b820, 0xf13ca8c9), TOBN(0xc8798114, 0x877793af), + TOBN(0x19e6125d, 0xec3f1dec), TOBN(0x07b1f040, 0x911178da), + TOBN(0xd93ededa, 0x904a6738), TOBN(0x55187a5a, 0x0bebedcd), + TOBN(0xf7d04722, 0xeb329d41), TOBN(0xf449099e, 0xf170b391), + TOBN(0xfd317a69, 0xca99f828), TOBN(0x50c3db2b, 0x34a4976d), + TOBN(0xe9ba7784, 0x3757b392), TOBN(0x326caefd, 0xaa3ca05a), + TOBN(0x78e5293b, 0xf1e593d4), TOBN(0x7842a937, 0x0d98fd13), + TOBN(0xe694bf96, 0x5f96b10d), TOBN(0x373a9df6, 0x06a8cd05), + TOBN(0x997d1e51, 0xe8f0c7fc), TOBN(0x1d019790, 0x63fd972e), + TOBN(0x0064d858, 0x5499fb32), TOBN(0x7b67bad9, 0x77a8aeb7), + TOBN(0x1d3eb977, 0x2d08eec5), TOBN(0x5fc047a6, 0xcbabae1d), + TOBN(0x0577d159, 0xe54a64bb), TOBN(0x8862201b, 0xc43497e4), + TOBN(0xad6b4e28, 0x2ce0608d), TOBN(0x8b687b7d, 0x0b167aac), + TOBN(0x6ed4d367, 0x8b2ecfa9), TOBN(0x24dfe62d, 0xa90c3c38), + TOBN(0xa1862e10, 0x3fe5c42b), TOBN(0x1ca73dca, 0xd5732a9f), + TOBN(0x35f038b7, 0x76bb87ad), TOBN(0x674976ab, 0xf242b81f), + TOBN(0x4f2bde7e, 0xb0fd90cd), TOBN(0x6efc172e, 0xa7fdf092), + TOBN(0x3806b69b, 0x92222f1f), TOBN(0x5a2459ca, 0x6cf7ae70), + TOBN(0x6789f69c, 0xa85217ee), TOBN(0x5f232b5e, 0xe3dc85ac), + TOBN(0x660e3ec5, 0x48e9e516), TOBN(0x124b4e47, 0x3197eb31), + TOBN(0x10a0cb13, 0xaafcca23), TOBN(0x7bd63ba4, 0x8213224f), + TOBN(0xaffad7cc, 0x290a7f4f), TOBN(0x6b409c9e, 0x0286b461), + TOBN(0x58ab809f, 0xffa407af), TOBN(0xc3122eed, 0xc68ac073), + TOBN(0x17bf9e50, 0x4ef24d7e), TOBN(0x5d929794, 0x3e2a5811), + TOBN(0x519bc867, 0x02902e01), TOBN(0x76bba5da, 0x39c8a851), + TOBN(0xe9f9669c, 0xda94951e), TOBN(0x4b6af58d, 0x66b8d418), + TOBN(0xfa321074, 0x17d426a4), TOBN(0xc78e66a9, 0x9dde6027), + TOBN(0x0516c083, 0x4a53b964), TOBN(0xfc659d38, 0xff602330), + TOBN(0x0ab55e5c, 0x58c5c897), TOBN(0x985099b2, 0x838bc5df), + TOBN(0x061d9efc, 0xc52fc238), TOBN(0x712b2728, 0x6ac1da3f), + TOBN(0xfb658149, 0x9283fe08), TOBN(0x4954ac94, 0xb8aaa2f7), + TOBN(0x85c0ada4, 0x7fb2e74f), TOBN(0xee8ba98e, 0xb89926b0), + TOBN(0xe4f9d37d, 0x23d1af5b), TOBN(0x14ccdbf9, 0xba9b015e), + TOBN(0xb674481b, 0x7bfe7178), TOBN(0x4e1debae, 0x65405868), + TOBN(0x061b2821, 0xc48c867d), TOBN(0x69c15b35, 0x513b30ea), + TOBN(0x3b4a1666, 0x36871088), TOBN(0xe5e29f5d, 0x1220b1ff), + TOBN(0x4b82bb35, 0x233d9f4d), TOBN(0x4e076333, 0x18cdc675)} + , + {TOBN(0x0d53f5c7, 0xa3e6fced), TOBN(0xe8cbbdd5, 0xf45fbdeb), + TOBN(0xf85c01df, 0x13339a70), TOBN(0x0ff71880, 0x142ceb81), + TOBN(0x4c4e8774, 0xbd70437a), TOBN(0x5fb32891, 0xba0bda6a), + TOBN(0x1cdbebd2, 0xf18bd26e), TOBN(0x2f9526f1, 0x03a9d522), + TOBN(0x40ce3051, 0x92c4d684), TOBN(0x8b04d725, 0x7612efcd), + TOBN(0xb9dcda36, 0x6f9cae20), TOBN(0x0edc4d24, 0xf058856c), + TOBN(0x64f2e6bf, 0x85427900), TOBN(0x3de81295, 0xdc09dfea), + TOBN(0xd41b4487, 0x379bf26c), TOBN(0x50b62c6d, 0x6df135a9), + TOBN(0xd4f8e3b4, 0xc72dfe67), TOBN(0xc416b0f6, 0x90e19fdf), + TOBN(0x18b9098d, 0x4c13bd35), TOBN(0xac11118a, 0x15b8cb9e), + TOBN(0xf598a318, 0xf0062841), TOBN(0xbfe0602f, 0x89f356f4), + TOBN(0x7ae3637e, 0x30177a0c), TOBN(0x34097747, 0x61136537), + TOBN(0x0db2fb5e, 0xd005832a), TOBN(0x5f5efd3b, 0x91042e4f), + TOBN(0x8c4ffdc6, 0xed70f8ca), TOBN(0xe4645d0b, 0xb52da9cc), + TOBN(0x9596f58b, 0xc9001d1f), TOBN(0x52c8f0bc, 0x4e117205), + TOBN(0xfd4aa0d2, 0xe398a084), TOBN(0x815bfe3a, 0x104f49de), + TOBN(0x97e5443f, 0x23885e5f), TOBN(0xf72f8f99, 0xe8433aab), + TOBN(0xbd00b154, 0xe4d4e604), TOBN(0xd0b35e6a, 0xe5e173ff), + TOBN(0x57b2a048, 0x9164722d), TOBN(0x3e3c665b, 0x88761ec8), + TOBN(0x6bdd1397, 0x3da83832), TOBN(0x3c8b1a1e, 0x73dafe3b), + TOBN(0x4497ace6, 0x54317cac), TOBN(0xbe600ab9, 0x521771b3), + TOBN(0xb42e409e, 0xb0dfe8b8), TOBN(0x386a67d7, 0x3942310f), + TOBN(0x25548d8d, 0x4431cc28), TOBN(0xa7cff142, 0x985dc524), + TOBN(0x4d60f5a1, 0x93c4be32), TOBN(0x83ebd5c8, 0xd071c6e1), + TOBN(0xba3a80a7, 0xb1fd2b0b), TOBN(0x9b3ad396, 0x5bec33e8), + TOBN(0xb3868d61, 0x79743fb3), TOBN(0xcfd169fc, 0xfdb462fa), + TOBN(0xd3b499d7, 0x9ce0a6af), TOBN(0x55dc1cf1, 0xe42d3ff8), + TOBN(0x04fb9e6c, 0xc6c3e1b2), TOBN(0x47e6961d, 0x6f69a474), + TOBN(0x54eb3acc, 0xe548b37b), TOBN(0xb38e7542, 0x84d40549), + TOBN(0x8c3daa51, 0x7b341b4f), TOBN(0x2f6928ec, 0x690bf7fa), + TOBN(0x0496b323, 0x86ce6c41), TOBN(0x01be1c55, 0x10adadcd), + TOBN(0xc04e67e7, 0x4bb5faf9), TOBN(0x3cbaf678, 0xe15c9985), + TOBN(0x8cd12145, 0x50ca4247), TOBN(0xba1aa47a, 0xe7dd30aa), + TOBN(0x2f81ddf1, 0xe58fee24), TOBN(0x03452936, 0xeec9b0e8), + TOBN(0x8bdc3b81, 0x243aea96), TOBN(0x9a2919af, 0x15c3d0e5), + TOBN(0x9ea640ec, 0x10948361), TOBN(0x5ac86d5b, 0x6e0bcccf), + TOBN(0xf892d918, 0xc36cf440), TOBN(0xaed3e837, 0xc939719c), + TOBN(0xb07b08d2, 0xc0218b64), TOBN(0x6f1bcbba, 0xce9790dd), + TOBN(0x4a84d6ed, 0x60919b8e), TOBN(0xd8900791, 0x8ac1f9eb), + TOBN(0xf84941aa, 0x0dd5daef), TOBN(0xb22fe40a, 0x67fd62c5), + TOBN(0x97e15ba2, 0x157f2db3), TOBN(0xbda2fc8f, 0x8e28ca9c), + TOBN(0x5d050da4, 0x37b9f454), TOBN(0x3d57eb57, 0x2379d72e), + TOBN(0xe9b5eba2, 0xfb5ee997), TOBN(0x01648ca2, 0xe11538ca), + TOBN(0x32bb76f6, 0xf6327974), TOBN(0x338f14b8, 0xff3f4bb7), + TOBN(0x524d226a, 0xd7ab9a2d), TOBN(0x9c00090d, 0x7dfae958), + TOBN(0x0ba5f539, 0x8751d8c2), TOBN(0x8afcbcdd, 0x3ab8262d), + TOBN(0x57392729, 0xe99d043b), TOBN(0xef51263b, 0xaebc943a), + TOBN(0x9feace93, 0x20862935), TOBN(0x639efc03, 0xb06c817b), + TOBN(0x1fe054b3, 0x66b4be7a), TOBN(0x3f25a9de, 0x84a37a1e), + TOBN(0xf39ef1ad, 0x78d75cd9), TOBN(0xd7b58f49, 0x5062c1b5), + TOBN(0x6f74f9a9, 0xff563436), TOBN(0xf718ff29, 0xe8af51e7), + TOBN(0x5234d313, 0x15e97fec), TOBN(0xb6a8e2b1, 0x292f1c0a), + TOBN(0xa7f53aa8, 0x327720c1), TOBN(0x956ca322, 0xba092cc8), + TOBN(0x8f03d64a, 0x28746c4d), TOBN(0x51fe1782, 0x66d0d392), + TOBN(0xd19b34db, 0x3c832c80), TOBN(0x60dccc5c, 0x6da2e3b4), + TOBN(0x245dd62e, 0x0a104ccc), TOBN(0xa7ab1de1, 0x620b21fd), + TOBN(0xb293ae0b, 0x3893d123), TOBN(0xf7b75783, 0xb15ee71c), + TOBN(0x5aa3c614, 0x42a9468b), TOBN(0xd686123c, 0xdb15d744), + TOBN(0x8c616891, 0xa7ab4116), TOBN(0x6fcd72c8, 0xa4e6a459), + TOBN(0xac219110, 0x77e5fad7), TOBN(0xfb6a20e7, 0x704fa46b), + TOBN(0xe839be7d, 0x341d81dc), TOBN(0xcddb6889, 0x32148379), + TOBN(0xda6211a1, 0xf7026ead), TOBN(0xf3b2575f, 0xf4d1cc5e), + TOBN(0x40cfc8f6, 0xa7a73ae6), TOBN(0x83879a5e, 0x61d5b483), + TOBN(0xc5acb1ed, 0x41a50ebc), TOBN(0x59a60cc8, 0x3c07d8fa), + TOBN(0x1b73bdce, 0xb1876262), TOBN(0x2b0d79f0, 0x12af4ee9), + TOBN(0x8bcf3b0b, 0xd46e1d07), TOBN(0x17d6af9d, 0xe45d152f), + TOBN(0x73520461, 0x6d736451), TOBN(0x43cbbd97, 0x56b0bf5a), + TOBN(0xb0833a5b, 0xd5999b9d), TOBN(0x702614f0, 0xeb72e398), + TOBN(0x0aadf01a, 0x59c3e9f8), TOBN(0x40200e77, 0xce6b3d16), + TOBN(0xda22bdd3, 0xdeddafad), TOBN(0x76dedaf4, 0x310d72e1), + TOBN(0x49ef807c, 0x4bc2e88f), TOBN(0x6ba81291, 0x146dd5a5), + TOBN(0xa1a4077a, 0x7d8d59e9), TOBN(0x87b6a2e7, 0x802db349), + TOBN(0xd5679997, 0x1b4e598e), TOBN(0xf499ef1f, 0x06fe4b1d), + TOBN(0x3978d3ae, 0xfcb267c5), TOBN(0xb582b557, 0x235786d0), + TOBN(0x32b3b2ca, 0x1715cb07), TOBN(0x4c3de6a2, 0x8480241d), + TOBN(0x63b5ffed, 0xcb571ecd), TOBN(0xeaf53900, 0xed2fe9a9), + TOBN(0xdec98d4a, 0xc3b81990), TOBN(0x1cb83722, 0x9e0cc8fe), + TOBN(0xfe0b0491, 0xd2b427b9), TOBN(0x0f2386ac, 0xe983a66c), + TOBN(0x930c4d1e, 0xb3291213), TOBN(0xa2f82b2e, 0x59a62ae4), + TOBN(0x77233853, 0xf93e89e3), TOBN(0x7f8063ac, 0x11777c7f), + TOBN(0xff0eb567, 0x59ad2877), TOBN(0x6f454642, 0x9865c754), + TOBN(0xe6fe701a, 0x236e9a84), TOBN(0xc586ef16, 0x06e40fc3), + TOBN(0x3f62b6e0, 0x24bafad9), TOBN(0xc8b42bd2, 0x64da906a), + TOBN(0xc98e1eb4, 0xda3276a0), TOBN(0x30d0e5fc, 0x06cbf852), + TOBN(0x1b6b2ae1, 0xe8b4dfd4), TOBN(0xd754d5c7, 0x8301cbac), + TOBN(0x66097629, 0x112a39ac), TOBN(0xf86b5999, 0x93ba4ab9), + TOBN(0x26c9dea7, 0x99f9d581), TOBN(0x0473b1a8, 0xc2fafeaa), + TOBN(0x1469af55, 0x3b2505a5), TOBN(0x227d16d7, 0xd6a43323), + TOBN(0x3316f73c, 0xad3d97f9), TOBN(0x52bf3bb5, 0x1f137455), + TOBN(0x953eafeb, 0x09954e7c), TOBN(0xa721dfed, 0xdd732411), + TOBN(0xb4929821, 0x141d4579), TOBN(0x3411321c, 0xaa3bd435), + TOBN(0xafb355aa, 0x17fa6015), TOBN(0xb4e7ef4a, 0x18e42f0e), + TOBN(0x604ac97c, 0x59371000), TOBN(0xe1c48c70, 0x7f759c18), + TOBN(0x3f62ecc5, 0xa5db6b65), TOBN(0x0a78b173, 0x38a21495), + TOBN(0x6be1819d, 0xbcc8ad94), TOBN(0x70dc04f6, 0xd89c3400), + TOBN(0x462557b4, 0xa6b4840a), TOBN(0x544c6ade, 0x60bd21c0), + TOBN(0x6a00f24e, 0x907a544b), TOBN(0xa7520dcb, 0x313da210), + TOBN(0xfe939b75, 0x11e4994b), TOBN(0x918b6ba6, 0xbc275d70), + TOBN(0xd3e5e0fc, 0x644be892), TOBN(0x707a9816, 0xfdaf6c42), + TOBN(0x60145567, 0xf15c13fe), TOBN(0x4818ebaa, 0xe130a54a), + TOBN(0x28aad3ad, 0x58d2f767), TOBN(0xdc5267fd, 0xd7e7c773), + TOBN(0x4919cc88, 0xc3afcc98), TOBN(0xaa2e6ab0, 0x2db8cd4b), + TOBN(0xd46fec04, 0xd0c63eaa), TOBN(0xa1cb92c5, 0x19ffa832), + TOBN(0x678dd178, 0xe43a631f), TOBN(0xfb5ae1cd, 0x3dc788b3), + TOBN(0x68b4fb90, 0x6e77de04), TOBN(0x7992bcf0, 0xf06dbb97), + TOBN(0x896e6a13, 0xc417c01d), TOBN(0x8d96332c, 0xb956be01), + TOBN(0x902fc93a, 0x413aa2b9), TOBN(0x99a4d915, 0xfc98c8a5), + TOBN(0x52c29407, 0x565f1137), TOBN(0x4072690f, 0x21e4f281), + TOBN(0x36e607cf, 0x02ff6072), TOBN(0xa47d2ca9, 0x8ad98cdc), + TOBN(0xbf471d1e, 0xf5f56609), TOBN(0xbcf86623, 0xf264ada0), + TOBN(0xb70c0687, 0xaa9e5cb6), TOBN(0xc98124f2, 0x17401c6c), + TOBN(0x8189635f, 0xd4a61435), TOBN(0xd28fb8af, 0xa9d98ea6), + TOBN(0xb9a67c2a, 0x40c251f8), TOBN(0x88cd5d87, 0xa2da44be), + TOBN(0x437deb96, 0xe09b5423), TOBN(0x150467db, 0x64287dc1), + TOBN(0xe161debb, 0xcdabb839), TOBN(0xa79e9742, 0xf1839a3e), + TOBN(0xbb8dd3c2, 0x652d202b), TOBN(0x7b3e67f7, 0xe9f97d96), + TOBN(0x5aa5d78f, 0xb1cb6ac9), TOBN(0xffa13e8e, 0xca1d0d45), + TOBN(0x369295dd, 0x2ba5bf95), TOBN(0xd68bd1f8, 0x39aff05e), + TOBN(0xaf0d86f9, 0x26d783f2), TOBN(0x543a59b3, 0xfc3aafc1), + TOBN(0x3fcf81d2, 0x7b7da97c), TOBN(0xc990a056, 0xd25dee46), + TOBN(0x3e6775b8, 0x519cce2c), TOBN(0xfc9af71f, 0xae13d863), + TOBN(0x774a4a6f, 0x47c1605c), TOBN(0x46ba4245, 0x2fd205e8), + TOBN(0xa06feea4, 0xd3fd524d), TOBN(0x1e724641, 0x6de1acc2), + TOBN(0xf53816f1, 0x334e2b42), TOBN(0x49e5918e, 0x922f0024), + TOBN(0x439530b6, 0x65c7322d), TOBN(0xcf12cc01, 0xb3c1b3fb), + TOBN(0xc70b0186, 0x0172f685), TOBN(0xb915ee22, 0x1b58391d), + TOBN(0x9afdf03b, 0xa317db24), TOBN(0x87dec659, 0x17b8ffc4), + TOBN(0x7f46597b, 0xe4d3d050), TOBN(0x80a1c1ed, 0x006500e7), + TOBN(0x84902a96, 0x78bf030e), TOBN(0xfb5e9c9a, 0x50560148), + TOBN(0x6dae0a92, 0x63362426), TOBN(0xdcaeecf4, 0xa9e30c40), + TOBN(0xc0d887bb, 0x518d0c6b), TOBN(0x99181152, 0xcb985b9d), + TOBN(0xad186898, 0xef7bc381), TOBN(0x18168ffb, 0x9ee46201), + TOBN(0x9a04cdaa, 0x2502753c), TOBN(0xbb279e26, 0x51407c41), + TOBN(0xeacb03aa, 0xf23564e5), TOBN(0x18336582, 0x71e61016), + TOBN(0x8684b8c4, 0xeb809877), TOBN(0xb336e18d, 0xea0e672e), + TOBN(0xefb601f0, 0x34ee5867), TOBN(0x2733edbe, 0x1341cfd1), + TOBN(0xb15e809a, 0x26025c3c), TOBN(0xe6e981a6, 0x9350df88), + TOBN(0x92376237, 0x8502fd8e), TOBN(0x4791f216, 0x0c12be9b), + TOBN(0xb7256789, 0x25f02425), TOBN(0xec863194, 0x7a974443), + TOBN(0x7c0ce882, 0xfb41cc52), TOBN(0xc266ff7e, 0xf25c07f2), + TOBN(0x3d4da8c3, 0x017025f3), TOBN(0xefcf628c, 0xfb9579b4), + TOBN(0x5c4d0016, 0x1f3716ec), TOBN(0x9c27ebc4, 0x6801116e), + TOBN(0x5eba0ea1, 0x1da1767e), TOBN(0xfe151452, 0x47004c57), + TOBN(0x3ace6df6, 0x8c2373b7), TOBN(0x75c3dffe, 0x5dbc37ac), + TOBN(0x3dc32a73, 0xddc925fc), TOBN(0xb679c841, 0x2f65ee0b), + TOBN(0x715a3295, 0x451cbfeb), TOBN(0xd9889768, 0xf76e9a29), + TOBN(0xec20ce7f, 0xb28ad247), TOBN(0xe99146c4, 0x00894d79), + TOBN(0x71457d7c, 0x9f5e3ea7), TOBN(0x097b2662, 0x38030031), + TOBN(0xdb7f6ae6, 0xcf9f82a8), TOBN(0x319decb9, 0x438f473a), + TOBN(0xa63ab386, 0x283856c3), TOBN(0x13e3172f, 0xb06a361b), + TOBN(0x2959f8dc, 0x7d5a006c), TOBN(0x2dbc27c6, 0x75fba752), + TOBN(0xc1227ab2, 0x87c22c9e), TOBN(0x06f61f75, 0x71a268b2), + TOBN(0x1b6bb971, 0x04779ce2), TOBN(0xaca83812, 0x0aadcb1d), + TOBN(0x297ae0bc, 0xaeaab2d5), TOBN(0xa5c14ee7, 0x5bfb9f13), + TOBN(0xaa00c583, 0xf17a62c7), TOBN(0x39eb962c, 0x173759f6), + TOBN(0x1eeba1d4, 0x86c9a88f), TOBN(0x0ab6c37a, 0xdf016c5e), + TOBN(0xa2a147db, 0xa28a0749), TOBN(0x246c20d6, 0xee519165), + TOBN(0x5068d1b1, 0xd3810715), TOBN(0xb1e7018c, 0x748160b9), + TOBN(0x03f5b1fa, 0xf380ff62), TOBN(0xef7fb1dd, 0xf3cb2c1e), + TOBN(0xeab539a8, 0xfc91a7da), TOBN(0x83ddb707, 0xf3f9b561), + TOBN(0xc550e211, 0xfe7df7a4), TOBN(0xa7cd07f2, 0x063f6f40), + TOBN(0xb0de3635, 0x2976879c), TOBN(0xb5f83f85, 0xe55741da), + TOBN(0x4ea9d25e, 0xf3d8ac3d), TOBN(0x6fe2066f, 0x62819f02), + TOBN(0x4ab2b9c2, 0xcef4a564), TOBN(0x1e155d96, 0x5ffa2de3), + TOBN(0x0eb0a19b, 0xc3a72d00), TOBN(0x4037665b, 0x8513c31b), + TOBN(0x2fb2b6bf, 0x04c64637), TOBN(0x45c34d6e, 0x08cdc639), + TOBN(0x56f1e10f, 0xf01fd796), TOBN(0x4dfb8101, 0xfe3667b8), + TOBN(0xe0eda253, 0x9021d0c0), TOBN(0x7a94e9ff, 0x8a06c6ab), + TOBN(0x2d3bb0d9, 0xbb9aa882), TOBN(0xea20e4e5, 0xec05fd10), + TOBN(0xed7eeb5f, 0x1a1ca64e), TOBN(0x2fa6b43c, 0xc6327cbd), + TOBN(0xb577e3cf, 0x3aa91121), TOBN(0x8c6bd5ea, 0x3a34079b), + TOBN(0xd7e5ba39, 0x60e02fc0), TOBN(0xf16dd2c3, 0x90141bf8), + TOBN(0xb57276d9, 0x80101b98), TOBN(0x760883fd, 0xb82f0f66), + TOBN(0x89d7de75, 0x4bc3eff3), TOBN(0x03b60643, 0x5dc2ab40), + TOBN(0xcd6e53df, 0xe05beeac), TOBN(0xf2f1e862, 0xbc3325cd), + TOBN(0xdd0f7921, 0x774f03c3), TOBN(0x97ca7221, 0x4552cc1b), + TOBN(0x5a0d6afe, 0x1cd19f72), TOBN(0xa20915dc, 0xf183fbeb), + TOBN(0x9fda4b40, 0x832c403c), TOBN(0x32738edd, 0xbe425442), + TOBN(0x469a1df6, 0xb5eccf1a), TOBN(0x4b5aff42, 0x28bbe1f0), + TOBN(0x31359d7f, 0x570dfc93), TOBN(0xa18be235, 0xf0088628), + TOBN(0xa5b30fba, 0xb00ed3a9), TOBN(0x34c61374, 0x73cdf8be), + TOBN(0x2c5c5f46, 0xabc56797), TOBN(0x5cecf93d, 0xb82a8ae2), + TOBN(0x7d3dbe41, 0xa968fbf0), TOBN(0xd23d4583, 0x1a5c7f3d), + TOBN(0xf28f69a0, 0xc087a9c7), TOBN(0xc2d75471, 0x474471ca), + TOBN(0x36ec9f4a, 0x4eb732ec), TOBN(0x6c943bbd, 0xb1ca6bed), + TOBN(0xd64535e1, 0xf2457892), TOBN(0x8b84a8ea, 0xf7e2ac06), + TOBN(0xe0936cd3, 0x2499dd5f), TOBN(0x12053d7e, 0x0ed04e57), + TOBN(0x4bdd0076, 0xe4305d9d), TOBN(0x34a527b9, 0x1f67f0a2), + TOBN(0xe79a4af0, 0x9cec46ea), TOBN(0xb15347a1, 0x658b9bc7), + TOBN(0x6bd2796f, 0x35af2f75), TOBN(0xac957990, 0x4051c435), + TOBN(0x2669dda3, 0xc33a655d), TOBN(0x5d503c2e, 0x88514aa3), + TOBN(0xdfa11337, 0x3753dd41), TOBN(0x3f054673, 0x0b754f78), + TOBN(0xbf185677, 0x496125bd), TOBN(0xfb0023c8, 0x3775006c), + TOBN(0xfa0f072f, 0x3a037899), TOBN(0x4222b6eb, 0x0e4aea57), + TOBN(0x3dde5e76, 0x7866d25a), TOBN(0xb6eb04f8, 0x4837aa6f), + TOBN(0x5315591a, 0x2cf1cdb8), TOBN(0x6dfb4f41, 0x2d4e683c), + TOBN(0x7e923ea4, 0x48ee1f3a), TOBN(0x9604d9f7, 0x05a2afd5), + TOBN(0xbe1d4a33, 0x40ea4948), TOBN(0x5b45f1f4, 0xb44cbd2f), + TOBN(0x5faf8376, 0x4acc757e), TOBN(0xa7cf9ab8, 0x63d68ff7), + TOBN(0x8ad62f69, 0xdf0e404b), TOBN(0xd65f33c2, 0x12bdafdf), + TOBN(0xc365de15, 0xa377b14e), TOBN(0x6bf5463b, 0x8e39f60c), + TOBN(0x62030d2d, 0x2ce68148), TOBN(0xd95867ef, 0xe6f843a8), + TOBN(0xd39a0244, 0xef5ab017), TOBN(0x0bd2d8c1, 0x4ab55d12), + TOBN(0xc9503db3, 0x41639169), TOBN(0x2d4e25b0, 0xf7660c8a), + TOBN(0x760cb3b5, 0xe224c5d7), TOBN(0xfa3baf8c, 0x68616919), + TOBN(0x9fbca113, 0x8d142552), TOBN(0x1ab18bf1, 0x7669ebf5), + TOBN(0x55e6f53e, 0x9bdf25dd), TOBN(0x04cc0bf3, 0xcb6cd154), + TOBN(0x595bef49, 0x95e89080), TOBN(0xfe9459a8, 0x104a9ac1), + TOBN(0xad2d89ca, 0xcce9bb32), TOBN(0xddea65e1, 0xf7de8285), + TOBN(0x62ed8c35, 0xb351bd4b), TOBN(0x4150ff36, 0x0c0e19a7), + TOBN(0x86e3c801, 0x345f4e47), TOBN(0x3bf21f71, 0x203a266c), + TOBN(0x7ae110d4, 0x855b1f13), TOBN(0x5d6aaf6a, 0x07262517), + TOBN(0x1e0f12e1, 0x813d28f1), TOBN(0x6000e11d, 0x7ad7a523), + TOBN(0xc7d8deef, 0xc744a17b), TOBN(0x1e990b48, 0x14c05a00), + TOBN(0x68fddaee, 0x93e976d5), TOBN(0x696241d1, 0x46610d63), + TOBN(0xb204e7c3, 0x893dda88), TOBN(0x8bccfa65, 0x6a3a6946), + TOBN(0xb59425b4, 0xc5cd1411), TOBN(0x701b4042, 0xff3658b1), + TOBN(0xe3e56bca, 0x4784cf93), TOBN(0x27de5f15, 0x8fe68d60), + TOBN(0x4ab9cfce, 0xf8d53f19), TOBN(0xddb10311, 0xa40a730d), + TOBN(0x6fa73cd1, 0x4eee0a8a), TOBN(0xfd548748, 0x5249719d), + TOBN(0x49d66316, 0xa8123ef0), TOBN(0x73c32db4, 0xe7f95438), + TOBN(0x2e2ed209, 0x0d9e7854), TOBN(0xf98a9329, 0x9d9f0507), + TOBN(0xc5d33cf6, 0x0c6aa20a), TOBN(0x9a32ba14, 0x75279bb2), + TOBN(0x7e3202cb, 0x774a7307), TOBN(0x64ed4bc4, 0xe8c42dbd), + TOBN(0xc20f1a06, 0xd4caed0d), TOBN(0xb8021407, 0x171d22b3), + TOBN(0xd426ca04, 0xd13268d7), TOBN(0x92377007, 0x25f4d126), + TOBN(0x4204cbc3, 0x71f21a85), TOBN(0x18461b7a, 0xf82369ba), + TOBN(0xc0c07d31, 0x3fc858f9), TOBN(0x5deb5a50, 0xe2bab569), + TOBN(0xd5959d46, 0xd5eea89e), TOBN(0xfdff8424, 0x08437f4b), + TOBN(0xf21071e4, 0x3cfe254f), TOBN(0x72417696, 0x95468321), + TOBN(0x5d8288b9, 0x102cae3e), TOBN(0x2d143e3d, 0xf1965dff), + TOBN(0x00c9a376, 0xa078d847), TOBN(0x6fc0da31, 0x26028731), + TOBN(0xa2baeadf, 0xe45083a2), TOBN(0x66bc7218, 0x5e5b4bcd), + TOBN(0x2c826442, 0xd04b8e7f), TOBN(0xc19f5451, 0x6c4b586b), + TOBN(0x60182c49, 0x5b7eeed5), TOBN(0xd9954ecd, 0x7aa9dfa1), + TOBN(0xa403a8ec, 0xc73884ad), TOBN(0x7fb17de2, 0x9bb39041), + TOBN(0x694b64c5, 0xabb020e8), TOBN(0x3d18c184, 0x19c4eec7), + TOBN(0x9c4673ef, 0x1c4793e5), TOBN(0xc7b8aeb5, 0x056092e6), + TOBN(0x3aa1ca43, 0xf0f8c16b), TOBN(0x224ed5ec, 0xd679b2f6), + TOBN(0x0d56eeaf, 0x55a205c9), TOBN(0xbfe115ba, 0x4b8e028b), + TOBN(0x97e60849, 0x3927f4fe), TOBN(0xf91fbf94, 0x759aa7c5), + TOBN(0x985af769, 0x6be90a51), TOBN(0xc1277b78, 0x78ccb823), + TOBN(0x395b656e, 0xe7a75952), TOBN(0x00df7de0, 0x928da5f5), + TOBN(0x09c23175, 0x4ca4454f), TOBN(0x4ec971f4, 0x7aa2d3c1), + TOBN(0x45c3c507, 0xe75d9ccc), TOBN(0x63b7be8a, 0x3dc90306), + TOBN(0x37e09c66, 0x5db44bdc), TOBN(0x50d60da1, 0x6841c6a2), + TOBN(0x6f9b65ee, 0x08df1b12), TOBN(0x38734879, 0x7ff089df), + TOBN(0x9c331a66, 0x3fe8013d), TOBN(0x017f5de9, 0x5f42fcc8), + TOBN(0x43077866, 0xe8e57567), TOBN(0xc9f781ce, 0xf9fcdb18), + TOBN(0x38131dda, 0x9b12e174), TOBN(0x25d84aa3, 0x8a03752a), + TOBN(0x45e09e09, 0x4d0c0ce2), TOBN(0x1564008b, 0x92bebba5), + TOBN(0xf7e8ad31, 0xa87284c7), TOBN(0xb7c4b46c, 0x97e7bbaa), + TOBN(0x3e22a7b3, 0x97acf4ec), TOBN(0x0426c400, 0x5ea8b640), + TOBN(0x5e3295a6, 0x4e969285), TOBN(0x22aabc59, 0xa6a45670), + TOBN(0xb929714c, 0x5f5942bc), TOBN(0x9a6168bd, 0xfa3182ed), + TOBN(0x2216a665, 0x104152ba), TOBN(0x46908d03, 0xb6926368)} + , + {TOBN(0xa9f5d874, 0x5a1251fb), TOBN(0x967747a8, 0xc72725c7), + TOBN(0x195c33e5, 0x31ffe89e), TOBN(0x609d210f, 0xe964935e), + TOBN(0xcafd6ca8, 0x2fe12227), TOBN(0xaf9b5b96, 0x0426469d), + TOBN(0x2e9ee04c, 0x5693183c), TOBN(0x1084a333, 0xc8146fef), + TOBN(0x96649933, 0xaed1d1f7), TOBN(0x566eaff3, 0x50563090), + TOBN(0x345057f0, 0xad2e39cf), TOBN(0x148ff65b, 0x1f832124), + TOBN(0x042e89d4, 0xcf94cf0d), TOBN(0x319bec84, 0x520c58b3), + TOBN(0x2a267626, 0x5361aa0d), TOBN(0xc86fa302, 0x8fbc87ad), + TOBN(0xfc83d2ab, 0x5c8b06d5), TOBN(0xb1a785a2, 0xfe4eac46), + TOBN(0xb99315bc, 0x846f7779), TOBN(0xcf31d816, 0xef9ea505), + TOBN(0x2391fe6a, 0x15d7dc85), TOBN(0x2f132b04, 0xb4016b33), + TOBN(0x29547fe3, 0x181cb4c7), TOBN(0xdb66d8a6, 0x650155a1), + TOBN(0x6b66d7e1, 0xadc1696f), TOBN(0x98ebe593, 0x0acd72d0), + TOBN(0x65f24550, 0xcc1b7435), TOBN(0xce231393, 0xb4b9a5ec), + TOBN(0x234a22d4, 0xdb067df9), TOBN(0x98dda095, 0xcaff9b00), + TOBN(0x1bbc75a0, 0x6100c9c1), TOBN(0x1560a9c8, 0x939cf695), + TOBN(0xcf006d3e, 0x99e0925f), TOBN(0x2dd74a96, 0x6322375a), + TOBN(0xc58b446a, 0xb56af5ba), TOBN(0x50292683, 0xe0b9b4f1), + TOBN(0xe2c34cb4, 0x1aeaffa3), TOBN(0x8b17203f, 0x9b9587c1), + TOBN(0x6d559207, 0xead1350c), TOBN(0x2b66a215, 0xfb7f9604), + TOBN(0x0850325e, 0xfe51bf74), TOBN(0x9c4f579e, 0x5e460094), + TOBN(0x5c87b92a, 0x76da2f25), TOBN(0x889de4e0, 0x6febef33), + TOBN(0x6900ec06, 0x646083ce), TOBN(0xbe2a0335, 0xbfe12773), + TOBN(0xadd1da35, 0xc5344110), TOBN(0x757568b7, 0xb802cd20), + TOBN(0x75559779, 0x00f7e6c8), TOBN(0x38e8b94f, 0x0facd2f0), + TOBN(0xfea1f3af, 0x03fde375), TOBN(0x5e11a1d8, 0x75881dfc), + TOBN(0xb3a6b02e, 0xc1e2f2ef), TOBN(0x193d2bbb, 0xc605a6c5), + TOBN(0x325ffeee, 0x339a0b2d), TOBN(0x27b6a724, 0x9e0c8846), + TOBN(0xe4050f1c, 0xf1c367ca), TOBN(0x9bc85a9b, 0xc90fbc7d), + TOBN(0xa373c4a2, 0xe1a11032), TOBN(0xb64232b7, 0xad0393a9), + TOBN(0xf5577eb0, 0x167dad29), TOBN(0x1604f301, 0x94b78ab2), + TOBN(0x0baa94af, 0xe829348b), TOBN(0x77fbd8dd, 0x41654342), + TOBN(0xdab50ea5, 0xb964e39a), TOBN(0xd4c29e3c, 0xd0d3c76e), + TOBN(0x80dae67c, 0x56d11964), TOBN(0x7307a8bf, 0xe5ffcc2f), + TOBN(0x65bbc1aa, 0x91708c3b), TOBN(0xa151e62c, 0x28bf0eeb), + TOBN(0x6cb53381, 0x6fa34db7), TOBN(0x5139e05c, 0xa29403a8), + TOBN(0x6ff651b4, 0x94a7cd2e), TOBN(0x5671ffd1, 0x0699336c), + TOBN(0x6f5fd2cc, 0x979a896a), TOBN(0x11e893a8, 0xd8148cef), + TOBN(0x988906a1, 0x65cf7b10), TOBN(0x81b67178, 0xc50d8485), + TOBN(0x7c0deb35, 0x8a35b3de), TOBN(0x423ac855, 0xc1d29799), + TOBN(0xaf580d87, 0xdac50b74), TOBN(0x28b2b89f, 0x5869734c), + TOBN(0x99a3b936, 0x874e28fb), TOBN(0xbb2c9190, 0x25f3f73a), + TOBN(0x199f6918, 0x84a9d5b7), TOBN(0x7ebe2325, 0x7e770374), + TOBN(0xf442e107, 0x0738efe2), TOBN(0xcf9f3f56, 0xcf9082d2), + TOBN(0x719f69e1, 0x09618708), TOBN(0xcc9e8364, 0xc183f9b1), + TOBN(0xec203a95, 0x366a21af), TOBN(0x6aec5d6d, 0x068b141f), + TOBN(0xee2df78a, 0x994f04e9), TOBN(0xb39ccae8, 0x271245b0), + TOBN(0xb875a4a9, 0x97e43f4f), TOBN(0x507dfe11, 0xdb2cea98), + TOBN(0x4fbf81cb, 0x489b03e9), TOBN(0xdb86ec5b, 0x6ec414fa), + TOBN(0xfad444f9, 0xf51b3ae5), TOBN(0xca7d33d6, 0x1914e3fe), + TOBN(0xa9c32f5c, 0x0ae6c4d0), TOBN(0xa9ca1d1e, 0x73969568), + TOBN(0x98043c31, 0x1aa7467e), TOBN(0xe832e75c, 0xe21b5ac6), + TOBN(0x314b7aea, 0x5232123d), TOBN(0x08307c8c, 0x65ae86db), + TOBN(0x06e7165c, 0xaa4668ed), TOBN(0xb170458b, 0xb4d3ec39), + TOBN(0x4d2e3ec6, 0xc19bb986), TOBN(0xc5f34846, 0xae0304ed), + TOBN(0x917695a0, 0x6c9f9722), TOBN(0x6c7f7317, 0x4cab1c0a), + TOBN(0x6295940e, 0x9d6d2e8b), TOBN(0xd318b8c1, 0x549f7c97), + TOBN(0x22453204, 0x97713885), TOBN(0x468d834b, 0xa8a440fe), + TOBN(0xd81fe5b2, 0xbfba796e), TOBN(0x152364db, 0x6d71f116), + TOBN(0xbb8c7c59, 0xb5b66e53), TOBN(0x0b12c61b, 0x2641a192), + TOBN(0x31f14802, 0xfcf0a7fd), TOBN(0x42fd0789, 0x5488b01e), + TOBN(0x71d78d6d, 0x9952b498), TOBN(0x8eb572d9, 0x07ac5201), + TOBN(0xe0a2a44c, 0x4d194a88), TOBN(0xd2b63fd9, 0xba017e66), + TOBN(0x78efc6c8, 0xf888aefc), TOBN(0xb76f6bda, 0x4a881a11), + TOBN(0x187f314b, 0xb46c2397), TOBN(0x004cf566, 0x5ded2819), + TOBN(0xa9ea5704, 0x38764d34), TOBN(0xbba45217, 0x78084709), + TOBN(0x06474571, 0x1171121e), TOBN(0xad7b7eb1, 0xe7c9b671), + TOBN(0xdacfbc40, 0x730f7507), TOBN(0x178cd8c6, 0xc7ad7bd1), + TOBN(0xbf0be101, 0xb2a67238), TOBN(0x3556d367, 0xaf9c14f2), + TOBN(0x104b7831, 0xa5662075), TOBN(0x58ca59bb, 0x79d9e60a), + TOBN(0x4bc45392, 0xa569a73b), TOBN(0x517a52e8, 0x5698f6c9), + TOBN(0x85643da5, 0xaeadd755), TOBN(0x1aed0cd5, 0x2a581b84), + TOBN(0xb9b4ff84, 0x80af1372), TOBN(0x244c3113, 0xf1ba5d1f), + TOBN(0x2a5dacbe, 0xf5f98d31), TOBN(0x2c3323e8, 0x4375bc2a), + TOBN(0x17a3ab4a, 0x5594b1dd), TOBN(0xa1928bfb, 0xceb4797e), + TOBN(0xe83af245, 0xe4886a19), TOBN(0x8979d546, 0x72b5a74a), + TOBN(0xa0f726bc, 0x19f9e967), TOBN(0xd9d03152, 0xe8fbbf4e), + TOBN(0xcfd6f51d, 0xb7707d40), TOBN(0x633084d9, 0x63f6e6e0), + TOBN(0xedcd9cdc, 0x55667eaf), TOBN(0x73b7f92b, 0x2e44d56f), + TOBN(0xfb2e39b6, 0x4e962b14), TOBN(0x7d408f6e, 0xf671fcbf), + TOBN(0xcc634ddc, 0x164a89bb), TOBN(0x74a42bb2, 0x3ef3bd05), + TOBN(0x1280dbb2, 0x428decbb), TOBN(0x6103f6bb, 0x402c8596), + TOBN(0xfa2bf581, 0x355a5752), TOBN(0x562f96a8, 0x00946674), + TOBN(0x4e4ca16d, 0x6da0223b), TOBN(0xfe47819f, 0x28d3aa25), + TOBN(0x9eea3075, 0xf8dfcf8a), TOBN(0xa284f0aa, 0x95669825), + TOBN(0xb3fca250, 0x867d3fd8), TOBN(0x20757b5f, 0x269d691e), + TOBN(0xf2c24020, 0x93b8a5de), TOBN(0xd3f93359, 0xebc06da6), + TOBN(0x1178293e, 0xb2739c33), TOBN(0xd2a3e770, 0xbcd686e5), + TOBN(0xa76f49f4, 0xcd941534), TOBN(0x0d37406b, 0xe3c71c0e), + TOBN(0x172d9397, 0x3b97f7e3), TOBN(0xec17e239, 0xbd7fd0de), + TOBN(0xe3290551, 0x6f496ba2), TOBN(0x6a693172, 0x36ad50e7), + TOBN(0xc4e539a2, 0x83e7eff5), TOBN(0x752737e7, 0x18e1b4cf), + TOBN(0xa2f7932c, 0x68af43ee), TOBN(0x5502468e, 0x703d00bd), + TOBN(0xe5dc978f, 0x2fb061f5), TOBN(0xc9a1904a, 0x28c815ad), + TOBN(0xd3af538d, 0x470c56a4), TOBN(0x159abc5f, 0x193d8ced), + TOBN(0x2a37245f, 0x20108ef3), TOBN(0xfa17081e, 0x223f7178), + TOBN(0x27b0fb2b, 0x10c8c0f5), TOBN(0x2102c3ea, 0x40650547), + TOBN(0x594564df, 0x8ac3bfa7), TOBN(0x98102033, 0x509dad96), + TOBN(0x6989643f, 0xf1d18a13), TOBN(0x35eebd91, 0xd7fc5af0), + TOBN(0x078d096a, 0xfaeaafd8), TOBN(0xb7a89341, 0xdef3de98), + TOBN(0x2a206e8d, 0xecf2a73a), TOBN(0x066a6397, 0x8e551994), + TOBN(0x3a6a088a, 0xb98d53a2), TOBN(0x0ce7c67c, 0x2d1124aa), + TOBN(0x48cec671, 0x759a113c), TOBN(0xe3b373d3, 0x4f6f67fa), + TOBN(0x5455d479, 0xfd36727b), TOBN(0xe5a428ee, 0xa13c0d81), + TOBN(0xb853dbc8, 0x1c86682b), TOBN(0xb78d2727, 0xb8d02b2a), + TOBN(0xaaf69bed, 0x8ebc329a), TOBN(0xdb6b40b3, 0x293b2148), + TOBN(0xe42ea77d, 0xb8c4961f), TOBN(0xb1a12f7c, 0x20e5e0ab), + TOBN(0xa0ec5274, 0x79e8b05e), TOBN(0x68027391, 0xfab60a80), + TOBN(0x6bfeea5f, 0x16b1bd5e), TOBN(0xf957e420, 0x4de30ad3), + TOBN(0xcbaf664e, 0x6a353b9e), TOBN(0x5c873312, 0x26d14feb), + TOBN(0x4e87f98c, 0xb65f57cb), TOBN(0xdb60a621, 0x5e0cdd41), + TOBN(0x67c16865, 0xa6881440), TOBN(0x1093ef1a, 0x46ab52aa), + TOBN(0xc095afb5, 0x3f4ece64), TOBN(0x6a6bb02e, 0x7604551a), + TOBN(0x55d44b4e, 0x0b26b8cd), TOBN(0xe5f9a999, 0xf971268a), + TOBN(0xc08ec425, 0x11a7de84), TOBN(0x83568095, 0xfda469dd), + TOBN(0x737bfba1, 0x6c6c90a2), TOBN(0x1cb9c4a0, 0xbe229831), + TOBN(0x93bccbba, 0xbb2eec64), TOBN(0xa0c23b64, 0xda03adbe), + TOBN(0x5f7aa00a, 0xe0e86ac4), TOBN(0x470b941e, 0xfc1401e6), + TOBN(0x5ad8d679, 0x9df43574), TOBN(0x4ccfb8a9, 0x0f65d810), + TOBN(0x1bce80e3, 0xaa7fbd81), TOBN(0x273291ad, 0x9508d20a), + TOBN(0xf5c4b46b, 0x42a92806), TOBN(0x810684ec, 0xa86ab44a), + TOBN(0x4591640b, 0xca0bc9f8), TOBN(0xb5efcdfc, 0x5c4b6054), + TOBN(0x16fc8907, 0x6e9edd12), TOBN(0xe29d0b50, 0xd4d792f9), + TOBN(0xa45fd01c, 0x9b03116d), TOBN(0x85035235, 0xc81765a4), + TOBN(0x1fe2a9b2, 0xb4b4b67c), TOBN(0xc1d10df0, 0xe8020604), + TOBN(0x9d64abfc, 0xbc8058d8), TOBN(0x8943b9b2, 0x712a0fbb), + TOBN(0x90eed914, 0x3b3def04), TOBN(0x85ab3aa2, 0x4ce775ff), + TOBN(0x605fd4ca, 0x7bbc9040), TOBN(0x8b34a564, 0xe2c75dfb), + TOBN(0x41ffc94a, 0x10358560), TOBN(0x2d8a5072, 0x9e5c28aa), + TOBN(0xe915a0fc, 0x4cc7eb15), TOBN(0xe9efab05, 0x8f6d0f5d), + TOBN(0xdbab47a9, 0xd19e9b91), TOBN(0x8cfed745, 0x0276154c), + TOBN(0x154357ae, 0x2cfede0d), TOBN(0x520630df, 0x19f5a4ef), + TOBN(0x25759f7c, 0xe382360f), TOBN(0xb6db05c9, 0x88bf5857), + TOBN(0x2917d61d, 0x6c58d46c), TOBN(0x14f8e491, 0xfd20cb7a), + TOBN(0xb68a727a, 0x11c20340), TOBN(0x0386f86f, 0xaf7ccbb6), + TOBN(0x5c8bc6cc, 0xfee09a20), TOBN(0x7d76ff4a, 0xbb7eea35), + TOBN(0xa7bdebe7, 0xdb15be7a), TOBN(0x67a08054, 0xd89f0302), + TOBN(0x56bf0ea9, 0xc1193364), TOBN(0xc8244467, 0x62837ebe), + TOBN(0x32bd8e8b, 0x20d841b8), TOBN(0x127a0548, 0xdbb8a54f), + TOBN(0x83dd4ca6, 0x63b20236), TOBN(0x87714718, 0x203491fa), + TOBN(0x4dabcaaa, 0xaa8a5288), TOBN(0x91cc0c8a, 0xaf23a1c9), + TOBN(0x34c72c6a, 0x3f220e0c), TOBN(0xbcc20bdf, 0x1232144a), + TOBN(0x6e2f42da, 0xa20ede1b), TOBN(0xc441f00c, 0x74a00515), + TOBN(0xbf46a5b6, 0x734b8c4b), TOBN(0x57409503, 0x7b56c9a4), + TOBN(0x9f735261, 0xe4585d45), TOBN(0x9231faed, 0x6734e642), + TOBN(0x1158a176, 0xbe70ee6c), TOBN(0x35f1068d, 0x7c3501bf), + TOBN(0x6beef900, 0xa2d26115), TOBN(0x649406f2, 0xef0afee3), + TOBN(0x3f43a60a, 0xbc2420a1), TOBN(0x509002a7, 0xd5aee4ac), + TOBN(0xb46836a5, 0x3ff3571b), TOBN(0x24f98b78, 0x837927c1), + TOBN(0x6254256a, 0x4533c716), TOBN(0xf27abb0b, 0xd07ee196), + TOBN(0xd7cf64fc, 0x5c6d5bfd), TOBN(0x6915c751, 0xf0cd7a77), + TOBN(0xd9f59012, 0x8798f534), TOBN(0x772b0da8, 0xf81d8b5f), + TOBN(0x1244260c, 0x2e03fa69), TOBN(0x36cf0e3a, 0x3be1a374), + TOBN(0x6e7c1633, 0xef06b960), TOBN(0xa71a4c55, 0x671f90f6), + TOBN(0x7a941251, 0x33c673db), TOBN(0xc0bea510, 0x73e8c131), + TOBN(0x61a8a699, 0xd4f6c734), TOBN(0x25e78c88, 0x341ed001), + TOBN(0x5c18acf8, 0x8e2f7d90), TOBN(0xfdbf33d7, 0x77be32cd), + TOBN(0x0a085cd7, 0xd2eb5ee9), TOBN(0x2d702cfb, 0xb3201115), + TOBN(0xb6e0ebdb, 0x85c88ce8), TOBN(0x23a3ce3c, 0x1e01d617), + TOBN(0x3041618e, 0x567333ac), TOBN(0x9dd0fd8f, 0x157edb6b), + TOBN(0x27f74702, 0xb57872b8), TOBN(0x2ef26b4f, 0x657d5fe1), + TOBN(0x95426f0a, 0x57cf3d40), TOBN(0x847e2ad1, 0x65a6067a), + TOBN(0xd474d9a0, 0x09996a74), TOBN(0x16a56acd, 0x2a26115c), + TOBN(0x02a615c3, 0xd16f4d43), TOBN(0xcc3fc965, 0xaadb85b7), + TOBN(0x386bda73, 0xce07d1b0), TOBN(0xd82910c2, 0x58ad4178), + TOBN(0x124f82cf, 0xcd2617f4), TOBN(0xcc2f5e8d, 0xef691770), + TOBN(0x82702550, 0xb8c30ccc), TOBN(0x7b856aea, 0x1a8e575a), + TOBN(0xbb822fef, 0xb1ab9459), TOBN(0x085928bc, 0xec24e38e), + TOBN(0x5d0402ec, 0xba8f4b4d), TOBN(0xc07cd4ba, 0x00b4d58b), + TOBN(0x5d8dffd5, 0x29227e7a), TOBN(0x61d44d0c, 0x31bf386f), + TOBN(0xe486dc2b, 0x135e6f4d), TOBN(0x680962eb, 0xe79410ef), + TOBN(0xa61bd343, 0xf10088b5), TOBN(0x6aa76076, 0xe2e28686), + TOBN(0x80463d11, 0x8fb98871), TOBN(0xcb26f5c3, 0xbbc76aff), + TOBN(0xd4ab8edd, 0xfbe03614), TOBN(0xc8eb579b, 0xc0cf2dee), + TOBN(0xcc004c15, 0xc93bae41), TOBN(0x46fbae5d, 0x3aeca3b2), + TOBN(0x671235cf, 0x0f1e9ab1), TOBN(0xadfba934, 0x9ec285c1), + TOBN(0x88ded013, 0xf216c980), TOBN(0xc8ac4fb8, 0xf79e0bc1), + TOBN(0xa29b89c6, 0xfb97a237), TOBN(0xb697b780, 0x9922d8e7), + TOBN(0x3142c639, 0xddb945b5), TOBN(0x447b06c7, 0xe094c3a9), + TOBN(0xcdcb3642, 0x72266c90), TOBN(0x633aad08, 0xa9385046), + TOBN(0xa36c936b, 0xb57c6477), TOBN(0x871f8b64, 0xe94dbcc6), + TOBN(0x28d0fb62, 0xa591a67b), TOBN(0x9d40e081, 0xc1d926f5), + TOBN(0x3111eaf6, 0xf2d84b5a), TOBN(0x228993f9, 0xa565b644), + TOBN(0x0ccbf592, 0x2c83188b), TOBN(0xf87b30ab, 0x3df3e197), + TOBN(0xb8658b31, 0x7642bca8), TOBN(0x1a032d7f, 0x52800f17), + TOBN(0x051dcae5, 0x79bf9445), TOBN(0xeba6b8ee, 0x54a2e253), + TOBN(0x5c8b9cad, 0xd4485692), TOBN(0x84bda40e, 0x8986e9be), + TOBN(0xd16d16a4, 0x2f0db448), TOBN(0x8ec80050, 0xa14d4188), + TOBN(0xb2b26107, 0x98fa7aaa), TOBN(0x41209ee4, 0xf073aa4e), + TOBN(0xf1570359, 0xf2d6b19b), TOBN(0xcbe6868c, 0xfc577caf), + TOBN(0x186c4bdc, 0x32c04dd3), TOBN(0xa6c35fae, 0xcfeee397), + TOBN(0xb4a1b312, 0xf086c0cf), TOBN(0xe0a5ccc6, 0xd9461fe2), + TOBN(0xc32278aa, 0x1536189f), TOBN(0x1126c55f, 0xba6df571), + TOBN(0x0f71a602, 0xb194560e), TOBN(0x8b2d7405, 0x324bd6e1), + TOBN(0x8481939e, 0x3738be71), TOBN(0xb5090b1a, 0x1a4d97a9), + TOBN(0x116c65a3, 0xf05ba915), TOBN(0x21863ad3, 0xaae448aa), + TOBN(0xd24e2679, 0xa7aae5d3), TOBN(0x7076013d, 0x0de5c1c4), + TOBN(0x2d50f8ba, 0xbb05b629), TOBN(0x73c1abe2, 0x6e66efbb), + TOBN(0xefd4b422, 0xf2488af7), TOBN(0xe4105d02, 0x663ba575), + TOBN(0x7eb60a8b, 0x53a69457), TOBN(0x62210008, 0xc945973b), + TOBN(0xfb255478, 0x77a50ec6), TOBN(0xbf0392f7, 0x0a37a72c), + TOBN(0xa0a7a19c, 0x4be18e7a), TOBN(0x90d8ea16, 0x25b1e0af), + TOBN(0x7582a293, 0xef953f57), TOBN(0x90a64d05, 0xbdc5465a), + TOBN(0xca79c497, 0xe2510717), TOBN(0x560dbb7c, 0x18cb641f), + TOBN(0x1d8e3286, 0x4b66abfb), TOBN(0xd26f52e5, 0x59030900), + TOBN(0x1ee3f643, 0x5584941a), TOBN(0x6d3b3730, 0x569f5958), + TOBN(0x9ff2a62f, 0x4789dba5), TOBN(0x91fcb815, 0x72b5c9b7), + TOBN(0xf446cb7d, 0x6c8f9a0e), TOBN(0x48f625c1, 0x39b7ecb5), + TOBN(0xbabae801, 0x1c6219b8), TOBN(0xe7a562d9, 0x28ac2f23), + TOBN(0xe1b48732, 0x26e20588), TOBN(0x06ee1cad, 0x775af051), + TOBN(0xda29ae43, 0xfaff79f7), TOBN(0xc141a412, 0x652ee9e0), + TOBN(0x1e127f6f, 0x195f4bd0), TOBN(0x29c6ab4f, 0x072f34f8), + TOBN(0x7b7c1477, 0x30448112), TOBN(0x82b51af1, 0xe4a38656), + TOBN(0x2bf2028a, 0x2f315010), TOBN(0xc9a4a01f, 0x6ea88cd4), + TOBN(0xf63e95d8, 0x257e5818), TOBN(0xdd8efa10, 0xb4519b16), + TOBN(0xed8973e0, 0x0da910bf), TOBN(0xed49d077, 0x5c0fe4a9), + TOBN(0xac3aac5e, 0xb7caee1e), TOBN(0x1033898d, 0xa7f4da57), + TOBN(0x42145c0e, 0x5c6669b9), TOBN(0x42daa688, 0xc1aa2aa0), + TOBN(0x629cc15c, 0x1a1d885a), TOBN(0x25572ec0, 0xf4b76817), + TOBN(0x8312e435, 0x9c8f8f28), TOBN(0x8107f8cd, 0x81965490), + TOBN(0x516ff3a3, 0x6fa6110c), TOBN(0x74fb1eb1, 0xfb93561f), + TOBN(0x6c0c9047, 0x8457522b), TOBN(0xcfd32104, 0x6bb8bdc6), + TOBN(0x2d6884a2, 0xcc80ad57), TOBN(0x7c27fc35, 0x86a9b637), + TOBN(0x3461baed, 0xadf4e8cd), TOBN(0x1d56251a, 0x617242f0), + TOBN(0x0b80d209, 0xc955bef4), TOBN(0xdf02cad2, 0x06adb047), + TOBN(0xf0d7cb91, 0x5ec74fee), TOBN(0xd2503375, 0x1111ba44), + TOBN(0x9671755e, 0xdf53cb36), TOBN(0x54dcb612, 0x3368551b), + TOBN(0x66d69aac, 0xc8a025a4), TOBN(0x6be946c6, 0xe77ef445), + TOBN(0x719946d1, 0xa995e094), TOBN(0x65e848f6, 0xe51e04d8), + TOBN(0xe62f3300, 0x6a1e3113), TOBN(0x1541c7c1, 0x501de503), + TOBN(0x4daac9fa, 0xf4acfade), TOBN(0x0e585897, 0x44cd0b71), + TOBN(0x544fd869, 0x0a51cd77), TOBN(0x60fc20ed, 0x0031016d), + TOBN(0x58b404ec, 0xa4276867), TOBN(0x46f6c3cc, 0x34f34993), + TOBN(0x477ca007, 0xc636e5bd), TOBN(0x8018f5e5, 0x7c458b47), + TOBN(0xa1202270, 0xe47b668f), TOBN(0xcef48ccd, 0xee14f203), + TOBN(0x23f98bae, 0x62ff9b4d), TOBN(0x55acc035, 0xc589eddd), + TOBN(0x3fe712af, 0x64db4444), TOBN(0x19e9d634, 0xbecdd480), + TOBN(0xe08bc047, 0xa930978a), TOBN(0x2dbf24ec, 0xa1280733), + TOBN(0x3c0ae38c, 0x2cd706b2), TOBN(0x5b012a5b, 0x359017b9), + TOBN(0x3943c38c, 0x72e0f5ae), TOBN(0x786167ea, 0x57176fa3), + TOBN(0xe5f9897d, 0x594881dc), TOBN(0x6b5efad8, 0xcfb820c1), + TOBN(0xb2179093, 0xd55018de), TOBN(0x39ad7d32, 0x0bac56ce), + TOBN(0xb55122e0, 0x2cfc0e81), TOBN(0x117c4661, 0xf6d89daa), + TOBN(0x362d01e1, 0xcb64fa09), TOBN(0x6a309b4e, 0x3e9c4ddd), + TOBN(0xfa979fb7, 0xabea49b1), TOBN(0xb4b1d27d, 0x10e2c6c5), + TOBN(0xbd61c2c4, 0x23afde7a), TOBN(0xeb6614f8, 0x9786d358), + TOBN(0x4a5d816b, 0x7f6f7459), TOBN(0xe431a44f, 0x09360e7b), + TOBN(0x8c27a032, 0xc309914c), TOBN(0xcea5d68a, 0xcaede3d8), + TOBN(0x3668f665, 0x3a0a3f95), TOBN(0x89369416, 0x7ceba27b), + TOBN(0x89981fad, 0xe4728fe9), TOBN(0x7102c8a0, 0x8a093562), + TOBN(0xbb80310e, 0x235d21c8), TOBN(0x505e55d1, 0xbefb7f7b), + TOBN(0xa0a90811, 0x12958a67), TOBN(0xd67e106a, 0x4d851fef), + TOBN(0xb84011a9, 0x431dd80e), TOBN(0xeb7c7cca, 0x73306cd9), + TOBN(0x20fadd29, 0xd1b3b730), TOBN(0x83858b5b, 0xfe37b3d3), + TOBN(0xbf4cd193, 0xb6251d5c), TOBN(0x1cca1fd3, 0x1352d952), + TOBN(0xc66157a4, 0x90fbc051), TOBN(0x7990a638, 0x89b98636),} + , + {TOBN(0xe5aa692a, 0x87dec0e1), TOBN(0x010ded8d, 0xf7b39d00), + TOBN(0x7b1b80c8, 0x54cfa0b5), TOBN(0x66beb876, 0xa0f8ea28), + TOBN(0x50d7f531, 0x3476cd0e), TOBN(0xa63d0e65, 0xb08d3949), + TOBN(0x1a09eea9, 0x53479fc6), TOBN(0x82ae9891, 0xf499e742), + TOBN(0xab58b910, 0x5ca7d866), TOBN(0x582967e2, 0x3adb3b34), + TOBN(0x89ae4447, 0xcceac0bc), TOBN(0x919c667c, 0x7bf56af5), + TOBN(0x9aec17b1, 0x60f5dcd7), TOBN(0xec697b9f, 0xddcaadbc), + TOBN(0x0b98f341, 0x463467f5), TOBN(0xb187f1f7, 0xa967132f), + TOBN(0x90fe7a1d, 0x214aeb18), TOBN(0x1506af3c, 0x741432f7), + TOBN(0xbb5565f9, 0xe591a0c4), TOBN(0x10d41a77, 0xb44f1bc3), + TOBN(0xa09d65e4, 0xa84bde96), TOBN(0x42f060d8, 0xf20a6a1c), + TOBN(0x652a3bfd, 0xf27f9ce7), TOBN(0xb6bdb65c, 0x3b3d739f), + TOBN(0xeb5ddcb6, 0xec7fae9f), TOBN(0x995f2714, 0xefb66e5a), + TOBN(0xdee95d8e, 0x69445d52), TOBN(0x1b6c2d46, 0x09e27620), + TOBN(0x32621c31, 0x8129d716), TOBN(0xb03909f1, 0x0958c1aa), + TOBN(0x8c468ef9, 0x1af4af63), TOBN(0x162c429f, 0xfba5cdf6), + TOBN(0x2f682343, 0x753b9371), TOBN(0x29cab45a, 0x5f1f9cd7), + TOBN(0x571623ab, 0xb245db96), TOBN(0xc507db09, 0x3fd79999), + TOBN(0x4e2ef652, 0xaf036c32), TOBN(0x86f0cc78, 0x05018e5c), + TOBN(0xc10a73d4, 0xab8be350), TOBN(0x6519b397, 0x7e826327), + TOBN(0xe8cb5eef, 0x9c053df7), TOBN(0x8de25b37, 0xb300ea6f), + TOBN(0xdb03fa92, 0xc849cffb), TOBN(0x242e43a7, 0xe84169bb), + TOBN(0xe4fa51f4, 0xdd6f958e), TOBN(0x6925a77f, 0xf4445a8d), + TOBN(0xe6e72a50, 0xe90d8949), TOBN(0xc66648e3, 0x2b1f6390), + TOBN(0xb2ab1957, 0x173e460c), TOBN(0x1bbbce75, 0x30704590), + TOBN(0xc0a90dbd, 0xdb1c7162), TOBN(0x505e399e, 0x15cdd65d), + TOBN(0x68434dcb, 0x57797ab7), TOBN(0x60ad35ba, 0x6a2ca8e8), + TOBN(0x4bfdb1e0, 0xde3336c1), TOBN(0xbbef99eb, 0xd8b39015), + TOBN(0x6c3b96f3, 0x1711ebec), TOBN(0x2da40f1f, 0xce98fdc4), + TOBN(0xb99774d3, 0x57b4411f), TOBN(0x87c8bdf4, 0x15b65bb6), + TOBN(0xda3a89e3, 0xc2eef12d), TOBN(0xde95bb9b, 0x3c7471f3), + TOBN(0x600f225b, 0xd812c594), TOBN(0x54907c5d, 0x2b75a56b), + TOBN(0xa93cc5f0, 0x8db60e35), TOBN(0x743e3cd6, 0xfa833319), + TOBN(0x7dad5c41, 0xf81683c9), TOBN(0x70c1e7d9, 0x9c34107e), + TOBN(0x0edc4a39, 0xa6be0907), TOBN(0x36d47035, 0x86d0b7d3), + TOBN(0x8c76da03, 0x272bfa60), TOBN(0x0b4a07ea, 0x0f08a414), + TOBN(0x699e4d29, 0x45c1dd53), TOBN(0xcadc5898, 0x231debb5), + TOBN(0xdf49fcc7, 0xa77f00e0), TOBN(0x93057bbf, 0xa73e5a0e), + TOBN(0x2f8b7ecd, 0x027a4cd1), TOBN(0x114734b3, 0xc614011a), + TOBN(0xe7a01db7, 0x67677c68), TOBN(0x89d9be5e, 0x7e273f4f), + TOBN(0xd225cb2e, 0x089808ef), TOBN(0xf1f7a27d, 0xd59e4107), + TOBN(0x53afc761, 0x8211b9c9), TOBN(0x0361bc67, 0xe6819159), + TOBN(0x2a865d0b, 0x7f071426), TOBN(0x6a3c1810, 0xe7072567), + TOBN(0x3e3bca1e, 0x0d6bcabd), TOBN(0xa1b02bc1, 0x408591bc), + TOBN(0xe0deee59, 0x31fba239), TOBN(0xf47424d3, 0x98bd91d1), + TOBN(0x0f8886f4, 0x071a3c1d), TOBN(0x3f7d41e8, 0xa819233b), + TOBN(0x708623c2, 0xcf6eb998), TOBN(0x86bb49af, 0x609a287f), + TOBN(0x942bb249, 0x63c90762), TOBN(0x0ef6eea5, 0x55a9654b), + TOBN(0x5f6d2d72, 0x36f5defe), TOBN(0xfa9922dc, 0x56f99176), + TOBN(0x6c8c5ece, 0xf78ce0c7), TOBN(0x7b44589d, 0xbe09b55e), + TOBN(0xe11b3bca, 0x9ea83770), TOBN(0xd7fa2c7f, 0x2ab71547), + TOBN(0x2a3dd6fa, 0x2a1ddcc0), TOBN(0x09acb430, 0x5a7b7707), + TOBN(0x4add4a2e, 0x649d4e57), TOBN(0xcd53a2b0, 0x1917526e), + TOBN(0xc5262330, 0x20b44ac4), TOBN(0x4028746a, 0xbaa2c31d), + TOBN(0x51318390, 0x64291d4c), TOBN(0xbf48f151, 0xee5ad909), + TOBN(0xcce57f59, 0x7b185681), TOBN(0x7c3ac1b0, 0x4854d442), + TOBN(0x65587dc3, 0xc093c171), TOBN(0xae7acb24, 0x24f42b65), + TOBN(0x5a338adb, 0x955996cb), TOBN(0xc8e65675, 0x6051f91b), + TOBN(0x66711fba, 0x28b8d0b1), TOBN(0x15d74137, 0xb6c10a90), + TOBN(0x70cdd7eb, 0x3a232a80), TOBN(0xc9e2f07f, 0x6191ed24), + TOBN(0xa80d1db6, 0xf79588c0), TOBN(0xfa52fc69, 0xb55768cc), + TOBN(0x0b4df1ae, 0x7f54438a), TOBN(0x0cadd1a7, 0xf9b46a4f), + TOBN(0xb40ea6b3, 0x1803dd6f), TOBN(0x488e4fa5, 0x55eaae35), + TOBN(0x9f047d55, 0x382e4e16), TOBN(0xc9b5b7e0, 0x2f6e0c98), + TOBN(0x6b1bd2d3, 0x95762649), TOBN(0xa9604ee7, 0xc7aea3f6), + TOBN(0x3646ff27, 0x6dc6f896), TOBN(0x9bf0e7f5, 0x2860bad1), + TOBN(0x2d92c821, 0x7cb44b92), TOBN(0xa2f5ce63, 0xaea9c182), + TOBN(0xd0a2afb1, 0x9154a5fd), TOBN(0x482e474c, 0x95801da6), + TOBN(0xc19972d0, 0xb611c24b), TOBN(0x1d468e65, 0x60a8f351), + TOBN(0xeb758069, 0x7bcf6421), TOBN(0xec9dd0ee, 0x88fbc491), + TOBN(0x5b59d2bf, 0x956c2e32), TOBN(0x73dc6864, 0xdcddf94e), + TOBN(0xfd5e2321, 0xbcee7665), TOBN(0xa7b4f8ef, 0x5e9a06c4), + TOBN(0xfba918dd, 0x7280f855), TOBN(0xbbaac260, 0x8baec688), + TOBN(0xa3b3f00f, 0x33400f42), TOBN(0x3d2dba29, 0x66f2e6e4), + TOBN(0xb6f71a94, 0x98509375), TOBN(0x8f33031f, 0xcea423cc), + TOBN(0x009b8dd0, 0x4807e6fb), TOBN(0x5163cfe5, 0x5cdb954c), + TOBN(0x03cc8f17, 0xcf41c6e8), TOBN(0xf1f03c2a, 0x037b925c), + TOBN(0xc39c19cc, 0x66d2427c), TOBN(0x823d24ba, 0x7b6c18e4), + TOBN(0x32ef9013, 0x901f0b4f), TOBN(0x684360f1, 0xf8941c2e), + TOBN(0x0ebaff52, 0x2c28092e), TOBN(0x7891e4e3, 0x256c932f), + TOBN(0x51264319, 0xac445e3d), TOBN(0x553432e7, 0x8ea74381), + TOBN(0xe6eeaa69, 0x67e9c50a), TOBN(0x27ced284, 0x62e628c7), + TOBN(0x3f96d375, 0x7a4afa57), TOBN(0xde0a14c3, 0xe484c150), + TOBN(0x364a24eb, 0x38bd9923), TOBN(0x1df18da0, 0xe5177422), + TOBN(0x174e8f82, 0xd8d38a9b), TOBN(0x2e97c600, 0xe7de1391), + TOBN(0xc5709850, 0xa1c175dd), TOBN(0x969041a0, 0x32ae5035), + TOBN(0xcbfd533b, 0x76a2086b), TOBN(0xd6bba71b, 0xd7c2e8fe), + TOBN(0xb2d58ee6, 0x099dfb67), TOBN(0x3a8b342d, 0x064a85d9), + TOBN(0x3bc07649, 0x522f9be3), TOBN(0x690c075b, 0xdf1f49a8), + TOBN(0x80e1aee8, 0x3854ec42), TOBN(0x2a7dbf44, 0x17689dc7), + TOBN(0xc004fc0e, 0x3faf4078), TOBN(0xb2f02e9e, 0xdf11862c), + TOBN(0xf10a5e0f, 0xa0a1b7b3), TOBN(0x30aca623, 0x8936ec80), + TOBN(0xf83cbf05, 0x02f40d9a), TOBN(0x4681c468, 0x2c318a4d), + TOBN(0x98575618, 0x0e9c2674), TOBN(0xbe79d046, 0x1847092e), + TOBN(0xaf1e480a, 0x78bd01e0), TOBN(0x6dd359e4, 0x72a51db9), + TOBN(0x62ce3821, 0xe3afbab6), TOBN(0xc5cee5b6, 0x17733199), + TOBN(0xe08b30d4, 0x6ffd9fbb), TOBN(0x6e5bc699, 0x36c610b7), + TOBN(0xf343cff2, 0x9ce262cf), TOBN(0xca2e4e35, 0x68b914c1), + TOBN(0x011d64c0, 0x16de36c5), TOBN(0xe0b10fdd, 0x42e2b829), + TOBN(0x78942981, 0x6685aaf8), TOBN(0xe7511708, 0x230ede97), + TOBN(0x671ed8fc, 0x3b922bf8), TOBN(0xe4d8c0a0, 0x4c29b133), + TOBN(0x87eb1239, 0x3b6e99c4), TOBN(0xaff3974c, 0x8793beba), + TOBN(0x03749405, 0x2c18df9b), TOBN(0xc5c3a293, 0x91007139), + TOBN(0x6a77234f, 0xe37a0b95), TOBN(0x02c29a21, 0xb661c96b), + TOBN(0xc3aaf1d6, 0x141ecf61), TOBN(0x9195509e, 0x3bb22f53), + TOBN(0x29597404, 0x22d51357), TOBN(0x1b083822, 0x537bed60), + TOBN(0xcd7d6e35, 0xe07289f0), TOBN(0x1f94c48c, 0x6dd86eff), + TOBN(0xc8bb1f82, 0xeb0f9cfa), TOBN(0x9ee0b7e6, 0x1b2eb97d), + TOBN(0x5a52fe2e, 0x34d74e31), TOBN(0xa352c310, 0x3bf79ab6), + TOBN(0x97ff6c5a, 0xabfeeb8f), TOBN(0xbfbe8fef, 0xf5c97305), + TOBN(0xd6081ce6, 0xa7904608), TOBN(0x1f812f3a, 0xc4fca249), + TOBN(0x9b24bc9a, 0xb9e5e200), TOBN(0x91022c67, 0x38012ee8), + TOBN(0xe83d9c5d, 0x30a713a1), TOBN(0x4876e3f0, 0x84ef0f93), + TOBN(0xc9777029, 0xc1fbf928), TOBN(0xef7a6bb3, 0xbce7d2a4), + TOBN(0xb8067228, 0xdfa2a659), TOBN(0xd5cd3398, 0xd877a48f), + TOBN(0xbea4fd8f, 0x025d0f3f), TOBN(0xd67d2e35, 0x2eae7c2b), + TOBN(0x184de7d7, 0xcc5f4394), TOBN(0xb5551b5c, 0x4536e142), + TOBN(0x2e89b212, 0xd34aa60a), TOBN(0x14a96fea, 0xf50051d5), + TOBN(0x4e21ef74, 0x0d12bb0b), TOBN(0xc522f020, 0x60b9677e), + TOBN(0x8b12e467, 0x2df7731d), TOBN(0x39f80382, 0x7b326d31), + TOBN(0xdfb8630c, 0x39024a94), TOBN(0xaacb96a8, 0x97319452), + TOBN(0xd68a3961, 0xeda3867c), TOBN(0x0c58e2b0, 0x77c4ffca), + TOBN(0x3d545d63, 0x4da919fa), TOBN(0xef79b69a, 0xf15e2289), + TOBN(0x54bc3d3d, 0x808bab10), TOBN(0xc8ab3007, 0x45f82c37), + TOBN(0xc12738b6, 0x7c4a658a), TOBN(0xb3c47639, 0x40e72182), + TOBN(0x3b77be46, 0x8798e44f), TOBN(0xdc047df2, 0x17a7f85f), + TOBN(0x2439d4c5, 0x5e59d92d), TOBN(0xcedca475, 0xe8e64d8d), + TOBN(0xa724cd0d, 0x87ca9b16), TOBN(0x35e4fd59, 0xa5540dfe), + TOBN(0xf8c1ff18, 0xe4bcf6b1), TOBN(0x856d6285, 0x295018fa), + TOBN(0x433f665c, 0x3263c949), TOBN(0xa6a76dd6, 0xa1f21409), + TOBN(0x17d32334, 0xcc7b4f79), TOBN(0xa1d03122, 0x06720e4a), + TOBN(0xadb6661d, 0x81d9bed5), TOBN(0xf0d6fb02, 0x11db15d1), + TOBN(0x7fd11ad5, 0x1fb747d2), TOBN(0xab50f959, 0x3033762b), + TOBN(0x2a7e711b, 0xfbefaf5a), TOBN(0xc7393278, 0x3fef2bbf), + TOBN(0xe29fa244, 0x0df6f9be), TOBN(0x9092757b, 0x71efd215), + TOBN(0xee60e311, 0x4f3d6fd9), TOBN(0x338542d4, 0x0acfb78b), + TOBN(0x44a23f08, 0x38961a0f), TOBN(0x1426eade, 0x986987ca), + TOBN(0x36e6ee2e, 0x4a863cc6), TOBN(0x48059420, 0x628b8b79), + TOBN(0x30303ad8, 0x7396e1de), TOBN(0x5c8bdc48, 0x38c5aad1), + TOBN(0x3e40e11f, 0x5c8f5066), TOBN(0xabd6e768, 0x8d246bbd), + TOBN(0x68aa40bb, 0x23330a01), TOBN(0xd23f5ee4, 0xc34eafa0), + TOBN(0x3bbee315, 0x5de02c21), TOBN(0x18dd4397, 0xd1d8dd06), + TOBN(0x3ba1939a, 0x122d7b44), TOBN(0xe6d3b40a, 0xa33870d6), + TOBN(0x8e620f70, 0x1c4fe3f8), TOBN(0xf6bba1a5, 0xd3a50cbf), + TOBN(0x4a78bde5, 0xcfc0aee0), TOBN(0x847edc46, 0xc08c50bd), + TOBN(0xbaa2439c, 0xad63c9b2), TOBN(0xceb4a728, 0x10fc2acb), + TOBN(0xa419e40e, 0x26da033d), TOBN(0x6cc3889d, 0x03e02683), + TOBN(0x1cd28559, 0xfdccf725), TOBN(0x0fd7e0f1, 0x8d13d208), + TOBN(0x01b9733b, 0x1f0df9d4), TOBN(0x8cc2c5f3, 0xa2b5e4f3), + TOBN(0x43053bfa, 0x3a304fd4), TOBN(0x8e87665c, 0x0a9f1aa7), + TOBN(0x087f29ec, 0xd73dc965), TOBN(0x15ace455, 0x3e9023db), + TOBN(0x2370e309, 0x2bce28b4), TOBN(0xf9723442, 0xb6b1e84a), + TOBN(0xbeee662e, 0xb72d9f26), TOBN(0xb19396de, 0xf0e47109), + TOBN(0x85b1fa73, 0xe13289d0), TOBN(0x436cf77e, 0x54e58e32), + TOBN(0x0ec833b3, 0xe990ef77), TOBN(0x7373e3ed, 0x1b11fc25), + TOBN(0xbe0eda87, 0x0fc332ce), TOBN(0xced04970, 0x8d7ea856), + TOBN(0xf85ff785, 0x7e977ca0), TOBN(0xb66ee8da, 0xdfdd5d2b), + TOBN(0xf5e37950, 0x905af461), TOBN(0x587b9090, 0x966d487c), + TOBN(0x6a198a1b, 0x32ba0127), TOBN(0xa7720e07, 0x141615ac), + TOBN(0xa23f3499, 0x996ef2f2), TOBN(0xef5f64b4, 0x470bcb3d), + TOBN(0xa526a962, 0x92b8c559), TOBN(0x0c14aac0, 0x69740a0f), + TOBN(0x0d41a9e3, 0xa6bdc0a5), TOBN(0x97d52106, 0x9c48aef4), + TOBN(0xcf16bd30, 0x3e7c253b), TOBN(0xcc834b1a, 0x47fdedc1), + TOBN(0x7362c6e5, 0x373aab2e), TOBN(0x264ed85e, 0xc5f590ff), + TOBN(0x7a46d9c0, 0x66d41870), TOBN(0xa50c20b1, 0x4787ba09), + TOBN(0x185e7e51, 0xe3d44635), TOBN(0xb3b3e080, 0x31e2d8dc), + TOBN(0xbed1e558, 0xa179e9d9), TOBN(0x2daa3f79, 0x74a76781), + TOBN(0x4372baf2, 0x3a40864f), TOBN(0x46900c54, 0x4fe75cb5), + TOBN(0xb95f171e, 0xf76765d0), TOBN(0x4ad726d2, 0x95c87502), + TOBN(0x2ec769da, 0x4d7c99bd), TOBN(0x5e2ddd19, 0xc36cdfa8), + TOBN(0xc22117fc, 0xa93e6dea), TOBN(0xe8a2583b, 0x93771123), + TOBN(0xbe2f6089, 0xfa08a3a2), TOBN(0x4809d5ed, 0x8f0e1112), + TOBN(0x3b414aa3, 0xda7a095e), TOBN(0x9049acf1, 0x26f5aadd), + TOBN(0x78d46a4d, 0x6be8b84a), TOBN(0xd66b1963, 0xb732b9b3), + TOBN(0x5c2ac2a0, 0xde6e9555), TOBN(0xcf52d098, 0xb5bd8770), + TOBN(0x15a15fa6, 0x0fd28921), TOBN(0x56ccb81e, 0x8b27536d), + TOBN(0x0f0d8ab8, 0x9f4ccbb8), TOBN(0xed5f44d2, 0xdb221729), + TOBN(0x43141988, 0x00bed10c), TOBN(0xc94348a4, 0x1d735b8b), + TOBN(0x79f3e9c4, 0x29ef8479), TOBN(0x4c13a4e3, 0x614c693f), + TOBN(0x32c9af56, 0x8e143a14), TOBN(0xbc517799, 0xe29ac5c4), + TOBN(0x05e17992, 0x2774856f), TOBN(0x6e52fb05, 0x6c1bf55f), + TOBN(0xaeda4225, 0xe4f19e16), TOBN(0x70f4728a, 0xaf5ccb26), + TOBN(0x5d2118d1, 0xb2947f22), TOBN(0xc827ea16, 0x281d6fb9), + TOBN(0x8412328d, 0x8cf0eabd), TOBN(0x45ee9fb2, 0x03ef9dcf), + TOBN(0x8e700421, 0xbb937d63), TOBN(0xdf8ff2d5, 0xcc4b37a6), + TOBN(0xa4c0d5b2, 0x5ced7b68), TOBN(0x6537c1ef, 0xc7308f59), + TOBN(0x25ce6a26, 0x3b37f8e8), TOBN(0x170e9a9b, 0xdeebc6ce), + TOBN(0xdd037952, 0x8728d72c), TOBN(0x445b0e55, 0x850154bc), + TOBN(0x4b7d0e06, 0x83a7337b), TOBN(0x1e3416d4, 0xffecf249), + TOBN(0x24840eff, 0x66a2b71f), TOBN(0xd0d9a50a, 0xb37cc26d), + TOBN(0xe2198150, 0x6fe28ef7), TOBN(0x3cc5ef16, 0x23324c7f), + TOBN(0x220f3455, 0x769b5263), TOBN(0xe2ade2f1, 0xa10bf475), + TOBN(0x28cd20fa, 0x458d3671), TOBN(0x1549722c, 0x2dc4847b), + TOBN(0x6dd01e55, 0x591941e3), TOBN(0x0e6fbcea, 0x27128ccb), + TOBN(0xae1a1e6b, 0x3bef0262), TOBN(0xfa8c472c, 0x8f54e103), + TOBN(0x7539c0a8, 0x72c052ec), TOBN(0xd7b27369, 0x5a3490e9), + TOBN(0x143fe1f1, 0x71684349), TOBN(0x36b4722e, 0x32e19b97), + TOBN(0xdc059227, 0x90980aff), TOBN(0x175c9c88, 0x9e13d674), + TOBN(0xa7de5b22, 0x6e6bfdb1), TOBN(0x5ea5b7b2, 0xbedb4b46), + TOBN(0xd5570191, 0xd34a6e44), TOBN(0xfcf60d2e, 0xa24ff7e6), + TOBN(0x614a392d, 0x677819e1), TOBN(0x7be74c7e, 0xaa5a29e8), + TOBN(0xab50fece, 0x63c85f3f), TOBN(0xaca2e2a9, 0x46cab337), + TOBN(0x7f700388, 0x122a6fe3), TOBN(0xdb69f703, 0x882a04a8), + TOBN(0x9a77935d, 0xcf7aed57), TOBN(0xdf16207c, 0x8d91c86f), + TOBN(0x2fca49ab, 0x63ed9998), TOBN(0xa3125c44, 0xa77ddf96), + TOBN(0x05dd8a86, 0x24344072), TOBN(0xa023dda2, 0xfec3fb56), + TOBN(0x421b41fc, 0x0c743032), TOBN(0x4f2120c1, 0x5e438639), + TOBN(0xfb7cae51, 0xc83c1b07), TOBN(0xb2370caa, 0xcac2171a), + TOBN(0x2eb2d962, 0x6cc820fb), TOBN(0x59feee5c, 0xb85a44bf), + TOBN(0x94620fca, 0x5b6598f0), TOBN(0x6b922cae, 0x7e314051), + TOBN(0xff8745ad, 0x106bed4e), TOBN(0x546e71f5, 0xdfa1e9ab), + TOBN(0x935c1e48, 0x1ec29487), TOBN(0x9509216c, 0x4d936530), + TOBN(0xc7ca3067, 0x85c9a2db), TOBN(0xd6ae5152, 0x6be8606f), + TOBN(0x09dbcae6, 0xe14c651d), TOBN(0xc9536e23, 0x9bc32f96), + TOBN(0xa90535a9, 0x34521b03), TOBN(0xf39c526c, 0x878756ff), + TOBN(0x383172ec, 0x8aedf03c), TOBN(0x20a8075e, 0xefe0c034), + TOBN(0xf22f9c62, 0x64026422), TOBN(0x8dd10780, 0x24b9d076), + TOBN(0x944c742a, 0x3bef2950), TOBN(0x55b9502e, 0x88a2b00b), + TOBN(0xa59e14b4, 0x86a09817), TOBN(0xa39dd3ac, 0x47bb4071), + TOBN(0x55137f66, 0x3be0592f), TOBN(0x07fcafd4, 0xc9e63f5b), + TOBN(0x963652ee, 0x346eb226), TOBN(0x7dfab085, 0xec2facb7), + TOBN(0x273bf2b8, 0x691add26), TOBN(0x30d74540, 0xf2b46c44), + TOBN(0x05e8e73e, 0xf2c2d065), TOBN(0xff9b8a00, 0xd42eeac9), + TOBN(0x2fcbd205, 0x97209d22), TOBN(0xeb740ffa, 0xde14ea2c), + TOBN(0xc71ff913, 0xa8aef518), TOBN(0x7bfc74bb, 0xfff4cfa2), + TOBN(0x1716680c, 0xb6b36048), TOBN(0x121b2cce, 0x9ef79af1), + TOBN(0xbff3c836, 0xa01eb3d3), TOBN(0x50eb1c6a, 0x5f79077b), + TOBN(0xa48c32d6, 0xa004bbcf), TOBN(0x47a59316, 0x7d64f61d), + TOBN(0x6068147f, 0x93102016), TOBN(0x12c5f654, 0x94d12576), + TOBN(0xefb071a7, 0xc9bc6b91), TOBN(0x7c2da0c5, 0x6e23ea95), + TOBN(0xf4fd45b6, 0xd4a1dd5d), TOBN(0x3e7ad9b6, 0x9122b13c), + TOBN(0x342ca118, 0xe6f57a48), TOBN(0x1c2e94a7, 0x06f8288f), + TOBN(0x99e68f07, 0x5a97d231), TOBN(0x7c80de97, 0x4d838758), + TOBN(0xbce0f5d0, 0x05872727), TOBN(0xbe5d95c2, 0x19c4d016), + TOBN(0x921d5cb1, 0x9c2492ee), TOBN(0x42192dc1, 0x404d6fb3), + TOBN(0x4c84dcd1, 0x32f988d3), TOBN(0xde26d61f, 0xa17b8e85), + TOBN(0xc466dcb6, 0x137c7408), TOBN(0x9a38d7b6, 0x36a266da), + TOBN(0x7ef5cb06, 0x83bebf1b), TOBN(0xe5cdcbbf, 0x0fd014e3), + TOBN(0x30aa376d, 0xf65965a0), TOBN(0x60fe88c2, 0xebb3e95e), + TOBN(0x33fd0b61, 0x66ee6f20), TOBN(0x8827dcdb, 0x3f41f0a0), + TOBN(0xbf8a9d24, 0x0c56c690), TOBN(0x40265dad, 0xddb7641d), + TOBN(0x522b05bf, 0x3a6b662b), TOBN(0x466d1dfe, 0xb1478c9b), + TOBN(0xaa616962, 0x1484469b), TOBN(0x0db60549, 0x02df8f9f), + TOBN(0xc37bca02, 0x3cb8bf51), TOBN(0x5effe346, 0x21371ce8), + TOBN(0xe8f65264, 0xff112c32), TOBN(0x8a9c736d, 0x7b971fb2), + TOBN(0xa4f19470, 0x7b75080d), TOBN(0xfc3f2c5a, 0x8839c59b), + TOBN(0x1d6c777e, 0x5aeb49c2), TOBN(0xf3db034d, 0xda1addfe), + TOBN(0xd76fee5a, 0x5535affc), TOBN(0x0853ac70, 0xb92251fd), + TOBN(0x37e3d594, 0x8b2a29d5), TOBN(0x28f1f457, 0x4de00ddb), + TOBN(0x8083c1b5, 0xf42c328b), TOBN(0xd8ef1d8f, 0xe493c73b), + TOBN(0x96fb6260, 0x41dc61bd), TOBN(0xf74e8a9d, 0x27ee2f8a), + TOBN(0x7c605a80, 0x2c946a5d), TOBN(0xeed48d65, 0x3839ccfd), + TOBN(0x9894344f, 0x3a29467a), TOBN(0xde81e949, 0xc51eba6d), + TOBN(0xdaea066b, 0xa5e5c2f2), TOBN(0x3fc8a614, 0x08c8c7b3), + TOBN(0x7adff88f, 0x06d0de9f), TOBN(0xbbc11cf5, 0x3b75ce0a), + TOBN(0x9fbb7acc, 0xfbbc87d5), TOBN(0xa1458e26, 0x7badfde2)} + , + {TOBN(0x1cb43668, 0xe039c256), TOBN(0x5f26fb8b, 0x7c17fd5d), + TOBN(0xeee426af, 0x79aa062b), TOBN(0x072002d0, 0xd78fbf04), + TOBN(0x4c9ca237, 0xe84fb7e3), TOBN(0xb401d8a1, 0x0c82133d), + TOBN(0xaaa52592, 0x6d7e4181), TOBN(0xe9430833, 0x73dbb152), + TOBN(0xf92dda31, 0xbe24319a), TOBN(0x03f7d28b, 0xe095a8e7), + TOBN(0xa52fe840, 0x98782185), TOBN(0x276ddafe, 0x29c24dbc), + TOBN(0x80cd5496, 0x1d7a64eb), TOBN(0xe4360889, 0x7f1dbe42), + TOBN(0x2f81a877, 0x8438d2d5), TOBN(0x7e4d52a8, 0x85169036), + TOBN(0x19e3d5b1, 0x1d59715d), TOBN(0xc7eaa762, 0xd788983e), + TOBN(0xe5a730b0, 0xabf1f248), TOBN(0xfbab8084, 0xfae3fd83), + TOBN(0x65e50d21, 0x53765b2f), TOBN(0xbdd4e083, 0xfa127f3d), + TOBN(0x9cf3c074, 0x397b1b10), TOBN(0x59f8090c, 0xb1b59fd3), + TOBN(0x7b15fd9d, 0x615faa8f), TOBN(0x8fa1eb40, 0x968554ed), + TOBN(0x7bb4447e, 0x7aa44882), TOBN(0x2bb2d0d1, 0x029fff32), + TOBN(0x075e2a64, 0x6caa6d2f), TOBN(0x8eb879de, 0x22e7351b), + TOBN(0xbcd5624e, 0x9a506c62), TOBN(0x218eaef0, 0xa87e24dc), + TOBN(0x37e56847, 0x44ddfa35), TOBN(0x9ccfc5c5, 0xdab3f747), + TOBN(0x9ac1df3f, 0x1ee96cf4), TOBN(0x0c0571a1, 0x3b480b8f), + TOBN(0x2fbeb3d5, 0x4b3a7b3c), TOBN(0x35c03669, 0x5dcdbb99), + TOBN(0x52a0f5dc, 0xb2415b3a), TOBN(0xd57759b4, 0x4413ed9a), + TOBN(0x1fe647d8, 0x3d30a2c5), TOBN(0x0857f77e, 0xf78a81dc), + TOBN(0x11d5a334, 0x131a4a9b), TOBN(0xc0a94af9, 0x29d393f5), + TOBN(0xbc3a5c0b, 0xdaa6ec1a), TOBN(0xba9fe493, 0x88d2d7ed), + TOBN(0xbb4335b4, 0xbb614797), TOBN(0x991c4d68, 0x72f83533), + TOBN(0x53258c28, 0xd2f01cb3), TOBN(0x93d6eaa3, 0xd75db0b1), + TOBN(0x419a2b0d, 0xe87d0db4), TOBN(0xa1e48f03, 0xd8fe8493), + TOBN(0xf747faf6, 0xc508b23a), TOBN(0xf137571a, 0x35d53549), + TOBN(0x9f5e58e2, 0xfcf9b838), TOBN(0xc7186cee, 0xa7fd3cf5), + TOBN(0x77b868ce, 0xe978a1d3), TOBN(0xe3a68b33, 0x7ab92d04), + TOBN(0x51029794, 0x87a5b862), TOBN(0x5f0606c3, 0x3a61d41d), + TOBN(0x2814be27, 0x6f9326f1), TOBN(0x2f521c14, 0xc6fe3c2e), + TOBN(0x17464d7d, 0xacdf7351), TOBN(0x10f5f9d3, 0x777f7e44), + TOBN(0xce8e616b, 0x269fb37d), TOBN(0xaaf73804, 0x7de62de5), + TOBN(0xaba11175, 0x4fdd4153), TOBN(0x515759ba, 0x3770b49b), + TOBN(0x8b09ebf8, 0xaa423a61), TOBN(0x592245a1, 0xcd41fb92), + TOBN(0x1cba8ec1, 0x9b4c8936), TOBN(0xa87e91e3, 0xaf36710e), + TOBN(0x1fd84ce4, 0x3d34a2e3), TOBN(0xee3759ce, 0xb43b5d61), + TOBN(0x895bc78c, 0x619186c7), TOBN(0xf19c3809, 0xcbb9725a), + TOBN(0xc0be21aa, 0xde744b1f), TOBN(0xa7d222b0, 0x60f8056b), + TOBN(0x74be6157, 0xb23efe11), TOBN(0x6fab2b4f, 0x0cd68253), + TOBN(0xad33ea5f, 0x4bf1d725), TOBN(0x9c1d8ee2, 0x4f6c950f), + TOBN(0x544ee78a, 0xa377af06), TOBN(0x54f489bb, 0x94a113e1), + TOBN(0x8f11d634, 0x992fb7e8), TOBN(0x0169a7aa, 0xa2a44347), + TOBN(0x1d49d4af, 0x95020e00), TOBN(0x95945722, 0xe08e120b), + TOBN(0xb6e33878, 0xa4d32282), TOBN(0xe36e029d, 0x48020ae7), + TOBN(0xe05847fb, 0x37a9b750), TOBN(0xf876812c, 0xb29e3819), + TOBN(0x84ad138e, 0xd23a17f0), TOBN(0x6d7b4480, 0xf0b3950e), + TOBN(0xdfa8aef4, 0x2fd67ae0), TOBN(0x8d3eea24, 0x52333af6), + TOBN(0x0d052075, 0xb15d5acc), TOBN(0xc6d9c79f, 0xbd815bc4), + TOBN(0x8dcafd88, 0xdfa36cf2), TOBN(0x908ccbe2, 0x38aa9070), + TOBN(0x638722c4, 0xba35afce), TOBN(0x5a3da8b0, 0xfd6abf0b), + TOBN(0x2dce252c, 0xc9c335c1), TOBN(0x84e7f0de, 0x65aa799b), + TOBN(0x2101a522, 0xb99a72cb), TOBN(0x06de6e67, 0x87618016), + TOBN(0x5ff8c7cd, 0xe6f3653e), TOBN(0x0a821ab5, 0xc7a6754a), + TOBN(0x7e3fa52b, 0x7cb0b5a2), TOBN(0xa7fb121c, 0xc9048790), + TOBN(0x1a725020, 0x06ce053a), TOBN(0xb490a31f, 0x04e929b0), + TOBN(0xe17be47d, 0x62dd61ad), TOBN(0x781a961c, 0x6be01371), + TOBN(0x1063bfd3, 0xdae3cbba), TOBN(0x35647406, 0x7f73c9ba), + TOBN(0xf50e957b, 0x2736a129), TOBN(0xa6313702, 0xed13f256), + TOBN(0x9436ee65, 0x3a19fcc5), TOBN(0xcf2bdb29, 0xe7a4c8b6), + TOBN(0xb06b1244, 0xc5f95cd8), TOBN(0xda8c8af0, 0xf4ab95f4), + TOBN(0x1bae59c2, 0xb9e5836d), TOBN(0x07d51e7e, 0x3acffffc), + TOBN(0x01e15e6a, 0xc2ccbcda), TOBN(0x3bc1923f, 0x8528c3e0), + TOBN(0x43324577, 0xa49fead4), TOBN(0x61a1b884, 0x2aa7a711), + TOBN(0xf9a86e08, 0x700230ef), TOBN(0x0af585a1, 0xbd19adf8), + TOBN(0x7645f361, 0xf55ad8f2), TOBN(0x6e676223, 0x46c3614c), + TOBN(0x23cb257c, 0x4e774d3f), TOBN(0x82a38513, 0xac102d1b), + TOBN(0x9bcddd88, 0x7b126aa5), TOBN(0xe716998b, 0xeefd3ee4), + TOBN(0x4239d571, 0xfb167583), TOBN(0xdd011c78, 0xd16c8f8a), + TOBN(0x271c2895, 0x69a27519), TOBN(0x9ce0a3b7, 0xd2d64b6a), + TOBN(0x8c977289, 0xd5ec6738), TOBN(0xa3b49f9a, 0x8840ef6b), + TOBN(0x808c14c9, 0x9a453419), TOBN(0x5c00295b, 0x0cf0a2d5), + TOBN(0x524414fb, 0x1d4bcc76), TOBN(0xb07691d2, 0x459a88f1), + TOBN(0x77f43263, 0xf70d110f), TOBN(0x64ada5e0, 0xb7abf9f3), + TOBN(0xafd0f94e, 0x5b544cf5), TOBN(0xb4a13a15, 0xfd2713fe), + TOBN(0xb99b7d6e, 0x250c74f4), TOBN(0x097f2f73, 0x20324e45), + TOBN(0x994b37d8, 0xaffa8208), TOBN(0xc3c31b0b, 0xdc29aafc), + TOBN(0x3da74651, 0x7a3a607f), TOBN(0xd8e1b8c1, 0xfe6955d6), + TOBN(0x716e1815, 0xc8418682), TOBN(0x541d487f, 0x7dc91d97), + TOBN(0x48a04669, 0xc6996982), TOBN(0xf39cab15, 0x83a6502e), + TOBN(0x025801a0, 0xe68db055), TOBN(0xf3569758, 0xba3338d5), + TOBN(0xb0c8c0aa, 0xee2afa84), TOBN(0x4f6985d3, 0xfb6562d1), + TOBN(0x351f1f15, 0x132ed17a), TOBN(0x510ed0b4, 0xc04365fe), + TOBN(0xa3f98138, 0xe5b1f066), TOBN(0xbc9d95d6, 0x32df03dc), + TOBN(0xa83ccf6e, 0x19abd09e), TOBN(0x0b4097c1, 0x4ff17edb), + TOBN(0x58a5c478, 0xd64a06ce), TOBN(0x2ddcc3fd, 0x544a58fd), + TOBN(0xd449503d, 0x9e8153b8), TOBN(0x3324fd02, 0x7774179b), + TOBN(0xaf5d47c8, 0xdbd9120c), TOBN(0xeb860162, 0x34fa94db), + TOBN(0x5817bdd1, 0x972f07f4), TOBN(0xe5579e2e, 0xd27bbceb), + TOBN(0x86847a1f, 0x5f11e5a6), TOBN(0xb39ed255, 0x7c3cf048), + TOBN(0xe1076417, 0xa2f62e55), TOBN(0x6b9ab38f, 0x1bcf82a2), + TOBN(0x4bb7c319, 0x7aeb29f9), TOBN(0xf6d17da3, 0x17227a46), + TOBN(0xab53ddbd, 0x0f968c00), TOBN(0xa03da7ec, 0x000c880b), + TOBN(0x7b239624, 0x6a9ad24d), TOBN(0x612c0401, 0x01ec60d0), + TOBN(0x70d10493, 0x109f5df1), TOBN(0xfbda4030, 0x80af7550), + TOBN(0x30b93f95, 0xc6b9a9b3), TOBN(0x0c74ec71, 0x007d9418), + TOBN(0x94175564, 0x6edb951f), TOBN(0x5f4a9d78, 0x7f22c282), + TOBN(0xb7870895, 0xb38d1196), TOBN(0xbc593df3, 0xa228ce7c), + TOBN(0xc78c5bd4, 0x6af3641a), TOBN(0x7802200b, 0x3d9b3dcc), + TOBN(0x0dc73f32, 0x8be33304), TOBN(0x847ed87d, 0x61ffb79a), + TOBN(0xf85c974e, 0x6d671192), TOBN(0x1e14100a, 0xde16f60f), + TOBN(0x45cb0d5a, 0x95c38797), TOBN(0x18923bba, 0x9b022da4), + TOBN(0xef2be899, 0xbbe7e86e), TOBN(0x4a1510ee, 0x216067bf), + TOBN(0xd98c8154, 0x84d5ce3e), TOBN(0x1af777f0, 0xf92a2b90), + TOBN(0x9fbcb400, 0x4ef65724), TOBN(0x3e04a4c9, 0x3c0ca6fe), + TOBN(0xfb3e2cb5, 0x55002994), TOBN(0x1f3a93c5, 0x5363ecab), + TOBN(0x1fe00efe, 0x3923555b), TOBN(0x744bedd9, 0x1e1751ea), + TOBN(0x3fb2db59, 0x6ab69357), TOBN(0x8dbd7365, 0xf5e6618b), + TOBN(0x99d53099, 0xdf1ea40e), TOBN(0xb3f24a0b, 0x57d61e64), + TOBN(0xd088a198, 0x596eb812), TOBN(0x22c8361b, 0x5762940b), + TOBN(0x66f01f97, 0xf9c0d95c), TOBN(0x88461172, 0x8e43cdae), + TOBN(0x11599a7f, 0xb72b15c3), TOBN(0x135a7536, 0x420d95cc), + TOBN(0x2dcdf0f7, 0x5f7ae2f6), TOBN(0x15fc6e1d, 0xd7fa6da2), + TOBN(0x81ca829a, 0xd1d441b6), TOBN(0x84c10cf8, 0x04a106b6), + TOBN(0xa9b26c95, 0xa73fbbd0), TOBN(0x7f24e0cb, 0x4d8f6ee8), + TOBN(0x48b45937, 0x1e25a043), TOBN(0xf8a74fca, 0x036f3dfe), + TOBN(0x1ed46585, 0xc9f84296), TOBN(0x7fbaa8fb, 0x3bc278b0), + TOBN(0xa8e96cd4, 0x6c4fcbd0), TOBN(0x940a1202, 0x73b60a5f), + TOBN(0x34aae120, 0x55a4aec8), TOBN(0x550e9a74, 0xdbd742f0), + TOBN(0x794456d7, 0x228c68ab), TOBN(0x492f8868, 0xa4e25ec6), + TOBN(0x682915ad, 0xb2d8f398), TOBN(0xf13b51cc, 0x5b84c953), + TOBN(0xcda90ab8, 0x5bb917d6), TOBN(0x4b615560, 0x4ea3dee1), + TOBN(0x578b4e85, 0x0a52c1c8), TOBN(0xeab1a695, 0x20b75fc4), + TOBN(0x60c14f3c, 0xaa0bb3c6), TOBN(0x220f448a, 0xb8216094), + TOBN(0x4fe7ee31, 0xb0e63d34), TOBN(0xf4600572, 0xa9e54fab), + TOBN(0xc0493334, 0xd5e7b5a4), TOBN(0x8589fb92, 0x06d54831), + TOBN(0xaa70f5cc, 0x6583553a), TOBN(0x0879094a, 0xe25649e5), + TOBN(0xcc904507, 0x10044652), TOBN(0xebb0696d, 0x02541c4f), + TOBN(0x5a171fde, 0xb9718710), TOBN(0x38f1bed8, 0xf374a9f5), + TOBN(0xc8c582e1, 0xba39bdc1), TOBN(0xfc457b0a, 0x908cc0ce), + TOBN(0x9a187fd4, 0x883841e2), TOBN(0x8ec25b39, 0x38725381), + TOBN(0x2553ed05, 0x96f84395), TOBN(0x095c7661, 0x6f6c6897), + TOBN(0x917ac85c, 0x4bdc5610), TOBN(0xb2885fe4, 0x179eb301), + TOBN(0x5fc65547, 0x8b78bdcc), TOBN(0x4a9fc893, 0xe59e4699), + TOBN(0xbb7ff0cd, 0x3ce299af), TOBN(0x195be9b3, 0xadf38b20), + TOBN(0x6a929c87, 0xd38ddb8f), TOBN(0x55fcc99c, 0xb21a51b9), + TOBN(0x2b695b4c, 0x721a4593), TOBN(0xed1e9a15, 0x768eaac2), + TOBN(0xfb63d71c, 0x7489f914), TOBN(0xf98ba31c, 0x78118910), + TOBN(0x80291373, 0x9b128eb4), TOBN(0x7801214e, 0xd448af4a), + TOBN(0xdbd2e22b, 0x55418dd3), TOBN(0xeffb3c0d, 0xd3998242), + TOBN(0xdfa6077c, 0xc7bf3827), TOBN(0xf2165bcb, 0x47f8238f), + TOBN(0xfe37cf68, 0x8564d554), TOBN(0xe5f825c4, 0x0a81fb98), + TOBN(0x43cc4f67, 0xffed4d6f), TOBN(0xbc609578, 0xb50a34b0), + TOBN(0x8aa8fcf9, 0x5041faf1), TOBN(0x5659f053, 0x651773b6), + TOBN(0xe87582c3, 0x6044d63b), TOBN(0xa6089409, 0x0cdb0ca0), + TOBN(0x8c993e0f, 0xbfb2bcf6), TOBN(0xfc64a719, 0x45985cfc), + TOBN(0x15c4da80, 0x83dbedba), TOBN(0x804ae112, 0x2be67df7), + TOBN(0xda4c9658, 0xa23defde), TOBN(0x12002ddd, 0x5156e0d3), + TOBN(0xe68eae89, 0x5dd21b96), TOBN(0x8b99f28b, 0xcf44624d), + TOBN(0x0ae00808, 0x1ec8897a), TOBN(0xdd0a9303, 0x6712f76e), + TOBN(0x96237522, 0x4e233de4), TOBN(0x192445b1, 0x2b36a8a5), + TOBN(0xabf9ff74, 0x023993d9), TOBN(0x21f37bf4, 0x2aad4a8f), + TOBN(0x340a4349, 0xf8bd2bbd), TOBN(0x1d902cd9, 0x4868195d), + TOBN(0x3d27bbf1, 0xe5fdb6f1), TOBN(0x7a5ab088, 0x124f9f1c), + TOBN(0xc466ab06, 0xf7a09e03), TOBN(0x2f8a1977, 0x31f2c123), + TOBN(0xda355dc7, 0x041b6657), TOBN(0xcb840d12, 0x8ece2a7c), + TOBN(0xb600ad9f, 0x7db32675), TOBN(0x78fea133, 0x07a06f1b), + TOBN(0x5d032269, 0xb31f6094), TOBN(0x07753ef5, 0x83ec37aa), + TOBN(0x03485aed, 0x9c0bea78), TOBN(0x41bb3989, 0xbc3f4524), + TOBN(0x09403761, 0x697f726d), TOBN(0x6109beb3, 0xdf394820), + TOBN(0x804111ea, 0x3b6d1145), TOBN(0xb6271ea9, 0xa8582654), + TOBN(0x619615e6, 0x24e66562), TOBN(0xa2554945, 0xd7b6ad9c), + TOBN(0xd9c4985e, 0x99bfe35f), TOBN(0x9770ccc0, 0x7b51cdf6), + TOBN(0x7c327013, 0x92881832), TOBN(0x8777d45f, 0x286b26d1), + TOBN(0x9bbeda22, 0xd847999d), TOBN(0x03aa33b6, 0xc3525d32), + TOBN(0x4b7b96d4, 0x28a959a1), TOBN(0xbb3786e5, 0x31e5d234), + TOBN(0xaeb5d3ce, 0x6961f247), TOBN(0x20aa85af, 0x02f93d3f), + TOBN(0x9cd1ad3d, 0xd7a7ae4f), TOBN(0xbf6688f0, 0x781adaa8), + TOBN(0xb1b40e86, 0x7469cead), TOBN(0x1904c524, 0x309fca48), + TOBN(0x9b7312af, 0x4b54bbc7), TOBN(0xbe24bf8f, 0x593affa2), + TOBN(0xbe5e0790, 0xbd98764b), TOBN(0xa0f45f17, 0xa26e299e), + TOBN(0x4af0d2c2, 0x6b8fe4c7), TOBN(0xef170db1, 0x8ae8a3e6), + TOBN(0x0e8d61a0, 0x29e0ccc1), TOBN(0xcd53e87e, 0x60ad36ca), + TOBN(0x328c6623, 0xc8173822), TOBN(0x7ee1767d, 0xa496be55), + TOBN(0x89f13259, 0x648945af), TOBN(0x9e45a5fd, 0x25c8009c), + TOBN(0xaf2febd9, 0x1f61ab8c), TOBN(0x43f6bc86, 0x8a275385), + TOBN(0x87792348, 0xf2142e79), TOBN(0x17d89259, 0xc6e6238a), + TOBN(0x7536d2f6, 0x4a839d9b), TOBN(0x1f428fce, 0x76a1fbdc), + TOBN(0x1c109601, 0x0db06dfe), TOBN(0xbfc16bc1, 0x50a3a3cc), + TOBN(0xf9cbd9ec, 0x9b30f41b), TOBN(0x5b5da0d6, 0x00138cce), + TOBN(0xec1d0a48, 0x56ef96a7), TOBN(0xb47eb848, 0x982bf842), + TOBN(0x66deae32, 0xec3f700d), TOBN(0x4e43c42c, 0xaa1181e0), + TOBN(0xa1d72a31, 0xd1a4aa2a), TOBN(0x440d4668, 0xc004f3ce), + TOBN(0x0d6a2d3b, 0x45fe8a7a), TOBN(0x820e52e2, 0xfb128365), + TOBN(0x29ac5fcf, 0x25e51b09), TOBN(0x180cd2bf, 0x2023d159), + TOBN(0xa9892171, 0xa1ebf90e), TOBN(0xf97c4c87, 0x7c132181), + TOBN(0x9f1dc724, 0xc03dbb7e), TOBN(0xae043765, 0x018cbbe4), + TOBN(0xfb0b2a36, 0x0767d153), TOBN(0xa8e2f4d6, 0x249cbaeb), + TOBN(0x172a5247, 0xd95ea168), TOBN(0x1758fada, 0x2970764a), + TOBN(0xac803a51, 0x1d978169), TOBN(0x299cfe2e, 0xde77e01b), + TOBN(0x652a1e17, 0xb0a98927), TOBN(0x2e26e1d1, 0x20014495), + TOBN(0x7ae0af9f, 0x7175b56a), TOBN(0xc2e22a80, 0xd64b9f95), + TOBN(0x4d0ff9fb, 0xd90a060a), TOBN(0x496a27db, 0xbaf38085), + TOBN(0x32305401, 0xda776bcf), TOBN(0xb8cdcef6, 0x725f209e), + TOBN(0x61ba0f37, 0x436a0bba), TOBN(0x263fa108, 0x76860049), + TOBN(0x92beb98e, 0xda3542cf), TOBN(0xa2d4d14a, 0xd5849538), + TOBN(0x989b9d68, 0x12e9a1bc), TOBN(0x61d9075c, 0x5f6e3268), + TOBN(0x352c6aa9, 0x99ace638), TOBN(0xde4e4a55, 0x920f43ff), + TOBN(0xe5e4144a, 0xd673c017), TOBN(0x667417ae, 0x6f6e05ea), + TOBN(0x613416ae, 0xdcd1bd56), TOBN(0x5eb36201, 0x86693711), + TOBN(0x2d7bc504, 0x3a1aa914), TOBN(0x175a1299, 0x76dc5975), + TOBN(0xe900e0f2, 0x3fc8125c), TOBN(0x569ef68c, 0x11198875), + TOBN(0x9012db63, 0x63a113b4), TOBN(0xe3bd3f56, 0x98835766), + TOBN(0xa5c94a52, 0x76412dea), TOBN(0xad9e2a09, 0xaa735e5c), + TOBN(0x405a984c, 0x508b65e9), TOBN(0xbde4a1d1, 0x6df1a0d1), + TOBN(0x1a9433a1, 0xdfba80da), TOBN(0xe9192ff9, 0x9440ad2e), + TOBN(0x9f649696, 0x5099fe92), TOBN(0x25ddb65c, 0x0b27a54a), + TOBN(0x178279dd, 0xc590da61), TOBN(0x5479a999, 0xfbde681a), + TOBN(0xd0e84e05, 0x013fe162), TOBN(0xbe11dc92, 0x632d471b), + TOBN(0xdf0b0c45, 0xfc0e089f), TOBN(0x04fb15b0, 0x4c144025), + TOBN(0xa61d5fc2, 0x13c99927), TOBN(0xa033e9e0, 0x3de2eb35), + TOBN(0xf8185d5c, 0xb8dacbb4), TOBN(0x9a88e265, 0x8644549d), + TOBN(0xf717af62, 0x54671ff6), TOBN(0x4bd4241b, 0x5fa58603), + TOBN(0x06fba40b, 0xe67773c0), TOBN(0xc1d933d2, 0x6a2847e9), + TOBN(0xf4f5acf3, 0x689e2c70), TOBN(0x92aab0e7, 0x46bafd31), + TOBN(0x798d76aa, 0x3473f6e5), TOBN(0xcc6641db, 0x93141934), + TOBN(0xcae27757, 0xd31e535e), TOBN(0x04cc43b6, 0x87c2ee11), + TOBN(0x8d1f9675, 0x2e029ffa), TOBN(0xc2150672, 0xe4cc7a2c), + TOBN(0x3b03c1e0, 0x8d68b013), TOBN(0xa9d6816f, 0xedf298f3), + TOBN(0x1bfbb529, 0xa2804464), TOBN(0x95a52fae, 0x5db22125), + TOBN(0x55b32160, 0x0e1cb64e), TOBN(0x004828f6, 0x7e7fc9fe), + TOBN(0x13394b82, 0x1bb0fb93), TOBN(0xb6293a2d, 0x35f1a920), + TOBN(0xde35ef21, 0xd145d2d9), TOBN(0xbe6225b3, 0xbb8fa603), + TOBN(0x00fc8f6b, 0x32cf252d), TOBN(0xa28e52e6, 0x117cf8c2), + TOBN(0x9d1dc89b, 0x4c371e6d), TOBN(0xcebe0675, 0x36ef0f28), + TOBN(0x5de05d09, 0xa4292f81), TOBN(0xa8303593, 0x353e3083), + TOBN(0xa1715b0a, 0x7e37a9bb), TOBN(0x8c56f61e, 0x2b8faec3), + TOBN(0x52507431, 0x33c9b102), TOBN(0x0130cefc, 0xa44431f0), + TOBN(0x56039fa0, 0xbd865cfb), TOBN(0x4b03e578, 0xbc5f1dd7), + TOBN(0x40edf2e4, 0xbabe7224), TOBN(0xc752496d, 0x3a1988f6), + TOBN(0xd1572d3b, 0x564beb6b), TOBN(0x0db1d110, 0x39a1c608), + TOBN(0x568d1934, 0x16f60126), TOBN(0x05ae9668, 0xf354af33), + TOBN(0x19de6d37, 0xc92544f2), TOBN(0xcc084353, 0xa35837d5), + TOBN(0xcbb6869c, 0x1a514ece), TOBN(0xb633e728, 0x2e1d1066), + TOBN(0xf15dd69f, 0x936c581c), TOBN(0x96e7b8ce, 0x7439c4f9), + TOBN(0x5e676f48, 0x2e448a5b), TOBN(0xb2ca7d5b, 0xfd916bbb), + TOBN(0xd55a2541, 0xf5024025), TOBN(0x47bc5769, 0xe4c2d937), + TOBN(0x7d31b92a, 0x0362189f), TOBN(0x83f3086e, 0xef7816f9), + TOBN(0xf9f46d94, 0xb587579a), TOBN(0xec2d22d8, 0x30e76c5f), + TOBN(0x27d57461, 0xb000ffcf), TOBN(0xbb7e65f9, 0x364ffc2c), + TOBN(0x7c7c9477, 0x6652a220), TOBN(0x61618f89, 0xd696c981), + TOBN(0x5021701d, 0x89effff3), TOBN(0xf2c8ff8e, 0x7c314163), + TOBN(0x2da413ad, 0x8efb4d3e), TOBN(0x937b5adf, 0xce176d95), + TOBN(0x22867d34, 0x2a67d51c), TOBN(0x262b9b10, 0x18eb3ac9), + TOBN(0x4e314fe4, 0xc43ff28b), TOBN(0x76476627, 0x6a664e7a), + TOBN(0x3e90e40b, 0xb7a565c2), TOBN(0x8588993a, 0xc1acf831), + TOBN(0xd7b501d6, 0x8f938829), TOBN(0x996627ee, 0x3edd7d4c), + TOBN(0x37d44a62, 0x90cd34c7), TOBN(0xa8327499, 0xf3833e8d), + TOBN(0x2e18917d, 0x4bf50353), TOBN(0x85dd726b, 0x556765fb), + TOBN(0x54fe65d6, 0x93d5ab66), TOBN(0x3ddbaced, 0x915c25fe), + TOBN(0xa799d9a4, 0x12f22e85), TOBN(0xe2a24867, 0x6d06f6bc), + TOBN(0xf4f1ee56, 0x43ca1637), TOBN(0xfda2828b, 0x61ece30a), + TOBN(0x758c1a3e, 0xa2dee7a6), TOBN(0xdcde2f3c, 0x734b2284), + TOBN(0xaba445d2, 0x4eaba6ad), TOBN(0x35aaf668, 0x76cee0a7), + TOBN(0x7e0b04a9, 0xe5aa049a), TOBN(0xe74083ad, 0x91103e84), + TOBN(0xbeb183ce, 0x40afecc3), TOBN(0x6b89de9f, 0xea043f7a),} + , + {TOBN(0x0e299d23, 0xfe67ba66), TOBN(0x91450760, 0x93cf2f34), + TOBN(0xf45b5ea9, 0x97fcf913), TOBN(0x5be00843, 0x8bd7ddda), + TOBN(0x358c3e05, 0xd53ff04d), TOBN(0xbf7ccdc3, 0x5de91ef7), + TOBN(0xad684dbf, 0xb69ec1a0), TOBN(0x367e7cf2, 0x801fd997), + TOBN(0x0ca1f3b7, 0xb0dc8595), TOBN(0x27de4608, 0x9f1d9f2e), + TOBN(0x1af3bf39, 0xbadd82a7), TOBN(0x79356a79, 0x65862448), + TOBN(0xc0602345, 0xf5f9a052), TOBN(0x1a8b0f89, 0x139a42f9), + TOBN(0xb53eee42, 0x844d40fc), TOBN(0x93b0bfe5, 0x4e5b6368), + TOBN(0x5434dd02, 0xc024789c), TOBN(0x90dca9ea, 0x41b57bfc), + TOBN(0x8aa898e2, 0x243398df), TOBN(0xf607c834, 0x894a94bb), + TOBN(0xbb07be97, 0xc2c99b76), TOBN(0x6576ba67, 0x18c29302), + TOBN(0x3d79efcc, 0xe703a88c), TOBN(0xf259ced7, 0xb6a0d106), + TOBN(0x0f893a5d, 0xc8de610b), TOBN(0xe8c515fb, 0x67e223ce), + TOBN(0x7774bfa6, 0x4ead6dc5), TOBN(0x89d20f95, 0x925c728f), + TOBN(0x7a1e0966, 0x098583ce), TOBN(0xa2eedb94, 0x93f2a7d7), + TOBN(0x1b282097, 0x4c304d4a), TOBN(0x0842e3da, 0xc077282d), + TOBN(0xe4d972a3, 0x3b9e2d7b), TOBN(0x7cc60b27, 0xc48218ff), + TOBN(0x8fc70838, 0x84149d91), TOBN(0x5c04346f, 0x2f461ecc), + TOBN(0xebe9fdf2, 0x614650a9), TOBN(0x5e35b537, 0xc1f666ac), + TOBN(0x645613d1, 0x88babc83), TOBN(0x88cace3a, 0xc5e1c93e), + TOBN(0x209ca375, 0x3de92e23), TOBN(0xccb03cc8, 0x5fbbb6e3), + TOBN(0xccb90f03, 0xd7b1487e), TOBN(0xfa9c2a38, 0xc710941f), + TOBN(0x756c3823, 0x6724ceed), TOBN(0x3a902258, 0x192d0323), + TOBN(0xb150e519, 0xea5e038e), TOBN(0xdcba2865, 0xc7427591), + TOBN(0xe549237f, 0x78890732), TOBN(0xc443bef9, 0x53fcb4d9), + TOBN(0x9884d8a6, 0xeb3480d6), TOBN(0x8a35b6a1, 0x3048b186), + TOBN(0xb4e44716, 0x65e9a90a), TOBN(0x45bf380d, 0x653006c0), + TOBN(0x8f3f820d, 0x4fe9ae3b), TOBN(0x244a35a0, 0x979a3b71), + TOBN(0xa1010e9d, 0x74cd06ff), TOBN(0x9c17c7df, 0xaca3eeac), + TOBN(0x74c86cd3, 0x8063aa2b), TOBN(0x8595c4b3, 0x734614ff), + TOBN(0xa3de00ca, 0x990f62cc), TOBN(0xd9bed213, 0xca0c3be5), + TOBN(0x7886078a, 0xdf8ce9f5), TOBN(0xddb27ce3, 0x5cd44444), + TOBN(0xed374a66, 0x58926ddd), TOBN(0x138b2d49, 0x908015b8), + TOBN(0x886c6579, 0xde1f7ab8), TOBN(0x888b9aa0, 0xc3020b7a), + TOBN(0xd3ec034e, 0x3a96e355), TOBN(0xba65b0b8, 0xf30fbe9a), + TOBN(0x064c8e50, 0xff21367a), TOBN(0x1f508ea4, 0x0b04b46e), + TOBN(0x98561a49, 0x747c866c), TOBN(0xbbb1e5fe, 0x0518a062), + TOBN(0x20ff4e8b, 0xecdc3608), TOBN(0x7f55cded, 0x20184027), + TOBN(0x8d73ec95, 0xf38c85f0), TOBN(0x5b589fdf, 0x8bc3b8c3), + TOBN(0xbe95dd98, 0x0f12b66f), TOBN(0xf5bd1a09, 0x0e338e01), + TOBN(0x65163ae5, 0x5e915918), TOBN(0x6158d6d9, 0x86f8a46b), + TOBN(0x8466b538, 0xeeebf99c), TOBN(0xca8761f6, 0xbca477ef), + TOBN(0xaf3449c2, 0x9ebbc601), TOBN(0xef3b0f41, 0xe0c3ae2f), + TOBN(0xaa6c577d, 0x5de63752), TOBN(0xe9166601, 0x64682a51), + TOBN(0x5a3097be, 0xfc15aa1e), TOBN(0x40d12548, 0xb54b0745), + TOBN(0x5bad4706, 0x519a5f12), TOBN(0xed03f717, 0xa439dee6), + TOBN(0x0794bb6c, 0x4a02c499), TOBN(0xf725083d, 0xcffe71d2), + TOBN(0x2cad7519, 0x0f3adcaf), TOBN(0x7f68ea1c, 0x43729310), + TOBN(0xe747c8c7, 0xb7ffd977), TOBN(0xec104c35, 0x80761a22), + TOBN(0x8395ebaf, 0x5a3ffb83), TOBN(0xfb3261f4, 0xe4b63db7), + TOBN(0x53544960, 0xd883e544), TOBN(0x13520d70, 0x8cc2eeb8), + TOBN(0x08f6337b, 0xd3d65f99), TOBN(0x83997db2, 0x781cf95b), + TOBN(0xce6ff106, 0x0dbd2c01), TOBN(0x4f8eea6b, 0x1f9ce934), + TOBN(0x546f7c4b, 0x0e993921), TOBN(0x6236a324, 0x5e753fc7), + TOBN(0x65a41f84, 0xa16022e9), TOBN(0x0c18d878, 0x43d1dbb2), + TOBN(0x73c55640, 0x2d4cef9c), TOBN(0xa0428108, 0x70444c74), + TOBN(0x68e4f15e, 0x9afdfb3c), TOBN(0x49a56143, 0x5bdfb6df), + TOBN(0xa9bc1bd4, 0x5f823d97), TOBN(0xbceb5970, 0xea111c2a), + TOBN(0x366b455f, 0xb269bbc4), TOBN(0x7cd85e1e, 0xe9bc5d62), + TOBN(0xc743c41c, 0x4f18b086), TOBN(0xa4b40990, 0x95294fb9), + TOBN(0x9c7c581d, 0x26ee8382), TOBN(0xcf17dcc5, 0x359d638e), + TOBN(0xee8273ab, 0xb728ae3d), TOBN(0x1d112926, 0xf821f047), + TOBN(0x11498477, 0x50491a74), TOBN(0x687fa761, 0xfde0dfb9), + TOBN(0x2c258022, 0x7ea435ab), TOBN(0x6b8bdb94, 0x91ce7e3f), + TOBN(0x4c5b5dc9, 0x3bf834aa), TOBN(0x04371819, 0x4f6c7e4b), + TOBN(0xc284e00a, 0x3736bcad), TOBN(0x0d881118, 0x21ae8f8d), + TOBN(0xf9cf0f82, 0xf48c8e33), TOBN(0xa11fd075, 0xa1bf40db), + TOBN(0xdceab0de, 0xdc2733e5), TOBN(0xc560a8b5, 0x8e986bd7), + TOBN(0x48dd1fe2, 0x3929d097), TOBN(0x3885b290, 0x92f188f1), + TOBN(0x0f2ae613, 0xda6fcdac), TOBN(0x9054303e, 0xb662a46c), + TOBN(0xb6871e44, 0x0738042a), TOBN(0x98e6a977, 0xbdaf6449), + TOBN(0xd8bc0650, 0xd1c9df1b), TOBN(0xef3d6451, 0x36e098f9), + TOBN(0x03fbae82, 0xb6d72d28), TOBN(0x77ca9db1, 0xf5d84080), + TOBN(0x8a112cff, 0xa58efc1c), TOBN(0x518d761c, 0xc564cb4a), + TOBN(0x69b5740e, 0xf0d1b5ce), TOBN(0x717039cc, 0xe9eb1785), + TOBN(0x3fe29f90, 0x22f53382), TOBN(0x8e54ba56, 0x6bc7c95c), + TOBN(0x9c806d8a, 0xf7f91d0f), TOBN(0x3b61b0f1, 0xa82a5728), + TOBN(0x4640032d, 0x94d76754), TOBN(0x273eb5de, 0x47d834c6), + TOBN(0x2988abf7, 0x7b4e4d53), TOBN(0xb7ce66bf, 0xde401777), + TOBN(0x9fba6b32, 0x715071b3), TOBN(0x82413c24, 0xad3a1a98), + TOBN(0x5b7fc8c4, 0xe0e8ad93), TOBN(0xb5679aee, 0x5fab868d), + TOBN(0xb1f9d2fa, 0x2b3946f3), TOBN(0x458897dc, 0x5685b50a), + TOBN(0x1e98c930, 0x89d0caf3), TOBN(0x39564c5f, 0x78642e92), + TOBN(0x1b77729a, 0x0dbdaf18), TOBN(0xf9170722, 0x579e82e6), + TOBN(0x680c0317, 0xe4515fa5), TOBN(0xf85cff84, 0xfb0c790f), + TOBN(0xc7a82aab, 0x6d2e0765), TOBN(0x7446bca9, 0x35c82b32), + TOBN(0x5de607aa, 0x6d63184f), TOBN(0x7c1a46a8, 0x262803a6), + TOBN(0xd218313d, 0xaebe8035), TOBN(0x92113ffd, 0xc73c51f8), + TOBN(0x4b38e083, 0x12e7e46c), TOBN(0x69d0a37a, 0x56126bd5), + TOBN(0xfb3f324b, 0x73c07e04), TOBN(0xa0c22f67, 0x8fda7267), + TOBN(0x8f2c0051, 0x4d2c7d8f), TOBN(0xbc45ced3, 0xcbe2cae5), + TOBN(0xe1c6cf07, 0xa8f0f277), TOBN(0xbc392312, 0x1eb99a98), + TOBN(0x75537b7e, 0x3cc8ac85), TOBN(0x8d725f57, 0xdd02753b), + TOBN(0xfd05ff64, 0xb737df2f), TOBN(0x55fe8712, 0xf6d2531d), + TOBN(0x57ce04a9, 0x6ab6b01c), TOBN(0x69a02a89, 0x7cd93724), + TOBN(0x4f82ac35, 0xcf86699b), TOBN(0x8242d3ad, 0x9cb4b232), + TOBN(0x713d0f65, 0xd62105e5), TOBN(0xbb222bfa, 0x2d29be61), + TOBN(0xf2f9a79e, 0x6cfbef09), TOBN(0xfc24d8d3, 0xd5d6782f), + TOBN(0x5db77085, 0xd4129967), TOBN(0xdb81c3cc, 0xdc3c2a43), + TOBN(0x9d655fc0, 0x05d8d9a3), TOBN(0x3f5d057a, 0x54298026), + TOBN(0x1157f56d, 0x88c54694), TOBN(0xb26baba5, 0x9b09573e), + TOBN(0x2cab03b0, 0x22adffd1), TOBN(0x60a412c8, 0xdd69f383), + TOBN(0xed76e98b, 0x54b25039), TOBN(0xd4ee67d3, 0x687e714d), + TOBN(0x87739648, 0x7b00b594), TOBN(0xce419775, 0xc9ef709b), + TOBN(0x40f76f85, 0x1c203a40), TOBN(0x30d352d6, 0xeafd8f91), + TOBN(0xaf196d3d, 0x95578dd2), TOBN(0xea4bb3d7, 0x77cc3f3d), + TOBN(0x42a5bd03, 0xb98e782b), TOBN(0xac958c40, 0x0624920d), + TOBN(0xb838134c, 0xfc56fcc8), TOBN(0x86ec4ccf, 0x89572e5e), + TOBN(0x69c43526, 0x9be47be0), TOBN(0x323b7dd8, 0xcb28fea1), + TOBN(0xfa5538ba, 0x3a6c67e5), TOBN(0xef921d70, 0x1d378e46), + TOBN(0xf92961fc, 0x3c4b880e), TOBN(0x3f6f914e, 0x98940a67), + TOBN(0xa990eb0a, 0xfef0ff39), TOBN(0xa6c2920f, 0xf0eeff9c), + TOBN(0xca804166, 0x51b8d9a3), TOBN(0x42531bc9, 0x0ffb0db1), + TOBN(0x72ce4718, 0xaa82e7ce), TOBN(0x6e199913, 0xdf574741), + TOBN(0xd5f1b13d, 0xd5d36946), TOBN(0x8255dc65, 0xf68f0194), + TOBN(0xdc9df4cd, 0x8710d230), TOBN(0x3453c20f, 0x138c1988), + TOBN(0x9af98dc0, 0x89a6ef01), TOBN(0x4dbcc3f0, 0x9857df85), + TOBN(0x34805601, 0x5c1ad924), TOBN(0x40448da5, 0xd0493046), + TOBN(0xf629926d, 0x4ee343e2), TOBN(0x6343f1bd, 0x90e8a301), + TOBN(0xefc93491, 0x40815b3f), TOBN(0xf882a423, 0xde8f66fb), + TOBN(0x3a12d5f4, 0xe7db9f57), TOBN(0x7dfba38a, 0x3c384c27), + TOBN(0x7a904bfd, 0x6fc660b1), TOBN(0xeb6c5db3, 0x2773b21c), + TOBN(0xc350ee66, 0x1cdfe049), TOBN(0x9baac0ce, 0x44540f29), + TOBN(0xbc57b6ab, 0xa5ec6aad), TOBN(0x167ce8c3, 0x0a7c1baa), + TOBN(0xb23a03a5, 0x53fb2b56), TOBN(0x6ce141e7, 0x4e057f78), + TOBN(0x796525c3, 0x89e490d9), TOBN(0x0bc95725, 0xa31a7e75), + TOBN(0x1ec56791, 0x1220fd06), TOBN(0x716e3a3c, 0x408b0bd6), + TOBN(0x31cd6bf7, 0xe8ebeba9), TOBN(0xa7326ca6, 0xbee6b670), + TOBN(0x3d9f851c, 0xcd090c43), TOBN(0x561e8f13, 0xf12c3988), + TOBN(0x50490b6a, 0x904b7be4), TOBN(0x61690ce1, 0x0410737b), + TOBN(0x299e9a37, 0x0f009052), TOBN(0x258758f0, 0xf026092e), + TOBN(0x9fa255f3, 0xfdfcdc0f), TOBN(0xdbc9fb1f, 0xc0e1bcd2), + TOBN(0x35f9dd6e, 0x24651840), TOBN(0xdca45a84, 0xa5c59abc), + TOBN(0x103d396f, 0xecca4938), TOBN(0x4532da0a, 0xb97b3f29), + TOBN(0xc4135ea5, 0x1999a6bf), TOBN(0x3aa9505a, 0x5e6bf2ee), + TOBN(0xf77cef06, 0x3f5be093), TOBN(0x97d1a0f8, 0xa943152e), + TOBN(0x2cb0ebba, 0x2e1c21dd), TOBN(0xf41b29fc, 0x2c6797c4), + TOBN(0xc6e17321, 0xb300101f), TOBN(0x4422b0e9, 0xd0d79a89), + TOBN(0x49e4901c, 0x92f1bfc4), TOBN(0x06ab1f8f, 0xe1e10ed9), + TOBN(0x84d35577, 0xdb2926b8), TOBN(0xca349d39, 0x356e8ec2), + TOBN(0x70b63d32, 0x343bf1a9), TOBN(0x8fd3bd28, 0x37d1a6b1), + TOBN(0x0454879c, 0x316865b4), TOBN(0xee959ff6, 0xc458efa2), + TOBN(0x0461dcf8, 0x9706dc3f), TOBN(0x737db0e2, 0x164e4b2e), + TOBN(0x09262680, 0x2f8843c8), TOBN(0x54498bbc, 0x7745e6f6), + TOBN(0x359473fa, 0xa29e24af), TOBN(0xfcc3c454, 0x70aa87a1), + TOBN(0xfd2c4bf5, 0x00573ace), TOBN(0xb65b514e, 0x28dd1965), + TOBN(0xe46ae7cf, 0x2193e393), TOBN(0x60e9a4e1, 0xf5444d97), + TOBN(0xe7594e96, 0x00ff38ed), TOBN(0x43d84d2f, 0x0a0e0f02), + TOBN(0x8b6db141, 0xee398a21), TOBN(0xb88a56ae, 0xe3bcc5be), + TOBN(0x0a1aa52f, 0x373460ea), TOBN(0x20da1a56, 0x160bb19b), + TOBN(0xfb54999d, 0x65bf0384), TOBN(0x71a14d24, 0x5d5a180e), + TOBN(0xbc44db7b, 0x21737b04), TOBN(0xd84fcb18, 0x01dd8e92), + TOBN(0x80de937b, 0xfa44b479), TOBN(0x53505499, 0x5c98fd4f), + TOBN(0x1edb12ab, 0x28f08727), TOBN(0x4c58b582, 0xa5f3ef53), + TOBN(0xbfb236d8, 0x8327f246), TOBN(0xc3a3bfaa, 0x4d7df320), + TOBN(0xecd96c59, 0xb96024f2), TOBN(0xfc293a53, 0x7f4e0433), + TOBN(0x5341352b, 0x5acf6e10), TOBN(0xc50343fd, 0xafe652c3), + TOBN(0x4af3792d, 0x18577a7f), TOBN(0xe1a4c617, 0xaf16823d), + TOBN(0x9b26d0cd, 0x33425d0a), TOBN(0x306399ed, 0x9b7bc47f), + TOBN(0x2a792f33, 0x706bb20b), TOBN(0x31219614, 0x98111055), + TOBN(0x864ec064, 0x87f5d28b), TOBN(0x11392d91, 0x962277fd), + TOBN(0xb5aa7942, 0xbb6aed5f), TOBN(0x080094dc, 0x47e799d9), + TOBN(0x4afa588c, 0x208ba19b), TOBN(0xd3e7570f, 0x8512f284), + TOBN(0xcbae64e6, 0x02f5799a), TOBN(0xdeebe7ef, 0x514b9492), + TOBN(0x30300f98, 0xe5c298ff), TOBN(0x17f561be, 0x3678361f), + TOBN(0xf52ff312, 0x98cb9a16), TOBN(0x6233c3bc, 0x5562d490), + TOBN(0x7bfa15a1, 0x92e3a2cb), TOBN(0x961bcfd1, 0xe6365119), + TOBN(0x3bdd29bf, 0x2c8c53b1), TOBN(0x739704df, 0x822844ba), + TOBN(0x7dacfb58, 0x7e7b754b), TOBN(0x23360791, 0xa806c9b9), + TOBN(0xe7eb88c9, 0x23504452), TOBN(0x2983e996, 0x852c1783), + TOBN(0xdd4ae529, 0x958d881d), TOBN(0x026bae03, 0x262c7b3c), + TOBN(0x3a6f9193, 0x960b52d1), TOBN(0xd0980f90, 0x92696cfb), + TOBN(0x4c1f428c, 0xd5f30851), TOBN(0x94dfed27, 0x2a4f6630), + TOBN(0x4df53772, 0xfc5d48a4), TOBN(0xdd2d5a2f, 0x933260ce), + TOBN(0x574115bd, 0xd44cc7a5), TOBN(0x4ba6b20d, 0xbd12533a), + TOBN(0x30e93cb8, 0x243057c9), TOBN(0x794c486a, 0x14de320e), + TOBN(0xe925d4ce, 0xf21496e4), TOBN(0xf951d198, 0xec696331), + TOBN(0x9810e2de, 0x3e8d812f), TOBN(0xd0a47259, 0x389294ab), + TOBN(0x513ba2b5, 0x0e3bab66), TOBN(0x462caff5, 0xabad306f), + TOBN(0xe2dc6d59, 0xaf04c49e), TOBN(0x1aeb8750, 0xe0b84b0b), + TOBN(0xc034f12f, 0x2f7d0ca2), TOBN(0x6d2e8128, 0xe06acf2f), + TOBN(0x801f4f83, 0x21facc2f), TOBN(0xa1170c03, 0xf40ef607), + TOBN(0xfe0a1d4f, 0x7805a99c), TOBN(0xbde56a36, 0xcc26aba5), + TOBN(0x5b1629d0, 0x35531f40), TOBN(0xac212c2b, 0x9afa6108), + TOBN(0x30a06bf3, 0x15697be5), TOBN(0x6f0545dc, 0x2c63c7c1), + TOBN(0x5d8cb842, 0x7ccdadaf), TOBN(0xd52e379b, 0xac7015bb), + TOBN(0xc4f56147, 0xf462c23e), TOBN(0xd44a4298, 0x46bc24b0), + TOBN(0xbc73d23a, 0xe2856d4f), TOBN(0x61cedd8c, 0x0832bcdf), + TOBN(0x60953556, 0x99f241d7), TOBN(0xee4adbd7, 0x001a349d), + TOBN(0x0b35bf6a, 0xaa89e491), TOBN(0x7f0076f4, 0x136f7546), + TOBN(0xd19a18ba, 0x9264da3d), TOBN(0x6eb2d2cd, 0x62a7a28b), + TOBN(0xcdba941f, 0x8761c971), TOBN(0x1550518b, 0xa3be4a5d), + TOBN(0xd0e8e2f0, 0x57d0b70c), TOBN(0xeea8612e, 0xcd133ba3), + TOBN(0x814670f0, 0x44416aec), TOBN(0x424db6c3, 0x30775061), + TOBN(0xd96039d1, 0x16213fd1), TOBN(0xc61e7fa5, 0x18a3478f), + TOBN(0xa805bdcc, 0xcb0c5021), TOBN(0xbdd6f3a8, 0x0cc616dd), + TOBN(0x06009667, 0x5d97f7e2), TOBN(0x31db0fc1, 0xaf0bf4b6), + TOBN(0x23680ed4, 0x5491627a), TOBN(0xb99a3c66, 0x7d741fb1), + TOBN(0xe9bb5f55, 0x36b1ff92), TOBN(0x29738577, 0x512b388d), + TOBN(0xdb8a2ce7, 0x50fcf263), TOBN(0x385346d4, 0x6c4f7b47), + TOBN(0xbe86c5ef, 0x31631f9e), TOBN(0xbf91da21, 0x03a57a29), + TOBN(0xc3b1f796, 0x7b23f821), TOBN(0x0f7d00d2, 0x770db354), + TOBN(0x8ffc6c3b, 0xd8fe79da), TOBN(0xcc5e8c40, 0xd525c996), + TOBN(0x4640991d, 0xcfff632a), TOBN(0x64d97e8c, 0x67112528), + TOBN(0xc232d973, 0x02f1cd1e), TOBN(0xce87eacb, 0x1dd212a4), + TOBN(0x6e4c8c73, 0xe69802f7), TOBN(0x12ef0290, 0x1fffddbd), + TOBN(0x941ec74e, 0x1bcea6e2), TOBN(0xd0b54024, 0x3cb92cbb), + TOBN(0x809fb9d4, 0x7e8f9d05), TOBN(0x3bf16159, 0xf2992aae), + TOBN(0xad40f279, 0xf8a7a838), TOBN(0x11aea631, 0x05615660), + TOBN(0xbf52e6f1, 0xa01f6fa1), TOBN(0xef046995, 0x3dc2aec9), + TOBN(0x785dbec9, 0xd8080711), TOBN(0xe1aec60a, 0x9fdedf76), + TOBN(0xece797b5, 0xfa21c126), TOBN(0xc66e898f, 0x05e52732), + TOBN(0x39bb69c4, 0x08811fdb), TOBN(0x8bfe1ef8, 0x2fc7f082), + TOBN(0xc8e7a393, 0x174f4138), TOBN(0xfba8ad1d, 0xd58d1f98), + TOBN(0xbc21d0ce, 0xbfd2fd5b), TOBN(0x0b839a82, 0x6ee60d61), + TOBN(0xaacf7658, 0xafd22253), TOBN(0xb526bed8, 0xaae396b3), + TOBN(0xccc1bbc2, 0x38564464), TOBN(0x9e3ff947, 0x8c45bc73), + TOBN(0xcde9bca3, 0x58188a78), TOBN(0x138b8ee0, 0xd73bf8f7), + TOBN(0x5c7e234c, 0x4123c489), TOBN(0x66e69368, 0xfa643297), + TOBN(0x0629eeee, 0x39a15fa3), TOBN(0x95fab881, 0xa9e2a927), + TOBN(0xb2497007, 0xeafbb1e1), TOBN(0xd75c9ce6, 0xe75b7a93), + TOBN(0x3558352d, 0xefb68d78), TOBN(0xa2f26699, 0x223f6396), + TOBN(0xeb911ecf, 0xe469b17a), TOBN(0x62545779, 0xe72d3ec2), + TOBN(0x8ea47de7, 0x82cb113f), TOBN(0xebe4b086, 0x4e1fa98d), + TOBN(0xec2d5ed7, 0x8cdfedb1), TOBN(0xa535c077, 0xfe211a74), + TOBN(0x9678109b, 0x11d244c5), TOBN(0xf17c8bfb, 0xbe299a76), + TOBN(0xb651412e, 0xfb11fbc4), TOBN(0xea0b5482, 0x94ab3f65), + TOBN(0xd8dffd95, 0x0cf78243), TOBN(0x2e719e57, 0xce0361d4), + TOBN(0x9007f085, 0x304ddc5b), TOBN(0x095e8c6d, 0x4daba2ea), + TOBN(0x5a33cdb4, 0x3f9d28a9), TOBN(0x85b95cd8, 0xe2283003), + TOBN(0xbcd6c819, 0xb9744733), TOBN(0x29c5f538, 0xfc7f5783), + TOBN(0x6c49b2fa, 0xd59038e4), TOBN(0x68349cc1, 0x3bbe1018), + TOBN(0xcc490c1d, 0x21830ee5), TOBN(0x36f9c4ee, 0xe9bfa297), + TOBN(0x58fd7294, 0x48de1a94), TOBN(0xaadb13a8, 0x4e8f2cdc), + TOBN(0x515eaaa0, 0x81313dba), TOBN(0xc76bb468, 0xc2152dd8), + TOBN(0x357f8d75, 0xa653dbf8), TOBN(0xe4d8c4d1, 0xb14ac143), + TOBN(0xbdb8e675, 0xb055cb40), TOBN(0x898f8e7b, 0x977b5167), + TOBN(0xecc65651, 0xb82fb863), TOBN(0x56544814, 0x6d88f01f), + TOBN(0xb0928e95, 0x263a75a9), TOBN(0xcfb6836f, 0x1a22fcda), + TOBN(0x651d14db, 0x3f3bd37c), TOBN(0x1d3837fb, 0xb6ad4664), + TOBN(0x7c5fb538, 0xff4f94ab), TOBN(0x7243c712, 0x6d7fb8f2), + TOBN(0xef13d60c, 0xa85c5287), TOBN(0x18cfb7c7, 0x4bb8dd1b), + TOBN(0x82f9bfe6, 0x72908219), TOBN(0x35c4592b, 0x9d5144ab), + TOBN(0x52734f37, 0x9cf4b42f), TOBN(0x6bac55e7, 0x8c60ddc4), + TOBN(0xb5cd811e, 0x94dea0f6), TOBN(0x259ecae4, 0xe18cc1a3), + TOBN(0x6a0e836e, 0x15e660f8), TOBN(0x6c639ea6, 0x0e02bff2), + TOBN(0x8721b8cb, 0x7e1026fd), TOBN(0x9e73b50b, 0x63261942), + TOBN(0xb8c70974, 0x77f01da3), TOBN(0x1839e6a6, 0x8268f57f), + TOBN(0x571b9415, 0x5150b805), TOBN(0x1892389e, 0xf92c7097), + TOBN(0x8d69c18e, 0x4a084b95), TOBN(0x7014c512, 0xbe5b495c), + TOBN(0x4780db36, 0x1b07523c), TOBN(0x2f6219ce, 0x2c1c64fa), + TOBN(0xc38b81b0, 0x602c105a), TOBN(0xab4f4f20, 0x5dc8e360), + TOBN(0x20d3c982, 0xcf7d62d2), TOBN(0x1f36e29d, 0x23ba8150), + TOBN(0x48ae0bf0, 0x92763f9e), TOBN(0x7a527e6b, 0x1d3a7007), + TOBN(0xb4a89097, 0x581a85e3), TOBN(0x1f1a520f, 0xdc158be5), + TOBN(0xf98db37d, 0x167d726e), TOBN(0x8802786e, 0x1113e862)} + , + {TOBN(0xefb2149e, 0x36f09ab0), TOBN(0x03f163ca, 0x4a10bb5b), + TOBN(0xd0297045, 0x06e20998), TOBN(0x56f0af00, 0x1b5a3bab), + TOBN(0x7af4cfec, 0x70880e0d), TOBN(0x7332a66f, 0xbe3d913f), + TOBN(0x32e6c84a, 0x7eceb4bd), TOBN(0xedc4a79a, 0x9c228f55), + TOBN(0xc37c7dd0, 0xc55c4496), TOBN(0xa6a96357, 0x25bbabd2), + TOBN(0x5b7e63f2, 0xadd7f363), TOBN(0x9dce3782, 0x2e73f1df), + TOBN(0xe1e5a16a, 0xb2b91f71), TOBN(0xe4489823, 0x5ba0163c), + TOBN(0xf2759c32, 0xf6e515ad), TOBN(0xa5e2f1f8, 0x8615eecf), + TOBN(0x74519be7, 0xabded551), TOBN(0x03d358b8, 0xc8b74410), + TOBN(0x4d00b10b, 0x0e10d9a9), TOBN(0x6392b0b1, 0x28da52b7), + TOBN(0x6744a298, 0x0b75c904), TOBN(0xc305b0ae, 0xa8f7f96c), + TOBN(0x042e421d, 0x182cf932), TOBN(0xf6fc5d50, 0x9e4636ca), + TOBN(0x795847c9, 0xd64cc78c), TOBN(0x6c50621b, 0x9b6cb27b), + TOBN(0x07099bf8, 0xdf8022ab), TOBN(0x48f862eb, 0xc04eda1d), + TOBN(0xd12732ed, 0xe1603c16), TOBN(0x19a80e0f, 0x5c9a9450), + TOBN(0xe2257f54, 0xb429b4fc), TOBN(0x66d3b2c6, 0x45460515), + TOBN(0x6ca4f87e, 0x822e37be), TOBN(0x73f237b4, 0x253bda4e), + TOBN(0xf747f3a2, 0x41190aeb), TOBN(0xf06fa36f, 0x804cf284), + TOBN(0x0a6bbb6e, 0xfc621c12), TOBN(0x5d624b64, 0x40b80ec6), + TOBN(0x4b072425, 0x7ba556f3), TOBN(0x7fa0c354, 0x3e2d20a8), + TOBN(0xe921fa31, 0xe3229d41), TOBN(0xa929c652, 0x94531bd4), + TOBN(0x84156027, 0xa6d38209), TOBN(0xf3d69f73, 0x6bdb97bd), + TOBN(0x8906d19a, 0x16833631), TOBN(0x68a34c2e, 0x03d51be3), + TOBN(0xcb59583b, 0x0e511cd8), TOBN(0x99ce6bfd, 0xfdc132a8), + TOBN(0x3facdaaa, 0xffcdb463), TOBN(0x658bbc1a, 0x34a38b08), + TOBN(0x12a801f8, 0xf1a9078d), TOBN(0x1567bcf9, 0x6ab855de), + TOBN(0xe08498e0, 0x3572359b), TOBN(0xcf0353e5, 0x8659e68b), + TOBN(0xbb86e9c8, 0x7d23807c), TOBN(0xbc08728d, 0x2198e8a2), + TOBN(0x8de2b7bc, 0x453cadd6), TOBN(0x203900a7, 0xbc0bc1f8), + TOBN(0xbcd86e47, 0xa6abd3af), TOBN(0x911cac12, 0x8502effb), + TOBN(0x2d550242, 0xec965469), TOBN(0x0e9f7692, 0x29e0017e), + TOBN(0x633f078f, 0x65979885), TOBN(0xfb87d449, 0x4cf751ef), + TOBN(0xe1790e4b, 0xfc25419a), TOBN(0x36467203, 0x4bff3cfd), + TOBN(0xc8db6386, 0x25b6e83f), TOBN(0x6cc69f23, 0x6cad6fd2), + TOBN(0x0219e45a, 0x6bc68bb9), TOBN(0xe43d79b6, 0x297f7334), + TOBN(0x7d445368, 0x465dc97c), TOBN(0x4b9eea32, 0x2a0b949a), + TOBN(0x1b96c6ba, 0x6102d021), TOBN(0xeaafac78, 0x2f4461ea), + TOBN(0xd4b85c41, 0xc49f19a8), TOBN(0x275c28e4, 0xcf538875), + TOBN(0x35451a9d, 0xdd2e54e0), TOBN(0x6991adb5, 0x0605618b), + TOBN(0x5b8b4bcd, 0x7b36cd24), TOBN(0x372a4f8c, 0x56f37216), + TOBN(0xc890bd73, 0xa6a5da60), TOBN(0x6f083da0, 0xdc4c9ff0), + TOBN(0xf4e14d94, 0xf0536e57), TOBN(0xf9ee1eda, 0xaaec8243), + TOBN(0x571241ec, 0x8bdcf8e7), TOBN(0xa5db8271, 0x0b041e26), + TOBN(0x9a0b9a99, 0xe3fff040), TOBN(0xcaaf21dd, 0x7c271202), + TOBN(0xb4e2b2e1, 0x4f0dd2e8), TOBN(0xe77e7c4f, 0x0a377ac7), + TOBN(0x69202c3f, 0x0d7a2198), TOBN(0xf759b7ff, 0x28200eb8), + TOBN(0xc87526ed, 0xdcfe314e), TOBN(0xeb84c524, 0x53d5cf99), + TOBN(0xb1b52ace, 0x515138b6), TOBN(0x5aa7ff8c, 0x23fca3f4), + TOBN(0xff0b13c3, 0xb9791a26), TOBN(0x960022da, 0xcdd58b16), + TOBN(0xdbd55c92, 0x57aad2de), TOBN(0x3baaaaa3, 0xf30fe619), + TOBN(0x9a4b2346, 0x0d881efd), TOBN(0x506416c0, 0x46325e2a), + TOBN(0x91381e76, 0x035c18d4), TOBN(0xb3bb68be, 0xf27817b0), + TOBN(0x15bfb8bf, 0x5116f937), TOBN(0x7c64a586, 0xc1268943), + TOBN(0x71e25cc3, 0x8419a2c8), TOBN(0x9fd6b0c4, 0x8335f463), + TOBN(0x4bf0ba3c, 0xe8ee0e0e), TOBN(0x6f6fba60, 0x298c21fa), + TOBN(0x57d57b39, 0xae66bee0), TOBN(0x292d5130, 0x22672544), + TOBN(0xf451105d, 0xbab093b3), TOBN(0x012f59b9, 0x02839986), + TOBN(0x8a915802, 0x3474a89c), TOBN(0x048c919c, 0x2de03e97), + TOBN(0xc476a2b5, 0x91071cd5), TOBN(0x791ed89a, 0x034970a5), + TOBN(0x89bd9042, 0xe1b7994b), TOBN(0x8eaf5179, 0xa1057ffd), + TOBN(0x6066e2a2, 0xd551ee10), TOBN(0x87a8f1d8, 0x727e09a6), + TOBN(0x00d08bab, 0x2c01148d), TOBN(0x6da8e4f1, 0x424f33fe), + TOBN(0x466d17f0, 0xcf9a4e71), TOBN(0xff502010, 0x3bf5cb19), + TOBN(0xdccf97d8, 0xd062ecc0), TOBN(0x80c0d9af, 0x81d80ac4), + TOBN(0xe87771d8, 0x033f2876), TOBN(0xb0186ec6, 0x7d5cc3db), + TOBN(0x58e8bb80, 0x3bc9bc1d), TOBN(0x4d1395cc, 0x6f6ef60e), + TOBN(0xa73c62d6, 0x186244a0), TOBN(0x918e5f23, 0x110a5b53), + TOBN(0xed4878ca, 0x741b7eab), TOBN(0x3038d71a, 0xdbe03e51), + TOBN(0x840204b7, 0xa93c3246), TOBN(0x21ab6069, 0xa0b9b4cd), + TOBN(0xf5fa6e2b, 0xb1d64218), TOBN(0x1de6ad0e, 0xf3d56191), + TOBN(0x570aaa88, 0xff1929c7), TOBN(0xc6df4c6b, 0x640e87b5), + TOBN(0xde8a74f2, 0xc65f0ccc), TOBN(0x8b972fd5, 0xe6f6cc01), + TOBN(0x3fff36b6, 0x0b846531), TOBN(0xba7e45e6, 0x10a5e475), + TOBN(0x84a1d10e, 0x4145b6c5), TOBN(0xf1f7f91a, 0x5e046d9d), + TOBN(0x0317a692, 0x44de90d7), TOBN(0x951a1d4a, 0xf199c15e), + TOBN(0x91f78046, 0xc9d73deb), TOBN(0x74c82828, 0xfab8224f), + TOBN(0xaa6778fc, 0xe7560b90), TOBN(0xb4073e61, 0xa7e824ce), + TOBN(0xff0d693c, 0xd642eba8), TOBN(0x7ce2e57a, 0x5dccef38), + TOBN(0x89c2c789, 0x1df1ad46), TOBN(0x83a06922, 0x098346fd), + TOBN(0x2d715d72, 0xda2fc177), TOBN(0x7b6dd71d, 0x85b6cf1d), + TOBN(0xc60a6d0a, 0x73fa9cb0), TOBN(0xedd3992e, 0x328bf5a9), + TOBN(0xc380ddd0, 0x832c8c82), TOBN(0xd182d410, 0xa2a0bf50), + TOBN(0x7d9d7438, 0xd9a528db), TOBN(0xe8b1a0e9, 0xcaf53994), + TOBN(0xddd6e5fe, 0x0e19987c), TOBN(0xacb8df03, 0x190b059d), + TOBN(0x53703a32, 0x8300129f), TOBN(0x1f637662, 0x68c43bfd), + TOBN(0xbcbd1913, 0x00e54051), TOBN(0x812fcc62, 0x7bf5a8c5), + TOBN(0x3f969d5f, 0x29fb85da), TOBN(0x72f4e00a, 0x694759e8), + TOBN(0x426b6e52, 0x790726b7), TOBN(0x617bbc87, 0x3bdbb209), + TOBN(0x511f8bb9, 0x97aee317), TOBN(0x812a4096, 0xe81536a8), + TOBN(0x137dfe59, 0x3ac09b9b), TOBN(0x0682238f, 0xba8c9a7a), + TOBN(0x7072ead6, 0xaeccb4bd), TOBN(0x6a34e9aa, 0x692ba633), + TOBN(0xc82eaec2, 0x6fff9d33), TOBN(0xfb753512, 0x1d4d2b62), + TOBN(0x1a0445ff, 0x1d7aadab), TOBN(0x65d38260, 0xd5f6a67c), + TOBN(0x6e62fb08, 0x91cfb26f), TOBN(0xef1e0fa5, 0x5c7d91d6), + TOBN(0x47e7c7ba, 0x33db72cd), TOBN(0x017cbc09, 0xfa7c74b2), + TOBN(0x3c931590, 0xf50a503c), TOBN(0xcac54f60, 0x616baa42), + TOBN(0x9b6cd380, 0xb2369f0f), TOBN(0x97d3a70d, 0x23c76151), + TOBN(0x5f9dd6fc, 0x9862a9c6), TOBN(0x044c4ab2, 0x12312f51), + TOBN(0x035ea0fd, 0x834a2ddc), TOBN(0x49e6b862, 0xcc7b826d), + TOBN(0xb03d6883, 0x62fce490), TOBN(0x62f2497a, 0xb37e36e9), + TOBN(0x04b005b6, 0xc6458293), TOBN(0x36bb5276, 0xe8d10af7), + TOBN(0xacf2dc13, 0x8ee617b8), TOBN(0x470d2d35, 0xb004b3d4), + TOBN(0x06790832, 0xfeeb1b77), TOBN(0x2bb75c39, 0x85657f9c), + TOBN(0xd70bd4ed, 0xc0f60004), TOBN(0xfe797ecc, 0x219b018b), + TOBN(0x9b5bec2a, 0x753aebcc), TOBN(0xdaf9f3dc, 0xc939eca5), + TOBN(0xd6bc6833, 0xd095ad09), TOBN(0x98abdd51, 0xdaa4d2fc), + TOBN(0xd9840a31, 0x8d168be5), TOBN(0xcf7c10e0, 0x2325a23c), + TOBN(0xa5c02aa0, 0x7e6ecfaf), TOBN(0x2462e7e6, 0xb5bfdf18), + TOBN(0xab2d8a8b, 0xa0cc3f12), TOBN(0x68dd485d, 0xbc672a29), + TOBN(0x72039752, 0x596f2cd3), TOBN(0x5d3eea67, 0xa0cf3d8d), + TOBN(0x810a1a81, 0xe6602671), TOBN(0x8f144a40, 0x14026c0c), + TOBN(0xbc753a6d, 0x76b50f85), TOBN(0xc4dc21e8, 0x645cd4a4), + TOBN(0xc5262dea, 0x521d0378), TOBN(0x802b8e0e, 0x05011c6f), + TOBN(0x1ba19cbb, 0x0b4c19ea), TOBN(0x21db64b5, 0xebf0aaec), + TOBN(0x1f394ee9, 0x70342f9d), TOBN(0x93a10aee, 0x1bc44a14), + TOBN(0xa7eed31b, 0x3efd0baa), TOBN(0x6e7c824e, 0x1d154e65), + TOBN(0xee23fa81, 0x9966e7ee), TOBN(0x64ec4aa8, 0x05b7920d), + TOBN(0x2d44462d, 0x2d90aad4), TOBN(0xf44dd195, 0xdf277ad5), + TOBN(0x8d6471f1, 0xbb46b6a1), TOBN(0x1e65d313, 0xfd885090), + TOBN(0x33a800f5, 0x13a977b4), TOBN(0xaca9d721, 0x0797e1ef), + TOBN(0x9a5a85a0, 0xfcff6a17), TOBN(0x9970a3f3, 0x1eca7cee), + TOBN(0xbb9f0d6b, 0xc9504be3), TOBN(0xe0c504be, 0xadd24ee2), + TOBN(0x7e09d956, 0x77fcc2f4), TOBN(0xef1a5227, 0x65bb5fc4), + TOBN(0x145d4fb1, 0x8b9286aa), TOBN(0x66fd0c5d, 0x6649028b), + TOBN(0x98857ceb, 0x1bf4581c), TOBN(0xe635e186, 0xaca7b166), + TOBN(0x278ddd22, 0x659722ac), TOBN(0xa0903c4c, 0x1db68007), + TOBN(0x366e4589, 0x48f21402), TOBN(0x31b49c14, 0xb96abda2), + TOBN(0x329c4b09, 0xe0403190), TOBN(0x97197ca3, 0xd29f43fe), + TOBN(0x8073dd1e, 0x274983d8), TOBN(0xda1a3bde, 0x55717c8f), + TOBN(0xfd3d4da2, 0x0361f9d1), TOBN(0x1332d081, 0x4c7de1ce), + TOBN(0x9b7ef7a3, 0xaa6d0e10), TOBN(0x17db2e73, 0xf54f1c4a), + TOBN(0xaf3dffae, 0x4cd35567), TOBN(0xaaa2f406, 0xe56f4e71), + TOBN(0x8966759e, 0x7ace3fc7), TOBN(0x9594eacf, 0x45a8d8c6), + TOBN(0x8de3bd8b, 0x91834e0e), TOBN(0xafe4ca53, 0x548c0421), + TOBN(0xfdd7e856, 0xe6ee81c6), TOBN(0x8f671beb, 0x6b891a3a), + TOBN(0xf7a58f2b, 0xfae63829), TOBN(0x9ab186fb, 0x9c11ac9f), + TOBN(0x8d6eb369, 0x10b5be76), TOBN(0x046b7739, 0xfb040bcd), + TOBN(0xccb4529f, 0xcb73de88), TOBN(0x1df0fefc, 0xcf26be03), + TOBN(0xad7757a6, 0xbcfcd027), TOBN(0xa8786c75, 0xbb3165ca), + TOBN(0xe9db1e34, 0x7e99a4d9), TOBN(0x99ee86df, 0xb06c504b), + TOBN(0x5b7c2ddd, 0xc15c9f0a), TOBN(0xdf87a734, 0x4295989e), + TOBN(0x59ece47c, 0x03d08fda), TOBN(0xb074d3dd, 0xad5fc702), + TOBN(0x20407903, 0x51a03776), TOBN(0x2bb1f77b, 0x2a608007), + TOBN(0x25c58f4f, 0xe1153185), TOBN(0xe6df62f6, 0x766e6447), + TOBN(0xefb3d1be, 0xed51275a), TOBN(0x5de47dc7, 0x2f0f483f), + TOBN(0x7932d98e, 0x97c2bedf), TOBN(0xd5c11927, 0x0219f8a1), + TOBN(0x9d751200, 0xa73a294e), TOBN(0x5f88434a, 0x9dc20172), + TOBN(0xd28d9fd3, 0xa26f506a), TOBN(0xa890cd31, 0x9d1dcd48), + TOBN(0x0aebaec1, 0x70f4d3b4), TOBN(0xfd1a1369, 0x0ffc8d00), + TOBN(0xb9d9c240, 0x57d57838), TOBN(0x45929d26, 0x68bac361), + TOBN(0x5a2cd060, 0x25b15ca6), TOBN(0x4b3c83e1, 0x6e474446), + TOBN(0x1aac7578, 0xee1e5134), TOBN(0xa418f5d6, 0xc91e2f41), + TOBN(0x6936fc8a, 0x213ed68b), TOBN(0x860ae7ed, 0x510a5224), + TOBN(0x63660335, 0xdef09b53), TOBN(0x641b2897, 0xcd79c98d), + TOBN(0x29bd38e1, 0x01110f35), TOBN(0x79c26f42, 0x648b1937), + TOBN(0x64dae519, 0x9d9164f4), TOBN(0xd85a2310, 0x0265c273), + TOBN(0x7173dd5d, 0x4b07e2b1), TOBN(0xd144c4cb, 0x8d9ea221), + TOBN(0xe8b04ea4, 0x1105ab14), TOBN(0x92dda542, 0xfe80d8f1), + TOBN(0xe9982fa8, 0xcf03dce6), TOBN(0x8b5ea965, 0x1a22cffc), + TOBN(0xf7f4ea7f, 0x3fad88c4), TOBN(0x62db773e, 0x6a5ba95c), + TOBN(0xd20f02fb, 0x93f24567), TOBN(0xfd46c69a, 0x315257ca), + TOBN(0x0ac74cc7, 0x8bcab987), TOBN(0x46f31c01, 0x5ceca2f5), + TOBN(0x40aedb59, 0x888b219e), TOBN(0xe50ecc37, 0xe1fccd02), + TOBN(0x1bcd9dad, 0x911f816c), TOBN(0x583cc1ec, 0x8db9b00c), + TOBN(0xf3cd2e66, 0xa483bf11), TOBN(0xfa08a6f5, 0xb1b2c169), + TOBN(0xf375e245, 0x4be9fa28), TOBN(0x99a7ffec, 0x5b6d011f), + TOBN(0x6a3ebddb, 0xc4ae62da), TOBN(0x6cea00ae, 0x374aef5d), + TOBN(0xab5fb98d, 0x9d4d05bc), TOBN(0x7cba1423, 0xd560f252), + TOBN(0x49b2cc21, 0x208490de), TOBN(0x1ca66ec3, 0xbcfb2879), + TOBN(0x7f1166b7, 0x1b6fb16f), TOBN(0xfff63e08, 0x65fe5db3), + TOBN(0xb8345abe, 0x8b2610be), TOBN(0xb732ed80, 0x39de3df4), + TOBN(0x0e24ed50, 0x211c32b4), TOBN(0xd10d8a69, 0x848ff27d), + TOBN(0xc1074398, 0xed4de248), TOBN(0xd7cedace, 0x10488927), + TOBN(0xa4aa6bf8, 0x85673e13), TOBN(0xb46bae91, 0x6daf30af), + TOBN(0x07088472, 0xfcef7ad8), TOBN(0x61151608, 0xd4b35e97), + TOBN(0xbcfe8f26, 0xdde29986), TOBN(0xeb84c4c7, 0xd5a34c79), + TOBN(0xc1eec55c, 0x164e1214), TOBN(0x891be86d, 0xa147bb03), + TOBN(0x9fab4d10, 0x0ba96835), TOBN(0xbf01e9b8, 0xa5c1ae9f), + TOBN(0x6b4de139, 0xb186ebc0), TOBN(0xd5c74c26, 0x85b91bca), + TOBN(0x5086a99c, 0xc2d93854), TOBN(0xeed62a7b, 0xa7a9dfbc), + TOBN(0x8778ed6f, 0x76b7618a), TOBN(0xbff750a5, 0x03b66062), + TOBN(0x4cb7be22, 0xb65186db), TOBN(0x369dfbf0, 0xcc3a6d13), + TOBN(0xc7dab26c, 0x7191a321), TOBN(0x9edac3f9, 0x40ed718e), + TOBN(0xbc142b36, 0xd0cfd183), TOBN(0xc8af82f6, 0x7c991693), + TOBN(0xb3d1e4d8, 0x97ce0b2a), TOBN(0xe6d7c87f, 0xc3a55cdf), + TOBN(0x35846b95, 0x68b81afe), TOBN(0x018d12af, 0xd3c239d8), + TOBN(0x2b2c6208, 0x01206e15), TOBN(0xe0e42453, 0xa3b882c6), + TOBN(0x854470a3, 0xa50162d5), TOBN(0x08157478, 0x7017a62a), + TOBN(0x18bd3fb4, 0x820357c7), TOBN(0x992039ae, 0x6f1458ad), + TOBN(0x9a1df3c5, 0x25b44aa1), TOBN(0x2d780357, 0xed3d5281), + TOBN(0x58cf7e4d, 0xc77ad4d4), TOBN(0xd49a7998, 0xf9df4fc4), + TOBN(0x4465a8b5, 0x1d71205e), TOBN(0xa0ee0ea6, 0x649254aa), + TOBN(0x4b5eeecf, 0xab7bd771), TOBN(0x6c873073, 0x35c262b9), + TOBN(0xdc5bd648, 0x3c9d61e7), TOBN(0x233d6d54, 0x321460d2), + TOBN(0xd20c5626, 0xfc195bcc), TOBN(0x25445958, 0x04d78b63), + TOBN(0xe03fcb3d, 0x17ec8ef3), TOBN(0x54b690d1, 0x46b8f781), + TOBN(0x82fa2c8a, 0x21230646), TOBN(0xf51aabb9, 0x084f418c), + TOBN(0xff4fbec1, 0x1a30ba43), TOBN(0x6a5acf73, 0x743c9df7), + TOBN(0x1da2b357, 0xd635b4d5), TOBN(0xc3de68dd, 0xecd5c1da), + TOBN(0xa689080b, 0xd61af0dd), TOBN(0xdea5938a, 0xd665bf99), + TOBN(0x0231d71a, 0xfe637294), TOBN(0x01968aa6, 0xa5a81cd8), + TOBN(0x11252d50, 0x048e63b5), TOBN(0xc446bc52, 0x6ca007e9), + TOBN(0xef8c50a6, 0x96d6134b), TOBN(0x9361fbf5, 0x9e09a05c), + TOBN(0xf17f85a6, 0xdca3291a), TOBN(0xb178d548, 0xff251a21), + TOBN(0x87f6374b, 0xa4df3915), TOBN(0x566ce1bf, 0x2fd5d608), + TOBN(0x425cba4d, 0x7de35102), TOBN(0x6b745f8f, 0x58c5d5e2), + TOBN(0x88402af6, 0x63122edf), TOBN(0x3190f9ed, 0x3b989a89), + TOBN(0x4ad3d387, 0xebba3156), TOBN(0xef385ad9, 0xc7c469a5), + TOBN(0xb08281de, 0x3f642c29), TOBN(0x20be0888, 0x910ffb88), + TOBN(0xf353dd4a, 0xd5292546), TOBN(0x3f1627de, 0x8377a262), + TOBN(0xa5faa013, 0xeefcd638), TOBN(0x8f3bf626, 0x74cc77c3), + TOBN(0x32618f65, 0xa348f55e), TOBN(0x5787c0dc, 0x9fefeb9e), + TOBN(0xf1673aa2, 0xd9a23e44), TOBN(0x88dfa993, 0x4e10690d), + TOBN(0x1ced1b36, 0x2bf91108), TOBN(0x9193ceca, 0x3af48649), + TOBN(0xfb34327d, 0x2d738fc5), TOBN(0x6697b037, 0x975fee6c), + TOBN(0x2f485da0, 0xc04079a5), TOBN(0x2cdf5735, 0x2feaa1ac), + TOBN(0x76944420, 0xbd55659e), TOBN(0x7973e32b, 0x4376090c), + TOBN(0x86bb4fe1, 0x163b591a), TOBN(0x10441aed, 0xc196f0ca), + TOBN(0x3b431f4a, 0x045ad915), TOBN(0x6c11b437, 0xa4afacb1), + TOBN(0x30b0c7db, 0x71fdbbd8), TOBN(0xb642931f, 0xeda65acd), + TOBN(0x4baae6e8, 0x9c92b235), TOBN(0xa73bbd0e, 0x6b3993a1), + TOBN(0xd06d60ec, 0x693dd031), TOBN(0x03cab91b, 0x7156881c), + TOBN(0xd615862f, 0x1db3574b), TOBN(0x485b0185, 0x64bb061a), + TOBN(0x27434988, 0xa0181e06), TOBN(0x2cd61ad4, 0xc1c0c757), + TOBN(0x3effed5a, 0x2ff9f403), TOBN(0x8dc98d8b, 0x62239029), + TOBN(0x2206021e, 0x1f17b70d), TOBN(0xafbec0ca, 0xbf510015), + TOBN(0x9fed7164, 0x80130dfa), TOBN(0x306dc2b5, 0x8a02dcf5), + TOBN(0x48f06620, 0xfeb10fc0), TOBN(0x78d1e1d5, 0x5a57cf51), + TOBN(0xadef8c5a, 0x192ef710), TOBN(0x88afbd4b, 0x3b7431f9), + TOBN(0x7e1f7407, 0x64250c9e), TOBN(0x6e31318d, 0xb58bec07), + TOBN(0xfd4fc4b8, 0x24f89b4e), TOBN(0x65a5dd88, 0x48c36a2a), + TOBN(0x4f1eccff, 0xf024baa7), TOBN(0x22a21cf2, 0xcba94650), + TOBN(0x95d29dee, 0x42a554f7), TOBN(0x828983a5, 0x002ec4ba), + TOBN(0x8112a1f7, 0x8badb73d), TOBN(0x79ea8897, 0xa27c1839), + TOBN(0x8969a5a7, 0xd065fd83), TOBN(0xf49af791, 0xb262a0bc), + TOBN(0xfcdea8b6, 0xaf2b5127), TOBN(0x10e913e1, 0x564c2dbc), + TOBN(0x51239d14, 0xbc21ef51), TOBN(0xe51c3ceb, 0x4ce57292), + TOBN(0x795ff068, 0x47bbcc3b), TOBN(0x86b46e1e, 0xbd7e11e6), + TOBN(0x0ea6ba23, 0x80041ef4), TOBN(0xd72fe505, 0x6262342e), + TOBN(0x8abc6dfd, 0x31d294d4), TOBN(0xbbe017a2, 0x1278c2c9), + TOBN(0xb1fcfa09, 0xb389328a), TOBN(0x322fbc62, 0xd01771b5), + TOBN(0x04c0d063, 0x60b045bf), TOBN(0xdb652edc, 0x10e52d01), + TOBN(0x50ef932c, 0x03ec6627), TOBN(0xde1b3b2d, 0xc1ee50e3), + TOBN(0x5ab7bdc5, 0xdc37a90d), TOBN(0xfea67213, 0x31e33a96), + TOBN(0x6482b5cb, 0x4f2999aa), TOBN(0x38476cc6, 0xb8cbf0dd), + TOBN(0x93ebfacb, 0x173405bb), TOBN(0x15cdafe7, 0xe52369ec), + TOBN(0xd42d5ba4, 0xd935b7db), TOBN(0x648b6004, 0x1c99a4cd), + TOBN(0x785101bd, 0xa3b5545b), TOBN(0x4bf2c38a, 0x9dd67faf), + TOBN(0xb1aadc63, 0x4442449c), TOBN(0xe0e9921a, 0x33ad4fb8), + TOBN(0x5c552313, 0xaa686d82), TOBN(0xdee635fa, 0x465d866c), + TOBN(0xbc3c224a, 0x18ee6e8a), TOBN(0xeed748a6, 0xed42e02f), + TOBN(0xe70f930a, 0xd474cd08), TOBN(0x774ea6ec, 0xfff24adf), + TOBN(0x03e2de1c, 0xf3480d4a), TOBN(0xf0d8edc7, 0xbc8acf1a), + TOBN(0xf23e3303, 0x68295a9c), TOBN(0xfadd5f68, 0xc546a97d), + TOBN(0x895597ad, 0x96f8acb1), TOBN(0xbddd49d5, 0x671bdae2), + TOBN(0x16fcd528, 0x21dd43f4), TOBN(0xa5a45412, 0x6619141a)} + , + {TOBN(0x8ce9b6bf, 0xc360e25a), TOBN(0xe6425195, 0x075a1a78), + TOBN(0x9dc756a8, 0x481732f4), TOBN(0x83c0440f, 0x5432b57a), + TOBN(0xc670b3f1, 0xd720281f), TOBN(0x2205910e, 0xd135e051), + TOBN(0xded14b0e, 0xdb052be7), TOBN(0x697b3d27, 0xc568ea39), + TOBN(0x2e599b9a, 0xfb3ff9ed), TOBN(0x28c2e0ab, 0x17f6515c), + TOBN(0x1cbee4fd, 0x474da449), TOBN(0x071279a4, 0x4f364452), + TOBN(0x97abff66, 0x01fbe855), TOBN(0x3ee394e8, 0x5fda51c4), + TOBN(0x190385f6, 0x67597c0b), TOBN(0x6e9fccc6, 0xa27ee34b), + TOBN(0x0b89de93, 0x14092ebb), TOBN(0xf17256bd, 0x428e240c), + TOBN(0xcf89a7f3, 0x93d2f064), TOBN(0x4f57841e, 0xe1ed3b14), + TOBN(0x4ee14405, 0xe708d855), TOBN(0x856aae72, 0x03f1c3d0), + TOBN(0xc8e5424f, 0xbdd7eed5), TOBN(0x3333e4ef, 0x73ab4270), + TOBN(0x3bc77ade, 0xdda492f8), TOBN(0xc11a3aea, 0x78297205), + TOBN(0x5e89a3e7, 0x34931b4c), TOBN(0x17512e2e, 0x9f5694bb), + TOBN(0x5dc349f3, 0x177bf8b6), TOBN(0x232ea4ba, 0x08c7ff3e), + TOBN(0x9c4f9d16, 0xf511145d), TOBN(0xccf109a3, 0x33b379c3), + TOBN(0xe75e7a88, 0xa1f25897), TOBN(0x7ac6961f, 0xa1b5d4d8), + TOBN(0xe3e10773, 0x08f3ed5c), TOBN(0x208a54ec, 0x0a892dfb), + TOBN(0xbe826e19, 0x78660710), TOBN(0x0cf70a97, 0x237df2c8), + TOBN(0x418a7340, 0xed704da5), TOBN(0xa3eeb9a9, 0x08ca33fd), + TOBN(0x49d96233, 0x169bca96), TOBN(0x04d286d4, 0x2da6aafb), + TOBN(0xc09606ec, 0xa0c2fa94), TOBN(0x8869d0d5, 0x23ff0fb3), + TOBN(0xa99937e5, 0xd0150d65), TOBN(0xa92e2503, 0x240c14c9), + TOBN(0x656bf945, 0x108e2d49), TOBN(0x152a733a, 0xa2f59e2b), + TOBN(0xb4323d58, 0x8434a920), TOBN(0xc0af8e93, 0x622103c5), + TOBN(0x667518ef, 0x938dbf9a), TOBN(0xa1843073, 0x83a9cdf2), + TOBN(0x350a94aa, 0x5447ab80), TOBN(0xe5e5a325, 0xc75a3d61), + TOBN(0x74ba507f, 0x68411a9e), TOBN(0x10581fc1, 0x594f70c5), + TOBN(0x60e28570, 0x80eb24a9), TOBN(0x7bedfb4d, 0x488e0cfd), + TOBN(0x721ebbd7, 0xc259cdb8), TOBN(0x0b0da855, 0xbc6390a9), + TOBN(0x2b4d04db, 0xde314c70), TOBN(0xcdbf1fbc, 0x6c32e846), + TOBN(0x33833eab, 0xb162fc9e), TOBN(0x9939b48b, 0xb0dd3ab7), + TOBN(0x5aaa98a7, 0xcb0c9c8c), TOBN(0x75105f30, 0x81c4375c), + TOBN(0xceee5057, 0x5ef1c90f), TOBN(0xb31e065f, 0xc23a17bf), + TOBN(0x5364d275, 0xd4b6d45a), TOBN(0xd363f3ad, 0x62ec8996), + TOBN(0xb5d21239, 0x4391c65b), TOBN(0x84564765, 0xebb41b47), + TOBN(0x20d18ecc, 0x37107c78), TOBN(0xacff3b6b, 0x570c2a66), + TOBN(0x22f975d9, 0x9bd0d845), TOBN(0xef0a0c46, 0xba178fa0), + TOBN(0x1a419651, 0x76b6028e), TOBN(0xc49ec674, 0x248612d4), + TOBN(0x5b6ac4f2, 0x7338af55), TOBN(0x06145e62, 0x7bee5a36), + TOBN(0x33e95d07, 0xe75746b5), TOBN(0x1c1e1f6d, 0xc40c78be), + TOBN(0x967833ef, 0x222ff8e2), TOBN(0x4bedcf6a, 0xb49180ad), + TOBN(0x6b37e9c1, 0x3d7a4c8a), TOBN(0x2748887c, 0x6ddfe760), + TOBN(0xf7055123, 0xaa3a5bbc), TOBN(0x954ff225, 0x7bbb8e74), + TOBN(0xc42b8ab1, 0x97c3dfb9), TOBN(0x55a549b0, 0xcf168154), + TOBN(0xad6748e7, 0xc1b50692), TOBN(0x2775780f, 0x6fc5cbcb), + TOBN(0x4eab80b8, 0xe1c9d7c8), TOBN(0x8c69dae1, 0x3fdbcd56), + TOBN(0x47e6b4fb, 0x9969eace), TOBN(0x002f1085, 0xa705cb5a), + TOBN(0x4e23ca44, 0x6d3fea55), TOBN(0xb4ae9c86, 0xf4810568), + TOBN(0x47bfb91b, 0x2a62f27d), TOBN(0x60deb4c9, 0xd9bac28c), + TOBN(0xa892d894, 0x7de6c34c), TOBN(0x4ee68259, 0x4494587d), + TOBN(0x914ee14e, 0x1a3f8a5b), TOBN(0xbb113eaa, 0x28700385), + TOBN(0x81ca03b9, 0x2115b4c9), TOBN(0x7c163d38, 0x8908cad1), + TOBN(0xc912a118, 0xaa18179a), TOBN(0xe09ed750, 0x886e3081), + TOBN(0xa676e3fa, 0x26f516ca), TOBN(0x753cacf7, 0x8e732f91), + TOBN(0x51592aea, 0x833da8b4), TOBN(0xc626f42f, 0x4cbea8aa), + TOBN(0xef9dc899, 0xa7b56eaf), TOBN(0x00c0e52c, 0x34ef7316), + TOBN(0x5b1e4e24, 0xfe818a86), TOBN(0x9d31e20d, 0xc538be47), + TOBN(0x22eb932d, 0x3ed68974), TOBN(0xe44bbc08, 0x7c4e87c4), + TOBN(0x4121086e, 0x0dde9aef), TOBN(0x8e6b9cff, 0x134f4345), + TOBN(0x96892c1f, 0x711b0eb9), TOBN(0xb905f2c8, 0x780ab954), + TOBN(0xace26309, 0xa20792db), TOBN(0xec8ac9b3, 0x0684e126), + TOBN(0x486ad8b6, 0xb40a2447), TOBN(0x60121fc1, 0x9fe3fb24), + TOBN(0x5626fccf, 0x1a8e3b3f), TOBN(0x4e568622, 0x6ad1f394), + TOBN(0xda7aae0d, 0x196aa5a1), TOBN(0xe0df8c77, 0x1041b5fb), + TOBN(0x451465d9, 0x26b318b7), TOBN(0xc29b6e55, 0x7ab136e9), + TOBN(0x2c2ab48b, 0x71148463), TOBN(0xb5738de3, 0x64454a76), + TOBN(0x54ccf9a0, 0x5a03abe4), TOBN(0x377c0296, 0x0427d58e), + TOBN(0x73f5f0b9, 0x2bb39c1f), TOBN(0x14373f2c, 0xe608d8c5), + TOBN(0xdcbfd314, 0x00fbb805), TOBN(0xdf18fb20, 0x83afdcfb), + TOBN(0x81a57f42, 0x42b3523f), TOBN(0xe958532d, 0x87f650fb), + TOBN(0xaa8dc8b6, 0x8b0a7d7c), TOBN(0x1b75dfb7, 0x150166be), + TOBN(0x90e4f7c9, 0x2d7d1413), TOBN(0x67e2d6b5, 0x9834f597), + TOBN(0x4fd4f4f9, 0xa808c3e8), TOBN(0xaf8237e0, 0xd5281ec1), + TOBN(0x25ab5fdc, 0x84687cee), TOBN(0xc5ded6b1, 0xa5b26c09), + TOBN(0x8e4a5aec, 0xc8ea7650), TOBN(0x23b73e5c, 0x14cc417f), + TOBN(0x2bfb4318, 0x3037bf52), TOBN(0xb61e6db5, 0x78c725d7), + TOBN(0x8efd4060, 0xbbb3e5d7), TOBN(0x2e014701, 0xdbac488e), + TOBN(0xac75cf9a, 0x360aa449), TOBN(0xb70cfd05, 0x79634d08), + TOBN(0xa591536d, 0xfffb15ef), TOBN(0xb2c37582, 0xd07c106c), + TOBN(0xb4293fdc, 0xf50225f9), TOBN(0xc52e175c, 0xb0e12b03), + TOBN(0xf649c3ba, 0xd0a8bf64), TOBN(0x745a8fef, 0xeb8ae3c6), + TOBN(0x30d7e5a3, 0x58321bc3), TOBN(0xb1732be7, 0x0bc4df48), + TOBN(0x1f217993, 0xe9ea5058), TOBN(0xf7a71cde, 0x3e4fd745), + TOBN(0x86cc533e, 0x894c5bbb), TOBN(0x6915c7d9, 0x69d83082), + TOBN(0xa6aa2d05, 0x5815c244), TOBN(0xaeeee592, 0x49b22ce5), + TOBN(0x89e39d13, 0x78135486), TOBN(0x3a275c1f, 0x16b76f2f), + TOBN(0xdb6bcc1b, 0xe036e8f5), TOBN(0x4df69b21, 0x5e4709f5), + TOBN(0xa188b250, 0x2d0f39aa), TOBN(0x622118bb, 0x15a85947), + TOBN(0x2ebf520f, 0xfde0f4fa), TOBN(0xa40e9f29, 0x4860e539), + TOBN(0x7b6a51eb, 0x22b57f0f), TOBN(0x849a33b9, 0x7e80644a), + TOBN(0x50e5d16f, 0x1cf095fe), TOBN(0xd754b54e, 0xec55f002), + TOBN(0x5cfbbb22, 0x236f4a98), TOBN(0x0b0c59e9, 0x066800bb), + TOBN(0x4ac69a8f, 0x5a9a7774), TOBN(0x2b33f804, 0xd6bec948), + TOBN(0xb3729295, 0x32e6c466), TOBN(0x68956d0f, 0x4e599c73), + TOBN(0xa47a249f, 0x155c31cc), TOBN(0x24d80f0d, 0xe1ce284e), + TOBN(0xcd821dfb, 0x988baf01), TOBN(0xe6331a7d, 0xdbb16647), + TOBN(0x1eb8ad33, 0x094cb960), TOBN(0x593cca38, 0xc91bbca5), + TOBN(0x384aac8d, 0x26567456), TOBN(0x40fa0309, 0xc04b6490), + TOBN(0x97834cd6, 0xdab6c8f6), TOBN(0x68a7318d, 0x3f91e55f), + TOBN(0xa00fd04e, 0xfc4d3157), TOBN(0xb56f8ab2, 0x2bf3bdea), + TOBN(0x014f5648, 0x4fa57172), TOBN(0x948c5860, 0x450abdb3), + TOBN(0x342b5df0, 0x0ebd4f08), TOBN(0x3e5168cd, 0x0e82938e), + TOBN(0x7aedc1ce, 0xb0df5dd0), TOBN(0x6bbbc6d9, 0xe5732516), + TOBN(0xc7bfd486, 0x605daaa6), TOBN(0x46fd72b7, 0xbb9a6c9e), + TOBN(0xe4847fb1, 0xa124fb89), TOBN(0x75959cbd, 0xa2d8ffbc), + TOBN(0x42579f65, 0xc8a588ee), TOBN(0x368c92e6, 0xb80b499d), + TOBN(0xea4ef6cd, 0x999a5df1), TOBN(0xaa73bb7f, 0x936fe604), + TOBN(0xf347a70d, 0x6457d188), TOBN(0x86eda86b, 0x8b7a388b), + TOBN(0xb7cdff06, 0x0ccd6013), TOBN(0xbeb1b6c7, 0xd0053fb2), + TOBN(0x0b022387, 0x99240a9f), TOBN(0x1bbb384f, 0x776189b2), + TOBN(0x8695e71e, 0x9066193a), TOBN(0x2eb50097, 0x06ffac7e), + TOBN(0x0654a9c0, 0x4a7d2caa), TOBN(0x6f3fb3d1, 0xa5aaa290), + TOBN(0x835db041, 0xff476e8f), TOBN(0x540b8b0b, 0xc42295e4), + TOBN(0xa5c73ac9, 0x05e214f5), TOBN(0x9a74075a, 0x56a0b638), + TOBN(0x2e4b1090, 0xce9e680b), TOBN(0x57a5b479, 0x6b8d9afa), + TOBN(0x0dca48e7, 0x26bfe65c), TOBN(0x097e391c, 0x7290c307), + TOBN(0x683c462e, 0x6669e72e), TOBN(0xf505be1e, 0x062559ac), + TOBN(0x5fbe3ea1, 0xe3a3035a), TOBN(0x6431ebf6, 0x9cd50da8), + TOBN(0xfd169d5c, 0x1f6407f2), TOBN(0x8d838a95, 0x60fce6b8), + TOBN(0x2a2bfa7f, 0x650006f0), TOBN(0xdfd7dad3, 0x50c0fbb2), + TOBN(0x92452495, 0xccf9ad96), TOBN(0x183bf494, 0xd95635f9), + TOBN(0x02d5df43, 0x4a7bd989), TOBN(0x505385cc, 0xa5431095), + TOBN(0xdd98e67d, 0xfd43f53e), TOBN(0xd61e1a6c, 0x500c34a9), + TOBN(0x5a4b46c6, 0x4a8a3d62), TOBN(0x8469c4d0, 0x247743d2), + TOBN(0x2bb3a13d, 0x88f7e433), TOBN(0x62b23a10, 0x01be5849), + TOBN(0xe83596b4, 0xa63d1a4c), TOBN(0x454e7fea, 0x7d183f3e), + TOBN(0x643fce61, 0x17afb01c), TOBN(0x4e65e5e6, 0x1c4c3638), + TOBN(0x41d85ea1, 0xef74c45b), TOBN(0x2cfbfa66, 0xae328506), + TOBN(0x98b078f5, 0x3ada7da9), TOBN(0xd985fe37, 0xec752fbb), + TOBN(0xeece68fe, 0x5a0148b4), TOBN(0x6f9a55c7, 0x2d78136d), + TOBN(0x232dccc4, 0xd2b729ce), TOBN(0xa27e0dfd, 0x90aafbc4), + TOBN(0x96474452, 0x12b4603e), TOBN(0xa876c551, 0x6b706d14), + TOBN(0xdf145fcf, 0x69a9d412), TOBN(0xe2ab75b7, 0x2d479c34), + TOBN(0x12df9a76, 0x1a23ff97), TOBN(0xc6138992, 0x5d359d10), + TOBN(0x6e51c7ae, 0xfa835f22), TOBN(0x69a79cb1, 0xc0fcc4d9), + TOBN(0xf57f350d, 0x594cc7e1), TOBN(0x3079ca63, 0x3350ab79), + TOBN(0x226fb614, 0x9aff594a), TOBN(0x35afec02, 0x6d59a62b), + TOBN(0x9bee46f4, 0x06ed2c6e), TOBN(0x58da1735, 0x7d939a57), + TOBN(0x44c50402, 0x8fd1797e), TOBN(0xd8853e7c, 0x5ccea6ca), + TOBN(0x4065508d, 0xa35fcd5f), TOBN(0x8965df8c, 0x495ccaeb), + TOBN(0x0f2da850, 0x12e1a962), TOBN(0xee471b94, 0xc1cf1cc4), + TOBN(0xcef19bc8, 0x0a08fb75), TOBN(0x704958f5, 0x81de3591), + TOBN(0x2867f8b2, 0x3aef4f88), TOBN(0x8d749384, 0xea9f9a5f), + TOBN(0x1b385537, 0x8c9049f4), TOBN(0x5be948f3, 0x7b92d8b6), + TOBN(0xd96f725d, 0xb6e2bd6b), TOBN(0x37a222bc, 0x958c454d), + TOBN(0xe7c61abb, 0x8809bf61), TOBN(0x46f07fbc, 0x1346f18d), + TOBN(0xfb567a7a, 0xe87c0d1c), TOBN(0x84a461c8, 0x7ef3d07a), + TOBN(0x0a5adce6, 0xd9278d98), TOBN(0x24d94813, 0x9dfc73e1), + TOBN(0x4f3528b6, 0x054321c3), TOBN(0x2e03fdde, 0x692ea706), + TOBN(0x10e60619, 0x47b533c0), TOBN(0x1a8bc73f, 0x2ca3c055), + TOBN(0xae58d4b2, 0x1bb62b8f), TOBN(0xb2045a73, 0x584a24e3), + TOBN(0x3ab3d5af, 0xbd76e195), TOBN(0x478dd1ad, 0x6938a810), + TOBN(0x6ffab393, 0x6ee3d5cb), TOBN(0xdfb693db, 0x22b361e4), + TOBN(0xf9694496, 0x51dbf1a7), TOBN(0xcab4b4ef, 0x08a2e762), + TOBN(0xe8c92f25, 0xd39bba9a), TOBN(0x850e61bc, 0xf1464d96), + TOBN(0xb7e830e3, 0xdc09508b), TOBN(0xfaf6d2cf, 0x74317655), + TOBN(0x72606ceb, 0xdf690355), TOBN(0x48bb92b3, 0xd0c3ded6), + TOBN(0x65b75484, 0x5c7cf892), TOBN(0xf6cd7ac9, 0xd5d5f01f), + TOBN(0xc2c30a59, 0x96401d69), TOBN(0x91268650, 0xed921878), + TOBN(0x380bf913, 0xb78c558f), TOBN(0x43c0baeb, 0xc8afdaa9), + TOBN(0x377f61d5, 0x54f169d3), TOBN(0xf8da07e3, 0xae5ff20b), + TOBN(0xb676c49d, 0xa8a90ea8), TOBN(0x81c1ff2b, 0x83a29b21), + TOBN(0x383297ac, 0x2ad8d276), TOBN(0x3001122f, 0xba89f982), + TOBN(0xe1d794be, 0x6718e448), TOBN(0x246c1482, 0x7c3e6e13), + TOBN(0x56646ef8, 0x5d26b5ef), TOBN(0x80f5091e, 0x88069cdd), + TOBN(0xc5992e2f, 0x724bdd38), TOBN(0x02e915b4, 0x8471e8c7), + TOBN(0x96ff320a, 0x0d0ff2a9), TOBN(0xbf886487, 0x4384d1a0), + TOBN(0xbbe1e6a6, 0xc93f72d6), TOBN(0xd5f75d12, 0xcad800ea), + TOBN(0xfa40a09f, 0xe7acf117), TOBN(0x32c8cdd5, 0x7581a355), + TOBN(0x74221992, 0x7023c499), TOBN(0xa8afe5d7, 0x38ec3901), + TOBN(0x5691afcb, 0xa90e83f0), TOBN(0x41bcaa03, 0x0b8f8eac), + TOBN(0xe38b5ff9, 0x8d2668d5), TOBN(0x0715281a, 0x7ad81965), + TOBN(0x1bc8fc7c, 0x03c6ce11), TOBN(0xcbbee6e2, 0x8b650436), + TOBN(0x06b00fe8, 0x0cdb9808), TOBN(0x17d6e066, 0xfe3ed315), + TOBN(0x2e9d38c6, 0x4d0b5018), TOBN(0xab8bfd56, 0x844dcaef), + TOBN(0x42894a59, 0x513aed8b), TOBN(0xf77f3b6d, 0x314bd07a), + TOBN(0xbbdecb8f, 0x8e42b582), TOBN(0xf10e2fa8, 0xd2390fe6), + TOBN(0xefb95022, 0x62a2f201), TOBN(0x4d59ea50, 0x50ee32b0), + TOBN(0xd87f7728, 0x6da789a8), TOBN(0xcf98a2cf, 0xf79492c4), + TOBN(0xf9577239, 0x720943c2), TOBN(0xba044cf5, 0x3990b9d0), + TOBN(0x5aa8e823, 0x95f2884a), TOBN(0x834de6ed, 0x0278a0af), + TOBN(0xc8e1ee9a, 0x5f25bd12), TOBN(0x9259ceaa, 0x6f7ab271), + TOBN(0x7e6d97a2, 0x77d00b76), TOBN(0x5c0c6eea, 0xa437832a), + TOBN(0x5232c20f, 0x5606b81d), TOBN(0xabd7b375, 0x0d991ee5), + TOBN(0x4d2bfe35, 0x8632d951), TOBN(0x78f85146, 0x98ed9364), + TOBN(0x951873f0, 0xf30c3282), TOBN(0x0da8ac80, 0xa789230b), + TOBN(0x3ac7789c, 0x5398967f), TOBN(0xa69b8f7f, 0xbdda0fb5), + TOBN(0xe5db7717, 0x6add8545), TOBN(0x1b71cb66, 0x72c49b66), + TOBN(0xd8560739, 0x68421d77), TOBN(0x03840fe8, 0x83e3afea), + TOBN(0xb391dad5, 0x1ec69977), TOBN(0xae243fb9, 0x307f6726), + TOBN(0xc88ac87b, 0xe8ca160c), TOBN(0x5174cced, 0x4ce355f4), + TOBN(0x98a35966, 0xe58ba37d), TOBN(0xfdcc8da2, 0x7817335d), + TOBN(0x5b752830, 0x83fbc7bf), TOBN(0x68e419d4, 0xd9c96984), + TOBN(0x409a39f4, 0x02a40380), TOBN(0x88940faf, 0x1fe977bc), + TOBN(0xc640a94b, 0x8f8edea6), TOBN(0x1e22cd17, 0xed11547d), + TOBN(0xe28568ce, 0x59ffc3e2), TOBN(0x60aa1b55, 0xc1dee4e7), + TOBN(0xc67497c8, 0x837cb363), TOBN(0x06fb438a, 0x105a2bf2), + TOBN(0x30357ec4, 0x500d8e20), TOBN(0x1ad9095d, 0x0670db10), + TOBN(0x7f589a05, 0xc73b7cfd), TOBN(0xf544607d, 0x880d6d28), + TOBN(0x17ba93b1, 0xa20ef103), TOBN(0xad859130, 0x6ba6577b), + TOBN(0x65c91cf6, 0x6fa214a0), TOBN(0xd7d49c6c, 0x27990da5), + TOBN(0xecd9ec8d, 0x20bb569d), TOBN(0xbd4b2502, 0xeeffbc33), + TOBN(0x2056ca5a, 0x6bed0467), TOBN(0x7916a1f7, 0x5b63728c), + TOBN(0xd4f9497d, 0x53a4f566), TOBN(0x89734664, 0x97b56810), + TOBN(0xf8e1da74, 0x0494a621), TOBN(0x82546a93, 0x8d011c68), + TOBN(0x1f3acb19, 0xc61ac162), TOBN(0x52f8fa9c, 0xabad0d3e), + TOBN(0x15356523, 0xb4b7ea43), TOBN(0x5a16ad61, 0xae608125), + TOBN(0xb0bcb87f, 0x4faed184), TOBN(0x5f236b1d, 0x5029f45f), + TOBN(0xd42c7607, 0x0bc6b1fc), TOBN(0xc644324e, 0x68aefce3), + TOBN(0x8e191d59, 0x5c5d8446), TOBN(0xc0208077, 0x13ae1979), + TOBN(0xadcaee55, 0x3ba59cc7), TOBN(0x20ed6d6b, 0xa2cb81ba), + TOBN(0x0952ba19, 0xb6efcffc), TOBN(0x60f12d68, 0x97c0b87c), + TOBN(0x4ee2c7c4, 0x9caa30bc), TOBN(0x767238b7, 0x97fbff4e), + TOBN(0xebc73921, 0x501b5d92), TOBN(0x3279e3df, 0xc2a37737), + TOBN(0x9fc12bc8, 0x6d197543), TOBN(0xfa94dc6f, 0x0a40db4e), + TOBN(0x7392b41a, 0x530ccbbd), TOBN(0x87c82146, 0xea823525), + TOBN(0xa52f984c, 0x05d98d0c), TOBN(0x2ae57d73, 0x5ef6974c), + TOBN(0x9377f7bf, 0x3042a6dd), TOBN(0xb1a007c0, 0x19647a64), + TOBN(0xfaa9079a, 0x0cca9767), TOBN(0x3d81a25b, 0xf68f72d5), + TOBN(0x752067f8, 0xff81578e), TOBN(0x78622150, 0x9045447d), + TOBN(0xc0c22fcf, 0x0505aa6f), TOBN(0x1030f0a6, 0x6bed1c77), + TOBN(0x31f29f15, 0x1f0bd739), TOBN(0x2d7989c7, 0xe6debe85), + TOBN(0x5c070e72, 0x8e677e98), TOBN(0x0a817bd3, 0x06e81fd5), + TOBN(0xc110d830, 0xb0f2ac95), TOBN(0x48d0995a, 0xab20e64e), + TOBN(0x0f3e00e1, 0x7729cd9a), TOBN(0x2a570c20, 0xdd556946), + TOBN(0x912dbcfd, 0x4e86214d), TOBN(0x2d014ee2, 0xcf615498), + TOBN(0x55e2b1e6, 0x3530d76e), TOBN(0xc5135ae4, 0xfd0fd6d1), + TOBN(0x0066273a, 0xd4f3049f), TOBN(0xbb8e9893, 0xe7087477), + TOBN(0x2dba1ddb, 0x14c6e5fd), TOBN(0xdba37886, 0x51f57e6c), + TOBN(0x5aaee0a6, 0x5a72f2cf), TOBN(0x1208bfbf, 0x7bea5642), + TOBN(0xf5c6aa3b, 0x67872c37), TOBN(0xd726e083, 0x43f93224), + TOBN(0x1854daa5, 0x061f1658), TOBN(0xc0016df1, 0xdf0cd2b3), + TOBN(0xc2a3f23e, 0x833d50de), TOBN(0x73b681d2, 0xbbbd3017), + TOBN(0x2f046dc4, 0x3ac343c0), TOBN(0x9c847e7d, 0x85716421), + TOBN(0xe1e13c91, 0x0917eed4), TOBN(0x3fc9eebd, 0x63a1b9c6), + TOBN(0x0f816a72, 0x7fe02299), TOBN(0x6335ccc2, 0x294f3319), + TOBN(0x3820179f, 0x4745c5be), TOBN(0xe647b782, 0x922f066e), + TOBN(0xc22e49de, 0x02cafb8a), TOBN(0x299bc2ff, 0xfcc2eccc), + TOBN(0x9a8feea2, 0x6e0e8282), TOBN(0xa627278b, 0xfe893205), + TOBN(0xa7e19733, 0x7933e47b), TOBN(0xf4ff6b13, 0x2e766402), + TOBN(0xa4d8be0a, 0x98440d9f), TOBN(0x658f5c2f, 0x38938808), + TOBN(0x90b75677, 0xc95b3b3e), TOBN(0xfa044269, 0x3137b6ff), + TOBN(0x077b039b, 0x43c47c29), TOBN(0xcca95dd3, 0x8a6445b2), + TOBN(0x0b498ba4, 0x2333fc4c), TOBN(0x274f8e68, 0xf736a1b1), + TOBN(0x6ca348fd, 0x5f1d4b2e), TOBN(0x24d3be78, 0xa8f10199), + TOBN(0x8535f858, 0xca14f530), TOBN(0xa6e7f163, 0x5b982e51), + TOBN(0x847c8512, 0x36e1bf62), TOBN(0xf6a7c58e, 0x03448418), + TOBN(0x583f3703, 0xf9374ab6), TOBN(0x864f9195, 0x6e564145), + TOBN(0x33bc3f48, 0x22526d50), TOBN(0x9f323c80, 0x1262a496), + TOBN(0xaa97a7ae, 0x3f046a9a), TOBN(0x70da183e, 0xdf8a039a), + TOBN(0x5b68f71c, 0x52aa0ba6), TOBN(0x9be0fe51, 0x21459c2d), + TOBN(0xc1e17eb6, 0xcbc613e5), TOBN(0x33131d55, 0x497ea61c), + TOBN(0x2f69d39e, 0xaf7eded5), TOBN(0x73c2f434, 0xde6af11b), + TOBN(0x4ca52493, 0xa4a375fa), TOBN(0x5f06787c, 0xb833c5c2), + TOBN(0x814e091f, 0x3e6e71cf), TOBN(0x76451f57, 0x8b746666)} + , + {TOBN(0x80f9bdef, 0x694db7e0), TOBN(0xedca8787, 0xb9fcddc6), + TOBN(0x51981c34, 0x03b8dce1), TOBN(0x4274dcf1, 0x70e10ba1), + TOBN(0xf72743b8, 0x6def6d1a), TOBN(0xd25b1670, 0xebdb1866), + TOBN(0xc4491e8c, 0x050c6f58), TOBN(0x2be2b2ab, 0x87fbd7f5), + TOBN(0x3e0e5c9d, 0xd111f8ec), TOBN(0xbcc33f8d, 0xb7c4e760), + TOBN(0x702f9a91, 0xbd392a51), TOBN(0x7da4a795, 0xc132e92d), + TOBN(0x1a0b0ae3, 0x0bb1151b), TOBN(0x54febac8, 0x02e32251), + TOBN(0xea3a5082, 0x694e9e78), TOBN(0xe58ffec1, 0xe4fe40b8), + TOBN(0xf85592fc, 0xd1e0cf9e), TOBN(0xdea75f0d, 0xc0e7b2e8), + TOBN(0xc04215cf, 0xc135584e), TOBN(0x174fc727, 0x2f57092a), + TOBN(0xe7277877, 0xeb930bea), TOBN(0x504caccb, 0x5eb02a5a), + TOBN(0xf9fe08f7, 0xf5241b9b), TOBN(0xe7fb62f4, 0x8d5ca954), + TOBN(0xfbb8349d, 0x29c4120b), TOBN(0x9f94391f, 0xc0d0d915), + TOBN(0xc4074fa7, 0x5410ba51), TOBN(0xa66adbf6, 0x150a5911), + TOBN(0xc164543c, 0x34bfca38), TOBN(0xe0f27560, 0xb9e1ccfc), + TOBN(0x99da0f53, 0xe820219c), TOBN(0xe8234498, 0xc6b4997a), + TOBN(0xcfb88b76, 0x9d4c5423), TOBN(0x9e56eb10, 0xb0521c49), + TOBN(0x418e0b5e, 0xbe8700a1), TOBN(0x00cbaad6, 0xf93cb58a), + TOBN(0xe923fbde, 0xd92a5e67), TOBN(0xca4979ac, 0x1f347f11), + TOBN(0x89162d85, 0x6bc0585b), TOBN(0xdd6254af, 0xac3c70e3), + TOBN(0x7b23c513, 0x516e19e4), TOBN(0x56e2e847, 0xc5c4d593), + TOBN(0x9f727d73, 0x5ce71ef6), TOBN(0x5b6304a6, 0xf79a44c5), + TOBN(0x6638a736, 0x3ab7e433), TOBN(0x1adea470, 0xfe742f83), + TOBN(0xe054b854, 0x5b7fc19f), TOBN(0xf935381a, 0xba1d0698), + TOBN(0x546eab2d, 0x799e9a74), TOBN(0x96239e0e, 0xa949f729), + TOBN(0xca274c6b, 0x7090055a), TOBN(0x835142c3, 0x9020c9b0), + TOBN(0xa405667a, 0xa2e8807f), TOBN(0x29f2c085, 0x1aa3d39e), + TOBN(0xcc555d64, 0x42fc72f5), TOBN(0xe856e0e7, 0xfbeacb3c), + TOBN(0xb5504f9d, 0x918e4936), TOBN(0x65035ef6, 0xb2513982), + TOBN(0x0553a0c2, 0x6f4d9cb9), TOBN(0x6cb10d56, 0xbea85509), + TOBN(0x48d957b7, 0xa242da11), TOBN(0x16a4d3dd, 0x672b7268), + TOBN(0x3d7e637c, 0x8502a96b), TOBN(0x27c7032b, 0x730d463b), + TOBN(0xbdc02b18, 0xe4136a14), TOBN(0xbacf969d, 0x678e32bf), + TOBN(0xc98d89a3, 0xdd9c3c03), TOBN(0x7b92420a, 0x23becc4f), + TOBN(0xd4b41f78, 0xc64d565c), TOBN(0x9f969d00, 0x10f28295), + TOBN(0xec7f7f76, 0xb13d051a), TOBN(0x08945e1e, 0xa92da585), + TOBN(0x55366b7d, 0x5846426f), TOBN(0xe7d09e89, 0x247d441d), + TOBN(0x510b404d, 0x736fbf48), TOBN(0x7fa003d0, 0xe784bd7d), + TOBN(0x25f7614f, 0x17fd9596), TOBN(0x49e0e0a1, 0x35cb98db), + TOBN(0x2c65957b, 0x2e83a76a), TOBN(0x5d40da8d, 0xcddbe0f8), + TOBN(0xf2b8c405, 0x050bad24), TOBN(0x8918426d, 0xc2aa4823), + TOBN(0x2aeab3dd, 0xa38365a7), TOBN(0x72031717, 0x7c91b690), + TOBN(0x8b00d699, 0x60a94120), TOBN(0x478a255d, 0xe99eaeec), + TOBN(0xbf656a5f, 0x6f60aafd), TOBN(0xdfd7cb75, 0x5dee77b3), + TOBN(0x37f68bb4, 0xa595939d), TOBN(0x03556479, 0x28740217), + TOBN(0x8e740e7c, 0x84ad7612), TOBN(0xd89bc843, 0x9044695f), + TOBN(0xf7f3da5d, 0x85a9184d), TOBN(0x562563bb, 0x9fc0b074), + TOBN(0x06d2e6aa, 0xf88a888e), TOBN(0x612d8643, 0x161fbe7c), + TOBN(0x465edba7, 0xf64085e7), TOBN(0xb230f304, 0x29aa8511), + TOBN(0x53388426, 0xcda2d188), TOBN(0x90885735, 0x4b666649), + TOBN(0x6f02ff9a, 0x652f54f6), TOBN(0x65c82294, 0x5fae2bf0), + TOBN(0x7816ade0, 0x62f5eee3), TOBN(0xdcdbdf43, 0xfcc56d70), + TOBN(0x9fb3bba3, 0x54530bb2), TOBN(0xbde3ef77, 0xcb0869ea), + TOBN(0x89bc9046, 0x0b431163), TOBN(0x4d03d7d2, 0xe4819a35), + TOBN(0x33ae4f9e, 0x43b6a782), TOBN(0x216db307, 0x9c88a686), + TOBN(0x91dd88e0, 0x00ffedd9), TOBN(0xb280da9f, 0x12bd4840), + TOBN(0x32a7cb8a, 0x1635e741), TOBN(0xfe14008a, 0x78be02a7), + TOBN(0x3fafb334, 0x1b7ae030), TOBN(0x7fd508e7, 0x5add0ce9), + TOBN(0x72c83219, 0xd607ad51), TOBN(0x0f229c0a, 0x8d40964a), + TOBN(0x1be2c336, 0x1c878da2), TOBN(0xe0c96742, 0xeab2ab86), + TOBN(0x458f8691, 0x3e538cd7), TOBN(0xa7001f6c, 0x8e08ad53), + TOBN(0x52b8c6e6, 0xbf5d15ff), TOBN(0x548234a4, 0x011215dd), + TOBN(0xff5a9d2d, 0x3d5b4045), TOBN(0xb0ffeeb6, 0x4a904190), + TOBN(0x55a3aca4, 0x48607f8b), TOBN(0x8cbd665c, 0x30a0672a), + TOBN(0x87f834e0, 0x42583068), TOBN(0x02da2aeb, 0xf3f6e683), + TOBN(0x6b763e5d, 0x05c12248), TOBN(0x7230378f, 0x65a8aefc), + TOBN(0x93bd80b5, 0x71e8e5ca), TOBN(0x53ab041c, 0xb3b62524), + TOBN(0x1b860513, 0x6c9c552e), TOBN(0xe84d402c, 0xd5524e66), + TOBN(0xa37f3573, 0xf37f5937), TOBN(0xeb0f6c7d, 0xd1e4fca5), + TOBN(0x2965a554, 0xac8ab0fc), TOBN(0x17fbf56c, 0x274676ac), + TOBN(0x2e2f6bd9, 0xacf7d720), TOBN(0x41fc8f88, 0x10224766), + TOBN(0x517a14b3, 0x85d53bef), TOBN(0xdae327a5, 0x7d76a7d1), + TOBN(0x6ad0a065, 0xc4818267), TOBN(0x33aa189b, 0x37c1bbc1), + TOBN(0x64970b52, 0x27392a92), TOBN(0x21699a1c, 0x2d1535ea), + TOBN(0xcd20779c, 0xc2d7a7fd), TOBN(0xe3186059, 0x99c83cf2), + TOBN(0x9b69440b, 0x72c0b8c7), TOBN(0xa81497d7, 0x7b9e0e4d), + TOBN(0x515d5c89, 0x1f5f82dc), TOBN(0x9a7f67d7, 0x6361079e), + TOBN(0xa8da81e3, 0x11a35330), TOBN(0xe44990c4, 0x4b18be1b), + TOBN(0xc7d5ed95, 0xaf103e59), TOBN(0xece8aba7, 0x8dac9261), + TOBN(0xbe82b099, 0x9394b8d3), TOBN(0x6830f09a, 0x16adfe83), + TOBN(0x250a29b4, 0x88172d01), TOBN(0x8b20bd65, 0xcaff9e02), + TOBN(0xb8a7661e, 0xe8a6329a), TOBN(0x4520304d, 0xd3fce920), + TOBN(0xae45da1f, 0x2b47f7ef), TOBN(0xe07f5288, 0x5bffc540), + TOBN(0xf7997009, 0x3464f874), TOBN(0x2244c2cd, 0xa6fa1f38), + TOBN(0x43c41ac1, 0x94d7d9b1), TOBN(0x5bafdd82, 0xc82e7f17), + TOBN(0xdf0614c1, 0x5fda0fca), TOBN(0x74b043a7, 0xa8ae37ad), + TOBN(0x3ba6afa1, 0x9e71734c), TOBN(0x15d5437e, 0x9c450f2e), + TOBN(0x4a5883fe, 0x67e242b1), TOBN(0x5143bdc2, 0x2c1953c2), + TOBN(0x542b8b53, 0xfc5e8920), TOBN(0x363bf9a8, 0x9a9cee08), + TOBN(0x02375f10, 0xc3486e08), TOBN(0x2037543b, 0x8c5e70d2), + TOBN(0x7109bccc, 0x625640b4), TOBN(0xcbc1051e, 0x8bc62c3b), + TOBN(0xf8455fed, 0x803f26ea), TOBN(0x6badceab, 0xeb372424), + TOBN(0xa2a9ce7c, 0x6b53f5f9), TOBN(0x64246595, 0x1b176d99), + TOBN(0xb1298d36, 0xb95c081b), TOBN(0x53505bb8, 0x1d9a9ee6), + TOBN(0x3f6f9e61, 0xf2ba70b0), TOBN(0xd07e16c9, 0x8afad453), + TOBN(0x9f1694bb, 0xe7eb4a6a), TOBN(0xdfebced9, 0x3cb0bc8e), + TOBN(0x92d3dcdc, 0x53868c8b), TOBN(0x174311a2, 0x386107a6), + TOBN(0x4109e07c, 0x689b4e64), TOBN(0x30e4587f, 0x2df3dcb6), + TOBN(0x841aea31, 0x0811b3b2), TOBN(0x6144d41d, 0x0cce43ea), + TOBN(0x464c4581, 0x2a9a7803), TOBN(0xd03d371f, 0x3e158930), + TOBN(0xc676d7f2, 0xb1f3390b), TOBN(0x9f7a1b8c, 0xa5b61272), + TOBN(0x4ebebfc9, 0xc2e127a9), TOBN(0x4602500c, 0x5dd997bf), + TOBN(0x7f09771c, 0x4711230f), TOBN(0x058eb37c, 0x020f09c1), + TOBN(0xab693d4b, 0xfee5e38b), TOBN(0x9289eb1f, 0x4653cbc0), + TOBN(0xbecf46ab, 0xd51b9cf5), TOBN(0xd2aa9c02, 0x9f0121af), + TOBN(0x36aaf7d2, 0xe90dc274), TOBN(0x909e4ea0, 0x48b95a3c), + TOBN(0xe6b70496, 0x6f32dbdb), TOBN(0x672188a0, 0x8b030b3e), + TOBN(0xeeffe5b3, 0xcfb617e2), TOBN(0x87e947de, 0x7c82709e), + TOBN(0xa44d2b39, 0x1770f5a7), TOBN(0xe4d4d791, 0x0e44eb82), + TOBN(0x42e69d1e, 0x3f69712a), TOBN(0xbf11c4d6, 0xac6a820e), + TOBN(0xb5e7f3e5, 0x42c4224c), TOBN(0xd6b4e81c, 0x449d941c), + TOBN(0x5d72bd16, 0x5450e878), TOBN(0x6a61e28a, 0xee25ac54), + TOBN(0x33272094, 0xe6f1cd95), TOBN(0x7512f30d, 0x0d18673f), + TOBN(0x32f7a4ca, 0x5afc1464), TOBN(0x2f095656, 0x6bbb977b), + TOBN(0x586f47ca, 0xa8226200), TOBN(0x02c868ad, 0x1ac07369), + TOBN(0x4ef2b845, 0xc613acbe), TOBN(0x43d7563e, 0x0386054c), + TOBN(0x54da9dc7, 0xab952578), TOBN(0xb5423df2, 0x26e84d0b), + TOBN(0xa8b64eeb, 0x9b872042), TOBN(0xac205782, 0x5990f6df), + TOBN(0x4ff696eb, 0x21f4c77a), TOBN(0x1a79c3e4, 0xaab273af), + TOBN(0x29bc922e, 0x9436b3f1), TOBN(0xff807ef8, 0xd6d9a27a), + TOBN(0x82acea3d, 0x778f22a0), TOBN(0xfb10b2e8, 0x5b5e7469), + TOBN(0xc0b16980, 0x2818ee7d), TOBN(0x011afff4, 0xc91c1a2f), + TOBN(0x95a6d126, 0xad124418), TOBN(0x31c081a5, 0xe72e295f), + TOBN(0x36bb283a, 0xf2f4db75), TOBN(0xd115540f, 0x7acef462), + TOBN(0xc7f3a8f8, 0x33f6746c), TOBN(0x21e46f65, 0xfea990ca), + TOBN(0x915fd5c5, 0xcaddb0a9), TOBN(0xbd41f016, 0x78614555), + TOBN(0x346f4434, 0x426ffb58), TOBN(0x80559436, 0x14dbc204), + TOBN(0xf3dd20fe, 0x5a969b7f), TOBN(0x9d59e956, 0xe899a39a), + TOBN(0xf1b0971c, 0x8ad4cf4b), TOBN(0x03448860, 0x2ffb8fb8), + TOBN(0xf071ac3c, 0x65340ba4), TOBN(0x408d0596, 0xb27fd758), + TOBN(0xe7c78ea4, 0x98c364b0), TOBN(0xa4aac4a5, 0x051e8ab5), + TOBN(0xb9e1d560, 0x485d9002), TOBN(0x9acd518a, 0x88844455), + TOBN(0xe4ca688f, 0xd06f56c0), TOBN(0xa48af70d, 0xdf027972), + TOBN(0x691f0f04, 0x5e9a609d), TOBN(0xa9dd82cd, 0xee61270e), + TOBN(0x8903ca63, 0xa0ef18d3), TOBN(0x9fb7ee35, 0x3d6ca3bd), + TOBN(0xa7b4a09c, 0xabf47d03), TOBN(0x4cdada01, 0x1c67de8e), + TOBN(0x52003749, 0x9355a244), TOBN(0xe77fd2b6, 0x4f2151a9), + TOBN(0x695d6cf6, 0x66b4efcb), TOBN(0xc5a0cacf, 0xda2cfe25), + TOBN(0x104efe5c, 0xef811865), TOBN(0xf52813e8, 0x9ea5cc3d), + TOBN(0x855683dc, 0x40b58dbc), TOBN(0x0338ecde, 0x175fcb11), + TOBN(0xf9a05637, 0x74921592), TOBN(0xb4f1261d, 0xb9bb9d31), + TOBN(0x551429b7, 0x4e9c5459), TOBN(0xbe182e6f, 0x6ea71f53), + TOBN(0xd3a3b07c, 0xdfc50573), TOBN(0x9ba1afda, 0x62be8d44), + TOBN(0x9bcfd2cb, 0x52ab65d3), TOBN(0xdf11d547, 0xa9571802), + TOBN(0x099403ee, 0x02a2404a), TOBN(0x497406f4, 0x21088a71), + TOBN(0x99479409, 0x5004ae71), TOBN(0xbdb42078, 0xa812c362), + TOBN(0x2b72a30f, 0xd8828442), TOBN(0x283add27, 0xfcb5ed1c), + TOBN(0xf7c0e200, 0x66a40015), TOBN(0x3e3be641, 0x08b295ef), + TOBN(0xac127dc1, 0xe038a675), TOBN(0x729deff3, 0x8c5c6320), + TOBN(0xb7df8fd4, 0xa90d2c53), TOBN(0x9b74b0ec, 0x681e7cd3), + TOBN(0x5cb5a623, 0xdab407e5), TOBN(0xcdbd3615, 0x76b340c6), + TOBN(0xa184415a, 0x7d28392c), TOBN(0xc184c1d8, 0xe96f7830), + TOBN(0xc3204f19, 0x81d3a80f), TOBN(0xfde0c841, 0xc8e02432), + TOBN(0x78203b3e, 0x8149e0c1), TOBN(0x5904bdbb, 0x08053a73), + TOBN(0x30fc1dd1, 0x101b6805), TOBN(0x43c223bc, 0x49aa6d49), + TOBN(0x9ed67141, 0x7a174087), TOBN(0x311469a0, 0xd5997008), + TOBN(0xb189b684, 0x5e43fc61), TOBN(0xf3282375, 0xe0d3ab57), + TOBN(0x4fa34b67, 0xb1181da8), TOBN(0x621ed0b2, 0x99ee52b8), + TOBN(0x9b178de1, 0xad990676), TOBN(0xd51de67b, 0x56d54065), + TOBN(0x2a2c27c4, 0x7538c201), TOBN(0x33856ec8, 0x38a40f5c), + TOBN(0x2522fc15, 0xbe6cdcde), TOBN(0x1e603f33, 0x9f0c6f89), + TOBN(0x7994edc3, 0x103e30a6), TOBN(0x033a00db, 0x220c853e), + TOBN(0xd3cfa409, 0xf7bb7fd7), TOBN(0x70f8781e, 0x462d18f6), + TOBN(0xbbd82980, 0x687fe295), TOBN(0x6eef4c32, 0x595669f3), + TOBN(0x86a9303b, 0x2f7e85c3), TOBN(0x5fce4621, 0x71988f9b), + TOBN(0x5b935bf6, 0xc138acb5), TOBN(0x30ea7d67, 0x25661212), + TOBN(0xef1eb5f4, 0xe51ab9a2), TOBN(0x0587c98a, 0xae067c78), + TOBN(0xb3ce1b3c, 0x77ca9ca6), TOBN(0x2a553d4d, 0x54b5f057), + TOBN(0xc7898236, 0x4da29ec2), TOBN(0xdbdd5d13, 0xb9c57316), + TOBN(0xc57d6e6b, 0x2cd80d47), TOBN(0x80b460cf, 0xfe9e7391), + TOBN(0x98648cab, 0xf963c31e), TOBN(0x67f9f633, 0xcc4d32fd), + TOBN(0x0af42a9d, 0xfdf7c687), TOBN(0x55f292a3, 0x0b015ea7), + TOBN(0x89e468b2, 0xcd21ab3d), TOBN(0xe504f022, 0xc393d392), + TOBN(0xab21e1d4, 0xa5013af9), TOBN(0xe3283f78, 0xc2c28acb), + TOBN(0xf38b35f6, 0x226bf99f), TOBN(0xe8354274, 0x0e291e69), + TOBN(0x61673a15, 0xb20c162d), TOBN(0xc101dc75, 0xb04fbdbe), + TOBN(0x8323b4c2, 0x255bd617), TOBN(0x6c969693, 0x6c2a9154), + TOBN(0xc6e65860, 0x62679387), TOBN(0x8e01db0c, 0xb8c88e23), + TOBN(0x33c42873, 0x893a5559), TOBN(0x7630f04b, 0x47a3e149), + TOBN(0xb5d80805, 0xddcf35f8), TOBN(0x582ca080, 0x77dfe732), + TOBN(0x2c7156e1, 0x0b1894a0), TOBN(0x92034001, 0xd81c68c0), + TOBN(0xed225d00, 0xc8b115b5), TOBN(0x237f9c22, 0x83b907f2), + TOBN(0x0ea2f32f, 0x4470e2c0), TOBN(0xb725f7c1, 0x58be4e95), + TOBN(0x0f1dcafa, 0xb1ae5463), TOBN(0x59ed5187, 0x1ba2fc04), + TOBN(0xf6e0f316, 0xd0115d4d), TOBN(0x5180b12f, 0xd3691599), + TOBN(0x157e32c9, 0x527f0a41), TOBN(0x7b0b081d, 0xa8e0ecc0), + TOBN(0x6dbaaa8a, 0xbf4f0dd0), TOBN(0x99b289c7, 0x4d252696), + TOBN(0x79b7755e, 0xdbf864fe), TOBN(0x6974e2b1, 0x76cad3ab), + TOBN(0x35dbbee2, 0x06ddd657), TOBN(0xe7cbdd11, 0x2ff3a96d), + TOBN(0x88381968, 0x076be758), TOBN(0x2d737e72, 0x08c91f5d), + TOBN(0x5f83ab62, 0x86ec3776), TOBN(0x98aa649d, 0x945fa7a1), + TOBN(0xf477ec37, 0x72ef0933), TOBN(0x66f52b1e, 0x098c17b1), + TOBN(0x9eec58fb, 0xd803738b), TOBN(0x91aaade7, 0xe4e86aa4), + TOBN(0x6b1ae617, 0xa5b51492), TOBN(0x63272121, 0xbbc45974), + TOBN(0x7e0e28f0, 0x862c5129), TOBN(0x0a8f79a9, 0x3321a4a0), + TOBN(0xe26d1664, 0x5041c88f), TOBN(0x0571b805, 0x53233e3a), + TOBN(0xd1b0ccde, 0xc9520711), TOBN(0x55a9e4ed, 0x3c8b84bf), + TOBN(0x9426bd39, 0xa1fef314), TOBN(0x4f5f638e, 0x6eb93f2b), + TOBN(0xba2a1ed3, 0x2bf9341b), TOBN(0xd63c1321, 0x4d42d5a9), + TOBN(0xd2964a89, 0x316dc7c5), TOBN(0xd1759606, 0xca511851), + TOBN(0xd8a9201f, 0xf9e6ed35), TOBN(0xb7b5ee45, 0x6736925a), + TOBN(0x0a83fbbc, 0x99581af7), TOBN(0x3076bc40, 0x64eeb051), + TOBN(0x5511c98c, 0x02dec312), TOBN(0x270de898, 0x238dcb78), + TOBN(0x2cf4cf9c, 0x539c08c9), TOBN(0xa70cb65e, 0x38d3b06e), + TOBN(0xb12ec10e, 0xcfe57bbd), TOBN(0x82c7b656, 0x35a0c2b5), + TOBN(0xddc7d5cd, 0x161c67bd), TOBN(0xe32e8985, 0xae3a32cc), + TOBN(0x7aba9444, 0xd11a5529), TOBN(0xe964ed02, 0x2427fa1a), + TOBN(0x1528392d, 0x24a1770a), TOBN(0xa152ce2c, 0x12c72fcd), + TOBN(0x714553a4, 0x8ec07649), TOBN(0x18b4c290, 0x459dd453), + TOBN(0xea32b714, 0x7b64b110), TOBN(0xb871bfa5, 0x2e6f07a2), + TOBN(0xb67112e5, 0x9e2e3c9b), TOBN(0xfbf250e5, 0x44aa90f6), + TOBN(0xf77aedb8, 0xbd539006), TOBN(0x3b0cdf9a, 0xd172a66f), + TOBN(0xedf69fea, 0xf8c51187), TOBN(0x05bb67ec, 0x741e4da7), + TOBN(0x47df0f32, 0x08114345), TOBN(0x56facb07, 0xbb9792b1), + TOBN(0xf3e007e9, 0x8f6229e4), TOBN(0x62d103f4, 0x526fba0f), + TOBN(0x4f33bef7, 0xb0339d79), TOBN(0x9841357b, 0xb59bfec1), + TOBN(0xfa8dbb59, 0xc34e6705), TOBN(0xc3c7180b, 0x7fdaa84c), + TOBN(0xf95872fc, 0xa4108537), TOBN(0x8750cc3b, 0x932a3e5a), + TOBN(0xb61cc69d, 0xb7275d7d), TOBN(0xffa0168b, 0x2e59b2e9), + TOBN(0xca032abc, 0x6ecbb493), TOBN(0x1d86dbd3, 0x2c9082d8), + TOBN(0xae1e0b67, 0xe28ef5ba), TOBN(0x2c9a4699, 0xcb18e169), + TOBN(0x0ecd0e33, 0x1e6bbd20), TOBN(0x571b360e, 0xaf5e81d2), + TOBN(0xcd9fea58, 0x101c1d45), TOBN(0x6651788e, 0x18880452), + TOBN(0xa9972635, 0x1f8dd446), TOBN(0x44bed022, 0xe37281d0), + TOBN(0x094b2b2d, 0x33da525d), TOBN(0xf193678e, 0x13144fd8), + TOBN(0xb8ab5ba4, 0xf4c1061d), TOBN(0x4343b5fa, 0xdccbe0f4), + TOBN(0xa8702371, 0x63812713), TOBN(0x47bf6d2d, 0xf7611d93), + TOBN(0x46729b8c, 0xbd21e1d7), TOBN(0x7484d4e0, 0xd629e77d), + TOBN(0x830e6eea, 0x60dbac1f), TOBN(0x23d8c484, 0xda06a2f7), + TOBN(0x896714b0, 0x50ca535b), TOBN(0xdc8d3644, 0xebd97a9b), + TOBN(0x106ef9fa, 0xb12177b4), TOBN(0xf79bf464, 0x534d5d9c), + TOBN(0x2537a349, 0xa6ab360b), TOBN(0xc7c54253, 0xa00c744f), + TOBN(0xb3c7a047, 0xe5911a76), TOBN(0x61ffa5c8, 0x647f1ee7), + TOBN(0x15aed36f, 0x8f56ab42), TOBN(0x6a0d41b0, 0xa3ff9ac9), + TOBN(0x68f469f5, 0xcc30d357), TOBN(0xbe9adf81, 0x6b72be96), + TOBN(0x1cd926fe, 0x903ad461), TOBN(0x7e89e38f, 0xcaca441b), + TOBN(0xf0f82de5, 0xfacf69d4), TOBN(0x363b7e76, 0x4775344c), + TOBN(0x6894f312, 0xb2e36d04), TOBN(0x3c6cb4fe, 0x11d1c9a5), + TOBN(0x85d9c339, 0x4008e1f2), TOBN(0x5e9a85ea, 0x249f326c), + TOBN(0xdc35c60a, 0x678c5e06), TOBN(0xc08b944f, 0x9f86fba9), + TOBN(0xde40c02c, 0x89f71f0f), TOBN(0xad8f3e31, 0xff3da3c0), + TOBN(0x3ea5096b, 0x42125ded), TOBN(0x13879cbf, 0xa7379183), + TOBN(0x6f4714a5, 0x6b306a0b), TOBN(0x359c2ea6, 0x67646c5e), + TOBN(0xfacf8943, 0x07726368), TOBN(0x07a58935, 0x65ff431e), + TOBN(0x24d661d1, 0x68754ab0), TOBN(0x801fce1d, 0x6f429a76), + TOBN(0xc068a85f, 0xa58ce769), TOBN(0xedc35c54, 0x5d5eca2b), + TOBN(0xea31276f, 0xa3f660d1), TOBN(0xa0184ebe, 0xb8fc7167), + TOBN(0x0f20f21a, 0x1d8db0ae), TOBN(0xd96d095f, 0x56c35e12), + TOBN(0xedf402b5, 0xf8c2a25b), TOBN(0x1bb772b9, 0x059204b6), + TOBN(0x50cbeae2, 0x19b4e34c), TOBN(0x93109d80, 0x3fa0845a), + TOBN(0x54f7ccf7, 0x8ef59fb5), TOBN(0x3b438fe2, 0x88070963), + TOBN(0x9e28c659, 0x31f3ba9b), TOBN(0x9cc31b46, 0xead9da92), + TOBN(0x3c2f0ba9, 0xb733aa5f), TOBN(0xdece47cb, 0xf05af235), + TOBN(0xf8e3f715, 0xa2ac82a5), TOBN(0xc97ba641, 0x2203f18a), + TOBN(0xc3af5504, 0x09c11060), TOBN(0x56ea2c05, 0x46af512d), + TOBN(0xfac28daf, 0xf3f28146), TOBN(0x87fab43a, 0x959ef494),} + , + {TOBN(0x09891641, 0xd4c5105f), TOBN(0x1ae80f8e, 0x6d7fbd65), + TOBN(0x9d67225f, 0xbee6bdb0), TOBN(0x3b433b59, 0x7fc4d860), + TOBN(0x44e66db6, 0x93e85638), TOBN(0xf7b59252, 0xe3e9862f), + TOBN(0xdb785157, 0x665c32ec), TOBN(0x702fefd7, 0xae362f50), + TOBN(0x3754475d, 0x0fefb0c3), TOBN(0xd48fb56b, 0x46d7c35d), + TOBN(0xa070b633, 0x363798a4), TOBN(0xae89f3d2, 0x8fdb98e6), + TOBN(0x970b89c8, 0x6363d14c), TOBN(0x89817521, 0x67abd27d), + TOBN(0x9bf7d474, 0x44d5a021), TOBN(0xb3083baf, 0xcac72aee), + TOBN(0x389741de, 0xbe949a44), TOBN(0x638e9388, 0x546a4fa5), + TOBN(0x3fe6419c, 0xa0047bdc), TOBN(0x7047f648, 0xaaea57ca), + TOBN(0x54e48a90, 0x41fbab17), TOBN(0xda8e0b28, 0x576bdba2), + TOBN(0xe807eebc, 0xc72afddc), TOBN(0x07d3336d, 0xf42577bf), + TOBN(0x62a8c244, 0xbfe20925), TOBN(0x91c19ac3, 0x8fdce867), + TOBN(0x5a96a5d5, 0xdd387063), TOBN(0x61d587d4, 0x21d324f6), + TOBN(0xe87673a2, 0xa37173ea), TOBN(0x23848008, 0x53778b65), + TOBN(0x10f8441e, 0x05bab43e), TOBN(0xfa11fe12, 0x4621efbe), + TOBN(0x047b772e, 0x81685d7b), TOBN(0x23f27d81, 0xbf34a976), + TOBN(0xc27608e2, 0x915f48ef), TOBN(0x3b0b43fa, 0xa521d5c3), + TOBN(0x7613fb26, 0x63ca7284), TOBN(0x7f5729b4, 0x1d4db837), + TOBN(0x87b14898, 0x583b526b), TOBN(0x00b732a6, 0xbbadd3d1), + TOBN(0x8e02f426, 0x2048e396), TOBN(0x436b50b6, 0x383d9de4), + TOBN(0xf78d3481, 0x471e85ad), TOBN(0x8b01ea6a, 0xd005c8d6), + TOBN(0xd3c7afee, 0x97015c07), TOBN(0x46cdf1a9, 0x4e3ba2ae), + TOBN(0x7a42e501, 0x83d3a1d2), TOBN(0xd54b5268, 0xb541dff4), + TOBN(0x3f24cf30, 0x4e23e9bc), TOBN(0x4387f816, 0x126e3624), + TOBN(0x26a46a03, 0x3b0b6d61), TOBN(0xaf1bc845, 0x8b2d777c), + TOBN(0x25c401ba, 0x527de79c), TOBN(0x0e1346d4, 0x4261bbb6), + TOBN(0x4b96c44b, 0x287b4bc7), TOBN(0x658493c7, 0x5254562f), + TOBN(0x23f949fe, 0xb8a24a20), TOBN(0x17ebfed1, 0xf52ca53f), + TOBN(0x9b691bbe, 0xbcfb4853), TOBN(0x5617ff6b, 0x6278a05d), + TOBN(0x241b34c5, 0xe3c99ebd), TOBN(0xfc64242e, 0x1784156a), + TOBN(0x4206482f, 0x695d67df), TOBN(0xb967ce0e, 0xee27c011), + TOBN(0x65db3751, 0x21c80b5d), TOBN(0x2e7a563c, 0xa31ecca0), + TOBN(0xe56ffc4e, 0x5238a07e), TOBN(0x3d6c2966, 0x32ced854), + TOBN(0xe99d7d1a, 0xaf70b885), TOBN(0xafc3bad9, 0x2d686459), + TOBN(0x9c78bf46, 0x0cc8ba5b), TOBN(0x5a439519, 0x18955aa3), + TOBN(0xf8b517a8, 0x5fe4e314), TOBN(0xe60234d0, 0xfcb8906f), + TOBN(0xffe542ac, 0xf2061b23), TOBN(0x287e191f, 0x6b4cb59c), + TOBN(0x21857ddc, 0x09d877d8), TOBN(0x1c23478c, 0x14678941), + TOBN(0xbbf0c056, 0xb6e05ea4), TOBN(0x82da4b53, 0xb01594fe), + TOBN(0xf7526791, 0xfadb8608), TOBN(0x049e832d, 0x7b74cdf6), + TOBN(0xa43581cc, 0xc2b90a34), TOBN(0x73639eb8, 0x9360b10c), + TOBN(0x4fba331f, 0xe1e4a71b), TOBN(0x6ffd6b93, 0x8072f919), + TOBN(0x6e53271c, 0x65679032), TOBN(0x67206444, 0xf14272ce), + TOBN(0xc0f734a3, 0xb2335834), TOBN(0x9526205a, 0x90ef6860), + TOBN(0xcb8be717, 0x04e2bb0d), TOBN(0x2418871e, 0x02f383fa), + TOBN(0xd7177681, 0x4082c157), TOBN(0xcc914ad0, 0x29c20073), + TOBN(0xf186c1eb, 0xe587e728), TOBN(0x6fdb3c22, 0x61bcd5fd), + TOBN(0x30d014a6, 0xf2f9f8e9), TOBN(0x963ece23, 0x4fec49d2), + TOBN(0x862025c5, 0x9605a8d9), TOBN(0x39874445, 0x19f8929a), + TOBN(0x01b6ff65, 0x12bf476a), TOBN(0x598a64d8, 0x09cf7d91), + TOBN(0xd7ec7749, 0x93be56ca), TOBN(0x10899785, 0xcbb33615), + TOBN(0xb8a092fd, 0x02eee3ad), TOBN(0xa86b3d35, 0x30145270), + TOBN(0x323d98c6, 0x8512b675), TOBN(0x4b8bc785, 0x62ebb40f), + TOBN(0x7d301f54, 0x413f9cde), TOBN(0xa5e4fb4f, 0x2bab5664), + TOBN(0x1d2b252d, 0x1cbfec23), TOBN(0xfcd576bb, 0xe177120d), + TOBN(0x04427d3e, 0x83731a34), TOBN(0x2bb9028e, 0xed836e8e), + TOBN(0xb36acff8, 0xb612ca7c), TOBN(0xb88fe5ef, 0xd3d9c73a), + TOBN(0xbe2a6bc6, 0xedea4eb3), TOBN(0x43b93133, 0x488eec77), + TOBN(0xf41ff566, 0xb17106e1), TOBN(0x469e9172, 0x654efa32), + TOBN(0xb4480f04, 0x41c23fa3), TOBN(0xb4712eb0, 0xc1989a2e), + TOBN(0x3ccbba0f, 0x93a29ca7), TOBN(0x6e205c14, 0xd619428c), + TOBN(0x90db7957, 0xb3641686), TOBN(0x0432691d, 0x45ac8b4e), + TOBN(0x07a759ac, 0xf64e0350), TOBN(0x0514d89c, 0x9c972517), + TOBN(0x1701147f, 0xa8e67fc3), TOBN(0x9e2e0b8b, 0xab2085be), + TOBN(0xd5651824, 0xac284e57), TOBN(0x890d4325, 0x74893664), + TOBN(0x8a7c5e6e, 0xc55e68a3), TOBN(0xbf12e90b, 0x4339c85a), + TOBN(0x31846b85, 0xf922b655), TOBN(0x9a54ce4d, 0x0bf4d700), + TOBN(0xd7f4e83a, 0xf1a14295), TOBN(0x916f955c, 0xb285d4f9), + TOBN(0xe57bb0e0, 0x99ffdaba), TOBN(0x28a43034, 0xeab0d152), + TOBN(0x0a36ffa2, 0xb8a9cef8), TOBN(0x5517407e, 0xb9ec051a), + TOBN(0x9c796096, 0xea68e672), TOBN(0x853db5fb, 0xfb3c77fb), + TOBN(0x21474ba9, 0xe864a51a), TOBN(0x6c267699, 0x6e8a1b8b), + TOBN(0x7c823626, 0x94120a28), TOBN(0xe61e9a48, 0x8383a5db), + TOBN(0x7dd75003, 0x9f84216d), TOBN(0xab020d07, 0xad43cd85), + TOBN(0x9437ae48, 0xda12c659), TOBN(0x6449c2eb, 0xe65452ad), + TOBN(0xcc7c4c1c, 0x2cf9d7c1), TOBN(0x1320886a, 0xee95e5ab), + TOBN(0xbb7b9056, 0xbeae170c), TOBN(0xc8a5b250, 0xdbc0d662), + TOBN(0x4ed81432, 0xc11d2303), TOBN(0x7da66912, 0x1f03769f), + TOBN(0x3ac7a5fd, 0x84539828), TOBN(0x14dada94, 0x3bccdd02), + TOBN(0x8b84c321, 0x7ef6b0d1), TOBN(0x52a9477a, 0x7c933f22), + TOBN(0x5ef6728a, 0xfd440b82), TOBN(0x5c3bd859, 0x6ce4bd5e), + TOBN(0x918b80f5, 0xf22c2d3e), TOBN(0x368d5040, 0xb7bb6cc5), + TOBN(0xb66142a1, 0x2695a11c), TOBN(0x60ac583a, 0xeb19ea70), + TOBN(0x317cbb98, 0x0eab2437), TOBN(0x8cc08c55, 0x5e2654c8), + TOBN(0xfe2d6520, 0xe6d8307f), TOBN(0xe9f147f3, 0x57428993), + TOBN(0x5f9c7d14, 0xd2fd6cf1), TOBN(0xa3ecd064, 0x2d4fcbb0), + TOBN(0xad83fef0, 0x8e7341f7), TOBN(0x643f23a0, 0x3a63115c), + TOBN(0xd38a78ab, 0xe65ab743), TOBN(0xbf7c75b1, 0x35edc89c), + TOBN(0x3dd8752e, 0x530df568), TOBN(0xf85c4a76, 0xe308c682), + TOBN(0x4c9955b2, 0xe68acf37), TOBN(0xa544df3d, 0xab32af85), + TOBN(0x4b8ec3f5, 0xa25cf493), TOBN(0x4d8f2764, 0x1a622feb), + TOBN(0x7bb4f7aa, 0xf0dcbc49), TOBN(0x7de551f9, 0x70bbb45b), + TOBN(0xcfd0f3e4, 0x9f2ca2e5), TOBN(0xece58709, 0x1f5c76ef), + TOBN(0x32920edd, 0x167d79ae), TOBN(0x039df8a2, 0xfa7d7ec1), + TOBN(0xf46206c0, 0xbb30af91), TOBN(0x1ff5e2f5, 0x22676b59), + TOBN(0x11f4a039, 0x6ea51d66), TOBN(0x506c1445, 0x807d7a26), + TOBN(0x60da5705, 0x755a9b24), TOBN(0x8fc8cc32, 0x1f1a319e), + TOBN(0x83642d4d, 0x9433d67d), TOBN(0x7fa5cb8f, 0x6a7dd296), + TOBN(0x576591db, 0x9b7bde07), TOBN(0x13173d25, 0x419716fb), + TOBN(0xea30599d, 0xd5b340ff), TOBN(0xfc6b5297, 0xb0fe76c5), + TOBN(0x1c6968c8, 0xab8f5adc), TOBN(0xf723c7f5, 0x901c928d), + TOBN(0x4203c321, 0x9773d402), TOBN(0xdf7c6aa3, 0x1b51dd47), + TOBN(0x3d49e37a, 0x552be23c), TOBN(0x57febee8, 0x0b5a6e87), + TOBN(0xc5ecbee4, 0x7bd8e739), TOBN(0x79d44994, 0xae63bf75), + TOBN(0x168bd00f, 0x38fb8923), TOBN(0x75d48ee4, 0xd0533130), + TOBN(0x554f77aa, 0xdb5cdf33), TOBN(0x3396e896, 0x3c696769), + TOBN(0x2fdddbf2, 0xd3fd674e), TOBN(0xbbb8f6ee, 0x99d0e3e5), + TOBN(0x51b90651, 0xcbae2f70), TOBN(0xefc4bc05, 0x93aaa8eb), + TOBN(0x8ecd8689, 0xdd1df499), TOBN(0x1aee99a8, 0x22f367a5), + TOBN(0x95d485b9, 0xae8274c5), TOBN(0x6c14d445, 0x7d30b39c), + TOBN(0xbafea90b, 0xbcc1ef81), TOBN(0x7c5f317a, 0xa459a2ed), + TOBN(0x01211075, 0x4ef44227), TOBN(0xa17bed6e, 0xdc20f496), + TOBN(0x0cdfe424, 0x819853cd), TOBN(0x13793298, 0xf71e2ce7), + TOBN(0x3c1f3078, 0xdbbe307b), TOBN(0x6dd1c20e, 0x76ee9936), + TOBN(0x23ee4b57, 0x423caa20), TOBN(0x4ac3793b, 0x8efb840e), + TOBN(0x934438eb, 0xed1f8ca0), TOBN(0x3e546658, 0x4ebb25a2), + TOBN(0xc415af0e, 0xc069896f), TOBN(0xc13eddb0, 0x9a5aa43d), + TOBN(0x7a04204f, 0xd49eb8f6), TOBN(0xd0d5bdfc, 0xd74f1670), + TOBN(0x3697e286, 0x56fc0558), TOBN(0x10207371, 0x01cebade), + TOBN(0x5f87e690, 0x0647a82b), TOBN(0x908e0ed4, 0x8f40054f), + TOBN(0xa9f633d4, 0x79853803), TOBN(0x8ed13c9a, 0x4a28b252), + TOBN(0x3e2ef676, 0x1f460f64), TOBN(0x53930b9b, 0x36d06336), + TOBN(0x347073ac, 0x8fc4979b), TOBN(0x84380e0e, 0x5ecd5597), + TOBN(0xe3b22c6b, 0xc4fe3c39), TOBN(0xba4a8153, 0x6c7bebdf), + TOBN(0xf23ab6b7, 0x25693459), TOBN(0x53bc3770, 0x14922b11), + TOBN(0x4645c8ab, 0x5afc60db), TOBN(0xaa022355, 0x20b9f2a3), + TOBN(0x52a2954c, 0xce0fc507), TOBN(0x8c2731bb, 0x7ce1c2e7), + TOBN(0xf39608ab, 0x18a0339d), TOBN(0xac7a658d, 0x3735436c), + TOBN(0xb22c2b07, 0xcd992b4f), TOBN(0x4e83daec, 0xf40dcfd4), + TOBN(0x8a34c7be, 0x2f39ea3e), TOBN(0xef0c005f, 0xb0a56d2e), + TOBN(0x62731f6a, 0x6edd8038), TOBN(0x5721d740, 0x4e3cb075), + TOBN(0x1ea41511, 0xfbeeee1b), TOBN(0xd1ef5e73, 0xef1d0c05), + TOBN(0x42feefd1, 0x73c07d35), TOBN(0xe530a00a, 0x8a329493), + TOBN(0x5d55b7fe, 0xf15ebfb0), TOBN(0x549de03c, 0xd322491a), + TOBN(0xf7b5f602, 0x745b3237), TOBN(0x3632a3a2, 0x1ab6e2b6), + TOBN(0x0d3bba89, 0x0ef59f78), TOBN(0x0dfc6443, 0xc9e52b9a), + TOBN(0x1dc79699, 0x72631447), TOBN(0xef033917, 0xb3be20b1), + TOBN(0x0c92735d, 0xb1383948), TOBN(0xc1fc29a2, 0xc0dd7d7d), + TOBN(0x6485b697, 0x403ed068), TOBN(0x13bfaab3, 0xaac93bdc), + TOBN(0x410dc6a9, 0x0deeaf52), TOBN(0xb003fb02, 0x4c641c15), + TOBN(0x1384978c, 0x5bc504c4), TOBN(0x37640487, 0x864a6a77), + TOBN(0x05991bc6, 0x222a77da), TOBN(0x62260a57, 0x5e47eb11), + TOBN(0xc7af6613, 0xf21b432c), TOBN(0x22f3acc9, 0xab4953e9), + TOBN(0x52934922, 0x8e41d155), TOBN(0x4d024568, 0x3ac059ef), + TOBN(0xb0201755, 0x4d884411), TOBN(0xce8055cf, 0xa59a178f), + TOBN(0xcd77d1af, 0xf6204549), TOBN(0xa0a00a3e, 0xc7066759), + TOBN(0x471071ef, 0x0272c229), TOBN(0x009bcf6b, 0xd3c4b6b0), + TOBN(0x2a2638a8, 0x22305177), TOBN(0xd51d59df, 0x41645bbf), + TOBN(0xa81142fd, 0xc0a7a3c0), TOBN(0xa17eca6d, 0x4c7063ee), + TOBN(0x0bb887ed, 0x60d9dcec), TOBN(0xd6d28e51, 0x20ad2455), + TOBN(0xebed6308, 0xa67102ba), TOBN(0x042c3114, 0x8bffa408), + TOBN(0xfd099ac5, 0x8aa68e30), TOBN(0x7a6a3d7c, 0x1483513e), + TOBN(0xffcc6b75, 0xba2d8f0c), TOBN(0x54dacf96, 0x1e78b954), + TOBN(0xf645696f, 0xa4a9af89), TOBN(0x3a411940, 0x06ac98ec), + TOBN(0x41b8b3f6, 0x22a67a20), TOBN(0x2d0b1e0f, 0x99dec626), + TOBN(0x27c89192, 0x40be34e8), TOBN(0xc7162b37, 0x91907f35), + TOBN(0x90188ec1, 0xa956702b), TOBN(0xca132f7d, 0xdf93769c), + TOBN(0x3ece44f9, 0x0e2025b4), TOBN(0x67aaec69, 0x0c62f14c), + TOBN(0xad741418, 0x22e3cc11), TOBN(0xcf9b75c3, 0x7ff9a50e), + TOBN(0x02fa2b16, 0x4d348272), TOBN(0xbd99d61a, 0x9959d56d), + TOBN(0xbc4f19db, 0x18762916), TOBN(0xcc7cce50, 0x49c1ac80), + TOBN(0x4d59ebaa, 0xd846bd83), TOBN(0x8775a9dc, 0xa9202849), + TOBN(0x07ec4ae1, 0x6e1f4ca9), TOBN(0x27eb5875, 0xba893f11), + TOBN(0x00284d51, 0x662cc565), TOBN(0x82353a6b, 0x0db4138d), + TOBN(0xd9c7aaaa, 0xaa32a594), TOBN(0xf5528b5e, 0xa5669c47), + TOBN(0xf3220231, 0x2f23c5ff), TOBN(0xe3e8147a, 0x6affa3a1), + TOBN(0xfb423d5c, 0x202ddda0), TOBN(0x3d6414ac, 0x6b871bd4), + TOBN(0x586f82e1, 0xa51a168a), TOBN(0xb712c671, 0x48ae5448), + TOBN(0x9a2e4bd1, 0x76233eb8), TOBN(0x0188223a, 0x78811ca9), + TOBN(0x553c5e21, 0xf7c18de1), TOBN(0x7682e451, 0xb27bb286), + TOBN(0x3ed036b3, 0x0e51e929), TOBN(0xf487211b, 0xec9cb34f), + TOBN(0x0d094277, 0x0c24efc8), TOBN(0x0349fd04, 0xbef737a4), + TOBN(0x6d1c9dd2, 0x514cdd28), TOBN(0x29c135ff, 0x30da9521), + TOBN(0xea6e4508, 0xf78b0b6f), TOBN(0x176f5dd2, 0x678c143c), + TOBN(0x08148418, 0x4be21e65), TOBN(0x27f7525c, 0xe7df38c4), + TOBN(0x1fb70e09, 0x748ab1a4), TOBN(0x9cba50a0, 0x5efe4433), + TOBN(0x7846c7a6, 0x15f75af2), TOBN(0x2a7c2c57, 0x5ee73ea8), + TOBN(0x42e566a4, 0x3f0a449a), TOBN(0x45474c3b, 0xad90fc3d), + TOBN(0x7447be3d, 0x8b61d057), TOBN(0x3e9d1cf1, 0x3a4ec092), + TOBN(0x1603e453, 0xf380a6e6), TOBN(0x0b86e431, 0x9b1437c2), + TOBN(0x7a4173f2, 0xef29610a), TOBN(0x8fa729a7, 0xf03d57f7), + TOBN(0x3e186f6e, 0x6c9c217e), TOBN(0xbe1d3079, 0x91919524), + TOBN(0x92a62a70, 0x153d4fb1), TOBN(0x32ed3e34, 0xd68c2f71), + TOBN(0xd785027f, 0x9eb1a8b7), TOBN(0xbc37eb77, 0xc5b22fe8), + TOBN(0x466b34f0, 0xb9d6a191), TOBN(0x008a89af, 0x9a05f816), + TOBN(0x19b028fb, 0x7d42c10a), TOBN(0x7fe8c92f, 0x49b3f6b8), + TOBN(0x58907cc0, 0xa5a0ade3), TOBN(0xb3154f51, 0x559d1a7c), + TOBN(0x5066efb6, 0xd9790ed6), TOBN(0xa77a0cbc, 0xa6aa793b), + TOBN(0x1a915f3c, 0x223e042e), TOBN(0x1c5def04, 0x69c5874b), + TOBN(0x0e830078, 0x73b6c1da), TOBN(0x55cf85d2, 0xfcd8557a), + TOBN(0x0f7c7c76, 0x0460f3b1), TOBN(0x87052acb, 0x46e58063), + TOBN(0x09212b80, 0x907eae66), TOBN(0x3cb068e0, 0x4d721c89), + TOBN(0xa87941ae, 0xdd45ac1c), TOBN(0xde8d5c0d, 0x0daa0dbb), + TOBN(0xda421fdc, 0xe3502e6e), TOBN(0xc8944201, 0x4d89a084), + TOBN(0x7307ba5e, 0xf0c24bfb), TOBN(0xda212beb, 0x20bde0ef), + TOBN(0xea2da24b, 0xf82ce682), TOBN(0x058d3816, 0x07f71fe4), + TOBN(0x35a02462, 0x5ffad8de), TOBN(0xcd7b05dc, 0xaadcefab), + TOBN(0xd442f8ed, 0x1d9f54ec), TOBN(0x8be3d618, 0xb2d3b5ca), + TOBN(0xe2220ed0, 0xe06b2ce2), TOBN(0x82699a5f, 0x1b0da4c0), + TOBN(0x3ff106f5, 0x71c0c3a7), TOBN(0x8f580f5a, 0x0d34180c), + TOBN(0x4ebb120e, 0x22d7d375), TOBN(0x5e5782cc, 0xe9513675), + TOBN(0x2275580c, 0x99c82a70), TOBN(0xe8359fbf, 0x15ea8c4c), + TOBN(0x53b48db8, 0x7b415e70), TOBN(0xaacf2240, 0x100c6014), + TOBN(0x9faaccf5, 0xe4652f1d), TOBN(0xbd6fdd2a, 0xd56157b2), + TOBN(0xa4f4fb1f, 0x6261ec50), TOBN(0x244e55ad, 0x476bcd52), + TOBN(0x881c9305, 0x047d320b), TOBN(0x1ca983d5, 0x6181263f), + TOBN(0x354e9a44, 0x278fb8ee), TOBN(0xad2dbc0f, 0x396e4964), + TOBN(0x723f3aa2, 0x9268b3de), TOBN(0x0d1ca29a, 0xe6e0609a), + TOBN(0x794866aa, 0x6cf44252), TOBN(0x0b59f3e3, 0x01af87ed), + TOBN(0xe234e5ff, 0x7f4a6c51), TOBN(0xa8768fd2, 0x61dc2f7e), + TOBN(0xdafc7332, 0x0a94d81f), TOBN(0xd7f84282, 0x06938ce1), + TOBN(0xae0b3c0e, 0x0546063e), TOBN(0x7fbadcb2, 0x5d61abc6), + TOBN(0xd5d7a2c9, 0x369ac400), TOBN(0xa5978d09, 0xae67d10c), + TOBN(0x290f211e, 0x4f85eaac), TOBN(0xe61e2ad1, 0xfacac681), + TOBN(0xae125225, 0x388384cd), TOBN(0xa7fb68e9, 0xccfde30f), + TOBN(0x7a59b936, 0x3daed4c2), TOBN(0x80a9aa40, 0x2606f789), + TOBN(0xb40c1ea5, 0xf6a6d90a), TOBN(0x948364d3, 0x514d5885), + TOBN(0x062ebc60, 0x70985182), TOBN(0xa6db5b0e, 0x33310895), + TOBN(0x64a12175, 0xe329c2f5), TOBN(0xc5f25bd2, 0x90ea237e), + TOBN(0x7915c524, 0x2d0a4c23), TOBN(0xeb5d26e4, 0x6bb3cc52), + TOBN(0x369a9116, 0xc09e2c92), TOBN(0x0c527f92, 0xcf182cf8), + TOBN(0x9e591938, 0x2aede0ac), TOBN(0xb2922208, 0x6cc34939), + TOBN(0x3c9d8962, 0x99a34361), TOBN(0x3c81836d, 0xc1905fe6), + TOBN(0x4bfeb57f, 0xa001ec5a), TOBN(0xe993f5bb, 0xa0dc5dba), + TOBN(0x47884109, 0x724a1380), TOBN(0x8a0369ab, 0x32fe9a04), + TOBN(0xea068d60, 0x8c927db8), TOBN(0xbf5f37cf, 0x94655741), + TOBN(0x47d402a2, 0x04b6c7ea), TOBN(0x4551c295, 0x6af259cb), + TOBN(0x698b71e7, 0xed77ee8b), TOBN(0xbddf7bd0, 0xf309d5c7), + TOBN(0x6201c22c, 0x34e780ca), TOBN(0xab04f7d8, 0x4c295ef4), + TOBN(0x1c947294, 0x4313a8ce), TOBN(0xe532e4ac, 0x92ca4cfe), + TOBN(0x89738f80, 0xd0a7a97a), TOBN(0xec088c88, 0xa580fd5b), + TOBN(0x612b1ecc, 0x42ce9e51), TOBN(0x8f9840fd, 0xb25fdd2a), + TOBN(0x3cda78c0, 0x01e7f839), TOBN(0x546b3d3a, 0xece05480), + TOBN(0x271719a9, 0x80d30916), TOBN(0x45497107, 0x584c20c4), + TOBN(0xaf8f9478, 0x5bc78608), TOBN(0x28c7d484, 0x277e2a4c), + TOBN(0xfce01767, 0x88a2ffe4), TOBN(0xdc506a35, 0x28e169a5), + TOBN(0x0ea10861, 0x7af9c93a), TOBN(0x1ed24361, 0x03fa0e08), + TOBN(0x96eaaa92, 0xa3d694e7), TOBN(0xc0f43b4d, 0xef50bc74), + TOBN(0xce6aa58c, 0x64114db4), TOBN(0x8218e8ea, 0x7c000fd4), + TOBN(0xac815dfb, 0x185f8844), TOBN(0xcd7e90cb, 0x1557abfb), + TOBN(0x23d16655, 0xafbfecdf), TOBN(0x80f3271f, 0x085cac4a), + TOBN(0x7fc39aa7, 0xd0e62f47), TOBN(0x88d519d1, 0x460a48e5), + TOBN(0x59559ac4, 0xd28f101e), TOBN(0x7981d9e9, 0xca9ae816), + TOBN(0x5c38652c, 0x9ac38203), TOBN(0x86eaf87f, 0x57657fe5), + TOBN(0x568fc472, 0xe21f5416), TOBN(0x2afff39c, 0xe7e597b5), + TOBN(0x3adbbb07, 0x256d4eab), TOBN(0x22598692, 0x8285ab89), + TOBN(0x35f8112a, 0x041caefe), TOBN(0x95df02e3, 0xa5064c8b), + TOBN(0x4d63356e, 0xc7004bf3), TOBN(0x230a08f4, 0xdb83c7de), + TOBN(0xca27b270, 0x8709a7b7), TOBN(0x0d1c4cc4, 0xcb9abd2d), + TOBN(0x8a0bc66e, 0x7550fee8), TOBN(0x369cd4c7, 0x9cf7247e), + TOBN(0x75562e84, 0x92b5b7e7), TOBN(0x8fed0da0, 0x5802af7b), + TOBN(0x6a7091c2, 0xe48fb889), TOBN(0x26882c13, 0x7b8a9d06), + TOBN(0xa2498663, 0x1b82a0e2), TOBN(0x844ed736, 0x3518152d), + TOBN(0x282f476f, 0xd86e27c7), TOBN(0xa04edaca, 0x04afefdc), + TOBN(0x8b256ebc, 0x6119e34d), TOBN(0x56a413e9, 0x0787d78b),} + , + {TOBN(0x82ee061d, 0x5a74be50), TOBN(0xe41781c4, 0xdea16ff5), + TOBN(0xe0b0c81e, 0x99bfc8a2), TOBN(0x624f4d69, 0x0b547e2d), + TOBN(0x3a83545d, 0xbdcc9ae4), TOBN(0x2573dbb6, 0x409b1e8e), + TOBN(0x482960c4, 0xa6c93539), TOBN(0xf01059ad, 0x5ae18798), + TOBN(0x715c9f97, 0x3112795f), TOBN(0xe8244437, 0x984e6ee1), + TOBN(0x55cb4858, 0xecb66bcd), TOBN(0x7c136735, 0xabaffbee), + TOBN(0x54661595, 0x5dbec38e), TOBN(0x51c0782c, 0x388ad153), + TOBN(0x9ba4c53a, 0xc6e0952f), TOBN(0x27e6782a, 0x1b21dfa8), + TOBN(0x682f903d, 0x4ed2dbc2), TOBN(0x0eba59c8, 0x7c3b2d83), + TOBN(0x8e9dc84d, 0x9c7e9335), TOBN(0x5f9b21b0, 0x0eb226d7), + TOBN(0xe33bd394, 0xaf267bae), TOBN(0xaa86cc25, 0xbe2e15ae), + TOBN(0x4f0bf67d, 0x6a8ec500), TOBN(0x5846aa44, 0xf9630658), + TOBN(0xfeb09740, 0xe2c2bf15), TOBN(0x627a2205, 0xa9e99704), + TOBN(0xec8d73d0, 0xc2fbc565), TOBN(0x223eed8f, 0xc20c8de8), + TOBN(0x1ee32583, 0xa8363b49), TOBN(0x1a0b6cb9, 0xc9c2b0a6), + TOBN(0x49f7c3d2, 0x90dbc85c), TOBN(0xa8dfbb97, 0x1ef4c1ac), + TOBN(0xafb34d4c, 0x65c7c2ab), TOBN(0x1d4610e7, 0xe2c5ea84), + TOBN(0x893f6d1b, 0x973c4ab5), TOBN(0xa3cdd7e9, 0x945ba5c4), + TOBN(0x60514983, 0x064417ee), TOBN(0x1459b23c, 0xad6bdf2b), + TOBN(0x23b2c341, 0x5cf726c3), TOBN(0x3a829635, 0x32d6354a), + TOBN(0x294f901f, 0xab192c18), TOBN(0xec5fcbfe, 0x7030164f), + TOBN(0xe2e2fcb7, 0xe2246ba6), TOBN(0x1e7c88b3, 0x221a1a0c), + TOBN(0x72c7dd93, 0xc92d88c5), TOBN(0x41c2148e, 0x1106fb59), + TOBN(0x547dd4f5, 0xa0f60f14), TOBN(0xed9b52b2, 0x63960f31), + TOBN(0x6c8349eb, 0xb0a5b358), TOBN(0xb154c5c2, 0x9e7e2ed6), + TOBN(0xcad5eccf, 0xeda462db), TOBN(0xf2d6dbe4, 0x2de66b69), + TOBN(0x426aedf3, 0x8665e5b2), TOBN(0x488a8513, 0x7b7f5723), + TOBN(0x15cc43b3, 0x8bcbb386), TOBN(0x27ad0af3, 0xd791d879), + TOBN(0xc16c236e, 0x846e364f), TOBN(0x7f33527c, 0xdea50ca0), + TOBN(0xc4810775, 0x0926b86d), TOBN(0x6c2a3609, 0x0598e70c), + TOBN(0xa6755e52, 0xf024e924), TOBN(0xe0fa07a4, 0x9db4afca), + TOBN(0x15c3ce7d, 0x66831790), TOBN(0x5b4ef350, 0xa6cbb0d6), + TOBN(0x2c4aafc4, 0xb6205969), TOBN(0x42563f02, 0xf6c7854f), + TOBN(0x016aced5, 0x1d983b48), TOBN(0xfeb356d8, 0x99949755), + TOBN(0x8c2a2c81, 0xd1a39bd7), TOBN(0x8f44340f, 0xe6934ae9), + TOBN(0x148cf91c, 0x447904da), TOBN(0x7340185f, 0x0f51a926), + TOBN(0x2f8f00fb, 0x7409ab46), TOBN(0x057e78e6, 0x80e289b2), + TOBN(0x03e5022c, 0xa888e5d1), TOBN(0x3c87111a, 0x9dede4e2), + TOBN(0x5b9b0e1c, 0x7809460b), TOBN(0xe751c852, 0x71c9abc7), + TOBN(0x8b944e28, 0xc7cc1dc9), TOBN(0x4f201ffa, 0x1d3cfa08), + TOBN(0x02fc905c, 0x3e6721ce), TOBN(0xd52d70da, 0xd0b3674c), + TOBN(0x5dc2e5ca, 0x18810da4), TOBN(0xa984b273, 0x5c69dd99), + TOBN(0x63b92527, 0x84de5ca4), TOBN(0x2f1c9872, 0xc852dec4), + TOBN(0x18b03593, 0xc2e3de09), TOBN(0x19d70b01, 0x9813dc2f), + TOBN(0x42806b2d, 0xa6dc1d29), TOBN(0xd3030009, 0xf871e144), + TOBN(0xa1feb333, 0xaaf49276), TOBN(0xb5583b9e, 0xc70bc04b), + TOBN(0x1db0be78, 0x95695f20), TOBN(0xfc841811, 0x89d012b5), + TOBN(0x6409f272, 0x05f61643), TOBN(0x40d34174, 0xd5883128), + TOBN(0xd79196f5, 0x67419833), TOBN(0x6059e252, 0x863b7b08), + TOBN(0x84da1817, 0x1c56700c), TOBN(0x5758ee56, 0xb28d3ec4), + TOBN(0x7da2771d, 0x013b0ea6), TOBN(0xfddf524b, 0x54c5e9b9), + TOBN(0x7df4faf8, 0x24305d80), TOBN(0x58f5c1bf, 0x3a97763f), + TOBN(0xa5af37f1, 0x7c696042), TOBN(0xd4cba22c, 0x4a2538de), + TOBN(0x211cb995, 0x9ea42600), TOBN(0xcd105f41, 0x7b069889), + TOBN(0xb1e1cf19, 0xddb81e74), TOBN(0x472f2d89, 0x5157b8ca), + TOBN(0x086fb008, 0xee9db885), TOBN(0x365cd570, 0x0f26d131), + TOBN(0x284b02bb, 0xa2be7053), TOBN(0xdcbbf7c6, 0x7ab9a6d6), + TOBN(0x4425559c, 0x20f7a530), TOBN(0x961f2dfa, 0x188767c8), + TOBN(0xe2fd9435, 0x70dc80c4), TOBN(0x104d6b63, 0xf0784120), + TOBN(0x7f592bc1, 0x53567122), TOBN(0xf6bc1246, 0xf688ad77), + TOBN(0x05214c05, 0x0f15dde9), TOBN(0xa47a76a8, 0x0d5f2b82), + TOBN(0xbb254d30, 0x62e82b62), TOBN(0x11a05fe0, 0x3ec955ee), + TOBN(0x7eaff46e, 0x9d529b36), TOBN(0x55ab1301, 0x8f9e3df6), + TOBN(0xc463e371, 0x99317698), TOBN(0xfd251438, 0xccda47ad), + TOBN(0xca9c3547, 0x23d695ea), TOBN(0x48ce626e, 0x16e589b5), + TOBN(0x6b5b64c7, 0xb187d086), TOBN(0xd02e1794, 0xb2207948), + TOBN(0x8b58e98f, 0x7198111d), TOBN(0x90ca6305, 0xdcf9c3cc), + TOBN(0x5691fe72, 0xf34089b0), TOBN(0x60941af1, 0xfc7c80ff), + TOBN(0xa09bc0a2, 0x22eb51e5), TOBN(0xc0bb7244, 0xaa9cf09a), + TOBN(0x36a8077f, 0x80159f06), TOBN(0x8b5c989e, 0xdddc560e), + TOBN(0x19d2f316, 0x512e1f43), TOBN(0x02eac554, 0xad08ff62), + TOBN(0x012ab84c, 0x07d20b4e), TOBN(0x37d1e115, 0xd6d4e4e1), + TOBN(0xb6443e1a, 0xab7b19a8), TOBN(0xf08d067e, 0xdef8cd45), + TOBN(0x63adf3e9, 0x685e03da), TOBN(0xcf15a10e, 0x4792b916), + TOBN(0xf44bcce5, 0xb738a425), TOBN(0xebe131d5, 0x9636b2fd), + TOBN(0x94068841, 0x7850d605), TOBN(0x09684eaa, 0xb40d749d), + TOBN(0x8c3c669c, 0x72ba075b), TOBN(0x89f78b55, 0xba469015), + TOBN(0x5706aade, 0x3e9f8ba8), TOBN(0x6d8bd565, 0xb32d7ed7), + TOBN(0x25f4e63b, 0x805f08d6), TOBN(0x7f48200d, 0xc3bcc1b5), + TOBN(0x4e801968, 0xb025d847), TOBN(0x74afac04, 0x87cbe0a8), + TOBN(0x43ed2c2b, 0x7e63d690), TOBN(0xefb6bbf0, 0x0223cdb8), + TOBN(0x4fec3cae, 0x2884d3fe), TOBN(0x065ecce6, 0xd75e25a4), + TOBN(0x6c2294ce, 0x69f79071), TOBN(0x0d9a8e5f, 0x044b8666), + TOBN(0x5009f238, 0x17b69d8f), TOBN(0x3c29f8fe, 0xc5dfdaf7), + TOBN(0x9067528f, 0xebae68c4), TOBN(0x5b385632, 0x30c5ba21), + TOBN(0x540df119, 0x1fdd1aec), TOBN(0xcf37825b, 0xcfba4c78), + TOBN(0x77eff980, 0xbeb11454), TOBN(0x40a1a991, 0x60c1b066), + TOBN(0xe8018980, 0xf889a1c7), TOBN(0xb9c52ae9, 0x76c24be0), + TOBN(0x05fbbcce, 0x45650ef4), TOBN(0xae000f10, 0x8aa29ac7), + TOBN(0x884b7172, 0x4f04c470), TOBN(0x7cd4fde2, 0x19bb5c25), + TOBN(0x6477b22a, 0xe8840869), TOBN(0xa8868859, 0x5fbd0686), + TOBN(0xf23cc02e, 0x1116dfba), TOBN(0x76cd563f, 0xd87d7776), + TOBN(0xe2a37598, 0xa9d82abf), TOBN(0x5f188ccb, 0xe6c170f5), + TOBN(0x81682200, 0x5066b087), TOBN(0xda22c212, 0xc7155ada), + TOBN(0x151e5d3a, 0xfbddb479), TOBN(0x4b606b84, 0x6d715b99), + TOBN(0x4a73b54b, 0xf997cb2e), TOBN(0x9a1bfe43, 0x3ecd8b66), + TOBN(0x1c312809, 0x2a67d48a), TOBN(0xcd6a671e, 0x031fa9e2), + TOBN(0xbec3312a, 0x0e43a34a), TOBN(0x1d935639, 0x55ef47d3), + TOBN(0x5ea02489, 0x8fea73ea), TOBN(0x8247b364, 0xa035afb2), + TOBN(0xb58300a6, 0x5265b54c), TOBN(0x3286662f, 0x722c7148), + TOBN(0xb77fd76b, 0xb4ec4c20), TOBN(0xf0a12fa7, 0x0f3fe3fd), + TOBN(0xf845bbf5, 0x41d8c7e8), TOBN(0xe4d969ca, 0x5ec10aa8), + TOBN(0x4c0053b7, 0x43e232a3), TOBN(0xdc7a3fac, 0x37f8a45a), + TOBN(0x3c4261c5, 0x20d81c8f), TOBN(0xfd4b3453, 0xb00eab00), + TOBN(0x76d48f86, 0xd36e3062), TOBN(0x626c5277, 0xa143ff02), + TOBN(0x538174de, 0xaf76f42e), TOBN(0x2267aa86, 0x6407ceac), + TOBN(0xfad76351, 0x72e572d5), TOBN(0xab861af7, 0xba7330eb), + TOBN(0xa0a1c8c7, 0x418d8657), TOBN(0x988821cb, 0x20289a52), + TOBN(0x79732522, 0xcccc18ad), TOBN(0xaadf3f8d, 0xf1a6e027), + TOBN(0xf7382c93, 0x17c2354d), TOBN(0x5ce1680c, 0xd818b689), + TOBN(0x359ebbfc, 0xd9ecbee9), TOBN(0x4330689c, 0x1cae62ac), + TOBN(0xb55ce5b4, 0xc51ac38a), TOBN(0x7921dfea, 0xfe238ee8), + TOBN(0x3972bef8, 0x271d1ca5), TOBN(0x3e423bc7, 0xe8aabd18), + TOBN(0x57b09f3f, 0x44a3e5e3), TOBN(0x5da886ae, 0x7b444d66), + TOBN(0x68206634, 0xa9964375), TOBN(0x356a2fa3, 0x699cd0ff), + TOBN(0xaf0faa24, 0xdba515e9), TOBN(0x536e1f5c, 0xb321d79a), + TOBN(0xd3b9913a, 0x5c04e4ea), TOBN(0xd549dcfe, 0xd6f11513), + TOBN(0xee227bf5, 0x79fd1d94), TOBN(0x9f35afee, 0xb43f2c67), + TOBN(0xd2638d24, 0xf1314f53), TOBN(0x62baf948, 0xcabcd822), + TOBN(0x5542de29, 0x4ef48db0), TOBN(0xb3eb6a04, 0xfc5f6bb2), + TOBN(0x23c110ae, 0x1208e16a), TOBN(0x1a4d15b5, 0xf8363e24), + TOBN(0x30716844, 0x164be00b), TOBN(0xa8e24824, 0xf6f4690d), + TOBN(0x548773a2, 0x90b170cf), TOBN(0xa1bef331, 0x42f191f4), + TOBN(0x70f418d0, 0x9247aa97), TOBN(0xea06028e, 0x48be9147), + TOBN(0xe13122f3, 0xdbfb894e), TOBN(0xbe9b79f6, 0xce274b18), + TOBN(0x85a49de5, 0xca58aadf), TOBN(0x24957758, 0x11487351), + TOBN(0x111def61, 0xbb939099), TOBN(0x1d6a974a, 0x26d13694), + TOBN(0x4474b4ce, 0xd3fc253b), TOBN(0x3a1485e6, 0x4c5db15e), + TOBN(0xe79667b4, 0x147c15b4), TOBN(0xe34f553b, 0x7bc61301), + TOBN(0x032b80f8, 0x17094381), TOBN(0x55d8bafd, 0x723eaa21), + TOBN(0x5a987995, 0xf1c0e74e), TOBN(0x5a9b292e, 0xebba289c), + TOBN(0x413cd4b2, 0xeb4c8251), TOBN(0x98b5d243, 0xd162db0a), + TOBN(0xbb47bf66, 0x68342520), TOBN(0x08d68949, 0xbaa862d1), + TOBN(0x11f349c7, 0xe906abcd), TOBN(0x454ce985, 0xed7bf00e), + TOBN(0xacab5c9e, 0xb55b803b), TOBN(0xb03468ea, 0x31e3c16d), + TOBN(0x5c24213d, 0xd273bf12), TOBN(0x211538eb, 0x71587887), + TOBN(0x198e4a2f, 0x731dea2d), TOBN(0xd5856cf2, 0x74ed7b2a), + TOBN(0x86a632eb, 0x13a664fe), TOBN(0x932cd909, 0xbda41291), + TOBN(0x850e95d4, 0xc0c4ddc0), TOBN(0xc0f422f8, 0x347fc2c9), + TOBN(0xe68cbec4, 0x86076bcb), TOBN(0xf9e7c0c0, 0xcd6cd286), + TOBN(0x65994ddb, 0x0f5f27ca), TOBN(0xe85461fb, 0xa80d59ff), + TOBN(0xff05481a, 0x66601023), TOBN(0xc665427a, 0xfc9ebbfb), + TOBN(0xb0571a69, 0x7587fd52), TOBN(0x935289f8, 0x8d49efce), + TOBN(0x61becc60, 0xea420688), TOBN(0xb22639d9, 0x13a786af), + TOBN(0x1a8e6220, 0x361ecf90), TOBN(0x001f23e0, 0x25506463), + TOBN(0xe4ae9b5d, 0x0a5c2b79), TOBN(0xebc9cdad, 0xd8149db5), + TOBN(0xb33164a1, 0x934aa728), TOBN(0x750eb00e, 0xae9b60f3), + TOBN(0x5a91615b, 0x9b9cfbfd), TOBN(0x97015cbf, 0xef45f7f6), + TOBN(0xb462c4a5, 0xbf5151df), TOBN(0x21adcc41, 0xb07118f2), + TOBN(0xd60c545b, 0x043fa42c), TOBN(0xfc21aa54, 0xe96be1ab), + TOBN(0xe84bc32f, 0x4e51ea80), TOBN(0x3dae45f0, 0x259b5d8d), + TOBN(0xbb73c7eb, 0xc38f1b5e), TOBN(0xe405a74a, 0xe8ae617d), + TOBN(0xbb1ae9c6, 0x9f1c56bd), TOBN(0x8c176b98, 0x49f196a4), + TOBN(0xc448f311, 0x6875092b), TOBN(0xb5afe3de, 0x9f976033), + TOBN(0xa8dafd49, 0x145813e5), TOBN(0x687fc4d9, 0xe2b34226), + TOBN(0xf2dfc92d, 0x4c7ff57f), TOBN(0x004e3fc1, 0x401f1b46), + TOBN(0x5afddab6, 0x1430c9ab), TOBN(0x0bdd41d3, 0x2238e997), + TOBN(0xf0947430, 0x418042ae), TOBN(0x71f9adda, 0xcdddc4cb), + TOBN(0x7090c016, 0xc52dd907), TOBN(0xd9bdf44d, 0x29e2047f), + TOBN(0xe6f1fe80, 0x1b1011a6), TOBN(0xb63accbc, 0xd9acdc78), + TOBN(0xcfc7e235, 0x1272a95b), TOBN(0x0c667717, 0xa6276ac8), + TOBN(0x3c0d3709, 0xe2d7eef7), TOBN(0x5add2b06, 0x9a685b3e), + TOBN(0x363ad32d, 0x14ea5d65), TOBN(0xf8e01f06, 0x8d7dd506), + TOBN(0xc9ea2213, 0x75b4aac6), TOBN(0xed2a2bf9, 0x0d353466), + TOBN(0x439d79b5, 0xe9d3a7c3), TOBN(0x8e0ee5a6, 0x81b7f34b), + TOBN(0xcf3dacf5, 0x1dc4ba75), TOBN(0x1d3d1773, 0xeb3310c7), + TOBN(0xa8e67112, 0x7747ae83), TOBN(0x31f43160, 0x197d6b40), + TOBN(0x0521ccee, 0xcd961400), TOBN(0x67246f11, 0xf6535768), + TOBN(0x702fcc5a, 0xef0c3133), TOBN(0x247cc45d, 0x7e16693b), + TOBN(0xfd484e49, 0xc729b749), TOBN(0x522cef7d, 0xb218320f), + TOBN(0xe56ef405, 0x59ab93b3), TOBN(0x225fba11, 0x9f181071), + TOBN(0x33bd6595, 0x15330ed0), TOBN(0xc4be69d5, 0x1ddb32f7), + TOBN(0x264c7668, 0x0448087c), TOBN(0xac30903f, 0x71432dae), + TOBN(0x3851b266, 0x00f9bf47), TOBN(0x400ed311, 0x6cdd6d03), + TOBN(0x045e79fe, 0xf8fd2424), TOBN(0xfdfd974a, 0xfa6da98b), + TOBN(0x45c9f641, 0x0c1e673a), TOBN(0x76f2e733, 0x5b2c5168), + TOBN(0x1adaebb5, 0x2a601753), TOBN(0xb286514c, 0xc57c2d49), + TOBN(0xd8769670, 0x1e0bfd24), TOBN(0x950c547e, 0x04478922), + TOBN(0xd1d41969, 0xe5d32bfe), TOBN(0x30bc1472, 0x750d6c3e), + TOBN(0x8f3679fe, 0xe0e27f3a), TOBN(0x8f64a7dc, 0xa4a6ee0c), + TOBN(0x2fe59937, 0x633dfb1f), TOBN(0xea82c395, 0x977f2547), + TOBN(0xcbdfdf1a, 0x661ea646), TOBN(0xc7ccc591, 0xb9085451), + TOBN(0x82177962, 0x81761e13), TOBN(0xda57596f, 0x9196885c), + TOBN(0xbc17e849, 0x28ffbd70), TOBN(0x1e6e0a41, 0x2671d36f), + TOBN(0x61ae872c, 0x4152fcf5), TOBN(0x441c87b0, 0x9e77e754), + TOBN(0xd0799dd5, 0xa34dff09), TOBN(0x766b4e44, 0x88a6b171), + TOBN(0xdc06a512, 0x11f1c792), TOBN(0xea02ae93, 0x4be35c3e), + TOBN(0xe5ca4d6d, 0xe90c469e), TOBN(0x4df4368e, 0x56e4ff5c), + TOBN(0x7817acab, 0x4baef62e), TOBN(0x9f5a2202, 0xa85b91e8), + TOBN(0x9666ebe6, 0x6ce57610), TOBN(0x32ad31f3, 0xf73bfe03), + TOBN(0x628330a4, 0x25bcf4d6), TOBN(0xea950593, 0x515056e6), + TOBN(0x59811c89, 0xe1332156), TOBN(0xc89cf1fe, 0x8c11b2d7), + TOBN(0x75b63913, 0x04e60cc0), TOBN(0xce811e8d, 0x4625d375), + TOBN(0x030e43fc, 0x2d26e562), TOBN(0xfbb30b4b, 0x608d36a0), + TOBN(0x634ff82c, 0x48528118), TOBN(0x7c6fe085, 0xcd285911), + TOBN(0x7f2830c0, 0x99358f28), TOBN(0x2e60a95e, 0x665e6c09), + TOBN(0x08407d3d, 0x9b785dbf), TOBN(0x530889ab, 0xa759bce7), + TOBN(0xf228e0e6, 0x52f61239), TOBN(0x2b6d1461, 0x6879be3c), + TOBN(0xe6902c04, 0x51a7bbf7), TOBN(0x30ad99f0, 0x76f24a64), + TOBN(0x66d9317a, 0x98bc6da0), TOBN(0xf4f877f3, 0xcb596ac0), + TOBN(0xb05ff62d, 0x4c44f119), TOBN(0x4555f536, 0xe9b77416), + TOBN(0xc7c0d059, 0x8caed63b), TOBN(0x0cd2b7ce, 0xc358b2a9), + TOBN(0x3f33287b, 0x46945fa3), TOBN(0xf8785b20, 0xd67c8791), + TOBN(0xc54a7a61, 0x9637bd08), TOBN(0x54d4598c, 0x18be79d7), + TOBN(0x889e5acb, 0xc46d7ce1), TOBN(0x9a515bb7, 0x8b085877), + TOBN(0xfac1a03d, 0x0b7a5050), TOBN(0x7d3e738a, 0xf2926035), + TOBN(0x861cc2ce, 0x2a6cb0eb), TOBN(0x6f2e2955, 0x8f7adc79), + TOBN(0x61c4d451, 0x33016376), TOBN(0xd9fd2c80, 0x5ad59090), + TOBN(0xe5a83738, 0xb2b836a1), TOBN(0x855b41a0, 0x7c0d6622), + TOBN(0x186fe317, 0x7cc19af1), TOBN(0x6465c1ff, 0xfdd99acb), + TOBN(0x46e5c23f, 0x6974b99e), TOBN(0x75a7cf8b, 0xa2717cbe), + TOBN(0x4d2ebc3f, 0x062be658), TOBN(0x094b4447, 0x5f209c98), + TOBN(0x4af285ed, 0xb940cb5a), TOBN(0x6706d792, 0x7cc82f10), + TOBN(0xc8c8776c, 0x030526fa), TOBN(0xfa8e6f76, 0xa0da9140), + TOBN(0x77ea9d34, 0x591ee4f0), TOBN(0x5f46e337, 0x40274166), + TOBN(0x1bdf98bb, 0xea671457), TOBN(0xd7c08b46, 0x862a1fe2), + TOBN(0x46cc303c, 0x1c08ad63), TOBN(0x99543440, 0x4c845e7b), + TOBN(0x1b8fbdb5, 0x48f36bf7), TOBN(0x5b82c392, 0x8c8273a7), + TOBN(0x08f712c4, 0x928435d5), TOBN(0x071cf0f1, 0x79330380), + TOBN(0xc74c2d24, 0xa8da054a), TOBN(0xcb0e7201, 0x43c46b5c), + TOBN(0x0ad7337a, 0xc0b7eff3), TOBN(0x8552225e, 0xc5e48b3c), + TOBN(0xe6f78b0c, 0x73f13a5f), TOBN(0x5e70062e, 0x82349cbe), + TOBN(0x6b8d5048, 0xe7073969), TOBN(0x392d2a29, 0xc33cb3d2), + TOBN(0xee4f727c, 0x4ecaa20f), TOBN(0xa068c99e, 0x2ccde707), + TOBN(0xfcd5651f, 0xb87a2913), TOBN(0xea3e3c15, 0x3cc252f0), + TOBN(0x777d92df, 0x3b6cd3e4), TOBN(0x7a414143, 0xc5a732e7), + TOBN(0xa895951a, 0xa71ff493), TOBN(0xfe980c92, 0xbbd37cf6), + TOBN(0x45bd5e64, 0xdecfeeff), TOBN(0x910dc2a9, 0xa44c43e9), + TOBN(0xcb403f26, 0xcca9f54d), TOBN(0x928bbdfb, 0x9303f6db), + TOBN(0x3c37951e, 0xa9eee67c), TOBN(0x3bd61a52, 0xf79961c3), + TOBN(0x09a238e6, 0x395c9a79), TOBN(0x6940ca2d, 0x61eb352d), + TOBN(0x7d1e5c5e, 0xc1875631), TOBN(0x1e19742c, 0x1e1b20d1), + TOBN(0x4633d908, 0x23fc2e6e), TOBN(0xa76e29a9, 0x08959149), + TOBN(0x61069d9c, 0x84ed7da5), TOBN(0x0baa11cf, 0x5dbcad51), + TOBN(0xd01eec64, 0x961849da), TOBN(0x93b75f1f, 0xaf3d8c28), + TOBN(0x57bc4f9f, 0x1ca2ee44), TOBN(0x5a26322d, 0x00e00558), + TOBN(0x1888d658, 0x61a023ef), TOBN(0x1d72aab4, 0xb9e5246e), + TOBN(0xa9a26348, 0xe5563ec0), TOBN(0xa0971963, 0xc3439a43), + TOBN(0x567dd54b, 0xadb9b5b7), TOBN(0x73fac1a1, 0xc45a524b), + TOBN(0x8fe97ef7, 0xfe38e608), TOBN(0x608748d2, 0x3f384f48), + TOBN(0xb0571794, 0xc486094f), TOBN(0x869254a3, 0x8bf3a8d6), + TOBN(0x148a8dd1, 0x310b0e25), TOBN(0x99ab9f3f, 0x9aa3f7d8), + TOBN(0x0927c68a, 0x6706c02e), TOBN(0x22b5e76c, 0x69790e6c), + TOBN(0x6c325260, 0x6c71376c), TOBN(0x53a57690, 0x09ef6657), + TOBN(0x8d63f852, 0xedffcf3a), TOBN(0xb4d2ed04, 0x3c0a6f55), + TOBN(0xdb3aa8de, 0x12519b9e), TOBN(0x5d38e9c4, 0x1e0a569a), + TOBN(0x871528bf, 0x303747e2), TOBN(0xa208e77c, 0xf5b5c18d), + TOBN(0x9d129c88, 0xca6bf923), TOBN(0xbcbf197f, 0xbf02839f), + TOBN(0x9b9bf030, 0x27323194), TOBN(0x3b055a8b, 0x339ca59d), + TOBN(0xb46b2312, 0x0f669520), TOBN(0x19789f1f, 0x497e5f24), + TOBN(0x9c499468, 0xaaf01801), TOBN(0x72ee1190, 0x8b69d59c), + TOBN(0x8bd39595, 0xacf4c079), TOBN(0x3ee11ece, 0x8e0cd048), + TOBN(0xebde86ec, 0x1ed66f18), TOBN(0x225d906b, 0xd61fce43), + TOBN(0x5cab07d6, 0xe8bed74d), TOBN(0x16e4617f, 0x27855ab7), + TOBN(0x6568aadd, 0xb2fbc3dd), TOBN(0xedb5484f, 0x8aeddf5b), + TOBN(0x878f20e8, 0x6dcf2fad), TOBN(0x3516497c, 0x615f5699),} + , + {TOBN(0xef0a3fec, 0xfa181e69), TOBN(0x9ea02f81, 0x30d69a98), + TOBN(0xb2e9cf8e, 0x66eab95d), TOBN(0x520f2beb, 0x24720021), + TOBN(0x621c540a, 0x1df84361), TOBN(0x12037721, 0x71fa6d5d), + TOBN(0x6e3c7b51, 0x0ff5f6ff), TOBN(0x817a069b, 0xabb2bef3), + TOBN(0x83572fb6, 0xb294cda6), TOBN(0x6ce9bf75, 0xb9039f34), + TOBN(0x20e012f0, 0x095cbb21), TOBN(0xa0aecc1b, 0xd063f0da), + TOBN(0x57c21c3a, 0xf02909e5), TOBN(0xc7d59ecf, 0x48ce9cdc), + TOBN(0x2732b844, 0x8ae336f8), TOBN(0x056e3723, 0x3f4f85f4), + TOBN(0x8a10b531, 0x89e800ca), TOBN(0x50fe0c17, 0x145208fd), + TOBN(0x9e43c0d3, 0xb714ba37), TOBN(0x427d200e, 0x34189acc), + TOBN(0x05dee24f, 0xe616e2c0), TOBN(0x9c25f4c8, 0xee1854c1), + TOBN(0x4d3222a5, 0x8f342a73), TOBN(0x0807804f, 0xa027c952), + TOBN(0xc222653a, 0x4f0d56f3), TOBN(0x961e4047, 0xca28b805), + TOBN(0x2c03f8b0, 0x4a73434b), TOBN(0x4c966787, 0xab712a19), + TOBN(0xcc196c42, 0x864fee42), TOBN(0xc1be93da, 0x5b0ece5c), + TOBN(0xa87d9f22, 0xc131c159), TOBN(0x2bb6d593, 0xdce45655), + TOBN(0x22c49ec9, 0xb809b7ce), TOBN(0x8a41486b, 0xe2c72c2c), + TOBN(0x813b9420, 0xfea0bf36), TOBN(0xb3d36ee9, 0xa66dac69), + TOBN(0x6fddc08a, 0x328cc987), TOBN(0x0a3bcd2c, 0x3a326461), + TOBN(0x7103c49d, 0xd810dbba), TOBN(0xf9d81a28, 0x4b78a4c4), + TOBN(0x3de865ad, 0xe4d55941), TOBN(0xdedafa5e, 0x30384087), + TOBN(0x6f414abb, 0x4ef18b9b), TOBN(0x9ee9ea42, 0xfaee5268), + TOBN(0x260faa16, 0x37a55a4a), TOBN(0xeb19a514, 0x015f93b9), + TOBN(0x51d7ebd2, 0x9e9c3598), TOBN(0x523fc56d, 0x1932178e), + TOBN(0x501d070c, 0xb98fe684), TOBN(0xd60fbe9a, 0x124a1458), + TOBN(0xa45761c8, 0x92bc6b3f), TOBN(0xf5384858, 0xfe6f27cb), + TOBN(0x4b0271f7, 0xb59e763b), TOBN(0x3d4606a9, 0x5b5a8e5e), + TOBN(0x1eda5d9b, 0x05a48292), TOBN(0xda7731d0, 0xe6fec446), + TOBN(0xa3e33693, 0x90d45871), TOBN(0xe9764040, 0x06166d8d), + TOBN(0xb5c33682, 0x89a90403), TOBN(0x4bd17983, 0x72f1d637), + TOBN(0xa616679e, 0xd5d2c53a), TOBN(0x5ec4bcd8, 0xfdcf3b87), + TOBN(0xae6d7613, 0xb66a694e), TOBN(0x7460fc76, 0xe3fc27e5), + TOBN(0x70469b82, 0x95caabee), TOBN(0xde024ca5, 0x889501e3), + TOBN(0x6bdadc06, 0x076ed265), TOBN(0x0cb1236b, 0x5a0ef8b2), + TOBN(0x4065ddbf, 0x0972ebf9), TOBN(0xf1dd3875, 0x22aca432), + TOBN(0xa88b97cf, 0x744aff76), TOBN(0xd1359afd, 0xfe8e3d24), + TOBN(0x52a3ba2b, 0x91502cf3), TOBN(0x2c3832a8, 0x084db75d), + TOBN(0x04a12ddd, 0xde30b1c9), TOBN(0x7802eabc, 0xe31fd60c), + TOBN(0x33707327, 0xa37fddab), TOBN(0x65d6f2ab, 0xfaafa973), + TOBN(0x3525c5b8, 0x11e6f91a), TOBN(0x76aeb0c9, 0x5f46530b), + TOBN(0xe8815ff6, 0x2f93a675), TOBN(0xa6ec9684, 0x05f48679), + TOBN(0x6dcbb556, 0x358ae884), TOBN(0x0af61472, 0xe19e3873), + TOBN(0x72334372, 0xa5f696be), TOBN(0xc65e57ea, 0x6f22fb70), + TOBN(0x268da30c, 0x946cea90), TOBN(0x136a8a87, 0x65681b2a), + TOBN(0xad5e81dc, 0x0f9f44d4), TOBN(0xf09a6960, 0x2c46585a), + TOBN(0xd1649164, 0xc447d1b1), TOBN(0x3b4b36c8, 0x879dc8b1), + TOBN(0x20d4177b, 0x3b6b234c), TOBN(0x096a2505, 0x1730d9d0), + TOBN(0x0611b9b8, 0xef80531d), TOBN(0xba904b3b, 0x64bb495d), + TOBN(0x1192d9d4, 0x93a3147a), TOBN(0x9f30a5dc, 0x9a565545), + TOBN(0x90b1f9cb, 0x6ef07212), TOBN(0x29958546, 0x0d87fc13), + TOBN(0xd3323eff, 0xc17db9ba), TOBN(0xcb18548c, 0xcb1644a8), + TOBN(0x18a306d4, 0x4f49ffbc), TOBN(0x28d658f1, 0x4c2e8684), + TOBN(0x44ba60cd, 0xa99f8c71), TOBN(0x67b7abdb, 0x4bf742ff), + TOBN(0x66310f9c, 0x914b3f99), TOBN(0xae430a32, 0xf412c161), + TOBN(0x1e6776d3, 0x88ace52f), TOBN(0x4bc0fa24, 0x52d7067d), + TOBN(0x03c286aa, 0x8f07cd1b), TOBN(0x4cb8f38c, 0xa985b2c1), + TOBN(0x83ccbe80, 0x8c3bff36), TOBN(0x005a0bd2, 0x5263e575), + TOBN(0x460d7dda, 0x259bdcd1), TOBN(0x4a1c5642, 0xfa5cab6b), + TOBN(0x2b7bdbb9, 0x9fe4fc88), TOBN(0x09418e28, 0xcc97bbb5), + TOBN(0xd8274fb4, 0xa12321ae), TOBN(0xb137007d, 0x5c87b64e), + TOBN(0x80531fe1, 0xc63c4962), TOBN(0x50541e89, 0x981fdb25), + TOBN(0xdc1291a1, 0xfd4c2b6b), TOBN(0xc0693a17, 0xa6df4fca), + TOBN(0xb2c4604e, 0x0117f203), TOBN(0x245f1963, 0x0a99b8d0), + TOBN(0xaedc20aa, 0xc6212c44), TOBN(0xb1ed4e56, 0x520f52a8), + TOBN(0xfe48f575, 0xf8547be3), TOBN(0x0a7033cd, 0xa9e45f98), + TOBN(0x4b45d3a9, 0x18c50100), TOBN(0xb2a6cd6a, 0xa61d41da), + TOBN(0x60bbb4f5, 0x57933c6b), TOBN(0xa7538ebd, 0x2b0d7ffc), + TOBN(0x9ea3ab8d, 0x8cd626b6), TOBN(0x8273a484, 0x3601625a), + TOBN(0x88859845, 0x0168e508), TOBN(0x8cbc9bb2, 0x99a94abd), + TOBN(0x713ac792, 0xfab0a671), TOBN(0xa3995b19, 0x6c9ebffc), + TOBN(0xe711668e, 0x1239e152), TOBN(0x56892558, 0xbbb8dff4), + TOBN(0x8bfc7dab, 0xdbf17963), TOBN(0x5b59fe5a, 0xb3de1253), + TOBN(0x7e3320eb, 0x34a9f7ae), TOBN(0xe5e8cf72, 0xd751efe4), + TOBN(0x7ea003bc, 0xd9be2f37), TOBN(0xc0f551a0, 0xb6c08ef7), + TOBN(0x56606268, 0x038f6725), TOBN(0x1dd38e35, 0x6d92d3b6), + TOBN(0x07dfce7c, 0xc3cbd686), TOBN(0x4e549e04, 0x651c5da8), + TOBN(0x4058f93b, 0x08b19340), TOBN(0xc2fae6f4, 0xcac6d89d), + TOBN(0x4bad8a8c, 0x8f159cc7), TOBN(0x0ddba4b3, 0xcb0b601c), + TOBN(0xda4fc7b5, 0x1dd95f8c), TOBN(0x1d163cd7, 0xcea5c255), + TOBN(0x30707d06, 0x274a8c4c), TOBN(0x79d9e008, 0x2802e9ce), + TOBN(0x02a29ebf, 0xe6ddd505), TOBN(0x37064e74, 0xb50bed1a), + TOBN(0x3f6bae65, 0xa7327d57), TOBN(0x3846f5f1, 0xf83920bc), + TOBN(0x87c37491, 0x60df1b9b), TOBN(0x4cfb2895, 0x2d1da29f), + TOBN(0x10a478ca, 0x4ed1743c), TOBN(0x390c6030, 0x3edd47c6), + TOBN(0x8f3e5312, 0x8c0a78de), TOBN(0xccd02bda, 0x1e85df70), + TOBN(0xd6c75c03, 0xa61b6582), TOBN(0x0762921c, 0xfc0eebd1), + TOBN(0xd34d0823, 0xd85010c0), TOBN(0xd73aaacb, 0x0044cf1f), + TOBN(0xfb4159bb, 0xa3b5e78a), TOBN(0x2287c7f7, 0xe5826f3f), + TOBN(0x4aeaf742, 0x580b1a01), TOBN(0xf080415d, 0x60423b79), + TOBN(0xe12622cd, 0xa7dea144), TOBN(0x49ea4996, 0x59d62472), + TOBN(0xb42991ef, 0x571f3913), TOBN(0x0610f214, 0xf5b25a8a), + TOBN(0x47adc585, 0x30b79e8f), TOBN(0xf90e3df6, 0x07a065a2), + TOBN(0x5d0a5deb, 0x43e2e034), TOBN(0x53fb5a34, 0x444024aa), + TOBN(0xa8628c68, 0x6b0c9f7f), TOBN(0x9c69c29c, 0xac563656), + TOBN(0x5a231feb, 0xbace47b6), TOBN(0xbdce0289, 0x9ea5a2ec), + TOBN(0x05da1fac, 0x9463853e), TOBN(0x96812c52, 0x509e78aa), + TOBN(0xd3fb5771, 0x57151692), TOBN(0xeb2721f8, 0xd98e1c44), + TOBN(0xc0506087, 0x32399be1), TOBN(0xda5a5511, 0xd979d8b8), + TOBN(0x737ed55d, 0xc6f56780), TOBN(0xe20d3004, 0x0dc7a7f4), + TOBN(0x02ce7301, 0xf5941a03), TOBN(0x91ef5215, 0xed30f83a), + TOBN(0x28727fc1, 0x4092d85f), TOBN(0x72d223c6, 0x5c49e41a), + TOBN(0xa7cf30a2, 0xba6a4d81), TOBN(0x7c086209, 0xb030d87d), + TOBN(0x04844c7d, 0xfc588b09), TOBN(0x728cd499, 0x5874bbb0), + TOBN(0xcc1281ee, 0xe84c0495), TOBN(0x0769b5ba, 0xec31958f), + TOBN(0x665c228b, 0xf99c2471), TOBN(0xf2d8a11b, 0x191eb110), + TOBN(0x4594f494, 0xd36d7024), TOBN(0x482ded8b, 0xcdcb25a1), + TOBN(0xc958a9d8, 0xdadd4885), TOBN(0x7004477e, 0xf1d2b547), + TOBN(0x0a45f6ef, 0x2a0af550), TOBN(0x4fc739d6, 0x2f8d6351), + TOBN(0x75cdaf27, 0x786f08a9), TOBN(0x8700bb26, 0x42c2737f), + TOBN(0x855a7141, 0x1c4e2670), TOBN(0x810188c1, 0x15076fef), + TOBN(0xc251d0c9, 0xabcd3297), TOBN(0xae4c8967, 0xf48108eb), + TOBN(0xbd146de7, 0x18ceed30), TOBN(0xf9d4f07a, 0xc986bced), + TOBN(0x5ad98ed5, 0x83fa1e08), TOBN(0x7780d33e, 0xbeabd1fb), + TOBN(0xe330513c, 0x903b1196), TOBN(0xba11de9e, 0xa47bc8c4), + TOBN(0x684334da, 0x02c2d064), TOBN(0x7ecf360d, 0xa48de23b), + TOBN(0x57a1b474, 0x0a9089d8), TOBN(0xf28fa439, 0xff36734c), + TOBN(0xf2a482cb, 0xea4570b3), TOBN(0xee65d68b, 0xa5ebcee9), + TOBN(0x988d0036, 0xb9694cd5), TOBN(0x53edd0e9, 0x37885d32), + TOBN(0xe37e3307, 0xbeb9bc6d), TOBN(0xe9abb907, 0x9f5c6768), + TOBN(0x4396ccd5, 0x51f2160f), TOBN(0x2500888c, 0x47336da6), + TOBN(0x383f9ed9, 0x926fce43), TOBN(0x809dd1c7, 0x04da2930), + TOBN(0x30f6f596, 0x8a4cb227), TOBN(0x0d700c7f, 0x73a56b38), + TOBN(0x1825ea33, 0xab64a065), TOBN(0xaab9b735, 0x1338df80), + TOBN(0x1516100d, 0x9b63f57f), TOBN(0x2574395a, 0x27a6a634), + TOBN(0xb5560fb6, 0x700a1acd), TOBN(0xe823fd73, 0xfd999681), + TOBN(0xda915d1f, 0x6cb4e1ba), TOBN(0x0d030118, 0x6ebe00a3), + TOBN(0x744fb0c9, 0x89fca8cd), TOBN(0x970d01db, 0xf9da0e0b), + TOBN(0x0ad8c564, 0x7931d76f), TOBN(0xb15737bf, 0xf659b96a), + TOBN(0xdc9933e8, 0xa8b484e7), TOBN(0xb2fdbdf9, 0x7a26dec7), + TOBN(0x2349e9a4, 0x9f1f0136), TOBN(0x7860368e, 0x70fddddb), + TOBN(0xd93d2c1c, 0xf9ad3e18), TOBN(0x6d6c5f17, 0x689f4e79), + TOBN(0x7a544d91, 0xb24ff1b6), TOBN(0x3e12a5eb, 0xfe16cd8c), + TOBN(0x543574e9, 0xa56b872f), TOBN(0xa1ad550c, 0xfcf68ea2), + TOBN(0x689e37d2, 0x3f560ef7), TOBN(0x8c54b9ca, 0xc9d47a8b), + TOBN(0x46d40a4a, 0x088ac342), TOBN(0xec450c7c, 0x1576c6d0), + TOBN(0xb589e31c, 0x1f9689e9), TOBN(0xdacf2602, 0xb8781718), + TOBN(0xa89237c6, 0xc8cb6b42), TOBN(0x1326fc93, 0xb96ef381), + TOBN(0x55d56c6d, 0xb5f07825), TOBN(0xacba2eea, 0x7449e22d), + TOBN(0x74e0887a, 0x633c3000), TOBN(0xcb6cd172, 0xd7cbcf71), + TOBN(0x309e81de, 0xc36cf1be), TOBN(0x07a18a6d, 0x60ae399b), + TOBN(0xb36c2679, 0x9edce57e), TOBN(0x52b892f4, 0xdf001d41), + TOBN(0xd884ae5d, 0x16a1f2c6), TOBN(0x9b329424, 0xefcc370a), + TOBN(0x3120daf2, 0xbd2e21df), TOBN(0x55298d2d, 0x02470a99), + TOBN(0x0b78af6c, 0xa05db32e), TOBN(0x5c76a331, 0x601f5636), + TOBN(0xaae861ff, 0xf8a4f29c), TOBN(0x70dc9240, 0xd68f8d49), + TOBN(0x960e649f, 0x81b1321c), TOBN(0x3d2c801b, 0x8792e4ce), + TOBN(0xf479f772, 0x42521876), TOBN(0x0bed93bc, 0x416c79b1), + TOBN(0xa67fbc05, 0x263e5bc9), TOBN(0x01e8e630, 0x521db049), + TOBN(0x76f26738, 0xc6f3431e), TOBN(0xe609cb02, 0xe3267541), + TOBN(0xb10cff2d, 0x818c877c), TOBN(0x1f0e75ce, 0x786a13cb), + TOBN(0xf4fdca64, 0x1158544d), TOBN(0x5d777e89, 0x6cb71ed0), + TOBN(0x3c233737, 0xa9aa4755), TOBN(0x7b453192, 0xe527ab40), + TOBN(0xdb59f688, 0x39f05ffe), TOBN(0x8f4f4be0, 0x6d82574e), + TOBN(0xcce3450c, 0xee292d1b), TOBN(0xaa448a12, 0x61ccd086), + TOBN(0xabce91b3, 0xf7914967), TOBN(0x4537f09b, 0x1908a5ed), + TOBN(0xa812421e, 0xf51042e7), TOBN(0xfaf5cebc, 0xec0b3a34), + TOBN(0x730ffd87, 0x4ca6b39a), TOBN(0x70fb72ed, 0x02efd342), + TOBN(0xeb4735f9, 0xd75c8edb), TOBN(0xc11f2157, 0xc278aa51), + TOBN(0xc459f635, 0xbf3bfebf), TOBN(0x3a1ff0b4, 0x6bd9601f), + TOBN(0xc9d12823, 0xc420cb73), TOBN(0x3e9af3e2, 0x3c2915a3), + TOBN(0xe0c82c72, 0xb41c3440), TOBN(0x175239e5, 0xe3039a5f), + TOBN(0xe1084b8a, 0x558795a3), TOBN(0x328d0a1d, 0xd01e5c60), + TOBN(0x0a495f2e, 0xd3788a04), TOBN(0x25d8ff16, 0x66c11a9f), + TOBN(0xf5155f05, 0x9ed692d6), TOBN(0x954fa107, 0x4f425fe4), + TOBN(0xd16aabf2, 0xe98aaa99), TOBN(0x90cd8ba0, 0x96b0f88a), + TOBN(0x957f4782, 0xc154026a), TOBN(0x54ee0734, 0x52af56d2), + TOBN(0xbcf89e54, 0x45b4147a), TOBN(0x3d102f21, 0x9a52816c), + TOBN(0x6808517e, 0x39b62e77), TOBN(0x92e25421, 0x69169ad8), + TOBN(0xd721d871, 0xbb608558), TOBN(0x60e4ebae, 0xf6d4ff9b), + TOBN(0x0ba10819, 0x41f2763e), TOBN(0xca2e45be, 0x51ee3247), + TOBN(0x66d172ec, 0x2bfd7a5f), TOBN(0x528a8f2f, 0x74d0b12d), + TOBN(0xe17f1e38, 0xdabe70dc), TOBN(0x1d5d7316, 0x9f93983c), + TOBN(0x51b2184a, 0xdf423e31), TOBN(0xcb417291, 0xaedb1a10), + TOBN(0x2054ca93, 0x625bcab9), TOBN(0x54396860, 0xa98998f0), + TOBN(0x4e53f6c4, 0xa54ae57e), TOBN(0x0ffeb590, 0xee648e9d), + TOBN(0xfbbdaadc, 0x6afaf6bc), TOBN(0xf88ae796, 0xaa3bfb8a), + TOBN(0x209f1d44, 0xd2359ed9), TOBN(0xac68dd03, 0xf3544ce2), + TOBN(0xf378da47, 0xfd51e569), TOBN(0xe1abd860, 0x2cc80097), + TOBN(0x23ca18d9, 0x343b6e3a), TOBN(0x480797e8, 0xb40a1bae), + TOBN(0xd1f0c717, 0x533f3e67), TOBN(0x44896970, 0x06e6cdfc), + TOBN(0x8ca21055, 0x52a82e8d), TOBN(0xb2caf785, 0x78460cdc), + TOBN(0x4c1b7b62, 0xe9037178), TOBN(0xefc09d2c, 0xdb514b58), + TOBN(0x5f2df9ee, 0x9113be5c), TOBN(0x2fbda78f, 0xb3f9271c), + TOBN(0xe09a81af, 0x8f83fc54), TOBN(0x06b13866, 0x8afb5141), + TOBN(0x38f6480f, 0x43e3865d), TOBN(0x72dd77a8, 0x1ddf47d9), + TOBN(0xf2a8e971, 0x4c205ff7), TOBN(0x46d449d8, 0x9d088ad8), + TOBN(0x926619ea, 0x185d706f), TOBN(0xe47e02eb, 0xc7dd7f62), + TOBN(0xe7f120a7, 0x8cbc2031), TOBN(0xc18bef00, 0x998d4ac9), + TOBN(0x18f37a9c, 0x6bdf22da), TOBN(0xefbc432f, 0x90dc82df), + TOBN(0xc52cef8e, 0x5d703651), TOBN(0x82887ba0, 0xd99881a5), + TOBN(0x7cec9dda, 0xb920ec1d), TOBN(0xd0d7e8c3, 0xec3e8d3b), + TOBN(0x445bc395, 0x4ca88747), TOBN(0xedeaa2e0, 0x9fd53535), + TOBN(0x461b1d93, 0x6cc87475), TOBN(0xd92a52e2, 0x6d2383bd), + TOBN(0xfabccb59, 0xd7903546), TOBN(0x6111a761, 0x3d14b112), + TOBN(0x0ae584fe, 0xb3d5f612), TOBN(0x5ea69b8d, 0x60e828ec), + TOBN(0x6c078985, 0x54087030), TOBN(0x649cab04, 0xac4821fe), + TOBN(0x25ecedcf, 0x8bdce214), TOBN(0xb5622f72, 0x86af7361), + TOBN(0x0e1227aa, 0x7038b9e2), TOBN(0xd0efb273, 0xac20fa77), + TOBN(0x817ff88b, 0x79df975b), TOBN(0x856bf286, 0x1999503e), + TOBN(0xb4d5351f, 0x5038ec46), TOBN(0x740a52c5, 0xfc42af6e), + TOBN(0x2e38bb15, 0x2cbb1a3f), TOBN(0xc3eb99fe, 0x17a83429), + TOBN(0xca4fcbf1, 0xdd66bb74), TOBN(0x880784d6, 0xcde5e8fc), + TOBN(0xddc84c1c, 0xb4e7a0be), TOBN(0x8780510d, 0xbd15a72f), + TOBN(0x44bcf1af, 0x81ec30e1), TOBN(0x141e50a8, 0x0a61073e), + TOBN(0x0d955718, 0x47be87ae), TOBN(0x68a61417, 0xf76a4372), + TOBN(0xf57e7e87, 0xc607c3d3), TOBN(0x043afaf8, 0x5252f332), + TOBN(0xcc14e121, 0x1552a4d2), TOBN(0xb6dee692, 0xbb4d4ab4), + TOBN(0xb6ab74c8, 0xa03816a4), TOBN(0x84001ae4, 0x6f394a29), + TOBN(0x5bed8344, 0xd795fb45), TOBN(0x57326e7d, 0xb79f55a5), + TOBN(0xc9533ce0, 0x4accdffc), TOBN(0x53473caf, 0x3993fa04), + TOBN(0x7906eb93, 0xa13df4c8), TOBN(0xa73e51f6, 0x97cbe46f), + TOBN(0xd1ab3ae1, 0x0ae4ccf8), TOBN(0x25614508, 0x8a5b3dbc), + TOBN(0x61eff962, 0x11a71b27), TOBN(0xdf71412b, 0x6bb7fa39), + TOBN(0xb31ba6b8, 0x2bd7f3ef), TOBN(0xb0b9c415, 0x69180d29), + TOBN(0xeec14552, 0x014cdde5), TOBN(0x702c624b, 0x227b4bbb), + TOBN(0x2b15e8c2, 0xd3e988f3), TOBN(0xee3bcc6d, 0xa4f7fd04), + TOBN(0x9d00822a, 0x42ac6c85), TOBN(0x2db0cea6, 0x1df9f2b7), + TOBN(0xd7cad2ab, 0x42de1e58), TOBN(0x346ed526, 0x2d6fbb61), + TOBN(0xb3962995, 0x1a2faf09), TOBN(0x2fa8a580, 0x7c25612e), + TOBN(0x30ae04da, 0x7cf56490), TOBN(0x75662908, 0x0eea3961), + TOBN(0x3609f5c5, 0x3d080847), TOBN(0xcb081d39, 0x5241d4f6), + TOBN(0xb4fb3810, 0x77961a63), TOBN(0xc20c5984, 0x2abb66fc), + TOBN(0x3d40aa7c, 0xf902f245), TOBN(0x9cb12736, 0x4e536b1e), + TOBN(0x5eda24da, 0x99b3134f), TOBN(0xafbd9c69, 0x5cd011af), + TOBN(0x9a16e30a, 0xc7088c7d), TOBN(0x5ab65710, 0x3207389f), + TOBN(0x1b09547f, 0xe7407a53), TOBN(0x2322f9d7, 0x4fdc6eab), + TOBN(0xc0f2f22d, 0x7430de4d), TOBN(0x19382696, 0xe68ca9a9), + TOBN(0x17f1eff1, 0x918e5868), TOBN(0xe3b5b635, 0x586f4204), + TOBN(0x146ef980, 0x3fbc4341), TOBN(0x359f2c80, 0x5b5eed4e), + TOBN(0x9f35744e, 0x7482e41d), TOBN(0x9a9ac3ec, 0xf3b224c2), + TOBN(0x9161a6fe, 0x91fc50ae), TOBN(0x89ccc66b, 0xc613fa7c), + TOBN(0x89268b14, 0xc732f15a), TOBN(0x7cd6f4e2, 0xb467ed03), + TOBN(0xfbf79869, 0xce56b40e), TOBN(0xf93e094c, 0xc02dde98), + TOBN(0xefe0c3a8, 0xedee2cd7), TOBN(0x90f3ffc0, 0xb268fd42), + TOBN(0x81a7fd56, 0x08241aed), TOBN(0x95ab7ad8, 0x00b1afe8), + TOBN(0x40127056, 0x3e310d52), TOBN(0xd3ffdeb1, 0x09d9fc43), + TOBN(0xc8f85c91, 0xd11a8594), TOBN(0x2e74d258, 0x31cf6db8), + TOBN(0x829c7ca3, 0x02b5dfd0), TOBN(0xe389cfbe, 0x69143c86), + TOBN(0xd01b6405, 0x941768d8), TOBN(0x45103995, 0x03bf825d), + TOBN(0xcc4ee166, 0x56cd17e2), TOBN(0xbea3c283, 0xba037e79), + TOBN(0x4e1ac06e, 0xd9a47520), TOBN(0xfbfe18aa, 0xaf852404), + TOBN(0x5615f8e2, 0x8087648a), TOBN(0x7301e47e, 0xb9d150d9), + TOBN(0x79f9f9dd, 0xb299b977), TOBN(0x76697a7b, 0xa5b78314), + TOBN(0x10d67468, 0x7d7c90e7), TOBN(0x7afffe03, 0x937210b5), + TOBN(0x5aef3e4b, 0x28c22cee), TOBN(0xefb0ecd8, 0x09fd55ae), + TOBN(0x4cea7132, 0x0d2a5d6a), TOBN(0x9cfb5fa1, 0x01db6357), + TOBN(0x395e0b57, 0xf36e1ac5), TOBN(0x008fa9ad, 0x36cafb7d), + TOBN(0x8f6cdf70, 0x5308c4db), TOBN(0x51527a37, 0x95ed2477), + TOBN(0xba0dee30, 0x5bd21311), TOBN(0x6ed41b22, 0x909c90d7), + TOBN(0xc5f6b758, 0x7c8696d3), TOBN(0x0db8eaa8, 0x3ce83a80), + TOBN(0xd297fe37, 0xb24b4b6f), TOBN(0xfe58afe8, 0x522d1f0d), + TOBN(0x97358736, 0x8c98dbd9), TOBN(0x6bc226ca, 0x9454a527), + TOBN(0xa12b384e, 0xce53c2d0), TOBN(0x779d897d, 0x5e4606da), + TOBN(0xa53e47b0, 0x73ec12b0), TOBN(0x462dbbba, 0x5756f1ad), + TOBN(0x69fe09f2, 0xcafe37b6), TOBN(0x273d1ebf, 0xecce2e17), + TOBN(0x8ac1d538, 0x3cf607fd), TOBN(0x8035f7ff, 0x12e10c25),} + , + {TOBN(0x854d34c7, 0x7e6c5520), TOBN(0xc27df9ef, 0xdcb9ea58), + TOBN(0x405f2369, 0xd686666d), TOBN(0x29d1febf, 0x0417aa85), + TOBN(0x9846819e, 0x93470afe), TOBN(0x3e6a9669, 0xe2a27f9e), + TOBN(0x24d008a2, 0xe31e6504), TOBN(0xdba7cecf, 0x9cb7680a), + TOBN(0xecaff541, 0x338d6e43), TOBN(0x56f7dd73, 0x4541d5cc), + TOBN(0xb5d426de, 0x96bc88ca), TOBN(0x48d94f6b, 0x9ed3a2c3), + TOBN(0x6354a3bb, 0x2ef8279c), TOBN(0xd575465b, 0x0b1867f2), + TOBN(0xef99b0ff, 0x95225151), TOBN(0xf3e19d88, 0xf94500d8), + TOBN(0x92a83268, 0xe32dd620), TOBN(0x913ec99f, 0x627849a2), + TOBN(0xedd8fdfa, 0x2c378882), TOBN(0xaf96f33e, 0xee6f8cfe), + TOBN(0xc06737e5, 0xdc3fa8a5), TOBN(0x236bb531, 0xb0b03a1d), + TOBN(0x33e59f29, 0x89f037b0), TOBN(0x13f9b5a7, 0xd9a12a53), + TOBN(0x0d0df6ce, 0x51efb310), TOBN(0xcb5b2eb4, 0x958df5be), + TOBN(0xd6459e29, 0x36158e59), TOBN(0x82aae2b9, 0x1466e336), + TOBN(0xfb658a39, 0x411aa636), TOBN(0x7152ecc5, 0xd4c0a933), + TOBN(0xf10c758a, 0x49f026b7), TOBN(0xf4837f97, 0xcb09311f), + TOBN(0xddfb02c4, 0xc753c45f), TOBN(0x18ca81b6, 0xf9c840fe), + TOBN(0x846fd09a, 0xb0f8a3e6), TOBN(0xb1162add, 0xe7733dbc), + TOBN(0x7070ad20, 0x236e3ab6), TOBN(0xf88cdaf5, 0xb2a56326), + TOBN(0x05fc8719, 0x997cbc7a), TOBN(0x442cd452, 0x4b665272), + TOBN(0x7807f364, 0xb71698f5), TOBN(0x6ba418d2, 0x9f7b605e), + TOBN(0xfd20b00f, 0xa03b2cbb), TOBN(0x883eca37, 0xda54386f), + TOBN(0xff0be43f, 0xf3437f24), TOBN(0xe910b432, 0xa48bb33c), + TOBN(0x4963a128, 0x329df765), TOBN(0xac1dd556, 0xbe2fe6f7), + TOBN(0x557610f9, 0x24a0a3fc), TOBN(0x38e17bf4, 0xe881c3f9), + TOBN(0x6ba84faf, 0xed0dac99), TOBN(0xd4a222c3, 0x59eeb918), + TOBN(0xc79c1dbe, 0x13f542b6), TOBN(0x1fc65e0d, 0xe425d457), + TOBN(0xeffb754f, 0x1debb779), TOBN(0x638d8fd0, 0x9e08af60), + TOBN(0x994f523a, 0x626332d5), TOBN(0x7bc38833, 0x5561bb44), + TOBN(0x005ed4b0, 0x3d845ea2), TOBN(0xd39d3ee1, 0xc2a1f08a), + TOBN(0x6561fdd3, 0xe7676b0d), TOBN(0x620e35ff, 0xfb706017), + TOBN(0x36ce424f, 0xf264f9a8), TOBN(0xc4c3419f, 0xda2681f7), + TOBN(0xfb6afd2f, 0x69beb6e8), TOBN(0x3a50b993, 0x6d700d03), + TOBN(0xc840b2ad, 0x0c83a14f), TOBN(0x573207be, 0x54085bef), + TOBN(0x5af882e3, 0x09fe7e5b), TOBN(0x957678a4, 0x3b40a7e1), + TOBN(0x172d4bdd, 0x543056e2), TOBN(0x9c1b26b4, 0x0df13c0a), + TOBN(0x1c30861c, 0xf405ff06), TOBN(0xebac86bd, 0x486e828b), + TOBN(0xe791a971, 0x636933fc), TOBN(0x50e7c2be, 0x7aeee947), + TOBN(0xc3d4a095, 0xfa90d767), TOBN(0xae60eb7b, 0xe670ab7b), + TOBN(0x17633a64, 0x397b056d), TOBN(0x93a21f33, 0x105012aa), + TOBN(0x663c370b, 0xabb88643), TOBN(0x91df36d7, 0x22e21599), + TOBN(0x183ba835, 0x8b761671), TOBN(0x381eea1d, 0x728f3bf1), + TOBN(0xb9b2f1ba, 0x39966e6c), TOBN(0x7c464a28, 0xe7295492), + TOBN(0x0fd5f70a, 0x09b26b7f), TOBN(0xa9aba1f9, 0xfbe009df), + TOBN(0x857c1f22, 0x369b87ad), TOBN(0x3c00e5d9, 0x32fca556), + TOBN(0x1ad74cab, 0x90b06466), TOBN(0xa7112386, 0x550faaf2), + TOBN(0x7435e198, 0x6d9bd5f5), TOBN(0x2dcc7e38, 0x59c3463f), + TOBN(0xdc7df748, 0xca7bd4b2), TOBN(0x13cd4c08, 0x9dec2f31), + TOBN(0x0d3b5df8, 0xe3237710), TOBN(0x0dadb26e, 0xcbd2f7b0), + TOBN(0x9f5966ab, 0xe4aa082b), TOBN(0x666ec8de, 0x350e966e), + TOBN(0x1bfd1ed5, 0xee524216), TOBN(0xcd93c59b, 0x41dab0b6), + TOBN(0x658a8435, 0xd186d6ba), TOBN(0x1b7d34d2, 0x159d1195), + TOBN(0x5936e460, 0x22caf46b), TOBN(0x6a45dd8f, 0x9a96fe4f), + TOBN(0xf7925434, 0xb98f474e), TOBN(0x41410412, 0x0053ef15), + TOBN(0x71cf8d12, 0x41de97bf), TOBN(0xb8547b61, 0xbd80bef4), + TOBN(0xb47d3970, 0xc4db0037), TOBN(0xf1bcd328, 0xfef20dff), + TOBN(0x31a92e09, 0x10caad67), TOBN(0x1f591960, 0x5531a1e1), + TOBN(0x3bb852e0, 0x5f4fc840), TOBN(0x63e297ca, 0x93a72c6c), + TOBN(0x3c2b0b2e, 0x49abad67), TOBN(0x6ec405fc, 0xed3db0d9), + TOBN(0xdc14a530, 0x7fef1d40), TOBN(0xccd19846, 0x280896fc), + TOBN(0x00f83176, 0x9bb81648), TOBN(0xd69eb485, 0x653120d0), + TOBN(0xd17d75f4, 0x4ccabc62), TOBN(0x34a07f82, 0xb749fcb1), + TOBN(0x2c3af787, 0xbbfb5554), TOBN(0xb06ed4d0, 0x62e283f8), + TOBN(0x5722889f, 0xa19213a0), TOBN(0x162b085e, 0xdcf3c7b4), + TOBN(0xbcaecb31, 0xe0dd3eca), TOBN(0xc6237fbc, 0xe52f13a5), + TOBN(0xcc2b6b03, 0x27bac297), TOBN(0x2ae1cac5, 0xb917f54a), + TOBN(0x474807d4, 0x7845ae4f), TOBN(0xfec7dd92, 0xce5972e0), + TOBN(0xc3bd2541, 0x1d7915bb), TOBN(0x66f85dc4, 0xd94907ca), + TOBN(0xd981b888, 0xbdbcf0ca), TOBN(0xd75f5da6, 0xdf279e9f), + TOBN(0x128bbf24, 0x7054e934), TOBN(0x3c6ff6e5, 0x81db134b), + TOBN(0x795b7cf4, 0x047d26e4), TOBN(0xf370f7b8, 0x5049ec37), + TOBN(0xc6712d4d, 0xced945af), TOBN(0xdf30b5ec, 0x095642bc), + TOBN(0x9b034c62, 0x4896246e), TOBN(0x5652c016, 0xee90bbd1), + TOBN(0xeb38636f, 0x87fedb73), TOBN(0x5e32f847, 0x0135a613), + TOBN(0x0703b312, 0xcf933c83), TOBN(0xd05bb76e, 0x1a7f47e6), + TOBN(0x825e4f0c, 0x949c2415), TOBN(0x569e5622, 0x7250d6f8), + TOBN(0xbbe9eb3a, 0x6568013e), TOBN(0x8dbd203f, 0x22f243fc), + TOBN(0x9dbd7694, 0xb342734a), TOBN(0x8f6d12f8, 0x46afa984), + TOBN(0xb98610a2, 0xc9eade29), TOBN(0xbab4f323, 0x47dd0f18), + TOBN(0x5779737b, 0x671c0d46), TOBN(0x10b6a7c6, 0xd3e0a42a), + TOBN(0xfb19ddf3, 0x3035b41c), TOBN(0xd336343f, 0x99c45895), + TOBN(0x61fe4938, 0x54c857e5), TOBN(0xc4d506be, 0xae4e57d5), + TOBN(0x3cd8c8cb, 0xbbc33f75), TOBN(0x7281f08a, 0x9262c77d), + TOBN(0x083f4ea6, 0xf11a2823), TOBN(0x8895041e, 0x9fba2e33), + TOBN(0xfcdfea49, 0x9c438edf), TOBN(0x7678dcc3, 0x91edba44), + TOBN(0xf07b3b87, 0xe2ba50f0), TOBN(0xc13888ef, 0x43948c1b), + TOBN(0xc2135ad4, 0x1140af42), TOBN(0x8e5104f3, 0x926ed1a7), + TOBN(0xf24430cb, 0x88f6695f), TOBN(0x0ce0637b, 0x6d73c120), + TOBN(0xb2db01e6, 0xfe631e8f), TOBN(0x1c5563d7, 0xd7bdd24b), + TOBN(0x8daea3ba, 0x369ad44f), TOBN(0x000c81b6, 0x8187a9f9), + TOBN(0x5f48a951, 0xaae1fd9a), TOBN(0xe35626c7, 0x8d5aed8a), + TOBN(0x20952763, 0x0498c622), TOBN(0x76d17634, 0x773aa504), + TOBN(0x36d90dda, 0xeb300f7a), TOBN(0x9dcf7dfc, 0xedb5e801), + TOBN(0x645cb268, 0x74d5244c), TOBN(0xa127ee79, 0x348e3aa2), + TOBN(0x488acc53, 0x575f1dbb), TOBN(0x95037e85, 0x80e6161e), + TOBN(0x57e59283, 0x292650d0), TOBN(0xabe67d99, 0x14938216), + TOBN(0x3c7f944b, 0x3f8e1065), TOBN(0xed908cb6, 0x330e8924), + TOBN(0x08ee8fd5, 0x6f530136), TOBN(0x2227b7d5, 0xd7ffc169), + TOBN(0x4f55c893, 0xb5cd6dd5), TOBN(0x82225e11, 0xa62796e8), + TOBN(0x5c6cead1, 0xcb18e12c), TOBN(0x4381ae0c, 0x84f5a51a), + TOBN(0x345913d3, 0x7fafa4c8), TOBN(0x3d918082, 0x0491aac0), + TOBN(0x9347871f, 0x3e69264c), TOBN(0xbea9dd3c, 0xb4f4f0cd), + TOBN(0xbda5d067, 0x3eadd3e7), TOBN(0x0033c1b8, 0x0573bcd8), + TOBN(0x25589379, 0x5da2486c), TOBN(0xcb89ee5b, 0x86abbee7), + TOBN(0x8fe0a8f3, 0x22532e5d), TOBN(0xb6410ff0, 0x727dfc4c), + TOBN(0x619b9d58, 0x226726db), TOBN(0x5ec25669, 0x7a2b2dc7), + TOBN(0xaf4d2e06, 0x4c3beb01), TOBN(0x852123d0, 0x7acea556), + TOBN(0x0e9470fa, 0xf783487a), TOBN(0x75a7ea04, 0x5664b3eb), + TOBN(0x4ad78f35, 0x6798e4ba), TOBN(0x9214e6e5, 0xc7d0e091), + TOBN(0xc420b488, 0xb1290403), TOBN(0x64049e0a, 0xfc295749), + TOBN(0x03ef5af1, 0x3ae9841f), TOBN(0xdbe4ca19, 0xb0b662a6), + TOBN(0x46845c5f, 0xfa453458), TOBN(0xf8dabf19, 0x10b66722), + TOBN(0xb650f0aa, 0xcce2793b), TOBN(0x71db851e, 0xc5ec47c1), + TOBN(0x3eb78f3e, 0x3b234fa9), TOBN(0xb0c60f35, 0xfc0106ce), + TOBN(0x05427121, 0x774eadbd), TOBN(0x25367faf, 0xce323863), + TOBN(0x7541b5c9, 0xcd086976), TOBN(0x4ff069e2, 0xdc507ad1), + TOBN(0x74145256, 0x8776e667), TOBN(0x6e76142c, 0xb23c6bb5), + TOBN(0xdbf30712, 0x1b3a8a87), TOBN(0x60e7363e, 0x98450836), + TOBN(0x5741450e, 0xb7366d80), TOBN(0xe4ee14ca, 0x4837dbdf), + TOBN(0xa765eb9b, 0x69d4316f), TOBN(0x04548dca, 0x8ef43825), + TOBN(0x9c9f4e4c, 0x5ae888eb), TOBN(0x733abb51, 0x56e9ac99), + TOBN(0xdaad3c20, 0xba6ac029), TOBN(0x9b8dd3d3, 0x2ba3e38e), + TOBN(0xa9bb4c92, 0x0bc5d11a), TOBN(0xf20127a7, 0x9c5f88a3), + TOBN(0x4f52b06e, 0x161d3cb8), TOBN(0x26c1ff09, 0x6afaf0a6), + TOBN(0x32670d2f, 0x7189e71f), TOBN(0xc6438748, 0x5ecf91e7), + TOBN(0x15758e57, 0xdb757a21), TOBN(0x427d09f8, 0x290a9ce5), + TOBN(0x846a308f, 0x38384a7a), TOBN(0xaac3acb4, 0xb0732b99), + TOBN(0x9e941009, 0x17845819), TOBN(0x95cba111, 0xa7ce5e03), + TOBN(0x6f3d4f7f, 0xb00009c4), TOBN(0xb8396c27, 0x8ff28b5f), + TOBN(0xb1a9ae43, 0x1c97975d), TOBN(0x9d7ba8af, 0xe5d9fed5), + TOBN(0x338cf09f, 0x34f485b6), TOBN(0xbc0ddacc, 0x64122516), + TOBN(0xa450da12, 0x05d471fe), TOBN(0x4c3a6250, 0x628dd8c9), + TOBN(0x69c7d103, 0xd1295837), TOBN(0xa2893e50, 0x3807eb2f), + TOBN(0xd6e1e1de, 0xbdb41491), TOBN(0xc630745b, 0x5e138235), + TOBN(0xc892109e, 0x48661ae1), TOBN(0x8d17e7eb, 0xea2b2674), + TOBN(0x00ec0f87, 0xc328d6b5), TOBN(0x6d858645, 0xf079ff9e), + TOBN(0x6cdf243e, 0x19115ead), TOBN(0x1ce1393e, 0x4bac4fcf), + TOBN(0x2c960ed0, 0x9c29f25b), TOBN(0x59be4d8e, 0x9d388a05), + TOBN(0x0d46e06c, 0xd0def72b), TOBN(0xb923db5d, 0xe0342748), + TOBN(0xf7d3aacd, 0x936d4a3d), TOBN(0x558519cc, 0x0b0b099e), + TOBN(0x3ea8ebf8, 0x827097ef), TOBN(0x259353db, 0xd054f55d), + TOBN(0x84c89abc, 0x6d2ed089), TOBN(0x5c548b69, 0x8e096a7c), + TOBN(0xd587f616, 0x994b995d), TOBN(0x4d1531f6, 0xa5845601), + TOBN(0x792ab31e, 0x451fd9f0), TOBN(0xc8b57bb2, 0x65adf6ca), + TOBN(0x68440fcb, 0x1cd5ad73), TOBN(0xb9c860e6, 0x6144da4f), + TOBN(0x2ab286aa, 0x8462beb8), TOBN(0xcc6b8fff, 0xef46797f), + TOBN(0xac820da4, 0x20c8a471), TOBN(0x69ae05a1, 0x77ff7faf), + TOBN(0xb9163f39, 0xbfb5da77), TOBN(0xbd03e590, 0x2c73ab7a), + TOBN(0x7e862b5e, 0xb2940d9e), TOBN(0x3c663d86, 0x4b9af564), + TOBN(0xd8309031, 0xbde3033d), TOBN(0x298231b2, 0xd42c5bc6), + TOBN(0x42090d2c, 0x552ad093), TOBN(0xa4799d1c, 0xff854695), + TOBN(0x0a88b5d6, 0xd31f0d00), TOBN(0xf8b40825, 0xa2f26b46), + TOBN(0xec29b1ed, 0xf1bd7218), TOBN(0xd491c53b, 0x4b24c86e), + TOBN(0xd2fe588f, 0x3395ea65), TOBN(0x6f3764f7, 0x4456ef15), + TOBN(0xdb43116d, 0xcdc34800), TOBN(0xcdbcd456, 0xc1e33955), + TOBN(0xefdb5540, 0x74ab286b), TOBN(0x948c7a51, 0xd18c5d7c), + TOBN(0xeb81aa37, 0x7378058e), TOBN(0x41c746a1, 0x04411154), + TOBN(0xa10c73bc, 0xfb828ac7), TOBN(0x6439be91, 0x9d972b29), + TOBN(0x4bf3b4b0, 0x43a2fbad), TOBN(0x39e6dadf, 0x82b5e840), + TOBN(0x4f716408, 0x6397bd4c), TOBN(0x0f7de568, 0x7f1eeccb), + TOBN(0x5865c5a1, 0xd2ffbfc1), TOBN(0xf74211fa, 0x4ccb6451), + TOBN(0x66368a88, 0xc0b32558), TOBN(0x5b539dc2, 0x9ad7812e), + TOBN(0x579483d0, 0x2f3af6f6), TOBN(0x52132078, 0x99934ece), + TOBN(0x50b9650f, 0xdcc9e983), TOBN(0xca989ec9, 0xaee42b8a), + TOBN(0x6a44c829, 0xd6f62f99), TOBN(0x8f06a309, 0x4c2a7c0c), + TOBN(0x4ea2b3a0, 0x98a0cb0a), TOBN(0x5c547b70, 0xbeee8364), + TOBN(0x461d40e1, 0x682afe11), TOBN(0x9e0fc77a, 0x7b41c0a8), + TOBN(0x79e4aefd, 0xe20d5d36), TOBN(0x2916e520, 0x32dd9f63), + TOBN(0xf59e52e8, 0x3f883faf), TOBN(0x396f9639, 0x2b868d35), + TOBN(0xc902a9df, 0x4ca19881), TOBN(0x0fc96822, 0xdb2401a6), + TOBN(0x41237587, 0x66f1c68d), TOBN(0x10fc6de3, 0xfb476c0d), + TOBN(0xf8b6b579, 0x841f5d90), TOBN(0x2ba8446c, 0xfa24f44a), + TOBN(0xa237b920, 0xef4a9975), TOBN(0x60bb6004, 0x2330435f), + TOBN(0xd6f4ab5a, 0xcfb7e7b5), TOBN(0xb2ac5097, 0x83435391), + TOBN(0xf036ee2f, 0xb0d1ea67), TOBN(0xae779a6a, 0x74c56230), + TOBN(0x59bff8c8, 0xab838ae6), TOBN(0xcd83ca99, 0x9b38e6f0), + TOBN(0xbb27bef5, 0xe33deed3), TOBN(0xe6356f6f, 0x001892a8), + TOBN(0xbf3be6cc, 0x7adfbd3e), TOBN(0xaecbc81c, 0x33d1ac9d), + TOBN(0xe4feb909, 0xe6e861dc), TOBN(0x90a247a4, 0x53f5f801), + TOBN(0x01c50acb, 0x27346e57), TOBN(0xce29242e, 0x461acc1b), + TOBN(0x04dd214a, 0x2f998a91), TOBN(0x271ee9b1, 0xd4baf27b), + TOBN(0x7e3027d1, 0xe8c26722), TOBN(0x21d1645c, 0x1820dce5), + TOBN(0x086f242c, 0x7501779c), TOBN(0xf0061407, 0xfa0e8009), + TOBN(0xf23ce477, 0x60187129), TOBN(0x05bbdedb, 0x0fde9bd0), + TOBN(0x682f4832, 0x25d98473), TOBN(0xf207fe85, 0x5c658427), + TOBN(0xb6fdd7ba, 0x4166ffa1), TOBN(0x0c314056, 0x9eed799d), + TOBN(0x0db8048f, 0x4107e28f), TOBN(0x74ed3871, 0x41216840), + TOBN(0x74489f8f, 0x56a3c06e), TOBN(0x1e1c005b, 0x12777134), + TOBN(0xdb332a73, 0xf37ec3c3), TOBN(0xc65259bd, 0xdd59eba0), + TOBN(0x2291709c, 0xdb4d3257), TOBN(0x9a793b25, 0xbd389390), + TOBN(0xf39fe34b, 0xe43756f0), TOBN(0x2f76bdce, 0x9afb56c9), + TOBN(0x9f37867a, 0x61208b27), TOBN(0xea1d4307, 0x089972c3), + TOBN(0x8c595330, 0x8bdf623a), TOBN(0x5f5accda, 0x8441fb7d), + TOBN(0xfafa9418, 0x32ddfd95), TOBN(0x6ad40c5a, 0x0fde9be7), + TOBN(0x43faba89, 0xaeca8709), TOBN(0xc64a7cf1, 0x2c248a9d), + TOBN(0x16620252, 0x72637a76), TOBN(0xaee1c791, 0x22b8d1bb), + TOBN(0xf0f798fd, 0x21a843b2), TOBN(0x56e4ed4d, 0x8d005cb1), + TOBN(0x355f7780, 0x1f0d8abe), TOBN(0x197b04cf, 0x34522326), + TOBN(0x41f9b31f, 0xfd42c13f), TOBN(0x5ef7feb2, 0xb40f933d), + TOBN(0x27326f42, 0x5d60bad4), TOBN(0x027ecdb2, 0x8c92cf89), + TOBN(0x04aae4d1, 0x4e3352fe), TOBN(0x08414d2f, 0x73591b90), + TOBN(0x5ed6124e, 0xb7da7d60), TOBN(0xb985b931, 0x4d13d4ec), + TOBN(0xa592d3ab, 0x96bf36f9), TOBN(0x012dbed5, 0xbbdf51df), + TOBN(0xa57963c0, 0xdf6c177d), TOBN(0x010ec869, 0x87ca29cf), + TOBN(0xba1700f6, 0xbf926dff), TOBN(0x7c9fdbd1, 0xf4bf6bc2), + TOBN(0xdc18dc8f, 0x64da11f5), TOBN(0xa6074b7a, 0xd938ae75), + TOBN(0x14270066, 0xe84f44a4), TOBN(0x99998d38, 0xd27b954e), + TOBN(0xc1be8ab2, 0xb4f38e9a), TOBN(0x8bb55bbf, 0x15c01016), + TOBN(0xf73472b4, 0x0ea2ab30), TOBN(0xd365a340, 0xf73d68dd), + TOBN(0xc01a7168, 0x19c2e1eb), TOBN(0x32f49e37, 0x34061719), + TOBN(0xb73c57f1, 0x01d8b4d6), TOBN(0x03c8423c, 0x26b47700), + TOBN(0x321d0bc8, 0xa4d8826a), TOBN(0x6004213c, 0x4bc0e638), + TOBN(0xf78c64a1, 0xc1c06681), TOBN(0x16e0a16f, 0xef018e50), + TOBN(0x31cbdf91, 0xdb42b2b3), TOBN(0xf8f4ffce, 0xe0d36f58), + TOBN(0xcdcc71cd, 0x4cc5e3e0), TOBN(0xd55c7cfa, 0xa129e3e0), + TOBN(0xccdb6ba0, 0x0fb2cbf1), TOBN(0x6aba0005, 0xc4bce3cb), + TOBN(0x501cdb30, 0xd232cfc4), TOBN(0x9ddcf12e, 0xd58a3cef), + TOBN(0x02d2cf9c, 0x87e09149), TOBN(0xdc5d7ec7, 0x2c976257), + TOBN(0x6447986e, 0x0b50d7dd), TOBN(0x88fdbaf7, 0x807f112a), + TOBN(0x58c9822a, 0xb00ae9f6), TOBN(0x6abfb950, 0x6d3d27e0), + TOBN(0xd0a74487, 0x8a429f4f), TOBN(0x0649712b, 0xdb516609), + TOBN(0xb826ba57, 0xe769b5df), TOBN(0x82335df2, 0x1fc7aaf2), + TOBN(0x2389f067, 0x5c93d995), TOBN(0x59ac367a, 0x68677be6), + TOBN(0xa77985ff, 0x21d9951b), TOBN(0x038956fb, 0x85011cce), + TOBN(0x608e48cb, 0xbb734e37), TOBN(0xc08c0bf2, 0x2be5b26f), + TOBN(0x17bbdd3b, 0xf9b1a0d9), TOBN(0xeac7d898, 0x10483319), + TOBN(0xc95c4baf, 0xbc1a6dea), TOBN(0xfdd0e2bf, 0x172aafdb), + TOBN(0x40373cbc, 0x8235c41a), TOBN(0x14303f21, 0xfb6f41d5), + TOBN(0xba063621, 0x0408f237), TOBN(0xcad3b09a, 0xecd2d1ed), + TOBN(0x4667855a, 0x52abb6a2), TOBN(0xba9157dc, 0xaa8b417b), + TOBN(0xfe7f3507, 0x4f013efb), TOBN(0x1b112c4b, 0xaa38c4a2), + TOBN(0xa1406a60, 0x9ba64345), TOBN(0xe53cba33, 0x6993c80b), + TOBN(0x45466063, 0xded40d23), TOBN(0x3d5f1f4d, 0x54908e25), + TOBN(0x9ebefe62, 0x403c3c31), TOBN(0x274ea0b5, 0x0672a624), + TOBN(0xff818d99, 0x451d1b71), TOBN(0x80e82643, 0x8f79cf79), + TOBN(0xa165df13, 0x73ce37f5), TOBN(0xa744ef4f, 0xfe3a21fd), + TOBN(0x73f1e7f5, 0xcf551396), TOBN(0xc616898e, 0x868c676b), + TOBN(0x671c28c7, 0x8c442c36), TOBN(0xcfe5e558, 0x5e0a317d), + TOBN(0x1242d818, 0x7051f476), TOBN(0x56fad2a6, 0x14f03442), + TOBN(0x262068bc, 0x0a44d0f6), TOBN(0xdfa2cd6e, 0xce6edf4e), + TOBN(0x0f43813a, 0xd15d1517), TOBN(0x61214cb2, 0x377d44f5), + TOBN(0xd399aa29, 0xc639b35f), TOBN(0x42136d71, 0x54c51c19), + TOBN(0x9774711b, 0x08417221), TOBN(0x0a5546b3, 0x52545a57), + TOBN(0x80624c41, 0x1150582d), TOBN(0x9ec5c418, 0xfbc555bc), + TOBN(0x2c87dcad, 0x771849f1), TOBN(0xb0c932c5, 0x01d7bf6f), + TOBN(0x6aa5cd3e, 0x89116eb2), TOBN(0xd378c25a, 0x51ca7bd3), + TOBN(0xc612a0da, 0x9e6e3e31), TOBN(0x0417a54d, 0xb68ad5d0), + TOBN(0x00451e4a, 0x22c6edb8), TOBN(0x9fbfe019, 0xb42827ce), + TOBN(0x2fa92505, 0xba9384a2), TOBN(0x21b8596e, 0x64ad69c1), + TOBN(0x8f4fcc49, 0x983b35a6), TOBN(0xde093760, 0x72754672), + TOBN(0x2f14ccc8, 0xf7bffe6d), TOBN(0x27566bff, 0x5d94263d), + TOBN(0xb5b4e9c6, 0x2df3ec30), TOBN(0x94f1d7d5, 0x3e6ea6ba), + TOBN(0x97b7851a, 0xaaca5e9b), TOBN(0x518aa521, 0x56713b97), + TOBN(0x3357e8c7, 0x150a61f6), TOBN(0x7842e7e2, 0xec2c2b69), + TOBN(0x8dffaf65, 0x6868a548), TOBN(0xd963bd82, 0xe068fc81), + TOBN(0x64da5c8b, 0x65917733), TOBN(0x927090ff, 0x7b247328),} + , + {TOBN(0x214bc9a7, 0xd298c241), TOBN(0xe3b697ba, 0x56807cfd), + TOBN(0xef1c7802, 0x4564eadb), TOBN(0xdde8cdcf, 0xb48149c5), + TOBN(0x946bf0a7, 0x5a4d2604), TOBN(0x27154d7f, 0x6c1538af), + TOBN(0x95cc9230, 0xde5b1fcc), TOBN(0xd88519e9, 0x66864f82), + TOBN(0xb828dd1a, 0x7cb1282c), TOBN(0xa08d7626, 0xbe46973a), + TOBN(0x6baf8d40, 0xe708d6b2), TOBN(0x72571fa1, 0x4daeb3f3), + TOBN(0x85b1732f, 0xf22dfd98), TOBN(0x87ab01a7, 0x0087108d), + TOBN(0xaaaafea8, 0x5988207a), TOBN(0xccc832f8, 0x69f00755), + TOBN(0x964d950e, 0x36ff3bf0), TOBN(0x8ad20f6f, 0xf0b34638), + TOBN(0x4d9177b3, 0xb5d7585f), TOBN(0xcf839760, 0xef3f019f), + TOBN(0x582fc5b3, 0x8288c545), TOBN(0x2f8e4e9b, 0x13116bd1), + TOBN(0xf91e1b2f, 0x332120ef), TOBN(0xcf568724, 0x2a17dd23), + TOBN(0x488f1185, 0xca8d9d1a), TOBN(0xadf2c77d, 0xd987ded2), + TOBN(0x5f3039f0, 0x60c46124), TOBN(0xe5d70b75, 0x71e095f4), + TOBN(0x82d58650, 0x6260e70f), TOBN(0x39d75ea7, 0xf750d105), + TOBN(0x8cf3d0b1, 0x75bac364), TOBN(0xf3a7564d, 0x21d01329), + TOBN(0x182f04cd, 0x2f52d2a7), TOBN(0x4fde149a, 0xe2df565a), + TOBN(0xb80c5eec, 0xa79fb2f7), TOBN(0xab491d7b, 0x22ddc897), + TOBN(0x99d76c18, 0xc6312c7f), TOBN(0xca0d5f3d, 0x6aa41a57), + TOBN(0x71207325, 0xd15363a0), TOBN(0xe82aa265, 0xbeb252c2), + TOBN(0x94ab4700, 0xec3128c2), TOBN(0x6c76d862, 0x8e383f49), + TOBN(0xdc36b150, 0xc03024eb), TOBN(0xfb439477, 0x53daac69), + TOBN(0xfc68764a, 0x8dc79623), TOBN(0x5b86995d, 0xb440fbb2), + TOBN(0xd66879bf, 0xccc5ee0d), TOBN(0x05228942, 0x95aa8bd3), + TOBN(0xb51a40a5, 0x1e6a75c1), TOBN(0x24327c76, 0x0ea7d817), + TOBN(0x06630182, 0x07774597), TOBN(0xd6fdbec3, 0x97fa7164), + TOBN(0x20c99dfb, 0x13c90f48), TOBN(0xd6ac5273, 0x686ef263), + TOBN(0xc6a50bdc, 0xfef64eeb), TOBN(0xcd87b281, 0x86fdfc32), + TOBN(0xb24aa43e, 0x3fcd3efc), TOBN(0xdd26c034, 0xb8088e9a), + TOBN(0xa5ef4dc9, 0xbd3d46ea), TOBN(0xa2f99d58, 0x8a4c6a6f), + TOBN(0xddabd355, 0x2f1da46c), TOBN(0x72c3f8ce, 0x1afacdd1), + TOBN(0xd90c4eee, 0x92d40578), TOBN(0xd28bb41f, 0xca623b94), + TOBN(0x50fc0711, 0x745edc11), TOBN(0x9dd9ad7d, 0x3dc87558), + TOBN(0xce6931fb, 0xb49d1e64), TOBN(0x6c77a0a2, 0xc98bd0f9), + TOBN(0x62b9a629, 0x6baf7cb1), TOBN(0xcf065f91, 0xccf72d22), + TOBN(0x7203cce9, 0x79639071), TOBN(0x09ae4885, 0xf9cb732f), + TOBN(0x5e7c3bec, 0xee8314f3), TOBN(0x1c068aed, 0xdbea298f), + TOBN(0x08d381f1, 0x7c80acec), TOBN(0x03b56be8, 0xe330495b), + TOBN(0xaeffb8f2, 0x9222882d), TOBN(0x95ff38f6, 0xc4af8bf7), + TOBN(0x50e32d35, 0x1fc57d8c), TOBN(0x6635be52, 0x17b444f0), + TOBN(0x04d15276, 0xa5177900), TOBN(0x4e1dbb47, 0xf6858752), + TOBN(0x5b475622, 0xc615796c), TOBN(0xa6fa0387, 0x691867bf), + TOBN(0xed7f5d56, 0x2844c6d0), TOBN(0xc633cf9b, 0x03a2477d), + TOBN(0xf6be5c40, 0x2d3721d6), TOBN(0xaf312eb7, 0xe9fd68e6), + TOBN(0x242792d2, 0xe7417ce1), TOBN(0xff42bc71, 0x970ee7f5), + TOBN(0x1ff4dc6d, 0x5c67a41e), TOBN(0x77709b7b, 0x20882a58), + TOBN(0x3554731d, 0xbe217f2c), TOBN(0x2af2a8cd, 0x5bb72177), + TOBN(0x58eee769, 0x591dd059), TOBN(0xbb2930c9, 0x4bba6477), + TOBN(0x863ee047, 0x7d930cfc), TOBN(0x4c262ad1, 0x396fd1f4), + TOBN(0xf4765bc8, 0x039af7e1), TOBN(0x2519834b, 0x5ba104f6), + TOBN(0x7cd61b4c, 0xd105f961), TOBN(0xa5415da5, 0xd63bca54), + TOBN(0x778280a0, 0x88a1f17c), TOBN(0xc4968949, 0x2329512c), + TOBN(0x174a9126, 0xcecdaa7a), TOBN(0xfc8c7e0e, 0x0b13247b), + TOBN(0x29c110d2, 0x3484c1c4), TOBN(0xf8eb8757, 0x831dfc3b), + TOBN(0x022f0212, 0xc0067452), TOBN(0x3f6f69ee, 0x7b9b926c), + TOBN(0x09032da0, 0xef42daf4), TOBN(0x79f00ade, 0x83f80de4), + TOBN(0x6210db71, 0x81236c97), TOBN(0x74f7685b, 0x3ee0781f), + TOBN(0x4df7da7b, 0xa3e41372), TOBN(0x2aae38b1, 0xb1a1553e), + TOBN(0x1688e222, 0xf6dd9d1b), TOBN(0x57695448, 0x5b8b6487), + TOBN(0x478d2127, 0x4b2edeaa), TOBN(0xb2818fa5, 0x1e85956a), + TOBN(0x1e6addda, 0xf176f2c0), TOBN(0x01ca4604, 0xe2572658), + TOBN(0x0a404ded, 0x85342ffb), TOBN(0x8cf60f96, 0x441838d6), + TOBN(0x9bbc691c, 0xc9071c4a), TOBN(0xfd588744, 0x34442803), + TOBN(0x97101c85, 0x809c0d81), TOBN(0xa7fb754c, 0x8c456f7f), + TOBN(0xc95f3c5c, 0xd51805e1), TOBN(0xab4ccd39, 0xb299dca8), + TOBN(0x3e03d20b, 0x47eaf500), TOBN(0xfa3165c1, 0xd7b80893), + TOBN(0x005e8b54, 0xe160e552), TOBN(0xdc4972ba, 0x9019d11f), + TOBN(0x21a6972e, 0x0c9a4a7a), TOBN(0xa52c258f, 0x37840fd7), + TOBN(0xf8559ff4, 0xc1e99d81), TOBN(0x08e1a7d6, 0xa3c617c0), + TOBN(0xb398fd43, 0x248c6ba7), TOBN(0x6ffedd91, 0xd1283794), + TOBN(0x8a6a59d2, 0xd629d208), TOBN(0xa9d141d5, 0x3490530e), + TOBN(0x42f6fc18, 0x38505989), TOBN(0x09bf250d, 0x479d94ee), + TOBN(0x223ad3b1, 0xb3822790), TOBN(0x6c5926c0, 0x93b8971c), + TOBN(0x609efc7e, 0x75f7fa62), TOBN(0x45d66a6d, 0x1ec2d989), + TOBN(0x4422d663, 0x987d2792), TOBN(0x4a73caad, 0x3eb31d2b), + TOBN(0xf06c2ac1, 0xa32cb9e6), TOBN(0xd9445c5f, 0x91aeba84), + TOBN(0x6af7a1d5, 0xaf71013f), TOBN(0xe68216e5, 0x0bedc946), + TOBN(0xf4cba30b, 0xd27370a0), TOBN(0x7981afbf, 0x870421cc), + TOBN(0x02496a67, 0x9449f0e1), TOBN(0x86cfc4be, 0x0a47edae), + TOBN(0x3073c936, 0xb1feca22), TOBN(0xf5694612, 0x03f8f8fb), + TOBN(0xd063b723, 0x901515ea), TOBN(0x4c6c77a5, 0x749cf038), + TOBN(0x6361e360, 0xab9e5059), TOBN(0x596cf171, 0xa76a37c0), + TOBN(0x800f53fa, 0x6530ae7a), TOBN(0x0f5e631e, 0x0792a7a6), + TOBN(0x5cc29c24, 0xefdb81c9), TOBN(0xa269e868, 0x3f9c40ba), + TOBN(0xec14f9e1, 0x2cb7191e), TOBN(0x78ea1bd8, 0xe5b08ea6), + TOBN(0x3c65aa9b, 0x46332bb9), TOBN(0x84cc22b3, 0xbf80ce25), + TOBN(0x0098e9e9, 0xd49d5bf1), TOBN(0xcd4ec1c6, 0x19087da4), + TOBN(0x3c9d07c5, 0xaef6e357), TOBN(0x839a0268, 0x9f8f64b8), + TOBN(0xc5e9eb62, 0xc6d8607f), TOBN(0x759689f5, 0x6aa995e4), + TOBN(0x70464669, 0xbbb48317), TOBN(0x921474bf, 0xe402417d), + TOBN(0xcabe135b, 0x2a354c8c), TOBN(0xd51e52d2, 0x812fa4b5), + TOBN(0xec741096, 0x53311fe8), TOBN(0x4f774535, 0xb864514b), + TOBN(0xbcadd671, 0x5bde48f8), TOBN(0xc9703873, 0x2189bc7d), + TOBN(0x5d45299e, 0xc709ee8a), TOBN(0xd1287ee2, 0x845aaff8), + TOBN(0x7d1f8874, 0xdb1dbf1f), TOBN(0xea46588b, 0x990c88d6), + TOBN(0x60ba649a, 0x84368313), TOBN(0xd5fdcbce, 0x60d543ae), + TOBN(0x90b46d43, 0x810d5ab0), TOBN(0x6739d8f9, 0x04d7e5cc), + TOBN(0x021c1a58, 0x0d337c33), TOBN(0x00a61162, 0x68e67c40), + TOBN(0x95ef413b, 0x379f0a1f), TOBN(0xfe126605, 0xe9e2ab95), + TOBN(0x67578b85, 0x2f5f199c), TOBN(0xf5c00329, 0x2cb84913), + TOBN(0xf7956430, 0x37577dd8), TOBN(0x83b82af4, 0x29c5fe88), + TOBN(0x9c1bea26, 0xcdbdc132), TOBN(0x589fa086, 0x9c04339e), + TOBN(0x033e9538, 0xb13799df), TOBN(0x85fa8b21, 0xd295d034), + TOBN(0xdf17f73f, 0xbd9ddcca), TOBN(0xf32bd122, 0xddb66334), + TOBN(0x55ef88a7, 0x858b044c), TOBN(0x1f0d69c2, 0x5aa9e397), + TOBN(0x55fd9cc3, 0x40d85559), TOBN(0xc774df72, 0x7785ddb2), + TOBN(0x5dcce9f6, 0xd3bd2e1c), TOBN(0xeb30da20, 0xa85dfed0), + TOBN(0x5ed7f5bb, 0xd3ed09c4), TOBN(0x7d42a35c, 0x82a9c1bd), + TOBN(0xcf3de995, 0x9890272d), TOBN(0x75f3432a, 0x3e713a10), + TOBN(0x5e13479f, 0xe28227b8), TOBN(0xb8561ea9, 0xfefacdc8), + TOBN(0xa6a297a0, 0x8332aafd), TOBN(0x9b0d8bb5, 0x73809b62), + TOBN(0xd2fa1cfd, 0x0c63036f), TOBN(0x7a16eb55, 0xbd64bda8), + TOBN(0x3f5cf5f6, 0x78e62ddc), TOBN(0x2267c454, 0x07fd752b), + TOBN(0x5e361b6b, 0x5e437bbe), TOBN(0x95c59501, 0x8354e075), + TOBN(0xec725f85, 0xf2b254d9), TOBN(0x844b617d, 0x2cb52b4e), + TOBN(0xed8554f5, 0xcf425fb5), TOBN(0xab67703e, 0x2af9f312), + TOBN(0x4cc34ec1, 0x3cf48283), TOBN(0xb09daa25, 0x9c8a705e), + TOBN(0xd1e9d0d0, 0x5b7d4f84), TOBN(0x4df6ef64, 0xdb38929d), + TOBN(0xe16b0763, 0xaa21ba46), TOBN(0xc6b1d178, 0xa293f8fb), + TOBN(0x0ff5b602, 0xd520aabf), TOBN(0x94d671bd, 0xc339397a), + TOBN(0x7c7d98cf, 0x4f5792fa), TOBN(0x7c5e0d67, 0x11215261), + TOBN(0x9b19a631, 0xa7c5a6d4), TOBN(0xc8511a62, 0x7a45274d), + TOBN(0x0c16621c, 0xa5a60d99), TOBN(0xf7fbab88, 0xcf5e48cb), + TOBN(0xab1e6ca2, 0xf7ddee08), TOBN(0x83bd08ce, 0xe7867f3c), + TOBN(0xf7e48e8a, 0x2ac13e27), TOBN(0x4494f6df, 0x4eb1a9f5), + TOBN(0xedbf84eb, 0x981f0a62), TOBN(0x49badc32, 0x536438f0), + TOBN(0x50bea541, 0x004f7571), TOBN(0xbac67d10, 0xdf1c94ee), + TOBN(0x253d73a1, 0xb727bc31), TOBN(0xb3d01cf2, 0x30686e28), + TOBN(0x51b77b1b, 0x55fd0b8b), TOBN(0xa099d183, 0xfeec3173), + TOBN(0x202b1fb7, 0x670e72b7), TOBN(0xadc88b33, 0xa8e1635f), + TOBN(0x34e8216a, 0xf989d905), TOBN(0xc2e68d20, 0x29b58d01), + TOBN(0x11f81c92, 0x6fe55a93), TOBN(0x15f1462a, 0x8f296f40), + TOBN(0x1915d375, 0xea3d62f2), TOBN(0xa17765a3, 0x01c8977d), + TOBN(0x7559710a, 0xe47b26f6), TOBN(0xe0bd29c8, 0x535077a5), + TOBN(0x615f976d, 0x08d84858), TOBN(0x370dfe85, 0x69ced5c1), + TOBN(0xbbc7503c, 0xa734fa56), TOBN(0xfbb9f1ec, 0x91ac4574), + TOBN(0x95d7ec53, 0x060dd7ef), TOBN(0xeef2dacd, 0x6e657979), + TOBN(0x54511af3, 0xe2a08235), TOBN(0x1e324aa4, 0x1f4aea3d), + TOBN(0x550e7e71, 0xe6e67671), TOBN(0xbccd5190, 0xbf52faf7), + TOBN(0xf880d316, 0x223cc62a), TOBN(0x0d402c7e, 0x2b32eb5d), + TOBN(0xa40bc039, 0x306a5a3b), TOBN(0x4e0a41fd, 0x96783a1b), + TOBN(0xa1e8d39a, 0x0253cdd4), TOBN(0x6480be26, 0xc7388638), + TOBN(0xee365e1d, 0x2285f382), TOBN(0x188d8d8f, 0xec0b5c36), + TOBN(0x34ef1a48, 0x1f0f4d82), TOBN(0x1a8f43e1, 0xa487d29a), + TOBN(0x8168226d, 0x77aefb3a), TOBN(0xf69a751e, 0x1e72c253), + TOBN(0x8e04359a, 0xe9594df1), TOBN(0x475ffd7d, 0xd14c0467), + TOBN(0xb5a2c2b1, 0x3844e95c), TOBN(0x85caf647, 0xdd12ef94), + TOBN(0x1ecd2a9f, 0xf1063d00), TOBN(0x1dd2e229, 0x23843311), + TOBN(0x38f0e09d, 0x73d17244), TOBN(0x3ede7746, 0x8fc653f1), + TOBN(0xae4459f5, 0xdc20e21c), TOBN(0x00db2ffa, 0x6a8599ea), + TOBN(0x11682c39, 0x30cfd905), TOBN(0x4934d074, 0xa5c112a6), + TOBN(0xbdf063c5, 0x568bfe95), TOBN(0x779a440a, 0x016c441a), + TOBN(0x0c23f218, 0x97d6fbdc), TOBN(0xd3a5cd87, 0xe0776aac), + TOBN(0xcee37f72, 0xd712e8db), TOBN(0xfb28c70d, 0x26f74e8d), + TOBN(0xffe0c728, 0xb61301a0), TOBN(0xa6282168, 0xd3724354), + TOBN(0x7ff4cb00, 0x768ffedc), TOBN(0xc51b3088, 0x03b02de9), + TOBN(0xa5a8147c, 0x3902dda5), TOBN(0x35d2f706, 0xfe6973b4), + TOBN(0x5ac2efcf, 0xc257457e), TOBN(0x933f48d4, 0x8700611b), + TOBN(0xc365af88, 0x4912beb2), TOBN(0x7f5a4de6, 0x162edf94), + TOBN(0xc646ba7c, 0x0c32f34b), TOBN(0x632c6af3, 0xb2091074), + TOBN(0x58d4f2e3, 0x753e43a9), TOBN(0x70e1d217, 0x24d4e23f), + TOBN(0xb24bf729, 0xafede6a6), TOBN(0x7f4a94d8, 0x710c8b60), + TOBN(0xaad90a96, 0x8d4faa6a), TOBN(0xd9ed0b32, 0xb066b690), + TOBN(0x52fcd37b, 0x78b6dbfd), TOBN(0x0b64615e, 0x8bd2b431), + TOBN(0x228e2048, 0xcfb9fad5), TOBN(0xbeaa386d, 0x240b76bd), + TOBN(0x2d6681c8, 0x90dad7bc), TOBN(0x3e553fc3, 0x06d38f5e), + TOBN(0xf27cdb9b, 0x9d5f9750), TOBN(0x3e85c52a, 0xd28c5b0e), + TOBN(0x190795af, 0x5247c39b), TOBN(0x547831eb, 0xbddd6828), + TOBN(0xf327a227, 0x4a82f424), TOBN(0x36919c78, 0x7e47f89d), + TOBN(0xe4783919, 0x43c7392c), TOBN(0xf101b9aa, 0x2316fefe), + TOBN(0xbcdc9e9c, 0x1c5009d2), TOBN(0xfb55ea13, 0x9cd18345), + TOBN(0xf5b5e231, 0xa3ce77c7), TOBN(0xde6b4527, 0xd2f2cb3d), + TOBN(0x10f6a333, 0x9bb26f5f), TOBN(0x1e85db8e, 0x044d85b6), + TOBN(0xc3697a08, 0x94197e54), TOBN(0x65e18cc0, 0xa7cb4ea8), + TOBN(0xa38c4f50, 0xa471fe6e), TOBN(0xf031747a, 0x2f13439c), + TOBN(0x53c4a6ba, 0xc007318b), TOBN(0xa8da3ee5, 0x1deccb3d), + TOBN(0x0555b31c, 0x558216b1), TOBN(0x90c7810c, 0x2f79e6c2), + TOBN(0x9b669f4d, 0xfe8eed3c), TOBN(0x70398ec8, 0xe0fac126), + TOBN(0xa96a449e, 0xf701b235), TOBN(0x0ceecdb3, 0xeb94f395), + TOBN(0x285fc368, 0xd0cb7431), TOBN(0x0d37bb52, 0x16a18c64), + TOBN(0x05110d38, 0xb880d2dd), TOBN(0xa60f177b, 0x65930d57), + TOBN(0x7da34a67, 0xf36235f5), TOBN(0x47f5e17c, 0x183816b9), + TOBN(0xc7664b57, 0xdb394af4), TOBN(0x39ba215d, 0x7036f789), + TOBN(0x46d2ca0e, 0x2f27b472), TOBN(0xc42647ee, 0xf73a84b7), + TOBN(0x44bc7545, 0x64488f1d), TOBN(0xaa922708, 0xf4cf85d5), + TOBN(0x721a01d5, 0x53e4df63), TOBN(0x649c0c51, 0x5db46ced), + TOBN(0x6bf0d64e, 0x3cffcb6c), TOBN(0xe3bf93fe, 0x50f71d96), + TOBN(0x75044558, 0xbcc194a0), TOBN(0x16ae3372, 0x6afdc554), + TOBN(0xbfc01adf, 0x5ca48f3f), TOBN(0x64352f06, 0xe22a9b84), + TOBN(0xcee54da1, 0xc1099e4a), TOBN(0xbbda54e8, 0xfa1b89c0), + TOBN(0x166a3df5, 0x6f6e55fb), TOBN(0x1ca44a24, 0x20176f88), + TOBN(0x936afd88, 0xdfb7b5ff), TOBN(0xe34c2437, 0x8611d4a0), + TOBN(0x7effbb75, 0x86142103), TOBN(0x6704ba1b, 0x1f34fc4d), + TOBN(0x7c2a468f, 0x10c1b122), TOBN(0x36b3a610, 0x8c6aace9), + TOBN(0xabfcc0a7, 0x75a0d050), TOBN(0x066f9197, 0x3ce33e32), + TOBN(0xce905ef4, 0x29fe09be), TOBN(0x89ee25ba, 0xa8376351), + TOBN(0x2a3ede22, 0xfd29dc76), TOBN(0x7fd32ed9, 0x36f17260), + TOBN(0x0cadcf68, 0x284b4126), TOBN(0x63422f08, 0xa7951fc8), + TOBN(0x562b24f4, 0x0807e199), TOBN(0xfe9ce5d1, 0x22ad4490), + TOBN(0xc2f51b10, 0x0db2b1b4), TOBN(0xeb3613ff, 0xe4541d0d), + TOBN(0xbd2c4a05, 0x2680813b), TOBN(0x527aa55d, 0x561b08d6), + TOBN(0xa9f8a40e, 0xa7205558), TOBN(0xe3eea56f, 0x243d0bec), + TOBN(0x7b853817, 0xa0ff58b3), TOBN(0xb67d3f65, 0x1a69e627), + TOBN(0x0b76bbb9, 0xa869b5d6), TOBN(0xa3afeb82, 0x546723ed), + TOBN(0x5f24416d, 0x3e554892), TOBN(0x8413b53d, 0x430e2a45), + TOBN(0x99c56aee, 0x9032a2a0), TOBN(0x09432bf6, 0xeec367b1), + TOBN(0x552850c6, 0xdaf0ecc1), TOBN(0x49ebce55, 0x5bc92048), + TOBN(0xdfb66ba6, 0x54811307), TOBN(0x1b84f797, 0x6f298597), + TOBN(0x79590481, 0x8d1d7a0d), TOBN(0xd9fabe03, 0x3a6fa556), + TOBN(0xa40f9c59, 0xba9e5d35), TOBN(0xcb1771c1, 0xf6247577), + TOBN(0x542a47ca, 0xe9a6312b), TOBN(0xa34b3560, 0x552dd8c5), + TOBN(0xfdf94de0, 0x0d794716), TOBN(0xd46124a9, 0x9c623094), + TOBN(0x56b7435d, 0x68afe8b4), TOBN(0x27f20540, 0x6c0d8ea1), + TOBN(0x12b77e14, 0x73186898), TOBN(0xdbc3dd46, 0x7479490f), + TOBN(0x951a9842, 0xc03b0c05), TOBN(0x8b1b3bb3, 0x7921bc96), + TOBN(0xa573b346, 0x2b202e0a), TOBN(0x77e4665d, 0x47254d56), + TOBN(0x08b70dfc, 0xd23e3984), TOBN(0xab86e8bc, 0xebd14236), + TOBN(0xaa3e07f8, 0x57114ba7), TOBN(0x5ac71689, 0xab0ef4f2), + TOBN(0x88fca384, 0x0139d9af), TOBN(0x72733f88, 0x76644af0), + TOBN(0xf122f72a, 0x65d74f4a), TOBN(0x13931577, 0xa5626c7a), + TOBN(0xd5b5d9eb, 0x70f8d5a4), TOBN(0x375adde7, 0xd7bbb228), + TOBN(0x31e88b86, 0x0c1c0b32), TOBN(0xd1f568c4, 0x173edbaa), + TOBN(0x1592fc83, 0x5459df02), TOBN(0x2beac0fb, 0x0fcd9a7e), + TOBN(0xb0a6fdb8, 0x1b473b0a), TOBN(0xe3224c6f, 0x0fe8fc48), + TOBN(0x680bd00e, 0xe87edf5b), TOBN(0x30385f02, 0x20e77cf5), + TOBN(0xe9ab98c0, 0x4d42d1b2), TOBN(0x72d191d2, 0xd3816d77), + TOBN(0x1564daca, 0x0917d9e5), TOBN(0x394eab59, 0x1f8fed7f), + TOBN(0xa209aa8d, 0x7fbb3896), TOBN(0x5564f3b9, 0xbe6ac98e), + TOBN(0xead21d05, 0xd73654ef), TOBN(0x68d1a9c4, 0x13d78d74), + TOBN(0x61e01708, 0x6d4973a0), TOBN(0x83da3500, 0x46e6d32a), + TOBN(0x6a3dfca4, 0x68ae0118), TOBN(0xa1b9a4c9, 0xd02da069), + TOBN(0x0b2ff9c7, 0xebab8302), TOBN(0x98af07c3, 0x944ba436), + TOBN(0x85997326, 0x995f0f9f), TOBN(0x467fade0, 0x71b58bc6), + TOBN(0x47e4495a, 0xbd625a2b), TOBN(0xfdd2d01d, 0x33c3b8cd), + TOBN(0x2c38ae28, 0xc693f9fa), TOBN(0x48622329, 0x348f7999), + TOBN(0x97bf738e, 0x2161f583), TOBN(0x15ee2fa7, 0x565e8cc9), + TOBN(0xa1a5c845, 0x5777e189), TOBN(0xcc10bee0, 0x456f2829), + TOBN(0x8ad95c56, 0xda762bd5), TOBN(0x152e2214, 0xe9d91da8), + TOBN(0x975b0e72, 0x7cb23c74), TOBN(0xfd5d7670, 0xa90c66df), + TOBN(0xb5b5b8ad, 0x225ffc53), TOBN(0xab6dff73, 0xfaded2ae), + TOBN(0xebd56781, 0x6f4cbe9d), TOBN(0x0ed8b249, 0x6a574bd7), + TOBN(0x41c246fe, 0x81a881fa), TOBN(0x91564805, 0xc3db9c70), + TOBN(0xd7c12b08, 0x5b862809), TOBN(0x1facd1f1, 0x55858d7b), + TOBN(0x7693747c, 0xaf09e92a), TOBN(0x3b69dcba, 0x189a425f), + TOBN(0x0be28e9f, 0x967365ef), TOBN(0x57300eb2, 0xe801f5c9), + TOBN(0x93b8ac6a, 0xd583352f), TOBN(0xa2cf1f89, 0xcd05b2b7), + TOBN(0x7c0c9b74, 0x4dcc40cc), TOBN(0xfee38c45, 0xada523fb), + TOBN(0xb49a4dec, 0x1099cc4d), TOBN(0x325c377f, 0x69f069c6), + TOBN(0xe12458ce, 0x476cc9ff), TOBN(0x580e0b6c, 0xc6d4cb63), + TOBN(0xd561c8b7, 0x9072289b), TOBN(0x0377f264, 0xa619e6da), + TOBN(0x26685362, 0x88e591a5), TOBN(0xa453a7bd, 0x7523ca2b), + TOBN(0x8a9536d2, 0xc1df4533), TOBN(0xc8e50f2f, 0xbe972f79), + TOBN(0xd433e50f, 0x6d3549cf), TOBN(0x6f33696f, 0xfacd665e), + TOBN(0x695bfdac, 0xce11fcb4), TOBN(0x810ee252, 0xaf7c9860), + TOBN(0x65450fe1, 0x7159bb2c), TOBN(0xf7dfbebe, 0x758b357b), + TOBN(0x2b057e74, 0xd69fea72), TOBN(0xd485717a, 0x92731745),} + , + {TOBN(0x896c42e8, 0xee36860c), TOBN(0xdaf04dfd, 0x4113c22d), + TOBN(0x1adbb7b7, 0x44104213), TOBN(0xe5fd5fa1, 0x1fd394ea), + TOBN(0x68235d94, 0x1a4e0551), TOBN(0x6772cfbe, 0x18d10151), + TOBN(0x276071e3, 0x09984523), TOBN(0xe4e879de, 0x5a56ba98), + TOBN(0xaaafafb0, 0x285b9491), TOBN(0x01a0be88, 0x1e4c705e), + TOBN(0xff1d4f5d, 0x2ad9caab), TOBN(0x6e349a4a, 0xc37a233f), + TOBN(0xcf1c1246, 0x4a1c6a16), TOBN(0xd99e6b66, 0x29383260), + TOBN(0xea3d4366, 0x5f6d5471), TOBN(0x36974d04, 0xff8cc89b), + TOBN(0xc26c49a1, 0xcfe89d80), TOBN(0xb42c026d, 0xda9c8371), + TOBN(0xca6c013a, 0xdad066d2), TOBN(0xfb8f7228, 0x56a4f3ee), + TOBN(0x08b579ec, 0xd850935b), TOBN(0x34c1a74c, 0xd631e1b3), + TOBN(0xcb5fe596, 0xac198534), TOBN(0x39ff21f6, 0xe1f24f25), + TOBN(0x27f29e14, 0x8f929057), TOBN(0x7a64ae06, 0xc0c853df), + TOBN(0x256cd183, 0x58e9c5ce), TOBN(0x9d9cce82, 0xded092a5), + TOBN(0xcc6e5979, 0x6e93b7c7), TOBN(0xe1e47092, 0x31bb9e27), + TOBN(0xb70b3083, 0xaa9e29a0), TOBN(0xbf181a75, 0x3785e644), + TOBN(0xf53f2c65, 0x8ead09f7), TOBN(0x1335e1d5, 0x9780d14d), + TOBN(0x69cc20e0, 0xcd1b66bc), TOBN(0x9b670a37, 0xbbe0bfc8), + TOBN(0xce53dc81, 0x28efbeed), TOBN(0x0c74e77c, 0x8326a6e5), + TOBN(0x3604e0d2, 0xb88e9a63), TOBN(0xbab38fca, 0x13dc2248), + TOBN(0x8ed6e8c8, 0x5c0a3f1e), TOBN(0xbcad2492, 0x7c87c37f), + TOBN(0xfdfb62bb, 0x9ee3b78d), TOBN(0xeba8e477, 0xcbceba46), + TOBN(0x37d38cb0, 0xeeaede4b), TOBN(0x0bc498e8, 0x7976deb6), + TOBN(0xb2944c04, 0x6b6147fb), TOBN(0x8b123f35, 0xf71f9609), + TOBN(0xa155dcc7, 0xde79dc24), TOBN(0xf1168a32, 0x558f69cd), + TOBN(0xbac21595, 0x0d1850df), TOBN(0x15c8295b, 0xb204c848), + TOBN(0xf661aa36, 0x7d8184ff), TOBN(0xc396228e, 0x30447bdb), + TOBN(0x11cd5143, 0xbde4a59e), TOBN(0xe3a26e3b, 0x6beab5e6), + TOBN(0xd3b3a13f, 0x1402b9d0), TOBN(0x573441c3, 0x2c7bc863), + TOBN(0x4b301ec4, 0x578c3e6e), TOBN(0xc26fc9c4, 0x0adaf57e), + TOBN(0x96e71bfd, 0x7493cea3), TOBN(0xd05d4b3f, 0x1af81456), + TOBN(0xdaca2a8a, 0x6a8c608f), TOBN(0x53ef07f6, 0x0725b276), + TOBN(0x07a5fbd2, 0x7824fc56), TOBN(0x34675218, 0x13289077), + TOBN(0x5bf69fd5, 0xe0c48349), TOBN(0xa613ddd3, 0xb6aa7875), + TOBN(0x7f78c19c, 0x5450d866), TOBN(0x46f4409c, 0x8f84a481), + TOBN(0x9f1d1928, 0x90fce239), TOBN(0x016c4168, 0xb2ce44b9), + TOBN(0xbae023f0, 0xc7435978), TOBN(0xb152c888, 0x20e30e19), + TOBN(0x9c241645, 0xe3fa6faf), TOBN(0x735d95c1, 0x84823e60), + TOBN(0x03197573, 0x03955317), TOBN(0x0b4b02a9, 0xf03b4995), + TOBN(0x076bf559, 0x70274600), TOBN(0x32c5cc53, 0xaaf57508), + TOBN(0xe8af6d1f, 0x60624129), TOBN(0xb7bc5d64, 0x9a5e2b5e), + TOBN(0x3814b048, 0x5f082d72), TOBN(0x76f267f2, 0xce19677a), + TOBN(0x626c630f, 0xb36eed93), TOBN(0x55230cd7, 0x3bf56803), + TOBN(0x78837949, 0xce2736a0), TOBN(0x0d792d60, 0xaa6c55f1), + TOBN(0x0318dbfd, 0xd5c7c5d2), TOBN(0xb38f8da7, 0x072b342d), + TOBN(0x3569bddc, 0x7b8de38a), TOBN(0xf25b5887, 0xa1c94842), + TOBN(0xb2d5b284, 0x2946ad60), TOBN(0x854f29ad, 0xe9d1707e), + TOBN(0xaa5159dc, 0x2c6a4509), TOBN(0x899f94c0, 0x57189837), + TOBN(0xcf6adc51, 0xf4a55b03), TOBN(0x261762de, 0x35e3b2d5), + TOBN(0x4cc43012, 0x04827b51), TOBN(0xcd22a113, 0xc6021442), + TOBN(0xce2fd61a, 0x247c9569), TOBN(0x59a50973, 0xd152beca), + TOBN(0x6c835a11, 0x63a716d4), TOBN(0xc26455ed, 0x187dedcf), + TOBN(0x27f536e0, 0x49ce89e7), TOBN(0x18908539, 0xcc890cb5), + TOBN(0x308909ab, 0xd83c2aa1), TOBN(0xecd3142b, 0x1ab73bd3), + TOBN(0x6a85bf59, 0xb3f5ab84), TOBN(0x3c320a68, 0xf2bea4c6), + TOBN(0xad8dc538, 0x6da4541f), TOBN(0xeaf34eb0, 0xb7c41186), + TOBN(0x1c780129, 0x977c97c4), TOBN(0x5ff9beeb, 0xc57eb9fa), + TOBN(0xa24d0524, 0xc822c478), TOBN(0xfd8eec2a, 0x461cd415), + TOBN(0xfbde194e, 0xf027458c), TOBN(0xb4ff5319, 0x1d1be115), + TOBN(0x63f874d9, 0x4866d6f4), TOBN(0x35c75015, 0xb21ad0c9), + TOBN(0xa6b5c9d6, 0x46ac49d2), TOBN(0x42c77c0b, 0x83137aa9), + TOBN(0x24d000fc, 0x68225a38), TOBN(0x0f63cfc8, 0x2fe1e907), + TOBN(0x22d1b01b, 0xc6441f95), TOBN(0x7d38f719, 0xec8e448f), + TOBN(0x9b33fa5f, 0x787fb1ba), TOBN(0x94dcfda1, 0x190158df), + TOBN(0xc47cb339, 0x5f6d4a09), TOBN(0x6b4f355c, 0xee52b826), + TOBN(0x3d100f5d, 0xf51b930a), TOBN(0xf4512fac, 0x9f668f69), + TOBN(0x546781d5, 0x206c4c74), TOBN(0xd021d4d4, 0xcb4d2e48), + TOBN(0x494a54c2, 0xca085c2d), TOBN(0xf1dbaca4, 0x520850a8), + TOBN(0x63c79326, 0x490a1aca), TOBN(0xcb64dd9c, 0x41526b02), + TOBN(0xbb772591, 0xa2979258), TOBN(0x3f582970, 0x48d97846), + TOBN(0xd66b70d1, 0x7c213ba7), TOBN(0xc28febb5, 0xe8a0ced4), + TOBN(0x6b911831, 0xc10338c1), TOBN(0x0d54e389, 0xbf0126f3), + TOBN(0x7048d460, 0x4af206ee), TOBN(0x786c88f6, 0x77e97cb9), + TOBN(0xd4375ae1, 0xac64802e), TOBN(0x469bcfe1, 0xd53ec11c), + TOBN(0xfc9b340d, 0x47062230), TOBN(0xe743bb57, 0xc5b4a3ac), + TOBN(0xfe00b4aa, 0x59ef45ac), TOBN(0x29a4ef23, 0x59edf188), + TOBN(0x40242efe, 0xb483689b), TOBN(0x2575d3f6, 0x513ac262), + TOBN(0xf30037c8, 0x0ca6db72), TOBN(0xc9fcce82, 0x98864be2), + TOBN(0x84a112ff, 0x0149362d), TOBN(0x95e57582, 0x1c4ae971), + TOBN(0x1fa4b1a8, 0x945cf86c), TOBN(0x4525a734, 0x0b024a2f), + TOBN(0xe76c8b62, 0x8f338360), TOBN(0x483ff593, 0x28edf32b), + TOBN(0x67e8e90a, 0x298b1aec), TOBN(0x9caab338, 0x736d9a21), + TOBN(0x5c09d2fd, 0x66892709), TOBN(0x2496b4dc, 0xb55a1d41), + TOBN(0x93f5fb1a, 0xe24a4394), TOBN(0x08c75049, 0x6fa8f6c1), + TOBN(0xcaead1c2, 0xc905d85f), TOBN(0xe9d7f790, 0x0733ae57), + TOBN(0x24c9a65c, 0xf07cdd94), TOBN(0x7389359c, 0xa4b55931), + TOBN(0xf58709b7, 0x367e45f7), TOBN(0x1f203067, 0xcb7e7adc), + TOBN(0x82444bff, 0xc7b72818), TOBN(0x07303b35, 0xbaac8033), + TOBN(0x1e1ee4e4, 0xd13b7ea1), TOBN(0xe6489b24, 0xe0e74180), + TOBN(0xa5f2c610, 0x7e70ef70), TOBN(0xa1655412, 0xbdd10894), + TOBN(0x555ebefb, 0x7af4194e), TOBN(0x533c1c3c, 0x8e89bd9c), + TOBN(0x735b9b57, 0x89895856), TOBN(0x15fb3cd2, 0x567f5c15), + TOBN(0x057fed45, 0x526f09fd), TOBN(0xe8a4f10c, 0x8128240a), + TOBN(0x9332efc4, 0xff2bfd8d), TOBN(0x214e77a0, 0xbd35aa31), + TOBN(0x32896d73, 0x14faa40e), TOBN(0x767867ec, 0x01e5f186), + TOBN(0xc9adf8f1, 0x17a1813e), TOBN(0xcb6cda78, 0x54741795), + TOBN(0xb7521b6d, 0x349d51aa), TOBN(0xf56b5a9e, 0xe3c7b8e9), + TOBN(0xc6f1e5c9, 0x32a096df), TOBN(0x083667c4, 0xa3635024), + TOBN(0x365ea135, 0x18087f2f), TOBN(0xf1b8eaac, 0xd136e45d), + TOBN(0xc8a0e484, 0x73aec989), TOBN(0xd75a324b, 0x142c9259), + TOBN(0xb7b4d001, 0x01dae185), TOBN(0x45434e0b, 0x9b7a94bc), + TOBN(0xf54339af, 0xfbd8cb0b), TOBN(0xdcc4569e, 0xe98ef49e), + TOBN(0x7789318a, 0x09a51299), TOBN(0x81b4d206, 0xb2b025d8), + TOBN(0xf64aa418, 0xfae85792), TOBN(0x3e50258f, 0xacd7baf7), + TOBN(0xdce84cdb, 0x2996864b), TOBN(0xa2e67089, 0x1f485fa4), + TOBN(0xb28b2bb6, 0x534c6a5a), TOBN(0x31a7ec6b, 0xc94b9d39), + TOBN(0x1d217766, 0xd6bc20da), TOBN(0x4acdb5ec, 0x86761190), + TOBN(0x68726328, 0x73701063), TOBN(0x4d24ee7c, 0x2128c29b), + TOBN(0xc072ebd3, 0xa19fd868), TOBN(0x612e481c, 0xdb8ddd3b), + TOBN(0xb4e1d754, 0x1a64d852), TOBN(0x00ef95ac, 0xc4c6c4ab), + TOBN(0x1536d2ed, 0xaa0a6c46), TOBN(0x61294086, 0x43774790), + TOBN(0x54af25e8, 0x343fda10), TOBN(0x9ff9d98d, 0xfd25d6f2), + TOBN(0x0746af7c, 0x468b8835), TOBN(0x977a31cb, 0x730ecea7), + TOBN(0xa5096b80, 0xc2cf4a81), TOBN(0xaa986833, 0x6458c37a), + TOBN(0x6af29bf3, 0xa6bd9d34), TOBN(0x6a62fe9b, 0x33c5d854), + TOBN(0x50e6c304, 0xb7133b5e), TOBN(0x04b60159, 0x7d6e6848), + TOBN(0x4cd296df, 0x5579bea4), TOBN(0x10e35ac8, 0x5ceedaf1), + TOBN(0x04c4c5fd, 0xe3bcc5b1), TOBN(0x95f9ee8a, 0x89412cf9), + TOBN(0x2c9459ee, 0x82b6eb0f), TOBN(0x2e845765, 0x95c2aadd), + TOBN(0x774a84ae, 0xd327fcfe), TOBN(0xd8c93722, 0x0368d476), + TOBN(0x0dbd5748, 0xf83e8a3b), TOBN(0xa579aa96, 0x8d2495f3), + TOBN(0x535996a0, 0xae496e9b), TOBN(0x07afbfe9, 0xb7f9bcc2), + TOBN(0x3ac1dc6d, 0x5b7bd293), TOBN(0x3b592cff, 0x7022323d), + TOBN(0xba0deb98, 0x9c0a3e76), TOBN(0x18e78e9f, 0x4b197acb), + TOBN(0x211cde10, 0x296c36ef), TOBN(0x7ee89672, 0x82c4da77), + TOBN(0xb617d270, 0xa57836da), TOBN(0xf0cd9c31, 0x9cb7560b), + TOBN(0x01fdcbf7, 0xe455fe90), TOBN(0x3fb53cbb, 0x7e7334f3), + TOBN(0x781e2ea4, 0x4e7de4ec), TOBN(0x8adab3ad, 0x0b384fd0), + TOBN(0x129eee2f, 0x53d64829), TOBN(0x7a471e17, 0xa261492b), + TOBN(0xe4f9adb9, 0xe4cb4a2c), TOBN(0x3d359f6f, 0x97ba2c2d), + TOBN(0x346c6786, 0x0aacd697), TOBN(0x92b444c3, 0x75c2f8a8), + TOBN(0xc79fa117, 0xd85df44e), TOBN(0x56782372, 0x398ddf31), + TOBN(0x60e690f2, 0xbbbab3b8), TOBN(0x4851f8ae, 0x8b04816b), + TOBN(0xc72046ab, 0x9c92e4d2), TOBN(0x518c74a1, 0x7cf3136b), + TOBN(0xff4eb50a, 0xf9877d4c), TOBN(0x14578d90, 0xa919cabb), + TOBN(0x8218f8c4, 0xac5eb2b6), TOBN(0xa3ccc547, 0x542016e4), + TOBN(0x025bf48e, 0x327f8349), TOBN(0xf3e97346, 0xf43cb641), + TOBN(0xdc2bafdf, 0x500f1085), TOBN(0x57167876, 0x2f063055), + TOBN(0x5bd914b9, 0x411925a6), TOBN(0x7c078d48, 0xa1123de5), + TOBN(0xee6bf835, 0x182b165d), TOBN(0xb11b5e5b, 0xba519727), + TOBN(0xe33ea76c, 0x1eea7b85), TOBN(0x2352b461, 0x92d4f85e), + TOBN(0xf101d334, 0xafe115bb), TOBN(0xfabc1294, 0x889175a3), + TOBN(0x7f6bcdc0, 0x5233f925), TOBN(0xe0a802db, 0xe77fec55), + TOBN(0xbdb47b75, 0x8069b659), TOBN(0x1c5e12de, 0xf98fbd74), + TOBN(0x869c58c6, 0x4b8457ee), TOBN(0xa5360f69, 0x4f7ea9f7), + TOBN(0xe576c09f, 0xf460b38f), TOBN(0x6b70d548, 0x22b7fb36), + TOBN(0x3fd237f1, 0x3bfae315), TOBN(0x33797852, 0xcbdff369), + TOBN(0x97df25f5, 0x25b516f9), TOBN(0x46f388f2, 0xba38ad2d), + TOBN(0x656c4658, 0x89d8ddbb), TOBN(0x8830b26e, 0x70f38ee8), + TOBN(0x4320fd5c, 0xde1212b0), TOBN(0xc34f30cf, 0xe4a2edb2), + TOBN(0xabb131a3, 0x56ab64b8), TOBN(0x7f77f0cc, 0xd99c5d26), + TOBN(0x66856a37, 0xbf981d94), TOBN(0x19e76d09, 0x738bd76e), + TOBN(0xe76c8ac3, 0x96238f39), TOBN(0xc0a482be, 0xa830b366), + TOBN(0xb7b8eaff, 0x0b4eb499), TOBN(0x8ecd83bc, 0x4bfb4865), + TOBN(0x971b2cb7, 0xa2f3776f), TOBN(0xb42176a4, 0xf4b88adf), + TOBN(0xb9617df5, 0xbe1fa446), TOBN(0x8b32d508, 0xcd031bd2), + TOBN(0x1c6bd47d, 0x53b618c0), TOBN(0xc424f46c, 0x6a227923), + TOBN(0x7303ffde, 0xdd92d964), TOBN(0xe9712878, 0x71b5abf2), + TOBN(0x8f48a632, 0xf815561d), TOBN(0x85f48ff5, 0xd3c055d1), + TOBN(0x222a1427, 0x7525684f), TOBN(0xd0d841a0, 0x67360cc3), + TOBN(0x4245a926, 0x0b9267c6), TOBN(0xc78913f1, 0xcf07f863), + TOBN(0xaa844c8e, 0x4d0d9e24), TOBN(0xa42ad522, 0x3d5f9017), + TOBN(0xbd371749, 0xa2c989d5), TOBN(0x928292df, 0xe1f5e78e), + TOBN(0x493b383e, 0x0a1ea6da), TOBN(0x5136fd8d, 0x13aee529), + TOBN(0x860c44b1, 0xf2c34a99), TOBN(0x3b00aca4, 0xbf5855ac), + TOBN(0xabf6aaa0, 0xfaaf37be), TOBN(0x65f43682, 0x2a53ec08), + TOBN(0x1d9a5801, 0xa11b12e1), TOBN(0x78a7ab2c, 0xe20ed475), + TOBN(0x0de1067e, 0x9a41e0d5), TOBN(0x30473f5f, 0x305023ea), + TOBN(0xdd3ae09d, 0x169c7d97), TOBN(0x5cd5baa4, 0xcfaef9cd), + TOBN(0x5cd7440b, 0x65a44803), TOBN(0xdc13966a, 0x47f364de), + TOBN(0x077b2be8, 0x2b8357c1), TOBN(0x0cb1b4c5, 0xe9d57c2a), + TOBN(0x7a4ceb32, 0x05ff363e), TOBN(0xf310fa4d, 0xca35a9ef), + TOBN(0xdbb7b352, 0xf97f68c6), TOBN(0x0c773b50, 0x0b02cf58), + TOBN(0xea2e4821, 0x3c1f96d9), TOBN(0xffb357b0, 0xeee01815), + TOBN(0xb9c924cd, 0xe0f28039), TOBN(0x0b36c95a, 0x46a3fbe4), + TOBN(0x1faaaea4, 0x5e46db6c), TOBN(0xcae575c3, 0x1928aaff), + TOBN(0x7f671302, 0xa70dab86), TOBN(0xfcbd12a9, 0x71c58cfc), + TOBN(0xcbef9acf, 0xbee0cb92), TOBN(0x573da0b9, 0xf8c1b583), + TOBN(0x4752fcfe, 0x0d41d550), TOBN(0xe7eec0e3, 0x2155cffe), + TOBN(0x0fc39fcb, 0x545ae248), TOBN(0x522cb8d1, 0x8065f44e), + TOBN(0x263c962a, 0x70cbb96c), TOBN(0xe034362a, 0xbcd124a9), + TOBN(0xf120db28, 0x3c2ae58d), TOBN(0xb9a38d49, 0xfef6d507), + TOBN(0xb1fd2a82, 0x1ff140fd), TOBN(0xbd162f30, 0x20aee7e0), + TOBN(0x4e17a5d4, 0xcb251949), TOBN(0x2aebcb83, 0x4f7e1c3d), + TOBN(0x608eb25f, 0x937b0527), TOBN(0xf42e1e47, 0xeb7d9997), + TOBN(0xeba699c4, 0xb8a53a29), TOBN(0x1f921c71, 0xe091b536), + TOBN(0xcce29e7b, 0x5b26bbd5), TOBN(0x7a8ef5ed, 0x3b61a680), + TOBN(0xe5ef8043, 0xba1f1c7e), TOBN(0x16ea8217, 0x18158dda), + TOBN(0x01778a2b, 0x599ff0f9), TOBN(0x68a923d7, 0x8104fc6b), + TOBN(0x5bfa44df, 0xda694ff3), TOBN(0x4f7199db, 0xf7667f12), + TOBN(0xc06d8ff6, 0xe46f2a79), TOBN(0x08b5dead, 0xe9f8131d), + TOBN(0x02519a59, 0xabb4ce7c), TOBN(0xc4f710bc, 0xb42aec3e), + TOBN(0x3d77b057, 0x78bde41a), TOBN(0x6474bf80, 0xb4186b5a), + TOBN(0x048b3f67, 0x88c65741), TOBN(0xc64519de, 0x03c7c154), + TOBN(0xdf073846, 0x0edfcc4f), TOBN(0x319aa737, 0x48f1aa6b), + TOBN(0x8b9f8a02, 0xca909f77), TOBN(0x90258139, 0x7580bfef), + TOBN(0xd8bfd3ca, 0xc0c22719), TOBN(0xc60209e4, 0xc9ca151e), + TOBN(0x7a744ab5, 0xd9a1a69c), TOBN(0x6de5048b, 0x14937f8f), + TOBN(0x171938d8, 0xe115ac04), TOBN(0x7df70940, 0x1c6b16d2), + TOBN(0xa6aeb663, 0x7f8e94e7), TOBN(0xc130388e, 0x2a2cf094), + TOBN(0x1850be84, 0x77f54e6e), TOBN(0x9f258a72, 0x65d60fe5), + TOBN(0xff7ff0c0, 0x6c9146d6), TOBN(0x039aaf90, 0xe63a830b), + TOBN(0x38f27a73, 0x9460342f), TOBN(0x4703148c, 0x3f795f8a), + TOBN(0x1bb5467b, 0x9681a97e), TOBN(0x00931ba5, 0xecaeb594), + TOBN(0xcdb6719d, 0x786f337c), TOBN(0xd9c01cd2, 0xe704397d), + TOBN(0x0f4a3f20, 0x555c2fef), TOBN(0x00452509, 0x7c0af223), + TOBN(0x54a58047, 0x84db8e76), TOBN(0x3bacf1aa, 0x93c8aa06), + TOBN(0x11ca957c, 0xf7919422), TOBN(0x50641053, 0x78cdaa40), + TOBN(0x7a303874, 0x9f7144ae), TOBN(0x170c963f, 0x43d4acfd), + TOBN(0x5e148149, 0x58ddd3ef), TOBN(0xa7bde582, 0x9e72dba8), + TOBN(0x0769da8b, 0x6fa68750), TOBN(0xfa64e532, 0x572e0249), + TOBN(0xfcaadf9d, 0x2619ad31), TOBN(0x87882daa, 0xa7b349cd), + TOBN(0x9f6eb731, 0x6c67a775), TOBN(0xcb10471a, 0xefc5d0b1), + TOBN(0xb433750c, 0xe1b806b2), TOBN(0x19c5714d, 0x57b1ae7e), + TOBN(0xc0dc8b7b, 0xed03fd3f), TOBN(0xdd03344f, 0x31bc194e), + TOBN(0xa66c52a7, 0x8c6320b5), TOBN(0x8bc82ce3, 0xd0b6fd93), + TOBN(0xf8e13501, 0xb35f1341), TOBN(0xe53156dd, 0x25a43e42), + TOBN(0xd3adf27e, 0x4daeb85c), TOBN(0xb81d8379, 0xbbeddeb5), + TOBN(0x1b0b546e, 0x2e435867), TOBN(0x9020eb94, 0xeba5dd60), + TOBN(0x37d91161, 0x8210cb9d), TOBN(0x4c596b31, 0x5c91f1cf), + TOBN(0xb228a90f, 0x0e0b040d), TOBN(0xbaf02d82, 0x45ff897f), + TOBN(0x2aac79e6, 0x00fa6122), TOBN(0x24828817, 0x8e36f557), + TOBN(0xb9521d31, 0x113ec356), TOBN(0x9e48861e, 0x15eff1f8), + TOBN(0x2aa1d412, 0xe0d41715), TOBN(0x71f86203, 0x53f131b8), + TOBN(0xf60da8da, 0x3fd19408), TOBN(0x4aa716dc, 0x278d9d99), + TOBN(0x394531f7, 0xa8c51c90), TOBN(0xb560b0e8, 0xf59db51c), + TOBN(0xa28fc992, 0xfa34bdad), TOBN(0xf024fa14, 0x9cd4f8bd), + TOBN(0x5cf530f7, 0x23a9d0d3), TOBN(0x615ca193, 0xe28c9b56), + TOBN(0x6d2a483d, 0x6f73c51e), TOBN(0xa4cb2412, 0xea0dc2dd), + TOBN(0x50663c41, 0x1eb917ff), TOBN(0x3d3a74cf, 0xeade299e), + TOBN(0x29b3990f, 0x4a7a9202), TOBN(0xa9bccf59, 0xa7b15c3d), + TOBN(0x66a3ccdc, 0xa5df9208), TOBN(0x48027c14, 0x43f2f929), + TOBN(0xd385377c, 0x40b557f0), TOBN(0xe001c366, 0xcd684660), + TOBN(0x1b18ed6b, 0xe2183a27), TOBN(0x879738d8, 0x63210329), + TOBN(0xa687c74b, 0xbda94882), TOBN(0xd1bbcc48, 0xa684b299), + TOBN(0xaf6f1112, 0x863b3724), TOBN(0x6943d1b4, 0x2c8ce9f8), + TOBN(0xe044a3bb, 0x098cafb4), TOBN(0x27ed2310, 0x60d48caf), + TOBN(0x542b5675, 0x3a31b84d), TOBN(0xcbf3dd50, 0xfcddbed7), + TOBN(0x25031f16, 0x41b1d830), TOBN(0xa7ec851d, 0xcb0c1e27), + TOBN(0xac1c8fe0, 0xb5ae75db), TOBN(0xb24c7557, 0x08c52120), + TOBN(0x57f811dc, 0x1d4636c3), TOBN(0xf8436526, 0x681a9939), + TOBN(0x1f6bc6d9, 0x9c81adb3), TOBN(0x840f8ac3, 0x5b7d80d4), + TOBN(0x731a9811, 0xf4387f1a), TOBN(0x7c501cd3, 0xb5156880), + TOBN(0xa5ca4a07, 0xdfe68867), TOBN(0xf123d8f0, 0x5fcea120), + TOBN(0x1fbb0e71, 0xd607039e), TOBN(0x2b70e215, 0xcd3a4546), + TOBN(0x32d2f01d, 0x53324091), TOBN(0xb796ff08, 0x180ab19b), + TOBN(0x32d87a86, 0x3c57c4aa), TOBN(0x2aed9caf, 0xb7c49a27), + TOBN(0x9fb35eac, 0x31630d98), TOBN(0x338e8cdf, 0x5c3e20a3), + TOBN(0x80f16182, 0x66cde8db), TOBN(0x4e159980, 0x2d72fd36), + TOBN(0xd7b8f13b, 0x9b6e5072), TOBN(0xf5213907, 0x3b7b5dc1), + TOBN(0x4d431f1d, 0x8ce4396e), TOBN(0x37a1a680, 0xa7ed2142), + TOBN(0xbf375696, 0xd01aaf6b), TOBN(0xaa1c0c54, 0xe63aab66), + TOBN(0x3014368b, 0x4ed80940), TOBN(0x67e6d056, 0x7a6fcedd), + TOBN(0x7c208c49, 0xca97579f), TOBN(0xfe3d7a81, 0xa23597f6), + TOBN(0x5e203202, 0x7e096ae2), TOBN(0xb1f3e1e7, 0x24b39366), + TOBN(0x26da26f3, 0x2fdcdffc), TOBN(0x79422f1d, 0x6097be83),} + , + {TOBN(0x263a2cfb, 0x9db3b381), TOBN(0x9c3a2dee, 0xd4df0a4b), + TOBN(0x728d06e9, 0x7d04e61f), TOBN(0x8b1adfbc, 0x42449325), + TOBN(0x6ec1d939, 0x7e053a1b), TOBN(0xee2be5c7, 0x66daf707), + TOBN(0x80ba1e14, 0x810ac7ab), TOBN(0xdd2ae778, 0xf530f174), + TOBN(0x0435d97a, 0x205b9d8b), TOBN(0x6eb8f064, 0x056756d4), + TOBN(0xd5e88a8b, 0xb6f8210e), TOBN(0x070ef12d, 0xec9fd9ea), + TOBN(0x4d849505, 0x3bcc876a), TOBN(0x12a75338, 0xa7404ce3), + TOBN(0xd22b49e1, 0xb8a1db5e), TOBN(0xec1f2051, 0x14bfa5ad), + TOBN(0xadbaeb79, 0xb6828f36), TOBN(0x9d7a0258, 0x01bd5b9e), + TOBN(0xeda01e0d, 0x1e844b0c), TOBN(0x4b625175, 0x887edfc9), + TOBN(0x14109fdd, 0x9669b621), TOBN(0x88a2ca56, 0xf6f87b98), + TOBN(0xfe2eb788, 0x170df6bc), TOBN(0x0cea06f4, 0xffa473f9), + TOBN(0x43ed81b5, 0xc4e83d33), TOBN(0xd9f35879, 0x5efd488b), + TOBN(0x164a620f, 0x9deb4d0f), TOBN(0xc6927bdb, 0xac6a7394), + TOBN(0x45c28df7, 0x9f9e0f03), TOBN(0x2868661e, 0xfcd7e1a9), + TOBN(0x7cf4e8d0, 0xffa348f1), TOBN(0x6bd4c284, 0x398538e0), + TOBN(0x2618a091, 0x289a8619), TOBN(0xef796e60, 0x6671b173), + TOBN(0x664e46e5, 0x9090c632), TOBN(0xa38062d4, 0x1e66f8fb), + TOBN(0x6c744a20, 0x0573274e), TOBN(0xd07b67e4, 0xa9271394), + TOBN(0x391223b2, 0x6bdc0e20), TOBN(0xbe2d93f1, 0xeb0a05a7), + TOBN(0xf23e2e53, 0x3f36d141), TOBN(0xe84bb3d4, 0x4dfca442), + TOBN(0xb804a48d, 0x6b7c023a), TOBN(0x1e16a8fa, 0x76431c3b), + TOBN(0x1b5452ad, 0xddd472e0), TOBN(0x7d405ee7, 0x0d1ee127), + TOBN(0x50fc6f1d, 0xffa27599), TOBN(0x351ac53c, 0xbf391b35), + TOBN(0x7efa14b8, 0x4444896b), TOBN(0x64974d2f, 0xf94027fb), + TOBN(0xefdcd0e8, 0xde84487d), TOBN(0x8c45b260, 0x2b48989b), + TOBN(0xa8fcbbc2, 0xd8463487), TOBN(0xd1b2b3f7, 0x3fbc476c), + TOBN(0x21d005b7, 0xc8f443c0), TOBN(0x518f2e67, 0x40c0139c), + TOBN(0x56036e8c, 0x06d75fc1), TOBN(0x2dcf7bb7, 0x3249a89f), + TOBN(0x81dd1d3d, 0xe245e7dd), TOBN(0xf578dc4b, 0xebd6e2a7), + TOBN(0x4c028903, 0xdf2ce7a0), TOBN(0xaee36288, 0x9c39afac), + TOBN(0xdc847c31, 0x146404ab), TOBN(0x6304c0d8, 0xa4e97818), + TOBN(0xae51dca2, 0xa91f6791), TOBN(0x2abe4190, 0x9baa9efc), + TOBN(0xd9d2e2f4, 0x559c7ac1), TOBN(0xe82f4b51, 0xfc9f773a), + TOBN(0xa7713027, 0x4073e81c), TOBN(0xc0276fac, 0xfbb596fc), + TOBN(0x1d819fc9, 0xa684f70c), TOBN(0x29b47fdd, 0xc9f7b1e0), + TOBN(0x358de103, 0x459b1940), TOBN(0xec881c59, 0x5b013e93), + TOBN(0x51574c93, 0x49532ad3), TOBN(0x2db1d445, 0xb37b46de), + TOBN(0xc6445b87, 0xdf239fd8), TOBN(0xc718af75, 0x151d24ee), + TOBN(0xaea1c4a4, 0xf43c6259), TOBN(0x40c0e5d7, 0x70be02f7), + TOBN(0x6a4590f4, 0x721b33f2), TOBN(0x2124f1fb, 0xfedf04ea), + TOBN(0xf8e53cde, 0x9745efe7), TOBN(0xe7e10432, 0x65f046d9), + TOBN(0xc3fca28e, 0xe4d0c7e6), TOBN(0x847e339a, 0x87253b1b), + TOBN(0x9b595348, 0x3743e643), TOBN(0xcb6a0a0b, 0x4fd12fc5), + TOBN(0xfb6836c3, 0x27d02dcc), TOBN(0x5ad00982, 0x7a68bcc2), + TOBN(0x1b24b44c, 0x005e912d), TOBN(0xcc83d20f, 0x811fdcfe), + TOBN(0x36527ec1, 0x666fba0c), TOBN(0x69948197, 0x14754635), + TOBN(0xfcdcb1a8, 0x556da9c2), TOBN(0xa5934267, 0x81a732b2), + TOBN(0xec1214ed, 0xa714181d), TOBN(0x609ac13b, 0x6067b341), + TOBN(0xff4b4c97, 0xa545df1f), TOBN(0xa1240501, 0x34d2076b), + TOBN(0x6efa0c23, 0x1409ca97), TOBN(0x254cc1a8, 0x20638c43), + TOBN(0xd4e363af, 0xdcfb46cd), TOBN(0x62c2adc3, 0x03942a27), + TOBN(0xc67b9df0, 0x56e46483), TOBN(0xa55abb20, 0x63736356), + TOBN(0xab93c098, 0xc551bc52), TOBN(0x382b49f9, 0xb15fe64b), + TOBN(0x9ec221ad, 0x4dff8d47), TOBN(0x79caf615, 0x437df4d6), + TOBN(0x5f13dc64, 0xbb456509), TOBN(0xe4c589d9, 0x191f0714), + TOBN(0x27b6a8ab, 0x3fd40e09), TOBN(0xe455842e, 0x77313ea9), + TOBN(0x8b51d1e2, 0x1f55988b), TOBN(0x5716dd73, 0x062bbbfc), + TOBN(0x633c11e5, 0x4e8bf3de), TOBN(0x9a0e77b6, 0x1b85be3b), + TOBN(0x56510729, 0x0911cca6), TOBN(0x27e76495, 0xefa6590f), + TOBN(0xe4ac8b33, 0x070d3aab), TOBN(0x2643672b, 0x9a2cd5e5), + TOBN(0x52eff79b, 0x1cfc9173), TOBN(0x665ca49b, 0x90a7c13f), + TOBN(0x5a8dda59, 0xb3efb998), TOBN(0x8a5b922d, 0x052f1341), + TOBN(0xae9ebbab, 0x3cf9a530), TOBN(0x35986e7b, 0xf56da4d7), + TOBN(0x3a636b5c, 0xff3513cc), TOBN(0xbb0cf8ba, 0x3198f7dd), + TOBN(0xb8d40522, 0x41f16f86), TOBN(0x760575d8, 0xde13a7bf), + TOBN(0x36f74e16, 0x9f7aa181), TOBN(0x163a3ecf, 0xf509ed1c), + TOBN(0x6aead61f, 0x3c40a491), TOBN(0x158c95fc, 0xdfe8fcaa), + TOBN(0xa3991b6e, 0x13cda46f), TOBN(0x79482415, 0x342faed0), + TOBN(0xf3ba5bde, 0x666b5970), TOBN(0x1d52e6bc, 0xb26ab6dd), + TOBN(0x768ba1e7, 0x8608dd3d), TOBN(0x4930db2a, 0xea076586), + TOBN(0xd9575714, 0xe7dc1afa), TOBN(0x1fc7bf7d, 0xf7c58817), + TOBN(0x6b47accd, 0xd9eee96c), TOBN(0x0ca277fb, 0xe58cec37), + TOBN(0x113fe413, 0xe702c42a), TOBN(0xdd1764ee, 0xc47cbe51), + TOBN(0x041e7cde, 0x7b3ed739), TOBN(0x50cb7459, 0x5ce9e1c0), + TOBN(0x35568513, 0x2925b212), TOBN(0x7cff95c4, 0x001b081c), + TOBN(0x63ee4cbd, 0x8088b454), TOBN(0xdb7f32f7, 0x9a9e0c8a), + TOBN(0xb377d418, 0x6b2447cb), TOBN(0xe3e982aa, 0xd370219b), + TOBN(0x06ccc1e4, 0xc2a2a593), TOBN(0x72c36865, 0x0773f24f), + TOBN(0xa13b4da7, 0x95859423), TOBN(0x8bbf1d33, 0x75040c8f), + TOBN(0x726f0973, 0xda50c991), TOBN(0x48afcd5b, 0x822d6ee2), + TOBN(0xe5fc718b, 0x20fd7771), TOBN(0xb9e8e77d, 0xfd0807a1), + TOBN(0x7f5e0f44, 0x99a7703d), TOBN(0x6972930e, 0x618e36f3), + TOBN(0x2b7c77b8, 0x23807bbe), TOBN(0xe5b82405, 0xcb27ff50), + TOBN(0xba8b8be3, 0xbd379062), TOBN(0xd64b7a1d, 0x2dce4a92), + TOBN(0x040a73c5, 0xb2952e37), TOBN(0x0a9e252e, 0xd438aeca), + TOBN(0xdd43956b, 0xc39d3bcb), TOBN(0x1a31ca00, 0xb32b2d63), + TOBN(0xd67133b8, 0x5c417a18), TOBN(0xd08e4790, 0x2ef442c8), + TOBN(0x98cb1ae9, 0x255c0980), TOBN(0x4bd86381, 0x2b4a739f), + TOBN(0x5a5c31e1, 0x1e4a45a1), TOBN(0x1e5d55fe, 0x9cb0db2f), + TOBN(0x74661b06, 0x8ff5cc29), TOBN(0x026b389f, 0x0eb8a4f4), + TOBN(0x536b21a4, 0x58848c24), TOBN(0x2e5bf8ec, 0x81dc72b0), + TOBN(0x03c187d0, 0xad886aac), TOBN(0x5c16878a, 0xb771b645), + TOBN(0xb07dfc6f, 0xc74045ab), TOBN(0x2c6360bf, 0x7800caed), + TOBN(0x24295bb5, 0xb9c972a3), TOBN(0xc9e6f88e, 0x7c9a6dba), + TOBN(0x90ffbf24, 0x92a79aa6), TOBN(0xde29d50a, 0x41c26ac2), + TOBN(0x9f0af483, 0xd309cbe6), TOBN(0x5b020d8a, 0xe0bced4f), + TOBN(0x606e986d, 0xb38023e3), TOBN(0xad8f2c9d, 0x1abc6933), + TOBN(0x19292e1d, 0xe7400e93), TOBN(0xfe3e18a9, 0x52be5e4d), + TOBN(0xe8e9771d, 0x2e0680bf), TOBN(0x8c5bec98, 0xc54db063), + TOBN(0x2af9662a, 0x74a55d1f), TOBN(0xe3fbf28f, 0x046f66d8), + TOBN(0xa3a72ab4, 0xd4dc4794), TOBN(0x09779f45, 0x5c7c2dd8), + TOBN(0xd893bdaf, 0xc3d19d8d), TOBN(0xd5a75094, 0x57d6a6df), + TOBN(0x8cf8fef9, 0x952e6255), TOBN(0x3da67cfb, 0xda9a8aff), + TOBN(0x4c23f62a, 0x2c160dcd), TOBN(0x34e6c5e3, 0x8f90eaef), + TOBN(0x35865519, 0xa9a65d5a), TOBN(0x07c48aae, 0x8fd38a3d), + TOBN(0xb7e7aeda, 0x50068527), TOBN(0x2c09ef23, 0x1c90936a), + TOBN(0x31ecfeb6, 0xe879324c), TOBN(0xa0871f6b, 0xfb0ec938), + TOBN(0xb1f0fb68, 0xd84d835d), TOBN(0xc90caf39, 0x861dc1e6), + TOBN(0x12e5b046, 0x7594f8d7), TOBN(0x26897ae2, 0x65012b92), + TOBN(0xbcf68a08, 0xa4d6755d), TOBN(0x403ee41c, 0x0991fbda), + TOBN(0x733e343e, 0x3bbf17e8), TOBN(0xd2c7980d, 0x679b3d65), + TOBN(0x33056232, 0xd2e11305), TOBN(0x966be492, 0xf3c07a6f), + TOBN(0x6a8878ff, 0xbb15509d), TOBN(0xff221101, 0x0a9b59a4), + TOBN(0x6c9f564a, 0xabe30129), TOBN(0xc6f2c940, 0x336e64cf), + TOBN(0x0fe75262, 0x8b0c8022), TOBN(0xbe0267e9, 0x6ae8db87), + TOBN(0x22e192f1, 0x93bc042b), TOBN(0xf085b534, 0xb237c458), + TOBN(0xa0d192bd, 0x832c4168), TOBN(0x7a76e9e3, 0xbdf6271d), + TOBN(0x52a882fa, 0xb88911b5), TOBN(0xc85345e4, 0xb4db0eb5), + TOBN(0xa3be02a6, 0x81a7c3ff), TOBN(0x51889c8c, 0xf0ec0469), + TOBN(0x9d031369, 0xa5e829e5), TOBN(0xcbb4c6fc, 0x1607aa41), + TOBN(0x75ac59a6, 0x241d84c1), TOBN(0xc043f2bf, 0x8829e0ee), + TOBN(0x82a38f75, 0x8ea5e185), TOBN(0x8bda40b9, 0xd87cbd9f), + TOBN(0x9e65e75e, 0x2d8fc601), TOBN(0x3d515f74, 0xa35690b3), + TOBN(0x534acf4f, 0xda79e5ac), TOBN(0x68b83b3a, 0x8630215f), + TOBN(0x5c748b2e, 0xd085756e), TOBN(0xb0317258, 0xe5d37cb2), + TOBN(0x6735841a, 0xc5ccc2c4), TOBN(0x7d7dc96b, 0x3d9d5069), + TOBN(0xa147e410, 0xfd1754bd), TOBN(0x65296e94, 0xd399ddd5), + TOBN(0xf6b5b2d0, 0xbc8fa5bc), TOBN(0x8a5ead67, 0x500c277b), + TOBN(0x214625e6, 0xdfa08a5d), TOBN(0x51fdfedc, 0x959cf047), + TOBN(0x6bc9430b, 0x289fca32), TOBN(0xe36ff0cf, 0x9d9bdc3f), + TOBN(0x2fe187cb, 0x58ea0ede), TOBN(0xed66af20, 0x5a900b3f), + TOBN(0x00e0968b, 0x5fa9f4d6), TOBN(0x2d4066ce, 0x37a362e7), + TOBN(0xa99a9748, 0xbd07e772), TOBN(0x710989c0, 0x06a4f1d0), + TOBN(0xd5dedf35, 0xce40cbd8), TOBN(0xab55c5f0, 0x1743293d), + TOBN(0x766f1144, 0x8aa24e2c), TOBN(0x94d874f8, 0x605fbcb4), + TOBN(0xa365f0e8, 0xa518001b), TOBN(0xee605eb6, 0x9d04ef0f), + TOBN(0x5a3915cd, 0xba8d4d25), TOBN(0x44c0e1b8, 0xb5113472), + TOBN(0xcbb024e8, 0x8b6740dc), TOBN(0x89087a53, 0xee1d4f0c), + TOBN(0xa88fa05c, 0x1fc4e372), TOBN(0x8bf395cb, 0xaf8b3af2), + TOBN(0x1e71c9a1, 0xdeb8568b), TOBN(0xa35daea0, 0x80fb3d32), + TOBN(0xe8b6f266, 0x2cf8fb81), TOBN(0x6d51afe8, 0x9490696a), + TOBN(0x81beac6e, 0x51803a19), TOBN(0xe3d24b7f, 0x86219080), + TOBN(0x727cfd9d, 0xdf6f463c), TOBN(0x8c6865ca, 0x72284ee8), + TOBN(0x32c88b7d, 0xb743f4ef), TOBN(0x3793909b, 0xe7d11dce), + TOBN(0xd398f922, 0x2ff2ebe8), TOBN(0x2c70ca44, 0xe5e49796), + TOBN(0xdf4d9929, 0xcb1131b1), TOBN(0x7826f298, 0x25888e79), + TOBN(0x4d3a112c, 0xf1d8740a), TOBN(0x00384cb6, 0x270afa8b), + TOBN(0xcb64125b, 0x3ab48095), TOBN(0x3451c256, 0x62d05106), + TOBN(0xd73d577d, 0xa4955845), TOBN(0x39570c16, 0xbf9f4433), + TOBN(0xd7dfaad3, 0xadecf263), TOBN(0xf1c3d8d1, 0xdc76e102), + TOBN(0x5e774a58, 0x54c6a836), TOBN(0xdad4b672, 0x3e92d47b), + TOBN(0xbe7e990f, 0xf0d796a0), TOBN(0x5fc62478, 0xdf0e8b02), + TOBN(0x8aae8bf4, 0x030c00ad), TOBN(0x3d2db93b, 0x9004ba0f), + TOBN(0xe48c8a79, 0xd85d5ddc), TOBN(0xe907caa7, 0x6bb07f34), + TOBN(0x58db343a, 0xa39eaed5), TOBN(0x0ea6e007, 0xadaf5724), + TOBN(0xe00df169, 0xd23233f3), TOBN(0x3e322796, 0x77cb637f), + TOBN(0x1f897c0e, 0x1da0cf6c), TOBN(0xa651f5d8, 0x31d6bbdd), + TOBN(0xdd61af19, 0x1a230c76), TOBN(0xbd527272, 0xcdaa5e4a), + TOBN(0xca753636, 0xd0abcd7e), TOBN(0x78bdd37c, 0x370bd8dc), + TOBN(0xc23916c2, 0x17cd93fe), TOBN(0x65b97a4d, 0xdadce6e2), + TOBN(0xe04ed4eb, 0x174e42f8), TOBN(0x1491ccaa, 0xbb21480a), + TOBN(0x145a8280, 0x23196332), TOBN(0x3c3862d7, 0x587b479a), + TOBN(0x9f4a88a3, 0x01dcd0ed), TOBN(0x4da2b7ef, 0x3ea12f1f), + TOBN(0xf8e7ae33, 0xb126e48e), TOBN(0x404a0b32, 0xf494e237), + TOBN(0x9beac474, 0xc55acadb), TOBN(0x4ee5cf3b, 0xcbec9fd9), + TOBN(0x336b33b9, 0x7df3c8c3), TOBN(0xbd905fe3, 0xb76808fd), + TOBN(0x8f436981, 0xaa45c16a), TOBN(0x255c5bfa, 0x3dd27b62), + TOBN(0x71965cbf, 0xc3dd9b4d), TOBN(0xce23edbf, 0xfc068a87), + TOBN(0xb78d4725, 0x745b029b), TOBN(0x74610713, 0xcefdd9bd), + TOBN(0x7116f75f, 0x1266bf52), TOBN(0x02046722, 0x18e49bb6), + TOBN(0xdf43df9f, 0x3d6f19e3), TOBN(0xef1bc7d0, 0xe685cb2f), + TOBN(0xcddb27c1, 0x7078c432), TOBN(0xe1961b9c, 0xb77fedb7), + TOBN(0x1edc2f5c, 0xc2290570), TOBN(0x2c3fefca, 0x19cbd886), + TOBN(0xcf880a36, 0xc2af389a), TOBN(0x96c610fd, 0xbda71cea), + TOBN(0xf03977a9, 0x32aa8463), TOBN(0x8eb7763f, 0x8586d90a), + TOBN(0x3f342454, 0x2a296e77), TOBN(0xc8718683, 0x42837a35), + TOBN(0x7dc71090, 0x6a09c731), TOBN(0x54778ffb, 0x51b816db), + TOBN(0x6b33bfec, 0xaf06defd), TOBN(0xfe3c105f, 0x8592b70b), + TOBN(0xf937fda4, 0x61da6114), TOBN(0x3c13e651, 0x4c266ad7), + TOBN(0xe363a829, 0x855938e8), TOBN(0x2eeb5d9e, 0x9de54b72), + TOBN(0xbeb93b0e, 0x20ccfab9), TOBN(0x3dffbb5f, 0x25e61a25), + TOBN(0x7f655e43, 0x1acc093d), TOBN(0x0cb6cc3d, 0x3964ce61), + TOBN(0x6ab283a1, 0xe5e9b460), TOBN(0x55d787c5, 0xa1c7e72d), + TOBN(0x4d2efd47, 0xdeadbf02), TOBN(0x11e80219, 0xac459068), + TOBN(0x810c7626, 0x71f311f0), TOBN(0xfa17ef8d, 0x4ab6ef53), + TOBN(0xaf47fd25, 0x93e43bff), TOBN(0x5cb5ff3f, 0x0be40632), + TOBN(0x54687106, 0x8ee61da3), TOBN(0x7764196e, 0xb08afd0f), + TOBN(0x831ab3ed, 0xf0290a8f), TOBN(0xcae81966, 0xcb47c387), + TOBN(0xaad7dece, 0x184efb4f), TOBN(0xdcfc53b3, 0x4749110e), + TOBN(0x6698f23c, 0x4cb632f9), TOBN(0xc42a1ad6, 0xb91f8067), + TOBN(0xb116a81d, 0x6284180a), TOBN(0xebedf5f8, 0xe901326f), + TOBN(0xf2274c9f, 0x97e3e044), TOBN(0x42018520, 0x11d09fc9), + TOBN(0x56a65f17, 0xd18e6e23), TOBN(0x2ea61e2a, 0x352b683c), + TOBN(0x27d291bc, 0x575eaa94), TOBN(0x9e7bc721, 0xb8ff522d), + TOBN(0x5f7268bf, 0xa7f04d6f), TOBN(0x5868c73f, 0xaba41748), + TOBN(0x9f85c2db, 0x7be0eead), TOBN(0x511e7842, 0xff719135), + TOBN(0x5a06b1e9, 0xc5ea90d7), TOBN(0x0c19e283, 0x26fab631), + TOBN(0x8af8f0cf, 0xe9206c55), TOBN(0x89389cb4, 0x3553c06a), + TOBN(0x39dbed97, 0xf65f8004), TOBN(0x0621b037, 0xc508991d), + TOBN(0x1c52e635, 0x96e78cc4), TOBN(0x5385c8b2, 0x0c06b4a8), + TOBN(0xd84ddfdb, 0xb0e87d03), TOBN(0xc49dfb66, 0x934bafad), + TOBN(0x7071e170, 0x59f70772), TOBN(0x3a073a84, 0x3a1db56b), + TOBN(0x03494903, 0x3b8af190), TOBN(0x7d882de3, 0xd32920f0), + TOBN(0x91633f0a, 0xb2cf8940), TOBN(0x72b0b178, 0x6f948f51), + TOBN(0x2d28dc30, 0x782653c8), TOBN(0x88829849, 0xdb903a05), + TOBN(0xb8095d0c, 0x6a19d2bb), TOBN(0x4b9e7f0c, 0x86f782cb), + TOBN(0x7af73988, 0x2d907064), TOBN(0xd12be0fe, 0x8b32643c), + TOBN(0x358ed23d, 0x0e165dc3), TOBN(0x3d47ce62, 0x4e2378ce), + TOBN(0x7e2bb0b9, 0xfeb8a087), TOBN(0x3246e8ae, 0xe29e10b9), + TOBN(0x459f4ec7, 0x03ce2b4d), TOBN(0xe9b4ca1b, 0xbbc077cf), + TOBN(0x2613b4f2, 0x0e9940c1), TOBN(0xfc598bb9, 0x047d1eb1), + TOBN(0x9744c62b, 0x45036099), TOBN(0xa9dee742, 0x167c65d8), + TOBN(0x0c511525, 0xdabe1943), TOBN(0xda110554, 0x93c6c624), + TOBN(0xae00a52c, 0x651a3be2), TOBN(0xcda5111d, 0x884449a6), + TOBN(0x063c06f4, 0xff33bed1), TOBN(0x73baaf9a, 0x0d3d76b4), + TOBN(0x52fb0c9d, 0x7fc63668), TOBN(0x6886c9dd, 0x0c039cde), + TOBN(0x602bd599, 0x55b22351), TOBN(0xb00cab02, 0x360c7c13), + TOBN(0x8cb616bc, 0x81b69442), TOBN(0x41486700, 0xb55c3cee), + TOBN(0x71093281, 0xf49ba278), TOBN(0xad956d9c, 0x64a50710), + TOBN(0x9561f28b, 0x638a7e81), TOBN(0x54155cdf, 0x5980ddc3), + TOBN(0xb2db4a96, 0xd26f247a), TOBN(0x9d774e4e, 0x4787d100), + TOBN(0x1a9e6e2e, 0x078637d2), TOBN(0x1c363e2d, 0x5e0ae06a), + TOBN(0x7493483e, 0xe9cfa354), TOBN(0x76843cb3, 0x7f74b98d), + TOBN(0xbaca6591, 0xd4b66947), TOBN(0xb452ce98, 0x04460a8c), + TOBN(0x6830d246, 0x43768f55), TOBN(0xf4197ed8, 0x7dff12df), + TOBN(0x6521b472, 0x400dd0f7), TOBN(0x59f5ca8f, 0x4b1e7093), + TOBN(0x6feff11b, 0x080338ae), TOBN(0x0ada31f6, 0xa29ca3c6), + TOBN(0x24794eb6, 0x94a2c215), TOBN(0xd83a43ab, 0x05a57ab4), + TOBN(0x264a543a, 0x2a6f89fe), TOBN(0x2c2a3868, 0xdd5ec7c2), + TOBN(0xd3373940, 0x8439d9b2), TOBN(0x715ea672, 0x0acd1f11), + TOBN(0x42c1d235, 0xe7e6cc19), TOBN(0x81ce6e96, 0xb990585c), + TOBN(0x04e5dfe0, 0xd809c7bd), TOBN(0xd7b2580c, 0x8f1050ab), + TOBN(0x6d91ad78, 0xd8a4176f), TOBN(0x0af556ee, 0x4e2e897c), + TOBN(0x162a8b73, 0x921de0ac), TOBN(0x52ac9c22, 0x7ea78400), + TOBN(0xee2a4eea, 0xefce2174), TOBN(0xbe61844e, 0x6d637f79), + TOBN(0x0491f1bc, 0x789a283b), TOBN(0x72d3ac3d, 0x880836f4), + TOBN(0xaa1c5ea3, 0x88e5402d), TOBN(0x1b192421, 0xd5cc473d), + TOBN(0x5c0b9998, 0x9dc84cac), TOBN(0xb0a8482d, 0x9c6e75b8), + TOBN(0x639961d0, 0x3a191ce2), TOBN(0xda3bc865, 0x6d837930), + TOBN(0xca990653, 0x056e6f8f), TOBN(0x84861c41, 0x64d133a7), + TOBN(0x8b403276, 0x746abe40), TOBN(0xb7b4d51a, 0xebf8e303), + TOBN(0x05b43211, 0x220a255d), TOBN(0xc997152c, 0x02419e6e), + TOBN(0x76ff47b6, 0x630c2fea), TOBN(0x50518677, 0x281fdade), + TOBN(0x3283b8ba, 0xcf902b0b), TOBN(0x8d4b4eb5, 0x37db303b), + TOBN(0xcc89f42d, 0x755011bc), TOBN(0xb43d74bb, 0xdd09d19b), + TOBN(0x65746bc9, 0x8adba350), TOBN(0x364eaf8c, 0xb51c1927), + TOBN(0x13c76596, 0x10ad72ec), TOBN(0x30045121, 0xf8d40c20), + TOBN(0x6d2d99b7, 0xea7b979b), TOBN(0xcd78cd74, 0xe6fb3bcd), + TOBN(0x11e45a9e, 0x86cffbfe), TOBN(0x78a61cf4, 0x637024f6), + TOBN(0xd06bc872, 0x3d502295), TOBN(0xf1376854, 0x458cb288), + TOBN(0xb9db26a1, 0x342f8586), TOBN(0xf33effcf, 0x4beee09e), + TOBN(0xd7e0c4cd, 0xb30cfb3a), TOBN(0x6d09b8c1, 0x6c9db4c8), + TOBN(0x40ba1a42, 0x07c8d9df), TOBN(0x6fd495f7, 0x1c52c66d), + TOBN(0xfb0e169f, 0x275264da), TOBN(0x80c2b746, 0xe57d8362), + TOBN(0xedd987f7, 0x49ad7222), TOBN(0xfdc229af, 0x4398ec7b),} + , + {TOBN(0xb0d1ed84, 0x52666a58), TOBN(0x4bcb6e00, 0xe6a9c3c2), + TOBN(0x3c57411c, 0x26906408), TOBN(0xcfc20755, 0x13556400), + TOBN(0xa08b1c50, 0x5294dba3), TOBN(0xa30ba286, 0x8b7dd31e), + TOBN(0xd70ba90e, 0x991eca74), TOBN(0x094e142c, 0xe762c2b9), + TOBN(0xb81d783e, 0x979f3925), TOBN(0x1efd130a, 0xaf4c89a7), + TOBN(0x525c2144, 0xfd1bf7fa), TOBN(0x4b296904, 0x1b265a9e), + TOBN(0xed8e9634, 0xb9db65b6), TOBN(0x35c82e32, 0x03599d8a), + TOBN(0xdaa7a54f, 0x403563f3), TOBN(0x9df088ad, 0x022c38ab), + TOBN(0xe5cfb066, 0xbb3fd30a), TOBN(0x429169da, 0xeff0354e), + TOBN(0x809cf852, 0x3524e36c), TOBN(0x136f4fb3, 0x0155be1d), + TOBN(0x4826af01, 0x1fbba712), TOBN(0x6ef0f0b4, 0x506ba1a1), + TOBN(0xd9928b31, 0x77aea73e), TOBN(0xe2bf6af2, 0x5eaa244e), + TOBN(0x8d084f12, 0x4237b64b), TOBN(0x688ebe99, 0xe3ecfd07), + TOBN(0x57b8a70c, 0xf6845dd8), TOBN(0x808fc59c, 0x5da4a325), + TOBN(0xa9032b2b, 0xa3585862), TOBN(0xb66825d5, 0xedf29386), + TOBN(0xb5a5a8db, 0x431ec29b), TOBN(0xbb143a98, 0x3a1e8dc8), + TOBN(0x35ee94ce, 0x12ae381b), TOBN(0x3a7f176c, 0x86ccda90), + TOBN(0xc63a657e, 0x4606eaca), TOBN(0x9ae5a380, 0x43cd04df), + TOBN(0x9bec8d15, 0xed251b46), TOBN(0x1f5d6d30, 0xcaca5e64), + TOBN(0x347b3b35, 0x9ff20f07), TOBN(0x4d65f034, 0xf7e4b286), + TOBN(0x9e93ba24, 0xf111661e), TOBN(0xedced484, 0xb105eb04), + TOBN(0x96dc9ba1, 0xf424b578), TOBN(0xbf8f66b7, 0xe83e9069), + TOBN(0x872d4df4, 0xd7ed8216), TOBN(0xbf07f377, 0x8e2cbecf), + TOBN(0x4281d899, 0x98e73754), TOBN(0xfec85fbb, 0x8aab8708), + TOBN(0x9a3c0dee, 0xa5ba5b0b), TOBN(0xe6a116ce, 0x42d05299), + TOBN(0xae9775fe, 0xe9b02d42), TOBN(0x72b05200, 0xa1545cb6), + TOBN(0xbc506f7d, 0x31a3b4ea), TOBN(0xe5893078, 0x8bbd9b32), + TOBN(0xc8bc5f37, 0xe4b12a97), TOBN(0x6b000c06, 0x4a73b671), + TOBN(0x13b5bf22, 0x765fa7d0), TOBN(0x59805bf0, 0x1d6a5370), + TOBN(0x67a5e29d, 0x4280db98), TOBN(0x4f53916f, 0x776b1ce3), + TOBN(0x714ff61f, 0x33ddf626), TOBN(0x4206238e, 0xa085d103), + TOBN(0x1c50d4b7, 0xe5809ee3), TOBN(0x999f450d, 0x85f8eb1d), + TOBN(0x658a6051, 0xe4c79e9b), TOBN(0x1394cb73, 0xc66a9fea), + TOBN(0x27f31ed5, 0xc6be7b23), TOBN(0xf4c88f36, 0x5aa6f8fe), + TOBN(0x0fb0721f, 0x4aaa499e), TOBN(0x68b3a7d5, 0xe3fb2a6b), + TOBN(0xa788097d, 0x3a92851d), TOBN(0x060e7f8a, 0xe96f4913), + TOBN(0x82eebe73, 0x1a3a93bc), TOBN(0x42bbf465, 0xa21adc1a), + TOBN(0xc10b6fa4, 0xef030efd), TOBN(0x247aa4c7, 0x87b097bb), + TOBN(0x8b8dc632, 0xf60c77da), TOBN(0x6ffbc26a, 0xc223523e), + TOBN(0xa4f6ff11, 0x344579cf), TOBN(0x5825653c, 0x980250f6), + TOBN(0xb2dd097e, 0xbc1aa2b9), TOBN(0x07889393, 0x37a0333a), + TOBN(0x1cf55e71, 0x37a0db38), TOBN(0x2648487f, 0x792c1613), + TOBN(0xdad01336, 0x3fcef261), TOBN(0x6239c81d, 0x0eabf129), + TOBN(0x8ee761de, 0x9d276be2), TOBN(0x406a7a34, 0x1eda6ad3), + TOBN(0x4bf367ba, 0x4a493b31), TOBN(0x54f20a52, 0x9bf7f026), + TOBN(0xb696e062, 0x9795914b), TOBN(0xcddab96d, 0x8bf236ac), + TOBN(0x4ff2c70a, 0xed25ea13), TOBN(0xfa1d09eb, 0x81cbbbe7), + TOBN(0x88fc8c87, 0x468544c5), TOBN(0x847a670d, 0x696b3317), + TOBN(0xf133421e, 0x64bcb626), TOBN(0xaea638c8, 0x26dee0b5), + TOBN(0xd6e7680b, 0xb310346c), TOBN(0xe06f4097, 0xd5d4ced3), + TOBN(0x09961452, 0x7512a30b), TOBN(0xf3d867fd, 0xe589a59a), + TOBN(0x2e73254f, 0x52d0c180), TOBN(0x9063d8a3, 0x333c74ac), + TOBN(0xeda6c595, 0xd314e7bc), TOBN(0x2ee7464b, 0x467899ed), + TOBN(0x1cef423c, 0x0a1ed5d3), TOBN(0x217e76ea, 0x69cc7613), + TOBN(0x27ccce1f, 0xe7cda917), TOBN(0x12d8016b, 0x8a893f16), + TOBN(0xbcd6de84, 0x9fc74f6b), TOBN(0xfa5817e2, 0xf3144e61), + TOBN(0x1f354164, 0x0821ee4c), TOBN(0x1583eab4, 0x0bc61992), + TOBN(0x7490caf6, 0x1d72879f), TOBN(0x998ad9f3, 0xf76ae7b2), + TOBN(0x1e181950, 0xa41157f7), TOBN(0xa9d7e1e6, 0xe8da3a7e), + TOBN(0x963784eb, 0x8426b95f), TOBN(0x0ee4ed6e, 0x542e2a10), + TOBN(0xb79d4cc5, 0xac751e7b), TOBN(0x93f96472, 0xfd4211bd), + TOBN(0x8c72d3d2, 0xc8de4fc6), TOBN(0x7b69cbf5, 0xdf44f064), + TOBN(0x3da90ca2, 0xf4bf94e1), TOBN(0x1a5325f8, 0xf12894e2), + TOBN(0x0a437f6c, 0x7917d60b), TOBN(0x9be70486, 0x96c9cb5d), + TOBN(0xb4d880bf, 0xe1dc5c05), TOBN(0xd738adda, 0xeebeeb57), + TOBN(0x6f0119d3, 0xdf0fe6a3), TOBN(0x5c686e55, 0x66eaaf5a), + TOBN(0x9cb10b50, 0xdfd0b7ec), TOBN(0xbdd0264b, 0x6a497c21), + TOBN(0xfc093514, 0x8c546c96), TOBN(0x58a947fa, 0x79dbf42a), + TOBN(0xc0b48d4e, 0x49ccd6d7), TOBN(0xff8fb02c, 0x88bd5580), + TOBN(0xc75235e9, 0x07d473b2), TOBN(0x4fab1ac5, 0xa2188af3), + TOBN(0x030fa3bc, 0x97576ec0), TOBN(0xe8c946e8, 0x0b7e7d2f), + TOBN(0x40a5c9cc, 0x70305600), TOBN(0x6d8260a9, 0xc8b013b4), + TOBN(0x0368304f, 0x70bba85c), TOBN(0xad090da1, 0xa4a0d311), + TOBN(0x7170e870, 0x2415eec1), TOBN(0xbfba35fe, 0x8461ea47), + TOBN(0x6279019a, 0xc1e91938), TOBN(0xa47638f3, 0x1afc415f), + TOBN(0x36c65cbb, 0xbcba0e0f), TOBN(0x02160efb, 0x034e2c48), + TOBN(0xe6c51073, 0x615cd9e4), TOBN(0x498ec047, 0xf1243c06), + TOBN(0x3e5a8809, 0xb17b3d8c), TOBN(0x5cd99e61, 0x0cc565f1), + TOBN(0x81e312df, 0x7851dafe), TOBN(0xf156f5ba, 0xa79061e2), + TOBN(0x80d62b71, 0x880c590e), TOBN(0xbec9746f, 0x0a39faa1), + TOBN(0x1d98a9c1, 0xc8ed1f7a), TOBN(0x09e43bb5, 0xa81d5ff2), + TOBN(0xd5f00f68, 0x0da0794a), TOBN(0x412050d9, 0x661aa836), + TOBN(0xa89f7c4e, 0x90747e40), TOBN(0x6dc05ebb, 0xb62a3686), + TOBN(0xdf4de847, 0x308e3353), TOBN(0x53868fbb, 0x9fb53bb9), + TOBN(0x2b09d2c3, 0xcfdcf7dd), TOBN(0x41a9fce3, 0x723fcab4), + TOBN(0x73d905f7, 0x07f57ca3), TOBN(0x080f9fb1, 0xac8e1555), + TOBN(0x7c088e84, 0x9ba7a531), TOBN(0x07d35586, 0xed9a147f), + TOBN(0x602846ab, 0xaf48c336), TOBN(0x7320fd32, 0x0ccf0e79), + TOBN(0xaa780798, 0xb18bd1ff), TOBN(0x52c2e300, 0xafdd2905), + TOBN(0xf27ea3d6, 0x434267cd), TOBN(0x8b96d16d, 0x15605b5f), + TOBN(0x7bb31049, 0x4b45706b), TOBN(0xe7f58b8e, 0x743d25f8), + TOBN(0xe9b5e45b, 0x87f30076), TOBN(0xd19448d6, 0x5d053d5a), + TOBN(0x1ecc8cb9, 0xd3210a04), TOBN(0x6bc7d463, 0xdafb5269), + TOBN(0x3e59b10a, 0x67c3489f), TOBN(0x1769788c, 0x65641e1b), + TOBN(0x8a53b82d, 0xbd6cb838), TOBN(0x7066d6e6, 0x236d5f22), + TOBN(0x03aa1c61, 0x6908536e), TOBN(0xc971da0d, 0x66ae9809), + TOBN(0x01b3a86b, 0xc49a2fac), TOBN(0x3b8420c0, 0x3092e77a), + TOBN(0x02057300, 0x7d6fb556), TOBN(0x6941b2a1, 0xbff40a87), + TOBN(0x140b6308, 0x0658ff2a), TOBN(0x87804363, 0x3424ab36), + TOBN(0x0253bd51, 0x5751e299), TOBN(0xc75bcd76, 0x449c3e3a), + TOBN(0x92eb4090, 0x7f8f875d), TOBN(0x9c9d754e, 0x56c26bbf), + TOBN(0x158cea61, 0x8110bbe7), TOBN(0x62a6b802, 0x745f91ea), + TOBN(0xa79c41aa, 0xc6e7394b), TOBN(0x445b6a83, 0xad57ef10), + TOBN(0x0c5277eb, 0x6ea6f40c), TOBN(0x319fe96b, 0x88633365), + TOBN(0x0b0fc61f, 0x385f63cb), TOBN(0x41250c84, 0x22bdd127), + TOBN(0x67d153f1, 0x09e942c2), TOBN(0x60920d08, 0xc021ad5d), + TOBN(0x229f5746, 0x724d81a5), TOBN(0xb7ffb892, 0x5bba3299), + TOBN(0x518c51a1, 0xde413032), TOBN(0x2a9bfe77, 0x3c2fd94c), + TOBN(0xcbcde239, 0x3191f4fd), TOBN(0x43093e16, 0xd3d6ada1), + TOBN(0x184579f3, 0x58769606), TOBN(0x2c94a8b3, 0xd236625c), + TOBN(0x6922b9c0, 0x5c437d8e), TOBN(0x3d4ae423, 0xd8d9f3c8), + TOBN(0xf72c31c1, 0x2e7090a2), TOBN(0x4ac3f5f3, 0xd76a55bd), + TOBN(0x342508fc, 0x6b6af991), TOBN(0x0d527100, 0x1b5cebbd), + TOBN(0xb84740d0, 0xdd440dd7), TOBN(0x748ef841, 0x780162fd), + TOBN(0xa8dbfe0e, 0xdfc6fafb), TOBN(0xeadfdf05, 0xf7300f27), + TOBN(0x7d06555f, 0xfeba4ec9), TOBN(0x12c56f83, 0x9e25fa97), + TOBN(0x77f84203, 0xd39b8c34), TOBN(0xed8b1be6, 0x3125eddb), + TOBN(0x5bbf2441, 0xf6e39dc5), TOBN(0xb00f6ee6, 0x6a5d678a), + TOBN(0xba456ecf, 0x57d0ea99), TOBN(0xdcae0f58, 0x17e06c43), + TOBN(0x01643de4, 0x0f5b4baa), TOBN(0x2c324341, 0xd161b9be), + TOBN(0x80177f55, 0xe126d468), TOBN(0xed325f1f, 0x76748e09), + TOBN(0x6116004a, 0xcfa9bdc2), TOBN(0x2d8607e6, 0x3a9fb468), + TOBN(0x0e573e27, 0x6009d660), TOBN(0x3a525d2e, 0x8d10c5a1), + TOBN(0xd26cb45c, 0x3b9009a0), TOBN(0xb6b0cdc0, 0xde9d7448), + TOBN(0x949c9976, 0xe1337c26), TOBN(0x6faadebd, 0xd73d68e5), + TOBN(0x9e158614, 0xf1b768d9), TOBN(0x22dfa557, 0x9cc4f069), + TOBN(0xccd6da17, 0xbe93c6d6), TOBN(0x24866c61, 0xa504f5b9), + TOBN(0x2121353c, 0x8d694da1), TOBN(0x1c6ca580, 0x0140b8c6), + TOBN(0xc245ad8c, 0xe964021e), TOBN(0xb83bffba, 0x032b82b3), + TOBN(0xfaa220c6, 0x47ef9898), TOBN(0x7e8d3ac6, 0x982c948a), + TOBN(0x1faa2091, 0xbc2d124a), TOBN(0xbd54c3dd, 0x05b15ff4), + TOBN(0x386bf3ab, 0xc87c6fb7), TOBN(0xfb2b0563, 0xfdeb6f66), + TOBN(0x4e77c557, 0x5b45afb4), TOBN(0xe9ded649, 0xefb8912d), + TOBN(0x7ec9bbf5, 0x42f6e557), TOBN(0x2570dfff, 0x62671f00), + TOBN(0x2b3bfb78, 0x88e084bd), TOBN(0xa024b238, 0xf37fe5b4), + TOBN(0x44e7dc04, 0x95649aee), TOBN(0x498ca255, 0x5e7ec1d8), + TOBN(0x3bc766ea, 0xaaa07e86), TOBN(0x0db6facb, 0xf3608586), + TOBN(0xbadd2549, 0xbdc259c8), TOBN(0x95af3c6e, 0x041c649f), + TOBN(0xb36a928c, 0x02e30afb), TOBN(0x9b5356ad, 0x008a88b8), + TOBN(0x4b67a5f1, 0xcf1d9e9d), TOBN(0xc6542e47, 0xa5d8d8ce), + TOBN(0x73061fe8, 0x7adfb6cc), TOBN(0xcc826fd3, 0x98678141), + TOBN(0x00e758b1, 0x3c80515a), TOBN(0x6afe3247, 0x41485083), + TOBN(0x0fcb08b9, 0xb6ae8a75), TOBN(0xb8cf388d, 0x4acf51e1), + TOBN(0x344a5560, 0x6961b9d6), TOBN(0x1a6778b8, 0x6a97fd0c), + TOBN(0xd840fdc1, 0xecc4c7e3), TOBN(0xde9fe47d, 0x16db68cc), + TOBN(0xe95f89de, 0xa3e216aa), TOBN(0x84f1a6a4, 0x9594a8be), + TOBN(0x7ddc7d72, 0x5a7b162b), TOBN(0xc5cfda19, 0xadc817a3), + TOBN(0x80a5d350, 0x78b58d46), TOBN(0x93365b13, 0x82978f19), + TOBN(0x2e44d225, 0x26a1fc90), TOBN(0x0d6d10d2, 0x4d70705d), + TOBN(0xd94b6b10, 0xd70c45f4), TOBN(0x0f201022, 0xb216c079), + TOBN(0xcec966c5, 0x658fde41), TOBN(0xa8d2bc7d, 0x7e27601d), + TOBN(0xbfcce3e1, 0xff230be7), TOBN(0x3394ff6b, 0x0033ffb5), + TOBN(0xd890c509, 0x8132c9af), TOBN(0xaac4b0eb, 0x361e7868), + TOBN(0x5194ded3, 0xe82d15aa), TOBN(0x4550bd2e, 0x23ae6b7d), + TOBN(0x3fda318e, 0xea5399d4), TOBN(0xd989bffa, 0x91638b80), + TOBN(0x5ea124d0, 0xa14aa12d), TOBN(0x1fb1b899, 0x3667b944), + TOBN(0x95ec7969, 0x44c44d6a), TOBN(0x91df144a, 0x57e86137), + TOBN(0x915fd620, 0x73adac44), TOBN(0x8f01732d, 0x59a83801), + TOBN(0xec579d25, 0x3aa0a633), TOBN(0x06de5e7c, 0xc9d6d59c), + TOBN(0xc132f958, 0xb1ef8010), TOBN(0x29476f96, 0xe65c1a02), + TOBN(0x336a77c0, 0xd34c3565), TOBN(0xef1105b2, 0x1b9f1e9e), + TOBN(0x63e6d08b, 0xf9e08002), TOBN(0x9aff2f21, 0xc613809e), + TOBN(0xb5754f85, 0x3a80e75d), TOBN(0xde71853e, 0x6bbda681), + TOBN(0x86f041df, 0x8197fd7a), TOBN(0x8b332e08, 0x127817fa), + TOBN(0x05d99be8, 0xb9c20cda), TOBN(0x89f7aad5, 0xd5cd0c98), + TOBN(0x7ef936fe, 0x5bb94183), TOBN(0x92ca0753, 0xb05cd7f2), + TOBN(0x9d65db11, 0x74a1e035), TOBN(0x02628cc8, 0x13eaea92), + TOBN(0xf2d9e242, 0x49e4fbf2), TOBN(0x94fdfd9b, 0xe384f8b7), + TOBN(0x65f56054, 0x63428c6b), TOBN(0x2f7205b2, 0x90b409a5), + TOBN(0xf778bb78, 0xff45ae11), TOBN(0xa13045be, 0xc5ee53b2), + TOBN(0xe00a14ff, 0x03ef77fe), TOBN(0x689cd59f, 0xffef8bef), + TOBN(0x3578f0ed, 0x1e9ade22), TOBN(0xe99f3ec0, 0x6268b6a8), + TOBN(0xa2057d91, 0xea1b3c3e), TOBN(0x2d1a7053, 0xb8823a4a), + TOBN(0xabbb336a, 0x2cca451e), TOBN(0xcd2466e3, 0x2218bb5d), + TOBN(0x3ac1f42f, 0xc8cb762d), TOBN(0x7e312aae, 0x7690211f), + TOBN(0xebb9bd73, 0x45d07450), TOBN(0x207c4b82, 0x46c2213f), + TOBN(0x99d425c1, 0x375913ec), TOBN(0x94e45e96, 0x67908220), + TOBN(0xc08f3087, 0xcd67dbf6), TOBN(0xa5670fbe, 0xc0887056), + TOBN(0x6717b64a, 0x66f5b8fc), TOBN(0xd5a56aea, 0x786fec28), + TOBN(0xa8c3f55f, 0xc0ff4952), TOBN(0xa77fefae, 0x457ac49b), + TOBN(0x29882d7c, 0x98379d44), TOBN(0xd000bdfb, 0x509edc8a), + TOBN(0xc6f95979, 0xe66fe464), TOBN(0x504a6115, 0xfa61bde0), + TOBN(0x56b3b871, 0xeffea31a), TOBN(0x2d3de26d, 0xf0c21a54), + TOBN(0x21dbff31, 0x834753bf), TOBN(0xe67ecf49, 0x69269d86), + TOBN(0x7a176952, 0x151fe690), TOBN(0x03515804, 0x7f2adb5f), + TOBN(0xee794b15, 0xd1b62a8d), TOBN(0xf004ceec, 0xaae454e6), + TOBN(0x0897ea7c, 0xf0386fac), TOBN(0x3b62ff12, 0xd1fca751), + TOBN(0x154181df, 0x1b7a04ec), TOBN(0x2008e04a, 0xfb5847ec), + TOBN(0xd147148e, 0x41dbd772), TOBN(0x2b419f73, 0x22942654), + TOBN(0x669f30d3, 0xe9c544f7), TOBN(0x52a2c223, 0xc8540149), + TOBN(0x5da9ee14, 0x634dfb02), TOBN(0x5f074ff0, 0xf47869f3), + TOBN(0x74ee878d, 0xa3933acc), TOBN(0xe6510651, 0x4fe35ed1), + TOBN(0xb3eb9482, 0xf1012e7a), TOBN(0x51013cc0, 0xa8a566ae), + TOBN(0xdd5e9243, 0x47c00d3b), TOBN(0x7fde089d, 0x946bb0e5), + TOBN(0x030754fe, 0xc731b4b3), TOBN(0x12a136a4, 0x99fda062), + TOBN(0x7c1064b8, 0x5a1a35bc), TOBN(0xbf1f5763, 0x446c84ef), + TOBN(0xed29a56d, 0xa16d4b34), TOBN(0x7fba9d09, 0xdca21c4f), + TOBN(0x66d7ac00, 0x6d8de486), TOBN(0x60061987, 0x73a2a5e1), + TOBN(0x8b400f86, 0x9da28ff0), TOBN(0x3133f708, 0x43c4599c), + TOBN(0x9911c9b8, 0xee28cb0d), TOBN(0xcd7e2874, 0x8e0af61d), + TOBN(0x5a85f0f2, 0x72ed91fc), TOBN(0x85214f31, 0x9cd4a373), + TOBN(0x881fe5be, 0x1925253c), TOBN(0xd8dc98e0, 0x91e8bc76), + TOBN(0x7120affe, 0x585cc3a2), TOBN(0x724952ed, 0x735bf97a), + TOBN(0x5581e7dc, 0x3eb34581), TOBN(0x5cbff4f2, 0xe52ee57d), + TOBN(0x8d320a0e, 0x87d8cc7b), TOBN(0x9beaa7f3, 0xf1d280d0), + TOBN(0x7a0b9571, 0x9beec704), TOBN(0x9126332e, 0x5b7f0057), + TOBN(0x01fbc1b4, 0x8ed3bd6d), TOBN(0x35bb2c12, 0xd945eb24), + TOBN(0x6404694e, 0x9a8ae255), TOBN(0xb6092eec, 0x8d6abfb3), + TOBN(0x4d76143f, 0xcc058865), TOBN(0x7b0a5af2, 0x6e249922), + TOBN(0x8aef9440, 0x6a50d353), TOBN(0xe11e4bcc, 0x64f0e07a), + TOBN(0x4472993a, 0xa14a90fa), TOBN(0x7706e20c, 0xba0c51d4), + TOBN(0xf403292f, 0x1532672d), TOBN(0x52573bfa, 0x21829382), + TOBN(0x6a7bb6a9, 0x3b5bdb83), TOBN(0x08da65c0, 0xa4a72318), + TOBN(0xc58d22aa, 0x63eb065f), TOBN(0x1717596c, 0x1b15d685), + TOBN(0x112df0d0, 0xb266d88b), TOBN(0xf688ae97, 0x5941945a), + TOBN(0x487386e3, 0x7c292cac), TOBN(0x42f3b50d, 0x57d6985c), + TOBN(0x6da4f998, 0x6a90fc34), TOBN(0xc8f257d3, 0x65ca8a8d), + TOBN(0xc2feabca, 0x6951f762), TOBN(0xe1bc81d0, 0x74c323ac), + TOBN(0x1bc68f67, 0x251a2a12), TOBN(0x10d86587, 0xbe8a70dc), + TOBN(0xd648af7f, 0xf0f84d2e), TOBN(0xf0aa9ebc, 0x6a43ac92), + TOBN(0x69e3be04, 0x27596893), TOBN(0xb6bb02a6, 0x45bf452b), + TOBN(0x0875c11a, 0xf4c698c8), TOBN(0x6652b5c7, 0xbece3794), + TOBN(0x7b3755fd, 0x4f5c0499), TOBN(0x6ea16558, 0xb5532b38), + TOBN(0xd1c69889, 0xa2e96ef7), TOBN(0x9c773c3a, 0x61ed8f48), + TOBN(0x2b653a40, 0x9b323abc), TOBN(0xe26605e1, 0xf0e1d791), + TOBN(0x45d41064, 0x4a87157a), TOBN(0x8f9a78b7, 0xcbbce616), + TOBN(0xcf1e44aa, 0xc407eddd), TOBN(0x81ddd1d8, 0xa35b964f), + TOBN(0x473e339e, 0xfd083999), TOBN(0x6c94bdde, 0x8e796802), + TOBN(0x5a304ada, 0x8545d185), TOBN(0x82ae44ea, 0x738bb8cb), + TOBN(0x628a35e3, 0xdf87e10e), TOBN(0xd3624f3d, 0xa15b9fe3), + TOBN(0xcc44209b, 0x14be4254), TOBN(0x7d0efcbc, 0xbdbc2ea5), + TOBN(0x1f603362, 0x04c37bbe), TOBN(0x21f363f5, 0x56a5852c), + TOBN(0xa1503d1c, 0xa8501550), TOBN(0x2251e0e1, 0xd8ab10bb), + TOBN(0xde129c96, 0x6961c51c), TOBN(0x1f7246a4, 0x81910f68), + TOBN(0x2eb744ee, 0x5f2591f2), TOBN(0x3c47d33f, 0x5e627157), + TOBN(0x4d6d62c9, 0x22f3bd68), TOBN(0x6120a64b, 0xcb8df856), + TOBN(0x3a9ac6c0, 0x7b5d07df), TOBN(0xa92b9558, 0x7ef39783), + TOBN(0xe128a134, 0xab3a9b4f), TOBN(0x41c18807, 0xb1252f05), + TOBN(0xfc7ed089, 0x80ba9b1c), TOBN(0xac8dc6de, 0xc532a9dd), + TOBN(0xbf829cef, 0x55246809), TOBN(0x101b784f, 0x5b4ee80f), + TOBN(0xc09945bb, 0xb6f11603), TOBN(0x57b09dbe, 0x41d2801e), + TOBN(0xfba5202f, 0xa97534a8), TOBN(0x7fd8ae5f, 0xc17b9614), + TOBN(0xa50ba666, 0x78308435), TOBN(0x9572f77c, 0xd3868c4d), + TOBN(0x0cef7bfd, 0x2dd7aab0), TOBN(0xe7958e08, 0x2c7c79ff), + TOBN(0x81262e42, 0x25346689), TOBN(0x716da290, 0xb07c7004), + TOBN(0x35f911ea, 0xb7950ee3), TOBN(0x6fd72969, 0x261d21b5), + TOBN(0x52389803, 0x08b640d3), TOBN(0x5b0026ee, 0x887f12a1), + TOBN(0x20e21660, 0x742e9311), TOBN(0x0ef6d541, 0x5ff77ff7), + TOBN(0x969127f0, 0xf9c41135), TOBN(0xf21d60c9, 0x68a64993), + TOBN(0x656e5d0c, 0xe541875c), TOBN(0xf1e0f84e, 0xa1d3c233), + TOBN(0x9bcca359, 0x06002d60), TOBN(0xbe2da60c, 0x06191552), + TOBN(0x5da8bbae, 0x61181ec3), TOBN(0x9f04b823, 0x65806f19), + TOBN(0xf1604a7d, 0xd4b79bb8), TOBN(0xaee806fb, 0x52c878c8), + TOBN(0x34144f11, 0x8d47b8e8), TOBN(0x72edf52b, 0x949f9054), + TOBN(0xebfca84e, 0x2127015a), TOBN(0x9051d0c0, 0x9cb7cef3), + TOBN(0x86e8fe58, 0x296deec8), TOBN(0x33b28188, 0x41010d74),} + , + {TOBN(0x01079383, 0x171b445f), TOBN(0x9bcf21e3, 0x8131ad4c), + TOBN(0x8cdfe205, 0xc93987e8), TOBN(0xe63f4152, 0xc92e8c8f), + TOBN(0x729462a9, 0x30add43d), TOBN(0x62ebb143, 0xc980f05a), + TOBN(0x4f3954e5, 0x3b06e968), TOBN(0xfe1d75ad, 0x242cf6b1), + TOBN(0x5f95c6c7, 0xaf8685c8), TOBN(0xd4c1c8ce, 0x2f8f01aa), + TOBN(0xc44bbe32, 0x2574692a), TOBN(0xb8003478, 0xd4a4a068), + TOBN(0x7c8fc6e5, 0x2eca3cdb), TOBN(0xea1db16b, 0xec04d399), + TOBN(0xb05bc82e, 0x8f2bc5cf), TOBN(0x763d517f, 0xf44793d2), + TOBN(0x4451c1b8, 0x08bd98d0), TOBN(0x644b1cd4, 0x6575f240), + TOBN(0x6907eb33, 0x7375d270), TOBN(0x56c8bebd, 0xfa2286bd), + TOBN(0xc713d2ac, 0xc4632b46), TOBN(0x17da427a, 0xafd60242), + TOBN(0x313065b7, 0xc95c7546), TOBN(0xf8239898, 0xbf17a3de), + TOBN(0xf3b7963f, 0x4c830320), TOBN(0x842c7aa0, 0x903203e3), + TOBN(0xaf22ca0a, 0xe7327afb), TOBN(0x38e13092, 0x967609b6), + TOBN(0x73b8fb62, 0x757558f1), TOBN(0x3cc3e831, 0xf7eca8c1), + TOBN(0xe4174474, 0xf6331627), TOBN(0xa77989ca, 0xc3c40234), + TOBN(0xe5fd17a1, 0x44a081e0), TOBN(0xd797fb7d, 0xb70e296a), + TOBN(0x2b472b30, 0x481f719c), TOBN(0x0e632a98, 0xfe6f8c52), + TOBN(0x89ccd116, 0xc5f0c284), TOBN(0xf51088af, 0x2d987c62), + TOBN(0x2a2bccda, 0x4c2de6cf), TOBN(0x810f9efe, 0xf679f0f9), + TOBN(0xb0f394b9, 0x7ffe4b3e), TOBN(0x0b691d21, 0xe5fa5d21), + TOBN(0xb0bd7747, 0x9dfbbc75), TOBN(0xd2830fda, 0xfaf78b00), + TOBN(0xf78c249c, 0x52434f57), TOBN(0x4b1f7545, 0x98096dab), + TOBN(0x73bf6f94, 0x8ff8c0b3), TOBN(0x34aef03d, 0x454e134c), + TOBN(0xf8d151f4, 0xb7ac7ec5), TOBN(0xd6ceb95a, 0xe50da7d5), + TOBN(0xa1b492b0, 0xdc3a0eb8), TOBN(0x75157b69, 0xb3dd2863), + TOBN(0xe2c4c74e, 0xc5413d62), TOBN(0xbe329ff7, 0xbc5fc4c7), + TOBN(0x835a2aea, 0x60fa9dda), TOBN(0xf117f5ad, 0x7445cb87), + TOBN(0xae8317f4, 0xb0166f7a), TOBN(0xfbd3e3f7, 0xceec74e6), + TOBN(0xfdb516ac, 0xe0874bfd), TOBN(0x3d846019, 0xc681f3a3), + TOBN(0x0b12ee5c, 0x7c1620b0), TOBN(0xba68b4dd, 0x2b63c501), + TOBN(0xac03cd32, 0x6668c51e), TOBN(0x2a6279f7, 0x4e0bcb5b), + TOBN(0x17bd69b0, 0x6ae85c10), TOBN(0x72946979, 0x1dfdd3a6), + TOBN(0xd9a03268, 0x2c078bec), TOBN(0x41c6a658, 0xbfd68a52), + TOBN(0xcdea1024, 0x0e023900), TOBN(0xbaeec121, 0xb10d144d), + TOBN(0x5a600e74, 0x058ab8dc), TOBN(0x1333af21, 0xbb89ccdd), + TOBN(0xdf25eae0, 0x3aaba1f1), TOBN(0x2cada16e, 0x3b7144cf), + TOBN(0x657ee27d, 0x71ab98bc), TOBN(0x99088b4c, 0x7a6fc96e), + TOBN(0x05d5c0a0, 0x3549dbd4), TOBN(0x42cbdf8f, 0xf158c3ac), + TOBN(0x3fb6b3b0, 0x87edd685), TOBN(0x22071cf6, 0x86f064d0), + TOBN(0xd2d6721f, 0xff2811e5), TOBN(0xdb81b703, 0xfe7fae8c), + TOBN(0x3cfb74ef, 0xd3f1f7bb), TOBN(0x0cdbcd76, 0x16cdeb5d), + TOBN(0x4f39642a, 0x566a808c), TOBN(0x02b74454, 0x340064d6), + TOBN(0xfabbadca, 0x0528fa6f), TOBN(0xe4c3074c, 0xd3fc0bb6), + TOBN(0xb32cb8b0, 0xb796d219), TOBN(0xc3e95f4f, 0x34741dd9), + TOBN(0x87212125, 0x68edf6f5), TOBN(0x7a03aee4, 0xa2b9cb8e), + TOBN(0x0cd3c376, 0xf53a89aa), TOBN(0x0d8af9b1, 0x948a28dc), + TOBN(0xcf86a3f4, 0x902ab04f), TOBN(0x8aacb62a, 0x7f42002d), + TOBN(0x106985eb, 0xf62ffd52), TOBN(0xe670b54e, 0x5797bf10), + TOBN(0x4b405209, 0xc5e30aef), TOBN(0x12c97a20, 0x4365b5e9), + TOBN(0x104646ce, 0x1fe32093), TOBN(0x13cb4ff6, 0x3907a8c9), + TOBN(0x8b9f30d1, 0xd46e726b), TOBN(0xe1985e21, 0xaba0f499), + TOBN(0xc573dea9, 0x10a230cd), TOBN(0x24f46a93, 0xcd30f947), + TOBN(0xf2623fcf, 0xabe2010a), TOBN(0x3f278cb2, 0x73f00e4f), + TOBN(0xed55c67d, 0x50b920eb), TOBN(0xf1cb9a2d, 0x8e760571), + TOBN(0x7c50d109, 0x0895b709), TOBN(0x4207cf07, 0x190d4369), + TOBN(0x3b027e81, 0xc4127fe1), TOBN(0xa9f8b9ad, 0x3ae9c566), + TOBN(0x5ab10851, 0xacbfbba5), TOBN(0xa747d648, 0x569556f5), + TOBN(0xcc172b5c, 0x2ba97bf7), TOBN(0x15e0f77d, 0xbcfa3324), + TOBN(0xa345b797, 0x7686279d), TOBN(0x5a723480, 0xe38003d3), + TOBN(0xfd8e139f, 0x8f5fcda8), TOBN(0xf3e558c4, 0xbdee5bfd), + TOBN(0xd76cbaf4, 0xe33f9f77), TOBN(0x3a4c97a4, 0x71771969), + TOBN(0xda27e84b, 0xf6dce6a7), TOBN(0xff373d96, 0x13e6c2d1), + TOBN(0xf115193c, 0xd759a6e9), TOBN(0x3f9b7025, 0x63d2262c), + TOBN(0xd9764a31, 0x317cd062), TOBN(0x30779d8e, 0x199f8332), + TOBN(0xd8074106, 0x16b11b0b), TOBN(0x7917ab9f, 0x78aeaed8), + TOBN(0xb67a9cbe, 0x28fb1d8e), TOBN(0x2e313563, 0x136eda33), + TOBN(0x010b7069, 0xa371a86c), TOBN(0x44d90fa2, 0x6744e6b7), + TOBN(0x68190867, 0xd6b3e243), TOBN(0x9fe6cd9d, 0x59048c48), + TOBN(0xb900b028, 0x95731538), TOBN(0xa012062f, 0x32cae04f), + TOBN(0x8107c8bc, 0x9399d082), TOBN(0x47e8c54a, 0x41df12e2), + TOBN(0x14ba5117, 0xb6ef3f73), TOBN(0x22260bea, 0x81362f0b), + TOBN(0x90ea261e, 0x1a18cc20), TOBN(0x2192999f, 0x2321d636), + TOBN(0xef64d314, 0xe311b6a0), TOBN(0xd7401e4c, 0x3b54a1f5), + TOBN(0x19019983, 0x6fbca2ba), TOBN(0x46ad3293, 0x8fbffc4b), + TOBN(0xa142d3f6, 0x3786bf40), TOBN(0xeb5cbc26, 0xb67039fc), + TOBN(0x9cb0ae6c, 0x252bd479), TOBN(0x05e0f88a, 0x12b5848f), + TOBN(0x78f6d2b2, 0xa5c97663), TOBN(0x6f6e149b, 0xc162225c), + TOBN(0xe602235c, 0xde601a89), TOBN(0xd17bbe98, 0xf373be1f), + TOBN(0xcaf49a5b, 0xa8471827), TOBN(0x7e1a0a85, 0x18aaa116), + TOBN(0x6c833196, 0x270580c3), TOBN(0x1e233839, 0xf1c98a14), + TOBN(0x67b2f7b4, 0xae34e0a5), TOBN(0x47ac8745, 0xd8ce7289), + TOBN(0x2b74779a, 0x100dd467), TOBN(0x274a4337, 0x4ee50d09), + TOBN(0x603dcf13, 0x83608bc9), TOBN(0xcd9da6c3, 0xc89e8388), + TOBN(0x2660199f, 0x355116ac), TOBN(0xcc38bb59, 0xb6d18eed), + TOBN(0x3075f31f, 0x2f4bc071), TOBN(0x9774457f, 0x265dc57e), + TOBN(0x06a6a9c8, 0xc6db88bb), TOBN(0x6429d07f, 0x4ec98e04), + TOBN(0x8d05e57b, 0x05ecaa8b), TOBN(0x20f140b1, 0x7872ea7b), + TOBN(0xdf8c0f09, 0xca494693), TOBN(0x48d3a020, 0xf252e909), + TOBN(0x4c5c29af, 0x57b14b12), TOBN(0x7e6fa37d, 0xbf47ad1c), + TOBN(0x66e7b506, 0x49a0c938), TOBN(0xb72c0d48, 0x6be5f41f), + TOBN(0x6a6242b8, 0xb2359412), TOBN(0xcd35c774, 0x8e859480), + TOBN(0x12536fea, 0x87baa627), TOBN(0x58c1fec1, 0xf72aa680), + TOBN(0x6c29b637, 0x601e5dc9), TOBN(0x9e3c3c1c, 0xde9e01b9), + TOBN(0xefc8127b, 0x2bcfe0b0), TOBN(0x35107102, 0x2a12f50d), + TOBN(0x6ccd6cb1, 0x4879b397), TOBN(0xf792f804, 0xf8a82f21), + TOBN(0x509d4804, 0xa9b46402), TOBN(0xedddf85d, 0xc10f0850), + TOBN(0x928410dc, 0x4b6208aa), TOBN(0xf6229c46, 0x391012dc), + TOBN(0xc5a7c41e, 0x7727b9b6), TOBN(0x289e4e4b, 0xaa444842), + TOBN(0x049ba1d9, 0xe9a947ea), TOBN(0x44f9e47f, 0x83c8debc), + TOBN(0xfa77a1fe, 0x611f8b8e), TOBN(0xfd2e416a, 0xf518f427), + TOBN(0xc5fffa70, 0x114ebac3), TOBN(0xfe57c4e9, 0x5d89697b), + TOBN(0xfdd053ac, 0xb1aaf613), TOBN(0x31df210f, 0xea585a45), + TOBN(0x318cc10e, 0x24985034), TOBN(0x1a38efd1, 0x5f1d6130), + TOBN(0xbf86f237, 0x0b1e9e21), TOBN(0xb258514d, 0x1dbe88aa), + TOBN(0x1e38a588, 0x90c1baf9), TOBN(0x2936a01e, 0xbdb9b692), + TOBN(0xd576de98, 0x6dd5b20c), TOBN(0xb586bf71, 0x70f98ecf), + TOBN(0xcccf0f12, 0xc42d2fd7), TOBN(0x8717e61c, 0xfb35bd7b), + TOBN(0x8b1e5722, 0x35e6fc06), TOBN(0x3477728f, 0x0b3e13d5), + TOBN(0x150c294d, 0xaa8a7372), TOBN(0xc0291d43, 0x3bfa528a), + TOBN(0xc6c8bc67, 0xcec5a196), TOBN(0xdeeb31e4, 0x5c2e8a7c), + TOBN(0xba93e244, 0xfb6e1c51), TOBN(0xb9f8b71b, 0x2e28e156), + TOBN(0xce65a287, 0x968a2ab9), TOBN(0xe3c5ce69, 0x46bbcb1f), + TOBN(0xf8c835b9, 0xe7ae3f30), TOBN(0x16bbee26, 0xff72b82b), + TOBN(0x665e2017, 0xfd42cd22), TOBN(0x1e139970, 0xf8b1d2a0), + TOBN(0x125cda29, 0x79204932), TOBN(0x7aee94a5, 0x49c3bee5), + TOBN(0x68c70160, 0x89821a66), TOBN(0xf7c37678, 0x8f981669), + TOBN(0xd90829fc, 0x48cc3645), TOBN(0x346af049, 0xd70addfc), + TOBN(0x2057b232, 0x370bf29c), TOBN(0xf90c73ce, 0x42e650ee), + TOBN(0xe03386ea, 0xa126ab90), TOBN(0x0e266e7e, 0x975a087b), + TOBN(0x80578eb9, 0x0fca65d9), TOBN(0x7e2989ea, 0x16af45b8), + TOBN(0x7438212d, 0xcac75a4e), TOBN(0x38c7ca39, 0x4fef36b8), + TOBN(0x8650c494, 0xd402676a), TOBN(0x26ab5a66, 0xf72c7c48), + TOBN(0x4e6cb426, 0xce3a464e), TOBN(0xf8f99896, 0x2b72f841), + TOBN(0x8c318491, 0x1a335cc8), TOBN(0x563459ba, 0x6a5913e4), + TOBN(0x1b920d61, 0xc7b32919), TOBN(0x805ab8b6, 0xa02425ad), + TOBN(0x2ac512da, 0x8d006086), TOBN(0x6ca4846a, 0xbcf5c0fd), + TOBN(0xafea51d8, 0xac2138d7), TOBN(0xcb647545, 0x344cd443), + TOBN(0x0429ee8f, 0xbd7d9040), TOBN(0xee66a2de, 0x819b9c96), + TOBN(0x54f9ec25, 0xdea7d744), TOBN(0x2ffea642, 0x671721bb), + TOBN(0x4f19dbd1, 0x114344ea), TOBN(0x04304536, 0xfd0dbc8b), + TOBN(0x014b50aa, 0x29ec7f91), TOBN(0xb5fc22fe, 0xbb06014d), + TOBN(0x60d963a9, 0x1ee682e0), TOBN(0xdf48abc0, 0xfe85c727), + TOBN(0x0cadba13, 0x2e707c2d), TOBN(0xde608d3a, 0xa645aeff), + TOBN(0x05f1c28b, 0xedafd883), TOBN(0x3c362ede, 0xbd94de1f), + TOBN(0x8dd0629d, 0x13593e41), TOBN(0x0a5e736f, 0x766d6eaf), + TOBN(0xbfa92311, 0xf68cf9d1), TOBN(0xa4f9ef87, 0xc1797556), + TOBN(0x10d75a1f, 0x5601c209), TOBN(0x651c374c, 0x09b07361), + TOBN(0x49950b58, 0x88b5cead), TOBN(0x0ef00058, 0x6fa9dbaa), + TOBN(0xf51ddc26, 0x4e15f33a), TOBN(0x1f8b5ca6, 0x2ef46140), + TOBN(0x343ac0a3, 0xee9523f0), TOBN(0xbb75eab2, 0x975ea978), + TOBN(0x1bccf332, 0x107387f4), TOBN(0x790f9259, 0x9ab0062e), + TOBN(0xf1a363ad, 0x1e4f6a5f), TOBN(0x06e08b84, 0x62519a50), + TOBN(0x60915187, 0x7265f1ee), TOBN(0x6a80ca34, 0x93ae985e), + TOBN(0x81b29768, 0xaaba4864), TOBN(0xb13cabf2, 0x8d52a7d6), + TOBN(0xb5c36348, 0x8ead03f1), TOBN(0xc932ad95, 0x81c7c1c0), + TOBN(0x5452708e, 0xcae1e27b), TOBN(0x9dac4269, 0x1b0df648), + TOBN(0x233e3f0c, 0xdfcdb8bc), TOBN(0xe6ceccdf, 0xec540174), + TOBN(0xbd0d845e, 0x95081181), TOBN(0xcc8a7920, 0x699355d5), + TOBN(0x111c0f6d, 0xc3b375a8), TOBN(0xfd95bc6b, 0xfd51e0dc), + TOBN(0x4a106a26, 0x6888523a), TOBN(0x4d142bd6, 0xcb01a06d), + TOBN(0x79bfd289, 0xadb9b397), TOBN(0x0bdbfb94, 0xe9863914), + TOBN(0x29d8a229, 0x1660f6a6), TOBN(0x7f6abcd6, 0x551c042d), + TOBN(0x13039deb, 0x0ac3ffe8), TOBN(0xa01be628, 0xec8523fb), + TOBN(0x6ea34103, 0x0ca1c328), TOBN(0xc74114bd, 0xb903928e), + TOBN(0x8aa4ff4e, 0x9e9144b0), TOBN(0x7064091f, 0x7f9a4b17), + TOBN(0xa3f4f521, 0xe447f2c4), TOBN(0x81b8da7a, 0x604291f0), + TOBN(0xd680bc46, 0x7d5926de), TOBN(0x84f21fd5, 0x34a1202f), + TOBN(0x1d1e3181, 0x4e9df3d8), TOBN(0x1ca4861a, 0x39ab8d34), + TOBN(0x809ddeec, 0x5b19aa4a), TOBN(0x59f72f7e, 0x4d329366), + TOBN(0xa2f93f41, 0x386d5087), TOBN(0x40bf739c, 0xdd67d64f), + TOBN(0xb4494205, 0x66702158), TOBN(0xc33c65be, 0x73b1e178), + TOBN(0xcdcd657c, 0x38ca6153), TOBN(0x97f4519a, 0xdc791976), + TOBN(0xcc7c7f29, 0xcd6e1f39), TOBN(0x38de9cfb, 0x7e3c3932), + TOBN(0xe448eba3, 0x7b793f85), TOBN(0xe9f8dbf9, 0xf067e914), + TOBN(0xc0390266, 0xf114ae87), TOBN(0x39ed75a7, 0xcd6a8e2a), + TOBN(0xadb14848, 0x7ffba390), TOBN(0x67f8cb8b, 0x6af9bc09), + TOBN(0x322c3848, 0x9c7476db), TOBN(0xa320fecf, 0x52a538d6), + TOBN(0xe0493002, 0xb2aced2b), TOBN(0xdfba1809, 0x616bd430), + TOBN(0x531c4644, 0xc331be70), TOBN(0xbc04d32e, 0x90d2e450), + TOBN(0x1805a0d1, 0x0f9f142d), TOBN(0x2c44a0c5, 0x47ee5a23), + TOBN(0x31875a43, 0x3989b4e3), TOBN(0x6b1949fd, 0x0c063481), + TOBN(0x2dfb9e08, 0xbe0f4492), TOBN(0x3ff0da03, 0xe9d5e517), + TOBN(0x03dbe9a1, 0xf79466a8), TOBN(0x0b87bcd0, 0x15ea9932), + TOBN(0xeb64fc83, 0xab1f58ab), TOBN(0x6d9598da, 0x817edc8a), + TOBN(0x699cff66, 0x1d3b67e5), TOBN(0x645c0f29, 0x92635853), + TOBN(0x253cdd82, 0xeabaf21c), TOBN(0x82b9602a, 0x2241659e), + TOBN(0x2cae07ec, 0x2d9f7091), TOBN(0xbe4c720c, 0x8b48cd9b), + TOBN(0x6ce5bc03, 0x6f08d6c9), TOBN(0x36e8a997, 0xaf10bf40), + TOBN(0x83422d21, 0x3e10ff12), TOBN(0x7b26d3eb, 0xbcc12494), + TOBN(0xb240d2d0, 0xc9469ad6), TOBN(0xc4a11b4d, 0x30afa05b), + TOBN(0x4b604ace, 0xdd6ba286), TOBN(0x18486600, 0x3ee2864c), + TOBN(0x5869d6ba, 0x8d9ce5be), TOBN(0x0d8f68c5, 0xff4bfb0d), + TOBN(0xb69f210b, 0x5700cf73), TOBN(0x61f6653a, 0x6d37c135), + TOBN(0xff3d432b, 0x5aff5a48), TOBN(0x0d81c4b9, 0x72ba3a69), + TOBN(0xee879ae9, 0xfa1899ef), TOBN(0xbac7e2a0, 0x2d6acafd), + TOBN(0xd6d93f6c, 0x1c664399), TOBN(0x4c288de1, 0x5bcb135d), + TOBN(0x83031dab, 0x9dab7cbf), TOBN(0xfe23feb0, 0x3abbf5f0), + TOBN(0x9f1b2466, 0xcdedca85), TOBN(0x140bb710, 0x1a09538c), + TOBN(0xac8ae851, 0x5e11115d), TOBN(0x0d63ff67, 0x6f03f59e), + TOBN(0x755e5551, 0x7d234afb), TOBN(0x61c2db4e, 0x7e208fc1), + TOBN(0xaa9859ce, 0xf28a4b5d), TOBN(0xbdd6d4fc, 0x34af030f), + TOBN(0xd1c4a26d, 0x3be01cb1), TOBN(0x9ba14ffc, 0x243aa07c), + TOBN(0xf95cd3a9, 0xb2503502), TOBN(0xe379bc06, 0x7d2a93ab), + TOBN(0x3efc18e9, 0xd4ca8d68), TOBN(0x083558ec, 0x80bb412a), + TOBN(0xd903b940, 0x9645a968), TOBN(0xa499f0b6, 0x9ba6054f), + TOBN(0x208b573c, 0xb8349abe), TOBN(0x3baab3e5, 0x30b4fc1c), + TOBN(0x87e978ba, 0xcb524990), TOBN(0x3524194e, 0xccdf0e80), + TOBN(0x62711725, 0x7d4bcc42), TOBN(0xe90a3d9b, 0xb90109ba), + TOBN(0x3b1bdd57, 0x1323e1e0), TOBN(0xb78e9bd5, 0x5eae1599), + TOBN(0x0794b746, 0x9e03d278), TOBN(0x80178605, 0xd70e6297), + TOBN(0x171792f8, 0x99c97855), TOBN(0x11b393ee, 0xf5a86b5c), + TOBN(0x48ef6582, 0xd8884f27), TOBN(0xbd44737a, 0xbf19ba5f), + TOBN(0x8698de4c, 0xa42062c6), TOBN(0x8975eb80, 0x61ce9c54), + TOBN(0xd50e57c7, 0xd7fe71f3), TOBN(0x15342190, 0xbc97ce38), + TOBN(0x51bda2de, 0x4df07b63), TOBN(0xba12aeae, 0x200eb87d), + TOBN(0xabe135d2, 0xa9b4f8f6), TOBN(0x04619d65, 0xfad6d99c), + TOBN(0x4a6683a7, 0x7994937c), TOBN(0x7a778c8b, 0x6f94f09a), + TOBN(0x8c508623, 0x20a71b89), TOBN(0x241a2aed, 0x1c229165), + TOBN(0x352be595, 0xaaf83a99), TOBN(0x9fbfee7f, 0x1562bac8), + TOBN(0xeaf658b9, 0x5c4017e3), TOBN(0x1dc7f9e0, 0x15120b86), + TOBN(0xd84f13dd, 0x4c034d6f), TOBN(0x283dd737, 0xeaea3038), + TOBN(0x197f2609, 0xcd85d6a2), TOBN(0x6ebbc345, 0xfae60177), + TOBN(0xb80f031b, 0x4e12fede), TOBN(0xde55d0c2, 0x07a2186b), + TOBN(0x1fb3e37f, 0x24dcdd5a), TOBN(0x8d602da5, 0x7ed191fb), + TOBN(0x108fb056, 0x76023e0d), TOBN(0x70178c71, 0x459c20c0), + TOBN(0xfad5a386, 0x3fe54cf0), TOBN(0xa4a3ec4f, 0x02bbb475), + TOBN(0x1aa5ec20, 0x919d94d7), TOBN(0x5d3b63b5, 0xa81e4ab3), + TOBN(0x7fa733d8, 0x5ad3d2af), TOBN(0xfbc586dd, 0xd1ac7a37), + TOBN(0x282925de, 0x40779614), TOBN(0xfe0ffffb, 0xe74a242a), + TOBN(0x3f39e67f, 0x906151e5), TOBN(0xcea27f5f, 0x55e10649), + TOBN(0xdca1d4e1, 0xc17cf7b7), TOBN(0x0c326d12, 0x2fe2362d), + TOBN(0x05f7ac33, 0x7dd35df3), TOBN(0x0c3b7639, 0xc396dbdf), + TOBN(0x0912f5ac, 0x03b7db1c), TOBN(0x9dea4b70, 0x5c9ed4a9), + TOBN(0x475e6e53, 0xaae3f639), TOBN(0xfaba0e7c, 0xfc278bac), + TOBN(0x16f9e221, 0x9490375f), TOBN(0xaebf9746, 0xa5a7ed0a), + TOBN(0x45f9af3f, 0xf41ad5d6), TOBN(0x03c4623c, 0xb2e99224), + TOBN(0x82c5bb5c, 0xb3cf56aa), TOBN(0x64311819, 0x34567ed3), + TOBN(0xec57f211, 0x8be489ac), TOBN(0x2821895d, 0xb9a1104b), + TOBN(0x610dc875, 0x6064e007), TOBN(0x8e526f3f, 0x5b20d0fe), + TOBN(0x6e71ca77, 0x5b645aee), TOBN(0x3d1dcb9f, 0x800e10ff), + TOBN(0x36b51162, 0x189cf6de), TOBN(0x2c5a3e30, 0x6bb17353), + TOBN(0xc186cd3e, 0x2a6c6fbf), TOBN(0xa74516fa, 0x4bf97906), + TOBN(0x5b4b8f4b, 0x279d6901), TOBN(0x0c4e57b4, 0x2b573743), + TOBN(0x75fdb229, 0xb6e386b6), TOBN(0xb46793fd, 0x99deac27), + TOBN(0xeeec47ea, 0xcf712629), TOBN(0xe965f3c4, 0xcbc3b2dd), + TOBN(0x8dd1fb83, 0x425c6559), TOBN(0x7fc00ee6, 0x0af06fda), + TOBN(0xe98c9225, 0x33d956df), TOBN(0x0f1ef335, 0x4fbdc8a2), + TOBN(0x2abb5145, 0xb79b8ea2), TOBN(0x40fd2945, 0xbdbff288), + TOBN(0x6a814ac4, 0xd7185db7), TOBN(0xc4329d6f, 0xc084609a), + TOBN(0xc9ba7b52, 0xed1be45d), TOBN(0x891dd20d, 0xe4cd2c74), + TOBN(0x5a4d4a7f, 0x824139b1), TOBN(0x66c17716, 0xb873c710), + TOBN(0x5e5bc141, 0x2843c4e0), TOBN(0xd5ac4817, 0xb97eb5bf), + TOBN(0xc0f8af54, 0x450c95c7), TOBN(0xc91b3fa0, 0x318406c5), + TOBN(0x360c340a, 0xab9d97f8), TOBN(0xfb57bd07, 0x90a2d611), + TOBN(0x4339ae3c, 0xa6a6f7e5), TOBN(0x9c1fcd2a, 0x2feb8a10), + TOBN(0x972bcca9, 0xc7ea7432), TOBN(0x1b0b924c, 0x308076f6), + TOBN(0x80b2814a, 0x2a5b4ca5), TOBN(0x2f78f55b, 0x61ef3b29), + TOBN(0xf838744a, 0xc18a414f), TOBN(0xc611eaae, 0x903d0a86), + TOBN(0x94dabc16, 0x2a453f55), TOBN(0xe6f2e3da, 0x14efb279), + TOBN(0x5b7a6017, 0x9320dc3c), TOBN(0x692e382f, 0x8df6b5a4), + TOBN(0x3f5e15e0, 0x2d40fa90), TOBN(0xc87883ae, 0x643dd318), + TOBN(0x511053e4, 0x53544774), TOBN(0x834d0ecc, 0x3adba2bc), + TOBN(0x4215d7f7, 0xbae371f5), TOBN(0xfcfd57bf, 0x6c8663bc), + TOBN(0xded2383d, 0xd6901b1d), TOBN(0x3b49fbb4, 0xb5587dc3), + TOBN(0xfd44a08d, 0x07625f62), TOBN(0x3ee4d65b, 0x9de9b762),} + , + {TOBN(0x64e5137d, 0x0d63d1fa), TOBN(0x658fc052, 0x02a9d89f), + TOBN(0x48894874, 0x50436309), TOBN(0xe9ae30f8, 0xd598da61), + TOBN(0x2ed710d1, 0x818baf91), TOBN(0xe27e9e06, 0x8b6a0c20), + TOBN(0x1e28dcfb, 0x1c1a6b44), TOBN(0x883acb64, 0xd6ac57dc), + TOBN(0x8735728d, 0xc2c6ff70), TOBN(0x79d6122f, 0xc5dc2235), + TOBN(0x23f5d003, 0x19e277f9), TOBN(0x7ee84e25, 0xdded8cc7), + TOBN(0x91a8afb0, 0x63cd880a), TOBN(0x3f3ea7c6, 0x3574af60), + TOBN(0x0cfcdc84, 0x02de7f42), TOBN(0x62d0792f, 0xb31aa152), + TOBN(0x8e1b4e43, 0x8a5807ce), TOBN(0xad283893, 0xe4109a7e), + TOBN(0xc30cc9cb, 0xafd59dda), TOBN(0xf65f36c6, 0x3d8d8093), + TOBN(0xdf31469e, 0xa60d32b2), TOBN(0xee93df4b, 0x3e8191c8), + TOBN(0x9c1017c5, 0x355bdeb5), TOBN(0xd2623185, 0x8616aa28), + TOBN(0xb02c83f9, 0xdec31a21), TOBN(0x988c8b23, 0x6ad9d573), + TOBN(0x53e983ae, 0xa57be365), TOBN(0xe968734d, 0x646f834e), + TOBN(0x9137ea8f, 0x5da6309b), TOBN(0x10f3a624, 0xc1f1ce16), + TOBN(0x782a9ea2, 0xca440921), TOBN(0xdf94739e, 0x5b46f1b5), + TOBN(0x9f9be006, 0xcce85c9b), TOBN(0x360e70d6, 0xa4c7c2d3), + TOBN(0x2cd5beea, 0xaefa1e60), TOBN(0x64cf63c0, 0x8c3d2b6d), + TOBN(0xfb107fa3, 0xe1cf6f90), TOBN(0xb7e937c6, 0xd5e044e6), + TOBN(0x74e8ca78, 0xce34db9f), TOBN(0x4f8b36c1, 0x3e210bd0), + TOBN(0x1df165a4, 0x34a35ea8), TOBN(0x3418e0f7, 0x4d4412f6), + TOBN(0x5af1f8af, 0x518836c3), TOBN(0x42ceef4d, 0x130e1965), + TOBN(0x5560ca0b, 0x543a1957), TOBN(0xc33761e5, 0x886cb123), + TOBN(0x66624b1f, 0xfe98ed30), TOBN(0xf772f4bf, 0x1090997d), + TOBN(0xf4e540bb, 0x4885d410), TOBN(0x7287f810, 0x9ba5f8d7), + TOBN(0x22d0d865, 0xde98dfb1), TOBN(0x49ff51a1, 0xbcfbb8a3), + TOBN(0xb6b6fa53, 0x6bc3012e), TOBN(0x3d31fd72, 0x170d541d), + TOBN(0x8018724f, 0x4b0f4966), TOBN(0x79e7399f, 0x87dbde07), + TOBN(0x56f8410e, 0xf4f8b16a), TOBN(0x97241afe, 0xc47b266a), + TOBN(0x0a406b8e, 0x6d9c87c1), TOBN(0x803f3e02, 0xcd42ab1b), + TOBN(0x7f0309a8, 0x04dbec69), TOBN(0xa83b85f7, 0x3bbad05f), + TOBN(0xc6097273, 0xad8e197f), TOBN(0xc097440e, 0x5067adc1), + TOBN(0x730eafb6, 0x3524ff16), TOBN(0xd7f9b51e, 0x823fc6ce), + TOBN(0x27bd0d32, 0x443e4ac0), TOBN(0x40c59ad9, 0x4d66f217), + TOBN(0x6c33136f, 0x17c387a4), TOBN(0x5043b8d5, 0xeb86804d), + TOBN(0x74970312, 0x675a73c9), TOBN(0x838fdb31, 0xf16669b6), + TOBN(0xc507b6dd, 0x418e7ddd), TOBN(0x39888d93, 0x472f19d6), + TOBN(0x7eae26be, 0x0c27eb4d), TOBN(0x17b53ed3, 0xfbabb884), + TOBN(0xfc27021b, 0x2b01ae4f), TOBN(0x88462e87, 0xcf488682), + TOBN(0xbee096ec, 0x215e2d87), TOBN(0xeb2fea9a, 0xd242e29b), + TOBN(0x5d985b5f, 0xb821fc28), TOBN(0x89d2e197, 0xdc1e2ad2), + TOBN(0x55b566b8, 0x9030ba62), TOBN(0xe3fd41b5, 0x4f41b1c6), + TOBN(0xb738ac2e, 0xb9a96d61), TOBN(0x7f8567ca, 0x369443f4), + TOBN(0x8698622d, 0xf803a440), TOBN(0x2b586236, 0x8fe2f4dc), + TOBN(0xbbcc00c7, 0x56b95bce), TOBN(0x5ec03906, 0x616da680), + TOBN(0x79162ee6, 0x72214252), TOBN(0x43132b63, 0x86a892d2), + TOBN(0x4bdd3ff2, 0x2f3263bf), TOBN(0xd5b3733c, 0x9cd0a142), + TOBN(0x592eaa82, 0x44415ccb), TOBN(0x663e8924, 0x8d5474ea), + TOBN(0x8058a25e, 0x5236344e), TOBN(0x82e8df9d, 0xbda76ee6), + TOBN(0xdcf6efd8, 0x11cc3d22), TOBN(0x00089cda, 0x3b4ab529), + TOBN(0x91d3a071, 0xbd38a3db), TOBN(0x4ea97fc0, 0xef72b925), + TOBN(0x0c9fc15b, 0xea3edf75), TOBN(0x5a6297cd, 0xa4348ed3), + TOBN(0x0d38ab35, 0xce7c42d4), TOBN(0x9fd493ef, 0x82feab10), + TOBN(0x46056b6d, 0x82111b45), TOBN(0xda11dae1, 0x73efc5c3), + TOBN(0xdc740278, 0x5545a7fb), TOBN(0xbdb2601c, 0x40d507e6), + TOBN(0x121dfeeb, 0x7066fa58), TOBN(0x214369a8, 0x39ae8c2a), + TOBN(0x195709cb, 0x06e0956c), TOBN(0x4c9d254f, 0x010cd34b), + TOBN(0xf51e13f7, 0x0471a532), TOBN(0xe19d6791, 0x1e73054d), + TOBN(0xf702a628, 0xdb5c7be3), TOBN(0xc7141218, 0xb24dde05), + TOBN(0xdc18233c, 0xf29b2e2e), TOBN(0x3a6bd1e8, 0x85342dba), + TOBN(0x3f747fa0, 0xb311898c), TOBN(0xe2a272e4, 0xcd0eac65), + TOBN(0x4bba5851, 0xf914d0bc), TOBN(0x7a1a9660, 0xc4a43ee3), + TOBN(0xe5a367ce, 0xa1c8cde9), TOBN(0x9d958ba9, 0x7271abe3), + TOBN(0xf3ff7eb6, 0x3d1615cd), TOBN(0xa2280dce, 0xf5ae20b0), + TOBN(0x56dba5c1, 0xcf640147), TOBN(0xea5a2e3d, 0x5e83d118), + TOBN(0x04cd6b6d, 0xda24c511), TOBN(0x1c0f4671, 0xe854d214), + TOBN(0x91a6b7a9, 0x69565381), TOBN(0xdc966240, 0xdecf1f5b), + TOBN(0x1b22d21c, 0xfcf5d009), TOBN(0x2a05f641, 0x9021dbd5), + TOBN(0x8c0ed566, 0xd4312483), TOBN(0x5179a95d, 0x643e216f), + TOBN(0xcc185fec, 0x17044493), TOBN(0xb3063339, 0x54991a21), + TOBN(0xd801ecdb, 0x0081a726), TOBN(0x0149b0c6, 0x4fa89bbb), + TOBN(0xafe9065a, 0x4391b6b9), TOBN(0xedc92786, 0xd633f3a3), + TOBN(0xe408c24a, 0xae6a8e13), TOBN(0x85833fde, 0x9f3897ab), + TOBN(0x43800e7e, 0xd81a0715), TOBN(0xde08e346, 0xb44ffc5f), + TOBN(0x7094184c, 0xcdeff2e0), TOBN(0x49f9387b, 0x165eaed1), + TOBN(0x635d6129, 0x777c468a), TOBN(0x8c0dcfd1, 0x538c2dd8), + TOBN(0xd6d9d9e3, 0x7a6a308b), TOBN(0x62375830, 0x4c2767d3), + TOBN(0x874a8bc6, 0xf38cbeb6), TOBN(0xd94d3f1a, 0xccb6fd9e), + TOBN(0x92a9735b, 0xba21f248), TOBN(0x272ad0e5, 0x6cd1efb0), + TOBN(0x7437b69c, 0x05b03284), TOBN(0xe7f04702, 0x6948c225), + TOBN(0x8a56c04a, 0xcba2ecec), TOBN(0x0c181270, 0xe3a73e41), + TOBN(0x6cb34e9d, 0x03e93725), TOBN(0xf77c8713, 0x496521a9), + TOBN(0x94569183, 0xfa7f9f90), TOBN(0xf2e7aa4c, 0x8c9707ad), + TOBN(0xced2c9ba, 0x26c1c9a3), TOBN(0x9109fe96, 0x40197507), + TOBN(0x9ae868a9, 0xe9adfe1c), TOBN(0x3984403d, 0x314e39bb), + TOBN(0xb5875720, 0xf2fe378f), TOBN(0x33f901e0, 0xba44a628), + TOBN(0xea1125fe, 0x3652438c), TOBN(0xae9ec4e6, 0x9dd1f20b), + TOBN(0x1e740d9e, 0xbebf7fbd), TOBN(0x6dbd3ddc, 0x42dbe79c), + TOBN(0x62082aec, 0xedd36776), TOBN(0xf612c478, 0xe9859039), + TOBN(0xa493b201, 0x032f7065), TOBN(0xebd4d8f2, 0x4ff9b211), + TOBN(0x3f23a0aa, 0xaac4cb32), TOBN(0xea3aadb7, 0x15ed4005), + TOBN(0xacf17ea4, 0xafa27e63), TOBN(0x56125c1a, 0xc11fd66c), + TOBN(0x266344a4, 0x3794f8dc), TOBN(0xdcca923a, 0x483c5c36), + TOBN(0x2d6b6bbf, 0x3f9d10a0), TOBN(0xb320c5ca, 0x81d9bdf3), + TOBN(0x620e28ff, 0x47b50a95), TOBN(0x933e3b01, 0xcef03371), + TOBN(0xf081bf85, 0x99100153), TOBN(0x183be9a0, 0xc3a8c8d6), + TOBN(0x4e3ddc5a, 0xd6bbe24d), TOBN(0xc6c74630, 0x53843795), + TOBN(0x78193dd7, 0x65ec2d4c), TOBN(0xb8df26cc, 0xcd3c89b2), + TOBN(0x98dbe399, 0x5a483f8d), TOBN(0x72d8a957, 0x7dd3313a), + TOBN(0x65087294, 0xab0bd375), TOBN(0xfcd89248, 0x7c259d16), + TOBN(0x8a9443d7, 0x7613aa81), TOBN(0x80100800, 0x85fe6584), + TOBN(0x70fc4dbc, 0x7fb10288), TOBN(0xf58280d3, 0xe86beee8), + TOBN(0x14fdd82f, 0x7c978c38), TOBN(0xdf1204c1, 0x0de44d7b), + TOBN(0xa08a1c84, 0x4160252f), TOBN(0x591554ca, 0xc17646a5), + TOBN(0x214a37d6, 0xa05bd525), TOBN(0x48d5f09b, 0x07957b3c), + TOBN(0x0247cdcb, 0xd7109bc9), TOBN(0x40f9e4bb, 0x30599ce7), + TOBN(0xc325fa03, 0xf46ad2ec), TOBN(0x00f766cf, 0xc3e3f9ee), + TOBN(0xab556668, 0xd43a4577), TOBN(0x68d30a61, 0x3ee03b93), + TOBN(0x7ddc81ea, 0x77b46a08), TOBN(0xcf5a6477, 0xc7480699), + TOBN(0x43a8cb34, 0x6633f683), TOBN(0x1b867e6b, 0x92363c60), + TOBN(0x43921114, 0x1f60558e), TOBN(0xcdbcdd63, 0x2f41450e), + TOBN(0x7fc04601, 0xcc630e8b), TOBN(0xea7c66d5, 0x97038b43), + TOBN(0x7259b8a5, 0x04e99fd8), TOBN(0x98a8dd12, 0x4785549a), + TOBN(0x0e459a7c, 0x840552e1), TOBN(0xcdfcf4d0, 0x4bb0909e), + TOBN(0x34a86db2, 0x53758da7), TOBN(0xe643bb83, 0xeac997e1), + TOBN(0x96400bd7, 0x530c5b7e), TOBN(0x9f97af87, 0xb41c8b52), + TOBN(0x34fc8820, 0xfbeee3f9), TOBN(0x93e53490, 0x49091afd), + TOBN(0x764b9be5, 0x9a31f35c), TOBN(0x71f37864, 0x57e3d924), + TOBN(0x02fb34e0, 0x943aa75e), TOBN(0xa18c9c58, 0xab8ff6e4), + TOBN(0x080f31b1, 0x33cf0d19), TOBN(0x5c9682db, 0x083518a7), + TOBN(0x873d4ca6, 0xb709c3de), TOBN(0x64a84262, 0x3575b8f0), + TOBN(0x6275da1f, 0x020154bb), TOBN(0x97678caa, 0xd17cf1ab), + TOBN(0x8779795f, 0x951a95c3), TOBN(0xdd35b163, 0x50fccc08), + TOBN(0x32709627, 0x33d8f031), TOBN(0x3c5ab10a, 0x498dd85c), + TOBN(0xb6c185c3, 0x41dca566), TOBN(0x7de7feda, 0xd8622aa3), + TOBN(0x99e84d92, 0x901b6dfb), TOBN(0x30a02b0e, 0x7c4ad288), + TOBN(0xc7c81daa, 0x2fd3cf36), TOBN(0xd1319547, 0xdf89e59f), + TOBN(0xb2be8184, 0xcd496733), TOBN(0xd5f449eb, 0x93d3412b), + TOBN(0x7ea41b1b, 0x25fe531d), TOBN(0xf9797432, 0x6a1d5646), + TOBN(0x86067f72, 0x2bde501a), TOBN(0xf91481c0, 0x0c85e89c), + TOBN(0xca8ee465, 0xf8b05bc6), TOBN(0x1844e1cf, 0x02e83cda), + TOBN(0xca82114a, 0xb4dbe33b), TOBN(0x0f9f8769, 0x4eabfde2), + TOBN(0x4936b1c0, 0x38b27fe2), TOBN(0x63b6359b, 0xaba402df), + TOBN(0x40c0ea2f, 0x656bdbab), TOBN(0x9c992a89, 0x6580c39c), + TOBN(0x600e8f15, 0x2a60aed1), TOBN(0xeb089ca4, 0xe0bf49df), + TOBN(0x9c233d7d, 0x2d42d99a), TOBN(0x648d3f95, 0x4c6bc2fa), + TOBN(0xdcc383a8, 0xe1add3f3), TOBN(0xf42c0c6a, 0x4f64a348), + TOBN(0x2abd176f, 0x0030dbdb), TOBN(0x4de501a3, 0x7d6c215e), + TOBN(0x4a107c1f, 0x4b9a64bc), TOBN(0xa77f0ad3, 0x2496cd59), + TOBN(0xfb78ac62, 0x7688dffb), TOBN(0x7025a2ca, 0x67937d8e), + TOBN(0xfde8b2d1, 0xd1a8f4e7), TOBN(0xf5b3da47, 0x7354927c), + TOBN(0xe48606a3, 0xd9205735), TOBN(0xac477cc6, 0xe177b917), + TOBN(0xfb1f73d2, 0xa883239a), TOBN(0xe12572f6, 0xcc8b8357), + TOBN(0x9d355e9c, 0xfb1f4f86), TOBN(0x89b795f8, 0xd9f3ec6e), + TOBN(0x27be56f1, 0xb54398dc), TOBN(0x1890efd7, 0x3fedeed5), + TOBN(0x62f77f1f, 0x9c6d0140), TOBN(0x7ef0e314, 0x596f0ee4), + TOBN(0x50ca6631, 0xcc61dab3), TOBN(0x4a39801d, 0xf4866e4f), + TOBN(0x66c8d032, 0xae363b39), TOBN(0x22c591e5, 0x2ead66aa), + TOBN(0x954ba308, 0xde02a53e), TOBN(0x2a6c060f, 0xd389f357), + TOBN(0xe6cfcde8, 0xfbf40b66), TOBN(0x8e02fc56, 0xc6340ce1), + TOBN(0xe4957795, 0x73adb4ba), TOBN(0x7b86122c, 0xa7b03805), + TOBN(0x63f83512, 0x0c8e6fa6), TOBN(0x83660ea0, 0x057d7804), + TOBN(0xbad79105, 0x21ba473c), TOBN(0xb6c50bee, 0xded5389d), + TOBN(0xee2caf4d, 0xaa7c9bc0), TOBN(0xd97b8de4, 0x8c4e98a7), + TOBN(0xa9f63e70, 0xab3bbddb), TOBN(0x3898aabf, 0x2597815a), + TOBN(0x7659af89, 0xac15b3d9), TOBN(0xedf7725b, 0x703ce784), + TOBN(0x25470fab, 0xe085116b), TOBN(0x04a43375, 0x87285310), + TOBN(0x4e39187e, 0xe2bfd52f), TOBN(0x36166b44, 0x7d9ebc74), + TOBN(0x92ad433c, 0xfd4b322c), TOBN(0x726aa817, 0xba79ab51), + TOBN(0xf96eacd8, 0xc1db15eb), TOBN(0xfaf71e91, 0x0476be63), + TOBN(0xdd69a640, 0x641fad98), TOBN(0xb7995918, 0x29622559), + TOBN(0x03c6daa5, 0xde4199dc), TOBN(0x92cadc97, 0xad545eb4), + TOBN(0x1028238b, 0x256534e4), TOBN(0x73e80ce6, 0x8595409a), + TOBN(0x690d4c66, 0xd05dc59b), TOBN(0xc95f7b8f, 0x981dee80), + TOBN(0xf4337014, 0xd856ac25), TOBN(0x441bd9dd, 0xac524dca), + TOBN(0x640b3d85, 0x5f0499f5), TOBN(0x39cf84a9, 0xd5fda182), + TOBN(0x04e7b055, 0xb2aa95a0), TOBN(0x29e33f0a, 0x0ddf1860), + TOBN(0x082e74b5, 0x423f6b43), TOBN(0x217edeb9, 0x0aaa2b0f), + TOBN(0x58b83f35, 0x83cbea55), TOBN(0xc485ee4d, 0xbc185d70), + TOBN(0x833ff03b, 0x1e5f6992), TOBN(0xb5b9b9cc, 0xcf0c0dd5), + TOBN(0x7caaee8e, 0x4e9e8a50), TOBN(0x462e907b, 0x6269dafd), + TOBN(0x6ed5cee9, 0xfbe791c6), TOBN(0x68ca3259, 0xed430790), + TOBN(0x2b72bdf2, 0x13b5ba88), TOBN(0x60294c8a, 0x35ef0ac4), + TOBN(0x9c3230ed, 0x19b99b08), TOBN(0x560fff17, 0x6c2589aa), + TOBN(0x552b8487, 0xd6770374), TOBN(0xa373202d, 0x9a56f685), + TOBN(0xd3e7f907, 0x45f175d9), TOBN(0x3c2f315f, 0xd080d810), + TOBN(0x1130e9dd, 0x7b9520e8), TOBN(0xc078f9e2, 0x0af037b5), + TOBN(0x38cd2ec7, 0x1e9c104c), TOBN(0x0f684368, 0xc472fe92), + TOBN(0xd3f1b5ed, 0x6247e7ef), TOBN(0xb32d33a9, 0x396dfe21), + TOBN(0x46f59cf4, 0x4a9aa2c2), TOBN(0x69cd5168, 0xff0f7e41), + TOBN(0x3f59da0f, 0x4b3234da), TOBN(0xcf0b0235, 0xb4579ebe), + TOBN(0x6d1cbb25, 0x6d2476c7), TOBN(0x4f0837e6, 0x9dc30f08), + TOBN(0x9a4075bb, 0x906f6e98), TOBN(0x253bb434, 0xc761e7d1), + TOBN(0xde2e645f, 0x6e73af10), TOBN(0xb89a4060, 0x0c5f131c), + TOBN(0xd12840c5, 0xb8cc037f), TOBN(0x3d093a5b, 0x7405bb47), + TOBN(0x6202c253, 0x206348b8), TOBN(0xbf5d57fc, 0xc55a3ca7), + TOBN(0x89f6c90c, 0x8c3bef48), TOBN(0x23ac7623, 0x5a0a960a), + TOBN(0xdfbd3d6b, 0x552b42ab), TOBN(0x3ef22458, 0x132061f6), + TOBN(0xd74e9bda, 0xc97e6516), TOBN(0x88779360, 0xc230f49e), + TOBN(0xa6ec1de3, 0x1e74ea49), TOBN(0x581dcee5, 0x3fb645a2), + TOBN(0xbaef2391, 0x8f483f14), TOBN(0x6d2dddfc, 0xd137d13b), + TOBN(0x54cde50e, 0xd2743a42), TOBN(0x89a34fc5, 0xe4d97e67), + TOBN(0x13f1f5b3, 0x12e08ce5), TOBN(0xa80540b8, 0xa7f0b2ca), + TOBN(0x854bcf77, 0x01982805), TOBN(0xb8653ffd, 0x233bea04), + TOBN(0x8e7b8787, 0x02b0b4c9), TOBN(0x2675261f, 0x9acb170a), + TOBN(0x061a9d90, 0x930c14e5), TOBN(0xb59b30e0, 0xdef0abea), + TOBN(0x1dc19ea6, 0x0200ec7d), TOBN(0xb6f4a3f9, 0x0bce132b), + TOBN(0xb8d5de90, 0xf13e27e0), TOBN(0xbaee5ef0, 0x1fade16f), + TOBN(0x6f406aaa, 0xe4c6cf38), TOBN(0xab4cfe06, 0xd1369815), + TOBN(0x0dcffe87, 0xefd550c6), TOBN(0x9d4f59c7, 0x75ff7d39), + TOBN(0xb02553b1, 0x51deb6ad), TOBN(0x812399a4, 0xb1877749), + TOBN(0xce90f71f, 0xca6006e1), TOBN(0xc32363a6, 0xb02b6e77), + TOBN(0x02284fbe, 0xdc36c64d), TOBN(0x86c81e31, 0xa7e1ae61), + TOBN(0x2576c7e5, 0xb909d94a), TOBN(0x8b6f7d02, 0x818b2bb0), + TOBN(0xeca3ed07, 0x56faa38a), TOBN(0xa3790e6c, 0x9305bb54), + TOBN(0xd784eeda, 0x7bc73061), TOBN(0xbd56d369, 0x6dd50614), + TOBN(0xd6575949, 0x229a8aa9), TOBN(0xdcca8f47, 0x4595ec28), + TOBN(0x814305c1, 0x06ab4fe6), TOBN(0xc8c39768, 0x24f43f16), + TOBN(0xe2a45f36, 0x523f2b36), TOBN(0x995c6493, 0x920d93bb), + TOBN(0xf8afdab7, 0x90f1632b), TOBN(0x79ebbecd, 0x1c295954), + TOBN(0xc7bb3ddb, 0x79592f48), TOBN(0x67216a7b, 0x5f88e998), + TOBN(0xd91f098b, 0xbc01193e), TOBN(0xf7d928a5, 0xb1db83fc), + TOBN(0x55e38417, 0xe991f600), TOBN(0x2a91113e, 0x2981a934), + TOBN(0xcbc9d648, 0x06b13bde), TOBN(0xb011b6ac, 0x0755ff44), + TOBN(0x6f4cb518, 0x045ec613), TOBN(0x522d2d31, 0xc2f5930a), + TOBN(0x5acae1af, 0x382e65de), TOBN(0x57643067, 0x27bc966f), + TOBN(0x5e12705d, 0x1c7193f0), TOBN(0xf0f32f47, 0x3be8858e), + TOBN(0x785c3d7d, 0x96c6dfc7), TOBN(0xd75b4a20, 0xbf31795d), + TOBN(0x91acf17b, 0x342659d4), TOBN(0xe596ea34, 0x44f0378f), + TOBN(0x4515708f, 0xce52129d), TOBN(0x17387e1e, 0x79f2f585), + TOBN(0x72cfd2e9, 0x49dee168), TOBN(0x1ae05223, 0x3e2af239), + TOBN(0x009e75be, 0x1d94066a), TOBN(0x6cca31c7, 0x38abf413), + TOBN(0xb50bd61d, 0x9bc49908), TOBN(0x4a9b4a8c, 0xf5e2bc1e), + TOBN(0xeb6cc5f7, 0x946f83ac), TOBN(0x27da93fc, 0xebffab28), + TOBN(0xea314c96, 0x4821c8c5), TOBN(0x8de49ded, 0xa83c15f4), + TOBN(0x7a64cf20, 0x7af33004), TOBN(0x45f1bfeb, 0xc9627e10), + TOBN(0x878b0626, 0x54b9df60), TOBN(0x5e4fdc3c, 0xa95c0b33), + TOBN(0xe54a37ca, 0xc2035d8e), TOBN(0x9087cda9, 0x80f20b8c), + TOBN(0x36f61c23, 0x8319ade4), TOBN(0x766f287a, 0xde8cfdf8), + TOBN(0x48821948, 0x346f3705), TOBN(0x49a7b853, 0x16e4f4a2), + TOBN(0xb9b3f8a7, 0x5cedadfd), TOBN(0x8f562815, 0x8db2a815), + TOBN(0xc0b7d554, 0x01f68f95), TOBN(0x12971e27, 0x688a208e), + TOBN(0xc9f8b696, 0xd0ff34fc), TOBN(0x20824de2, 0x1222718c), + TOBN(0x7213cf9f, 0x0c95284d), TOBN(0xe2ad741b, 0xdc158240), + TOBN(0x0ee3a6df, 0x54043ccf), TOBN(0x16ff479b, 0xd84412b3), + TOBN(0xf6c74ee0, 0xdfc98af0), TOBN(0xa78a169f, 0x52fcd2fb), + TOBN(0xd8ae8746, 0x99c930e9), TOBN(0x1d33e858, 0x49e117a5), + TOBN(0x7581fcb4, 0x6624759f), TOBN(0xde50644f, 0x5bedc01d), + TOBN(0xbeec5d00, 0xcaf3155e), TOBN(0x672d66ac, 0xbc73e75f), + TOBN(0x86b9d8c6, 0x270b01db), TOBN(0xd249ef83, 0x50f55b79), + TOBN(0x6131d6d4, 0x73978fe3), TOBN(0xcc4e4542, 0x754b00a1), + TOBN(0x4e05df05, 0x57dfcfe9), TOBN(0x94b29cdd, 0x51ef6bf0), + TOBN(0xe4530cff, 0x9bc7edf2), TOBN(0x8ac236fd, 0xd3da65f3), + TOBN(0x0faf7d5f, 0xc8eb0b48), TOBN(0x4d2de14c, 0x660eb039), + TOBN(0xc006bba7, 0x60430e54), TOBN(0x10a2d0d6, 0xda3289ab), + TOBN(0x9c037a5d, 0xd7979c59), TOBN(0x04d1f3d3, 0xa116d944), + TOBN(0x9ff22473, 0x8a0983cd), TOBN(0x28e25b38, 0xc883cabb), + TOBN(0xe968dba5, 0x47a58995), TOBN(0x2c80b505, 0x774eebdf), + TOBN(0xee763b71, 0x4a953beb), TOBN(0x502e223f, 0x1642e7f6), + TOBN(0x6fe4b641, 0x61d5e722), TOBN(0x9d37c5b0, 0xdbef5316), + TOBN(0x0115ed70, 0xf8330bc7), TOBN(0x139850e6, 0x75a72789), + TOBN(0x27d7faec, 0xffceccc2), TOBN(0x3016a860, 0x4fd9f7f6), + TOBN(0xc492ec64, 0x4cd8f64c), TOBN(0x58a2d790, 0x279d7b51), + TOBN(0x0ced1fc5, 0x1fc75256), TOBN(0x3e658aed, 0x8f433017), + TOBN(0x0b61942e, 0x05da59eb), TOBN(0xba3d60a3, 0x0ddc3722), + TOBN(0x7c311cd1, 0x742e7f87), TOBN(0x6473ffee, 0xf6b01b6e),} + , + {TOBN(0x8303604f, 0x692ac542), TOBN(0xf079ffe1, 0x227b91d3), + TOBN(0x19f63e63, 0x15aaf9bd), TOBN(0xf99ee565, 0xf1f344fb), + TOBN(0x8a1d661f, 0xd6219199), TOBN(0x8c883bc6, 0xd48ce41c), + TOBN(0x1065118f, 0x3c74d904), TOBN(0x713889ee, 0x0faf8b1b), + TOBN(0x972b3f8f, 0x81a1b3be), TOBN(0x4f3ce145, 0xce2764a0), + TOBN(0xe2d0f1cc, 0x28c4f5f7), TOBN(0xdeee0c0d, 0xc7f3985b), + TOBN(0x7df4adc0, 0xd39e25c3), TOBN(0x40619820, 0xc467a080), + TOBN(0x440ebc93, 0x61cf5a58), TOBN(0x527729a6, 0x422ad600), + TOBN(0xca6c0937, 0xb1b76ba6), TOBN(0x1a2eab85, 0x4d2026dc), + TOBN(0xb1715e15, 0x19d9ae0a), TOBN(0xf1ad9199, 0xbac4a026), + TOBN(0x35b3dfb8, 0x07ea7b0e), TOBN(0xedf5496f, 0x3ed9eb89), + TOBN(0x8932e5ff, 0x2d6d08ab), TOBN(0xf314874e, 0x25bd2731), + TOBN(0xefb26a75, 0x3f73f449), TOBN(0x1d1c94f8, 0x8d44fc79), + TOBN(0x49f0fbc5, 0x3bc0dc4d), TOBN(0xb747ea0b, 0x3698a0d0), + TOBN(0x5218c3fe, 0x228d291e), TOBN(0x35b804b5, 0x43c129d6), + TOBN(0xfac859b8, 0xd1acc516), TOBN(0x6c10697d, 0x95d6e668), + TOBN(0xc38e438f, 0x0876fd4e), TOBN(0x45f0c307, 0x83d2f383), + TOBN(0x203cc2ec, 0xb10934cb), TOBN(0x6a8f2439, 0x2c9d46ee), + TOBN(0xf16b431b, 0x65ccde7b), TOBN(0x41e2cd18, 0x27e76a6f), + TOBN(0xb9c8cf8f, 0x4e3484d7), TOBN(0x64426efd, 0x8315244a), + TOBN(0x1c0a8e44, 0xfc94dea3), TOBN(0x34c8cdbf, 0xdad6a0b0), + TOBN(0x919c3840, 0x04113cef), TOBN(0xfd32fba4, 0x15490ffa), + TOBN(0x58d190f6, 0x795dcfb7), TOBN(0xfef01b03, 0x83588baf), + TOBN(0x9e6d1d63, 0xca1fc1c0), TOBN(0x53173f96, 0xf0a41ac9), + TOBN(0x2b1d402a, 0xba16f73b), TOBN(0x2fb31014, 0x8cf9b9fc), + TOBN(0x2d51e60e, 0x446ef7bf), TOBN(0xc731021b, 0xb91e1745), + TOBN(0x9d3b4724, 0x4fee99d4), TOBN(0x4bca48b6, 0xfac5c1ea), + TOBN(0x70f5f514, 0xbbea9af7), TOBN(0x751f55a5, 0x974c283a), + TOBN(0x6e30251a, 0xcb452fdb), TOBN(0x31ee6965, 0x50f30650), + TOBN(0xb0b3e508, 0x933548d9), TOBN(0xb8949a4f, 0xf4b0ef5b), + TOBN(0x208b8326, 0x3c88f3bd), TOBN(0xab147c30, 0xdb1d9989), + TOBN(0xed6515fd, 0x44d4df03), TOBN(0x17a12f75, 0xe72eb0c5), + TOBN(0x3b59796d, 0x36cf69db), TOBN(0x1219eee9, 0x56670c18), + TOBN(0xfe3341f7, 0x7a070d8e), TOBN(0x9b70130b, 0xa327f90c), + TOBN(0x36a32462, 0x0ae18e0e), TOBN(0x2021a623, 0x46c0a638), + TOBN(0x251b5817, 0xc62eb0d4), TOBN(0x87bfbcdf, 0x4c762293), + TOBN(0xf78ab505, 0xcdd61d64), TOBN(0x8c7a53fc, 0xc8c18857), + TOBN(0xa653ce6f, 0x16147515), TOBN(0x9c923aa5, 0xea7d52d5), + TOBN(0xc24709cb, 0x5c18871f), TOBN(0x7d53bec8, 0x73b3cc74), + TOBN(0x59264aff, 0xfdd1d4c4), TOBN(0x5555917e, 0x240da582), + TOBN(0xcae8bbda, 0x548f5a0e), TOBN(0x1910eaba, 0x3bbfbbe1), + TOBN(0xae579685, 0x7677afc3), TOBN(0x49ea61f1, 0x73ff0b5c), + TOBN(0x78655478, 0x4f7c3922), TOBN(0x95d337cd, 0x20c68eef), + TOBN(0x68f1e1e5, 0xdf779ab9), TOBN(0x14b491b0, 0xb5cf69a8), + TOBN(0x7a6cbbe0, 0x28e3fe89), TOBN(0xe7e1fee4, 0xc5aac0eb), + TOBN(0x7f47eda5, 0x697e5140), TOBN(0x4f450137, 0xb454921f), + TOBN(0xdb625f84, 0x95cd8185), TOBN(0x74be0ba1, 0xcdb2e583), + TOBN(0xaee4fd7c, 0xdd5e6de4), TOBN(0x4251437d, 0xe8101739), + TOBN(0x686d72a0, 0xac620366), TOBN(0x4be3fb9c, 0xb6d59344), + TOBN(0x6e8b44e7, 0xa1eb75b9), TOBN(0x84e39da3, 0x91a5c10c), + TOBN(0x37cc1490, 0xb38f0409), TOBN(0x02951943, 0x2c2ade82), + TOBN(0x9b688783, 0x1190a2d8), TOBN(0x25627d14, 0x231182ba), + TOBN(0x6eb550aa, 0x658a6d87), TOBN(0x1405aaa7, 0xcf9c7325), + TOBN(0xd147142e, 0x5c8748c9), TOBN(0x7f637e4f, 0x53ede0e0), + TOBN(0xf8ca2776, 0x14ffad2c), TOBN(0xe58fb1bd, 0xbafb6791), + TOBN(0x17158c23, 0xbf8f93fc), TOBN(0x7f15b373, 0x0a4a4655), + TOBN(0x39d4add2, 0xd842ca72), TOBN(0xa71e4391, 0x3ed96305), + TOBN(0x5bb09cbe, 0x6700be14), TOBN(0x68d69d54, 0xd8befcf6), + TOBN(0xa45f5367, 0x37183bcf), TOBN(0x7152b7bb, 0x3370dff7), + TOBN(0xcf887baa, 0xbf12525b), TOBN(0xe7ac7bdd, 0xd6d1e3cd), + TOBN(0x25914f78, 0x81fdad90), TOBN(0xcf638f56, 0x0d2cf6ab), + TOBN(0xb90bc03f, 0xcc054de5), TOBN(0x932811a7, 0x18b06350), + TOBN(0x2f00b330, 0x9bbd11ff), TOBN(0x76108a6f, 0xb4044974), + TOBN(0x801bb9e0, 0xa851d266), TOBN(0x0dd099be, 0xbf8990c1), + TOBN(0x58c5aaaa, 0xabe32986), TOBN(0x0fe9dd2a, 0x50d59c27), + TOBN(0x84951ff4, 0x8d307305), TOBN(0x6c23f829, 0x86529b78), + TOBN(0x50bb2218, 0x0b136a79), TOBN(0x7e2174de, 0x77a20996), + TOBN(0x6f00a4b9, 0xc0bb4da6), TOBN(0x89a25a17, 0xefdde8da), + TOBN(0xf728a27e, 0xc11ee01d), TOBN(0xf900553a, 0xe5f10dfb), + TOBN(0x189a83c8, 0x02ec893c), TOBN(0x3ca5bdc1, 0x23f66d77), + TOBN(0x98781537, 0x97eada9f), TOBN(0x59c50ab3, 0x10256230), + TOBN(0x346042d9, 0x323c69b3), TOBN(0x1b715a6d, 0x2c460449), + TOBN(0xa41dd476, 0x6ae06e0b), TOBN(0xcdd7888e, 0x9d42e25f), + TOBN(0x0f395f74, 0x56b25a20), TOBN(0xeadfe0ae, 0x8700e27e), + TOBN(0xb09d52a9, 0x69950093), TOBN(0x3525d9cb, 0x327f8d40), + TOBN(0xb8235a94, 0x67df886a), TOBN(0x77e4b0dd, 0x035faec2), + TOBN(0x115eb20a, 0x517d7061), TOBN(0x77fe3433, 0x6c2df683), + TOBN(0x6870ddc7, 0xcdc6fc67), TOBN(0xb1610588, 0x0b87de83), + TOBN(0x343584ca, 0xd9c4ddbe), TOBN(0xb3164f1c, 0x3d754be2), + TOBN(0x0731ed3a, 0xc1e6c894), TOBN(0x26327dec, 0x4f6b904c), + TOBN(0x9d49c6de, 0x97b5cd32), TOBN(0x40835dae, 0xb5eceecd), + TOBN(0xc66350ed, 0xd9ded7fe), TOBN(0x8aeebb5c, 0x7a678804), + TOBN(0x51d42fb7, 0x5b8ee9ec), TOBN(0xd7a17bdd, 0x8e3ca118), + TOBN(0x40d7511a, 0x2ef4400e), TOBN(0xc48990ac, 0x875a66f4), + TOBN(0x8de07d2a, 0x2199e347), TOBN(0xbee75556, 0x2a39e051), + TOBN(0x56918786, 0x916e51dc), TOBN(0xeb191313, 0x4a2d89ec), + TOBN(0x6679610d, 0x37d341ed), TOBN(0x434fbb41, 0x56d51c2b), + TOBN(0xe54b7ee7, 0xd7492dba), TOBN(0xaa33a79a, 0x59021493), + TOBN(0x49fc5054, 0xe4bd6d3d), TOBN(0x09540f04, 0x5ab551d0), + TOBN(0x8acc9085, 0x4942d3a6), TOBN(0x231af02f, 0x2d28323b), + TOBN(0x93458cac, 0x0992c163), TOBN(0x1fef8e71, 0x888e3bb4), + TOBN(0x27578da5, 0xbe8c268c), TOBN(0xcc8be792, 0xe805ec00), + TOBN(0x29267bae, 0xc61c3855), TOBN(0xebff429d, 0x58c1fd3b), + TOBN(0x22d886c0, 0x8c0b93b8), TOBN(0xca5e00b2, 0x2ddb8953), + TOBN(0xcf330117, 0xc3fed8b7), TOBN(0xd49ac6fa, 0x819c01f6), + TOBN(0x6ddaa6bd, 0x3c0fbd54), TOBN(0x91743068, 0x8049a2cf), + TOBN(0xd67f981e, 0xaff2ef81), TOBN(0xc3654d35, 0x2818ae80), + TOBN(0x81d05044, 0x1b2aa892), TOBN(0x2db067bf, 0x3d099328), + TOBN(0xe7c79e86, 0x703dcc97), TOBN(0xe66f9b37, 0xe133e215), + TOBN(0xcdf119a6, 0xe39a7a5c), TOBN(0x47c60de3, 0x876f1b61), + TOBN(0x6e405939, 0xd860f1b2), TOBN(0x3e9a1dbc, 0xf5ed4d4a), + TOBN(0x3f23619e, 0xc9b6bcbd), TOBN(0x5ee790cf, 0x734e4497), + TOBN(0xf0a834b1, 0x5bdaf9bb), TOBN(0x02cedda7, 0x4ca295f0), + TOBN(0x4619aa2b, 0xcb8e378c), TOBN(0xe5613244, 0xcc987ea4), + TOBN(0x0bc022cc, 0x76b23a50), TOBN(0x4a2793ad, 0x0a6c21ce), + TOBN(0x38328780, 0x89cac3f5), TOBN(0x29176f1b, 0xcba26d56), + TOBN(0x06296187, 0x4f6f59eb), TOBN(0x86e9bca9, 0x8bdc658e), + TOBN(0x2ca9c4d3, 0x57e30402), TOBN(0x5438b216, 0x516a09bb), + TOBN(0x0a6a063c, 0x7672765a), TOBN(0x37a3ce64, 0x0547b9bf), + TOBN(0x42c099c8, 0x98b1a633), TOBN(0xb5ab800d, 0x05ee6961), + TOBN(0xf1963f59, 0x11a5acd6), TOBN(0xbaee6157, 0x46201063), + TOBN(0x36d9a649, 0xa596210a), TOBN(0xaed04363, 0x1ba7138c), + TOBN(0xcf817d1c, 0xa4a82b76), TOBN(0x5586960e, 0xf3806be9), + TOBN(0x7ab67c89, 0x09dc6bb5), TOBN(0x52ace7a0, 0x114fe7eb), + TOBN(0xcd987618, 0xcbbc9b70), TOBN(0x4f06fd5a, 0x604ca5e1), + TOBN(0x90af14ca, 0x6dbde133), TOBN(0x1afe4322, 0x948a3264), + TOBN(0xa70d2ca6, 0xc44b2c6c), TOBN(0xab726799, 0x0ef87dfe), + TOBN(0x310f64dc, 0x2e696377), TOBN(0x49b42e68, 0x4c8126a0), + TOBN(0x0ea444c3, 0xcea0b176), TOBN(0x53a8ddf7, 0xcb269182), + TOBN(0xf3e674eb, 0xbbba9dcb), TOBN(0x0d2878a8, 0xd8669d33), + TOBN(0x04b935d5, 0xd019b6a3), TOBN(0xbb5cf88e, 0x406f1e46), + TOBN(0xa1912d16, 0x5b57c111), TOBN(0x9803fc21, 0x19ebfd78), + TOBN(0x4f231c9e, 0xc07764a9), TOBN(0xd93286ee, 0xb75bd055), + TOBN(0x83a9457d, 0x8ee6c9de), TOBN(0x04695915, 0x6087ec90), + TOBN(0x14c6dd8a, 0x58d6cd46), TOBN(0x9cb633b5, 0x8e6634d2), + TOBN(0xc1305047, 0xf81bc328), TOBN(0x12ede0e2, 0x26a177e5), + TOBN(0x332cca62, 0x065a6f4f), TOBN(0xc3a47ecd, 0x67be487b), + TOBN(0x741eb187, 0x0f47ed1c), TOBN(0x99e66e58, 0xe7598b14), + TOBN(0x6f0544ca, 0x63d0ff12), TOBN(0xe5efc784, 0xb610a05f), + TOBN(0xf72917b1, 0x7cad7b47), TOBN(0x3ff6ea20, 0xf2cac0c0), + TOBN(0xcc23791b, 0xf21db8b7), TOBN(0x7dac70b1, 0xd7d93565), + TOBN(0x682cda1d, 0x694bdaad), TOBN(0xeb88bb8c, 0x1023516d), + TOBN(0xc4c634b4, 0xdfdbeb1b), TOBN(0x22f5ca72, 0xb4ee4dea), + TOBN(0x1045a368, 0xe6524821), TOBN(0xed9e8a3f, 0x052b18b2), + TOBN(0x9b7f2cb1, 0xb961f49a), TOBN(0x7fee2ec1, 0x7b009670), + TOBN(0x350d8754, 0x22507a6d), TOBN(0x561bd711, 0x4db55f1d), + TOBN(0x4c189ccc, 0x320bbcaf), TOBN(0x568434cf, 0xdf1de48c), + TOBN(0x6af1b00e, 0x0fa8f128), TOBN(0xf0ba9d02, 0x8907583c), + TOBN(0x735a4004, 0x32ff9f60), TOBN(0x3dd8e4b6, 0xc25dcf33), + TOBN(0xf2230f16, 0x42c74cef), TOBN(0xd8117623, 0x013fa8ad), + TOBN(0x36822876, 0xf51fe76e), TOBN(0x8a6811cc, 0x11d62589), + TOBN(0xc3fc7e65, 0x46225718), TOBN(0xb7df2c9f, 0xc82fdbcd), + TOBN(0x3b1d4e52, 0xdd7b205b), TOBN(0xb6959478, 0x47a2e414), + TOBN(0x05e4d793, 0xefa91148), TOBN(0xb47ed446, 0xfd2e9675), + TOBN(0x1a7098b9, 0x04c9d9bf), TOBN(0x661e2881, 0x1b793048), + TOBN(0xb1a16966, 0xb01ee461), TOBN(0xbc521308, 0x2954746f), + TOBN(0xc909a0fc, 0x2477de50), TOBN(0xd80bb41c, 0x7dbd51ef), + TOBN(0xa85be7ec, 0x53294905), TOBN(0x6d465b18, 0x83958f97), + TOBN(0x16f6f330, 0xfb6840fd), TOBN(0xfaaeb214, 0x3401e6c8), + TOBN(0xaf83d30f, 0xccb5b4f8), TOBN(0x22885739, 0x266dec4b), + TOBN(0x51b4367c, 0x7bc467df), TOBN(0x926562e3, 0xd842d27a), + TOBN(0xdfcb6614, 0x0fea14a6), TOBN(0xeb394dae, 0xf2734cd9), + TOBN(0x3eeae5d2, 0x11c0be98), TOBN(0xb1e6ed11, 0x814e8165), + TOBN(0x191086bc, 0xe52bce1c), TOBN(0x14b74cc6, 0xa75a04da), + TOBN(0x63cf1186, 0x8c060985), TOBN(0x071047de, 0x2dbd7f7c), + TOBN(0x4e433b8b, 0xce0942ca), TOBN(0xecbac447, 0xd8fec61d), + TOBN(0x8f0ed0e2, 0xebf3232f), TOBN(0xfff80f9e, 0xc52a2edd), + TOBN(0xad9ab433, 0x75b55fdb), TOBN(0x73ca7820, 0xe42e0c11), + TOBN(0x6dace0a0, 0xe6251b46), TOBN(0x89bc6b5c, 0x4c0d932d), + TOBN(0x3438cd77, 0x095da19a), TOBN(0x2f24a939, 0x8d48bdfb), + TOBN(0x99b47e46, 0x766561b7), TOBN(0x736600e6, 0x0ed0322a), + TOBN(0x06a47cb1, 0x638e1865), TOBN(0x927c1c2d, 0xcb136000), + TOBN(0x29542337, 0x0cc5df69), TOBN(0x99b37c02, 0x09d649a9), + TOBN(0xc5f0043c, 0x6aefdb27), TOBN(0x6cdd9987, 0x1be95c27), + TOBN(0x69850931, 0x390420d2), TOBN(0x299c40ac, 0x0983efa4), + TOBN(0x3a05e778, 0xaf39aead), TOBN(0x84274408, 0x43a45193), + TOBN(0x6bcd0fb9, 0x91a711a0), TOBN(0x461592c8, 0x9f52ab17), + TOBN(0xb49302b4, 0xda3c6ed6), TOBN(0xc51fddc7, 0x330d7067), + TOBN(0x94babeb6, 0xda50d531), TOBN(0x521b840d, 0xa6a7b9da), + TOBN(0x5305151e, 0x404bdc89), TOBN(0x1bcde201, 0xd0d07449), + TOBN(0xf427a78b, 0x3b76a59a), TOBN(0xf84841ce, 0x07791a1b), + TOBN(0xebd314be, 0xbf91ed1c), TOBN(0x8e61d34c, 0xbf172943), + TOBN(0x1d5dc451, 0x5541b892), TOBN(0xb186ee41, 0xfc9d9e54), + TOBN(0x9d9f345e, 0xd5bf610d), TOBN(0x3e7ba65d, 0xf6acca9f), + TOBN(0x9dda787a, 0xa8369486), TOBN(0x09f9dab7, 0x8eb5ba53), + TOBN(0x5afb2033, 0xd6481bc3), TOBN(0x76f4ce30, 0xafa62104), + TOBN(0xa8fa00cf, 0xf4f066b5), TOBN(0x89ab5143, 0x461dafc2), + TOBN(0x44339ed7, 0xa3389998), TOBN(0x2ff862f1, 0xbc214903), + TOBN(0x2c88f985, 0xb05556e3), TOBN(0xcd96058e, 0x3467081e), + TOBN(0x7d6a4176, 0xedc637ea), TOBN(0xe1743d09, 0x36a5acdc), + TOBN(0x66fd72e2, 0x7eb37726), TOBN(0xf7fa264e, 0x1481a037), + TOBN(0x9fbd3bde, 0x45f4aa79), TOBN(0xed1e0147, 0x767c3e22), + TOBN(0x7621f979, 0x82e7abe2), TOBN(0x19eedc72, 0x45f633f8), + TOBN(0xe69b155e, 0x6137bf3a), TOBN(0xa0ad13ce, 0x414ee94e), + TOBN(0x93e3d524, 0x1c0e651a), TOBN(0xab1a6e2a, 0x02ce227e), + TOBN(0xe7af1797, 0x4ab27eca), TOBN(0x245446de, 0xbd444f39), + TOBN(0x59e22a21, 0x56c07613), TOBN(0x43deafce, 0xf4275498), + TOBN(0x10834ccb, 0x67fd0946), TOBN(0xa75841e5, 0x47406edf), + TOBN(0xebd6a677, 0x7b0ac93d), TOBN(0xa6e37b0d, 0x78f5e0d7), + TOBN(0x2516c096, 0x76f5492b), TOBN(0x1e4bf888, 0x9ac05f3a), + TOBN(0xcdb42ce0, 0x4df0ba2b), TOBN(0x935d5cfd, 0x5062341b), + TOBN(0x8a303333, 0x82acac20), TOBN(0x429438c4, 0x5198b00e), + TOBN(0x1d083bc9, 0x049d33fa), TOBN(0x58b82dda, 0x946f67ff), + TOBN(0xac3e2db8, 0x67a1d6a3), TOBN(0x62e6bead, 0x1798aac8), + TOBN(0xfc85980f, 0xde46c58c), TOBN(0xa7f69379, 0x69c8d7be), + TOBN(0x23557927, 0x837b35ec), TOBN(0x06a933d8, 0xe0790c0c), + TOBN(0x827c0e9b, 0x077ff55d), TOBN(0x53977798, 0xbb26e680), + TOBN(0x59530874, 0x1d9cb54f), TOBN(0xcca3f449, 0x4aac53ef), + TOBN(0x11dc5c87, 0xa07eda0f), TOBN(0xc138bccf, 0xfd6400c8), + TOBN(0x549680d3, 0x13e5da72), TOBN(0xc93eed82, 0x4540617e), + TOBN(0xfd3db157, 0x4d0b75c0), TOBN(0x9716eb42, 0x6386075b), + TOBN(0x0639605c, 0x817b2c16), TOBN(0x09915109, 0xf1e4f201), + TOBN(0x35c9a928, 0x5cca6c3b), TOBN(0xb25f7d1a, 0x3505c900), + TOBN(0xeb9f7d20, 0x630480c4), TOBN(0xc3c7b8c6, 0x2a1a501c), + TOBN(0x3f99183c, 0x5a1f8e24), TOBN(0xfdb118fa, 0x9dd255f0), + TOBN(0xb9b18b90, 0xc27f62a6), TOBN(0xe8f732f7, 0x396ec191), + TOBN(0x524a2d91, 0x0be786ab), TOBN(0x5d32adef, 0x0ac5a0f5), + TOBN(0x9b53d4d6, 0x9725f694), TOBN(0x032a76c6, 0x0510ba89), + TOBN(0x840391a3, 0xebeb1544), TOBN(0x44b7b88c, 0x3ed73ac3), + TOBN(0xd24bae7a, 0x256cb8b3), TOBN(0x7ceb151a, 0xe394cb12), + TOBN(0xbd6b66d0, 0x5bc1e6a8), TOBN(0xec70cecb, 0x090f07bf), + TOBN(0x270644ed, 0x7d937589), TOBN(0xee9e1a3d, 0x5f1dccfe), + TOBN(0xb0d40a84, 0x745b98d2), TOBN(0xda429a21, 0x2556ed40), + TOBN(0xf676eced, 0x85148cb9), TOBN(0x5a22d40c, 0xded18936), + TOBN(0x3bc4b9e5, 0x70e8a4ce), TOBN(0xbfd1445b, 0x9eae0379), + TOBN(0xf23f2c0c, 0x1a0bd47e), TOBN(0xa9c0bb31, 0xe1845531), + TOBN(0x9ddc4d60, 0x0a4c3f6b), TOBN(0xbdfaad79, 0x2c15ef44), + TOBN(0xce55a236, 0x7f484acc), TOBN(0x08653ca7, 0x055b1f15), + TOBN(0x2efa8724, 0x538873a3), TOBN(0x09299e5d, 0xace1c7e7), + TOBN(0x07afab66, 0xade332ba), TOBN(0x9be1fdf6, 0x92dd71b7), + TOBN(0xa49b5d59, 0x5758b11c), TOBN(0x0b852893, 0xc8654f40), + TOBN(0xb63ef6f4, 0x52379447), TOBN(0xd4957d29, 0x105e690c), + TOBN(0x7d484363, 0x646559b0), TOBN(0xf4a8273c, 0x49788a8e), + TOBN(0xee406cb8, 0x34ce54a9), TOBN(0x1e1c260f, 0xf86fda9b), + TOBN(0xe150e228, 0xcf6a4a81), TOBN(0x1fa3b6a3, 0x1b488772), + TOBN(0x1e6ff110, 0xc5a9c15b), TOBN(0xc6133b91, 0x8ad6aa47), + TOBN(0x8ac5d55c, 0x9dffa978), TOBN(0xba1d1c1d, 0x5f3965f2), + TOBN(0xf969f4e0, 0x7732b52f), TOBN(0xfceecdb5, 0xa5172a07), + TOBN(0xb0120a5f, 0x10f2b8f5), TOBN(0xc83a6cdf, 0x5c4c2f63), + TOBN(0x4d47a491, 0xf8f9c213), TOBN(0xd9e1cce5, 0xd3f1bbd5), + TOBN(0x0d91bc7c, 0xaba7e372), TOBN(0xfcdc74c8, 0xdfd1a2db), + TOBN(0x05efa800, 0x374618e5), TOBN(0x11216969, 0x15a7925e), + TOBN(0xd4c89823, 0xf6021c5d), TOBN(0x880d5e84, 0xeff14423), + TOBN(0x6523bc5a, 0x6dcd1396), TOBN(0xd1acfdfc, 0x113c978b), + TOBN(0xb0c164e8, 0xbbb66840), TOBN(0xf7f4301e, 0x72b58459), + TOBN(0xc29ad4a6, 0xa638e8ec), TOBN(0xf5ab8961, 0x46b78699), + TOBN(0x9dbd7974, 0x0e954750), TOBN(0x0121de88, 0x64f9d2c6), + TOBN(0x2e597b42, 0xd985232e), TOBN(0x55b6c3c5, 0x53451777), + TOBN(0xbb53e547, 0x519cb9fb), TOBN(0xf134019f, 0x8428600d), + TOBN(0x5a473176, 0xe081791a), TOBN(0x2f3e2263, 0x35fb0c08), + TOBN(0xb28c3017, 0x73d273b0), TOBN(0xccd21076, 0x7721ef9a), + TOBN(0x054cc292, 0xb650dc39), TOBN(0x662246de, 0x6188045e), + TOBN(0x904b52fa, 0x6b83c0d1), TOBN(0xa72df267, 0x97e9cd46), + TOBN(0x886b43cd, 0x899725e4), TOBN(0x2b651688, 0xd849ff22), + TOBN(0x60479b79, 0x02f34533), TOBN(0x5e354c14, 0x0c77c148), + TOBN(0xb4bb7581, 0xa8537c78), TOBN(0x188043d7, 0xefe1495f), + TOBN(0x9ba12f42, 0x8c1d5026), TOBN(0x2e0c8a26, 0x93d4aaab), + TOBN(0xbdba7b8b, 0xaa57c450), TOBN(0x140c9ad6, 0x9bbdafef), + TOBN(0x2067aa42, 0x25ac0f18), TOBN(0xf7b1295b, 0x04d1fbf3), + TOBN(0x14829111, 0xa4b04824), TOBN(0x2ce3f192, 0x33bd5e91), + TOBN(0x9c7a1d55, 0x8f2e1b72), TOBN(0xfe932286, 0x302aa243), + TOBN(0x497ca7b4, 0xd4be9554), TOBN(0xb8e821b8, 0xe0547a6e), + TOBN(0xfb2838be, 0x67e573e0), TOBN(0x05891db9, 0x4084c44b), + TOBN(0x91311373, 0x96c1c2c5), TOBN(0x6aebfa3f, 0xd958444b), + TOBN(0xac9cdce9, 0xe56e55c1), TOBN(0x7148ced3, 0x2caa46d0), + TOBN(0x2e10c7ef, 0xb61fe8eb), TOBN(0x9fd835da, 0xff97cf4d),} + , + {TOBN(0xa36da109, 0x081e9387), TOBN(0xfb9780d7, 0x8c935828), + TOBN(0xd5940332, 0xe540b015), TOBN(0xc9d7b51b, 0xe0f466fa), + TOBN(0xfaadcd41, 0xd6d9f671), TOBN(0xba6c1e28, 0xb1a2ac17), + TOBN(0x066a7833, 0xed201e5f), TOBN(0x19d99719, 0xf90f462b), + TOBN(0xf431f462, 0x060b5f61), TOBN(0xa56f46b4, 0x7bd057c2), + TOBN(0x348dca6c, 0x47e1bf65), TOBN(0x9a38783e, 0x41bcf1ff), + TOBN(0x7a5d33a9, 0xda710718), TOBN(0x5a779987, 0x2e0aeaf6), + TOBN(0xca87314d, 0x2d29d187), TOBN(0xfa0edc3e, 0xc687d733), + TOBN(0x9df33621, 0x6a31e09b), TOBN(0xde89e44d, 0xc1350e35), + TOBN(0x29214871, 0x4ca0cf52), TOBN(0xdf379672, 0x0b88a538), + TOBN(0xc92a510a, 0x2591d61b), TOBN(0x79aa87d7, 0x585b447b), + TOBN(0xf67db604, 0xe5287f77), TOBN(0x1697c8bf, 0x5efe7a80), + TOBN(0x1c894849, 0xcb198ac7), TOBN(0xa884a93d, 0x0f264665), + TOBN(0x2da964ef, 0x9b200678), TOBN(0x3c351b87, 0x009834e6), + TOBN(0xafb2ef9f, 0xe2c4b44b), TOBN(0x580f6c47, 0x3326790c), + TOBN(0xb8480521, 0x0b02264a), TOBN(0x8ba6f9e2, 0x42a194e2), + TOBN(0xfc87975f, 0x8fb54738), TOBN(0x35160788, 0x27c3ead3), + TOBN(0x834116d2, 0xb74a085a), TOBN(0x53c99a73, 0xa62fe996), + TOBN(0x87585be0, 0x5b81c51b), TOBN(0x925bafa8, 0xbe0852b7), + TOBN(0x76a4fafd, 0xa84d19a7), TOBN(0x39a45982, 0x585206d4), + TOBN(0x499b6ab6, 0x5eb03c0e), TOBN(0xf19b7954, 0x72bc3fde), + TOBN(0xa86b5b9c, 0x6e3a80d2), TOBN(0xe4377508, 0x6d42819f), + TOBN(0xc1663650, 0xbb3ee8a3), TOBN(0x75eb14fc, 0xb132075f), + TOBN(0xa8ccc906, 0x7ad834f6), TOBN(0xea6a2474, 0xe6e92ffd), + TOBN(0x9d72fd95, 0x0f8d6758), TOBN(0xcb84e101, 0x408c07dd), + TOBN(0xb9114bfd, 0xa5e23221), TOBN(0x358b5fe2, 0xe94e742c), + TOBN(0x1c0577ec, 0x95f40e75), TOBN(0xf0155451, 0x3d73f3d6), + TOBN(0x9d55cd67, 0xbd1b9b66), TOBN(0x63e86e78, 0xaf8d63c7), + TOBN(0x39d934ab, 0xd3c095f1), TOBN(0x04b261be, 0xe4b76d71), + TOBN(0x1d2e6970, 0xe73e6984), TOBN(0x879fb23b, 0x5e5fcb11), + TOBN(0x11506c72, 0xdfd75490), TOBN(0x3a97d085, 0x61bcf1c1), + TOBN(0x43201d82, 0xbf5e7007), TOBN(0x7f0ac52f, 0x798232a7), + TOBN(0x2715cbc4, 0x6eb564d4), TOBN(0x8d6c752c, 0x9e570e29), + TOBN(0xf80247c8, 0x9ef5fd5d), TOBN(0xc3c66b46, 0xd53eb514), + TOBN(0x9666b401, 0x0f87de56), TOBN(0xce62c06f, 0xc6c603b5), + TOBN(0xae7b4c60, 0x7e4fc942), TOBN(0x38ac0b77, 0x663a9c19), + TOBN(0xcb4d20ee, 0x4b049136), TOBN(0x8b63bf12, 0x356a4613), + TOBN(0x1221aef6, 0x70e08128), TOBN(0xe62d8c51, 0x4acb6b16), + TOBN(0x71f64a67, 0x379e7896), TOBN(0xb25237a2, 0xcafd7fa5), + TOBN(0xf077bd98, 0x3841ba6a), TOBN(0xc4ac0244, 0x3cd16e7e), + TOBN(0x548ba869, 0x21fea4ca), TOBN(0xd36d0817, 0xf3dfdac1), + TOBN(0x09d8d71f, 0xf4685faf), TOBN(0x8eff66be, 0xc52c459a), + TOBN(0x182faee7, 0x0b57235e), TOBN(0xee3c39b1, 0x0106712b), + TOBN(0x5107331f, 0xc0fcdcb0), TOBN(0x669fb9dc, 0xa51054ba), + TOBN(0xb25101fb, 0x319d7682), TOBN(0xb0293129, 0x0a982fee), + TOBN(0x51c1c9b9, 0x0261b344), TOBN(0x0e008c5b, 0xbfd371fa), + TOBN(0xd866dd1c, 0x0278ca33), TOBN(0x666f76a6, 0xe5aa53b1), + TOBN(0xe5cfb779, 0x6013a2cf), TOBN(0x1d3a1aad, 0xa3521836), + TOBN(0xcedd2531, 0x73faa485), TOBN(0xc8ee6c4f, 0xc0a76878), + TOBN(0xddbccfc9, 0x2a11667d), TOBN(0x1a418ea9, 0x1c2f695a), + TOBN(0xdb11bd92, 0x51f73971), TOBN(0x3e4b3c82, 0xda2ed89f), + TOBN(0x9a44f3f4, 0xe73e0319), TOBN(0xd1e3de0f, 0x303431af), + TOBN(0x3c5604ff, 0x50f75f9c), TOBN(0x1d8eddf3, 0x7e752b22), + TOBN(0x0ef074dd, 0x3c9a1118), TOBN(0xd0ffc172, 0xccb86d7b), + TOBN(0xabd1ece3, 0x037d90f2), TOBN(0xe3f307d6, 0x6055856c), + TOBN(0x422f9328, 0x7e4c6daf), TOBN(0x902aac66, 0x334879a0), + TOBN(0xb6a1e7bf, 0x94cdfade), TOBN(0x6c97e1ed, 0x7fc6d634), + TOBN(0x662ad24d, 0xa2fb63f8), TOBN(0xf81be1b9, 0xa5928405), + TOBN(0x86d765e4, 0xd14b4206), TOBN(0xbecc2e0e, 0x8fa0db65), + TOBN(0xa28838e0, 0xb17fc76c), TOBN(0xe49a602a, 0xe37cf24e), + TOBN(0x76b4131a, 0x567193ec), TOBN(0xaf3c305a, 0xe5f6e70b), + TOBN(0x9587bd39, 0x031eebdd), TOBN(0x5709def8, 0x71bbe831), + TOBN(0x57059983, 0x0eb2b669), TOBN(0x4d80ce1b, 0x875b7029), + TOBN(0x838a7da8, 0x0364ac16), TOBN(0x2f431d23, 0xbe1c83ab), + TOBN(0xe56812a6, 0xf9294dd3), TOBN(0xb448d01f, 0x9b4b0d77), + TOBN(0xf3ae6061, 0x04e8305c), TOBN(0x2bead645, 0x94d8c63e), + TOBN(0x0a85434d, 0x84fd8b07), TOBN(0x537b983f, 0xf7a9dee5), + TOBN(0xedcc5f18, 0xef55bd85), TOBN(0x2041af62, 0x21c6cf8b), + TOBN(0x8e52874c, 0xb940c71e), TOBN(0x211935a9, 0xdb5f4b3a), + TOBN(0x94350492, 0x301b1dc3), TOBN(0x33d2646d, 0x29958620), + TOBN(0x16b0d64b, 0xef911404), TOBN(0x9d1f25ea, 0x9a3c5ef4), + TOBN(0x20f200eb, 0x4a352c78), TOBN(0x43929f2c, 0x4bd0b428), + TOBN(0xa5656667, 0xc7196e29), TOBN(0x7992c2f0, 0x9391be48), + TOBN(0xaaa97cbd, 0x9ee0cd6e), TOBN(0x51b0310c, 0x3dc8c9bf), + TOBN(0x237f8acf, 0xdd9f22cb), TOBN(0xbb1d81a1, 0xb585d584), + TOBN(0x8d5d85f5, 0x8c416388), TOBN(0x0d6e5a5a, 0x42fe474f), + TOBN(0xe7812766, 0x38235d4e), TOBN(0x1c62bd67, 0x496e3298), + TOBN(0x8378660c, 0x3f175bc8), TOBN(0x4d04e189, 0x17afdd4d), + TOBN(0x32a81601, 0x85a8068c), TOBN(0xdb58e4e1, 0x92b29a85), + TOBN(0xe8a65b86, 0xc70d8a3b), TOBN(0x5f0e6f4e, 0x98a0403b), + TOBN(0x08129684, 0x69ed2370), TOBN(0x34dc30bd, 0x0871ee26), + TOBN(0x3a5ce948, 0x7c9c5b05), TOBN(0x7d487b80, 0x43a90c87), + TOBN(0x4089ba37, 0xdd0e7179), TOBN(0x45f80191, 0xb4041811), + TOBN(0x1c3e1058, 0x98747ba5), TOBN(0x98c4e13a, 0x6e1ae592), + TOBN(0xd44636e6, 0xe82c9f9e), TOBN(0x711db87c, 0xc33a1043), + TOBN(0x6f431263, 0xaa8aec05), TOBN(0x43ff120d, 0x2744a4aa), + TOBN(0xd3bd892f, 0xae77779b), TOBN(0xf0fe0cc9, 0x8cdc9f82), + TOBN(0xca5f7fe6, 0xf1c5b1bc), TOBN(0xcc63a682, 0x44929a72), + TOBN(0xc7eaba0c, 0x09dbe19a), TOBN(0x2f3585ad, 0x6b5c73c2), + TOBN(0x8ab8924b, 0x0ae50c30), TOBN(0x17fcd27a, 0x638b30ba), + TOBN(0xaf414d34, 0x10b3d5a5), TOBN(0x09c107d2, 0x2a9accf1), + TOBN(0x15dac49f, 0x946a6242), TOBN(0xaec3df2a, 0xd707d642), + TOBN(0x2c2492b7, 0x3f894ae0), TOBN(0xf59df3e5, 0xb75f18ce), + TOBN(0x7cb740d2, 0x8f53cad0), TOBN(0x3eb585fb, 0xc4f01294), + TOBN(0x17da0c86, 0x32c7f717), TOBN(0xeb8c795b, 0xaf943f4c), + TOBN(0x4ee23fb5, 0xf67c51d2), TOBN(0xef187575, 0x68889949), + TOBN(0xa6b4bdb2, 0x0389168b), TOBN(0xc4ecd258, 0xea577d03), + TOBN(0x3a63782b, 0x55743082), TOBN(0x6f678f4c, 0xc72f08cd), + TOBN(0x553511cf, 0x65e58dd8), TOBN(0xd53b4e3e, 0xd402c0cd), + TOBN(0x37de3e29, 0xa037c14c), TOBN(0x86b6c516, 0xc05712aa), + TOBN(0x2834da3e, 0xb38dff6f), TOBN(0xbe012c52, 0xea636be8), + TOBN(0x292d238c, 0x61dd37f8), TOBN(0x0e54523f, 0x8f8142db), + TOBN(0xe31eb436, 0x036a05d8), TOBN(0x83e3cdff, 0x1e93c0ff), + TOBN(0x3fd2fe0f, 0x50821ddf), TOBN(0xc8e19b0d, 0xff9eb33b), + TOBN(0xc8cc943f, 0xb569a5fe), TOBN(0xad0090d4, 0xd4342d75), + TOBN(0x82090b4b, 0xcaeca000), TOBN(0xca39687f, 0x1bd410eb), + TOBN(0xe7bb0df7, 0x65959d77), TOBN(0x39d78218, 0x9c964999), + TOBN(0xd87f62e8, 0xb2415451), TOBN(0xe5efb774, 0xbed76108), + TOBN(0x3ea011a4, 0xe822f0d0), TOBN(0xbc647ad1, 0x5a8704f8), + TOBN(0xbb315b35, 0x50c6820f), TOBN(0x863dec3d, 0xb7e76bec), + TOBN(0x01ff5d3a, 0xf017bfc7), TOBN(0x20054439, 0x976b8229), + TOBN(0x067fca37, 0x0bbd0d3b), TOBN(0xf63dde64, 0x7f5e3d0f), + TOBN(0x22dbefb3, 0x2a4c94e9), TOBN(0xafbff0fe, 0x96f8278a), + TOBN(0x80aea0b1, 0x3503793d), TOBN(0xb2238029, 0x5f06cd29), + TOBN(0x65703e57, 0x8ec3feca), TOBN(0x06c38314, 0x393e7053), + TOBN(0xa0b751eb, 0x7c6734c4), TOBN(0xd2e8a435, 0xc59f0f1e), + TOBN(0x147d9052, 0x5e9ca895), TOBN(0x2f4dd31e, 0x972072df), + TOBN(0xa16fda8e, 0xe6c6755c), TOBN(0xc66826ff, 0xcf196558), + TOBN(0x1f1a76a3, 0x0cf43895), TOBN(0xa9d604e0, 0x83c3097b), + TOBN(0xe1908309, 0x66390e0e), TOBN(0xa50bf753, 0xb3c85eff), + TOBN(0x0696bdde, 0xf6a70251), TOBN(0x548b801b, 0x3c6ab16a), + TOBN(0x37fcf704, 0xa4d08762), TOBN(0x090b3def, 0xdff76c4e), + TOBN(0x87e8cb89, 0x69cb9158), TOBN(0x44a90744, 0x995ece43), + TOBN(0xf85395f4, 0x0ad9fbf5), TOBN(0x49b0f6c5, 0x4fb0c82d), + TOBN(0x75d9bc15, 0xadf7cccf), TOBN(0x81a3e5d6, 0xdfa1e1b0), + TOBN(0x8c39e444, 0x249bc17e), TOBN(0xf37dccb2, 0x8ea7fd43), + TOBN(0xda654873, 0x907fba12), TOBN(0x35daa6da, 0x4a372904), + TOBN(0x0564cfc6, 0x6283a6c5), TOBN(0xd09fa4f6, 0x4a9395bf), + TOBN(0x688e9ec9, 0xaeb19a36), TOBN(0xd913f1ce, 0xc7bfbfb4), + TOBN(0x797b9a3c, 0x61c2faa6), TOBN(0x2f979bec, 0x6a0a9c12), + TOBN(0xb5969d0f, 0x359679ec), TOBN(0xebcf523d, 0x079b0460), + TOBN(0xfd6b0008, 0x10fab870), TOBN(0x3f2edcda, 0x9373a39c), + TOBN(0x0d64f9a7, 0x6f568431), TOBN(0xf848c27c, 0x02f8898c), + TOBN(0xf418ade1, 0x260b5bd5), TOBN(0xc1f3e323, 0x6973dee8), + TOBN(0x46e9319c, 0x26c185dd), TOBN(0x6d85b7d8, 0x546f0ac4), + TOBN(0x427965f2, 0x247f9d57), TOBN(0xb519b636, 0xb0035f48), + TOBN(0x6b6163a9, 0xab87d59c), TOBN(0xff9f58c3, 0x39caaa11), + TOBN(0x4ac39cde, 0x3177387b), TOBN(0x5f6557c2, 0x873e77f9), + TOBN(0x67504006, 0x36a83041), TOBN(0x9b1c96ca, 0x75ef196c), + TOBN(0xf34283de, 0xb08c7940), TOBN(0x7ea09644, 0x1128c316), + TOBN(0xb510b3b5, 0x6aa39dff), TOBN(0x59b43da2, 0x9f8e4d8c), + TOBN(0xa8ce31fd, 0x9e4c4b9f), TOBN(0x0e20be26, 0xc1303c01), + TOBN(0x18187182, 0xe8ee47c9), TOBN(0xd9687cdb, 0x7db98101), + TOBN(0x7a520e4d, 0xa1e14ff6), TOBN(0x429808ba, 0x8836d572), + TOBN(0xa37ca60d, 0x4944b663), TOBN(0xf901f7a9, 0xa3f91ae5), + TOBN(0xe4e3e76e, 0x9e36e3b1), TOBN(0x9aa219cf, 0x29d93250), + TOBN(0x347fe275, 0x056a2512), TOBN(0xa4d643d9, 0xde65d95c), + TOBN(0x9669d396, 0x699fc3ed), TOBN(0xb598dee2, 0xcf8c6bbe), + TOBN(0x682ac1e5, 0xdda9e5c6), TOBN(0x4e0d3c72, 0xcaa9fc95), + TOBN(0x17faaade, 0x772bea44), TOBN(0x5ef8428c, 0xab0009c8), + TOBN(0xcc4ce47a, 0x460ff016), TOBN(0xda6d12bf, 0x725281cb), + TOBN(0x44c67848, 0x0223aad2), TOBN(0x6e342afa, 0x36256e28), + TOBN(0x1400bb0b, 0x93a37c04), TOBN(0x62b1bc9b, 0xdd10bd96), + TOBN(0x7251adeb, 0x0dac46b7), TOBN(0x7d33b92e, 0x7be4ef51), + TOBN(0x28b2a94b, 0xe61fa29a), TOBN(0x4b2be13f, 0x06422233), + TOBN(0x36d6d062, 0x330d8d37), TOBN(0x5ef80e1e, 0xb28ca005), + TOBN(0x174d4699, 0x6d16768e), TOBN(0x9fc4ff6a, 0x628bf217), + TOBN(0x77705a94, 0x154e490d), TOBN(0x9d96dd28, 0x8d2d997a), + TOBN(0x77e2d9d8, 0xce5d72c4), TOBN(0x9d06c5a4, 0xc11c714f), + TOBN(0x02aa5136, 0x79e4a03e), TOBN(0x1386b3c2, 0x030ff28b), + TOBN(0xfe82e8a6, 0xfb283f61), TOBN(0x7df203e5, 0xf3abc3fb), + TOBN(0xeec7c351, 0x3a4d3622), TOBN(0xf7d17dbf, 0xdf762761), + TOBN(0xc3956e44, 0x522055f0), TOBN(0xde3012db, 0x8fa748db), + TOBN(0xca9fcb63, 0xbf1dcc14), TOBN(0xa56d9dcf, 0xbe4e2f3a), + TOBN(0xb86186b6, 0x8bcec9c2), TOBN(0x7cf24df9, 0x680b9f06), + TOBN(0xc46b45ea, 0xc0d29281), TOBN(0xfff42bc5, 0x07b10e12), + TOBN(0x12263c40, 0x4d289427), TOBN(0x3d5f1899, 0xb4848ec4), + TOBN(0x11f97010, 0xd040800c), TOBN(0xb4c5f529, 0x300feb20), + TOBN(0xcc543f8f, 0xde94fdcb), TOBN(0xe96af739, 0xc7c2f05e), + TOBN(0xaa5e0036, 0x882692e1), TOBN(0x09c75b68, 0x950d4ae9), + TOBN(0x62f63df2, 0xb5932a7a), TOBN(0x2658252e, 0xde0979ad), + TOBN(0x2a19343f, 0xb5e69631), TOBN(0x718c7501, 0x525b666b), + TOBN(0x26a42d69, 0xea40dc3a), TOBN(0xdc84ad22, 0xaecc018f), + TOBN(0x25c36c7b, 0x3270f04a), TOBN(0x46ba6d47, 0x50fa72ed), + TOBN(0x6c37d1c5, 0x93e58a8e), TOBN(0xa2394731, 0x120c088c), + TOBN(0xc3be4263, 0xcb6e86da), TOBN(0x2c417d36, 0x7126d038), + TOBN(0x5b70f9c5, 0x8b6f8efa), TOBN(0x671a2faa, 0x37718536), + TOBN(0xd3ced3c6, 0xb539c92b), TOBN(0xe56f1bd9, 0xa31203c2), + TOBN(0x8b096ec4, 0x9ff3c8eb), TOBN(0x2deae432, 0x43491cea), + TOBN(0x2465c6eb, 0x17943794), TOBN(0x5d267e66, 0x20586843), + TOBN(0x9d3d116d, 0xb07159d0), TOBN(0xae07a67f, 0xc1896210), + TOBN(0x8fc84d87, 0xbb961579), TOBN(0x30009e49, 0x1c1f8dd6), + TOBN(0x8a8caf22, 0xe3132819), TOBN(0xcffa197c, 0xf23ab4ff), + TOBN(0x58103a44, 0x205dd687), TOBN(0x57b796c3, 0x0ded67a2), + TOBN(0x0b9c3a6c, 0xa1779ad7), TOBN(0xa33cfe2e, 0x357c09c5), + TOBN(0x2ea29315, 0x3db4a57e), TOBN(0x91959695, 0x8ebeb52e), + TOBN(0x118db9a6, 0xe546c879), TOBN(0x8e996df4, 0x6295c8d6), + TOBN(0xdd990484, 0x55ec806b), TOBN(0x24f291ca, 0x165c1035), + TOBN(0xcca523bb, 0x440e2229), TOBN(0x324673a2, 0x73ef4d04), + TOBN(0xaf3adf34, 0x3e11ec39), TOBN(0x6136d7f1, 0xdc5968d3), + TOBN(0x7a7b2899, 0xb053a927), TOBN(0x3eaa2661, 0xae067ecd), + TOBN(0x8549b9c8, 0x02779cd9), TOBN(0x061d7940, 0xc53385ea), + TOBN(0x3e0ba883, 0xf06d18bd), TOBN(0x4ba6de53, 0xb2700843), + TOBN(0xb966b668, 0x591a9e4d), TOBN(0x93f67567, 0x7f4fa0ed), + TOBN(0x5a02711b, 0x4347237b), TOBN(0xbc041e2f, 0xe794608e), + TOBN(0x55af10f5, 0x70f73d8c), TOBN(0xd2d4d4f7, 0xbb7564f7), + TOBN(0xd7d27a89, 0xb3e93ce7), TOBN(0xf7b5a875, 0x5d3a2c1b), + TOBN(0xb29e68a0, 0x255b218a), TOBN(0xb533837e, 0x8af76754), + TOBN(0xd1b05a73, 0x579fab2e), TOBN(0xb41055a1, 0xecd74385), + TOBN(0xb2369274, 0x445e9115), TOBN(0x2972a7c4, 0xf520274e), + TOBN(0x6c08334e, 0xf678e68a), TOBN(0x4e4160f0, 0x99b057ed), + TOBN(0x3cfe11b8, 0x52ccb69a), TOBN(0x2fd1823a, 0x21c8f772), + TOBN(0xdf7f072f, 0x3298f055), TOBN(0x8c0566f9, 0xfec74a6e), + TOBN(0xe549e019, 0x5bb4d041), TOBN(0x7c3930ba, 0x9208d850), + TOBN(0xe07141fc, 0xaaa2902b), TOBN(0x539ad799, 0xe4f69ad3), + TOBN(0xa6453f94, 0x813f9ffd), TOBN(0xc58d3c48, 0x375bc2f7), + TOBN(0xb3326fad, 0x5dc64e96), TOBN(0x3aafcaa9, 0xb240e354), + TOBN(0x1d1b0903, 0xaca1e7a9), TOBN(0x4ceb9767, 0x1211b8a0), + TOBN(0xeca83e49, 0xe32a858e), TOBN(0x4c32892e, 0xae907bad), + TOBN(0xd5b42ab6, 0x2eb9b494), TOBN(0x7fde3ee2, 0x1eabae1b), + TOBN(0x13b5ab09, 0xcaf54957), TOBN(0xbfb028be, 0xe5f5d5d5), + TOBN(0x928a0650, 0x2003e2c0), TOBN(0x90793aac, 0x67476843), + TOBN(0x5e942e79, 0xc81710a0), TOBN(0x557e4a36, 0x27ccadd4), + TOBN(0x72a2bc56, 0x4bcf6d0c), TOBN(0x09ee5f43, 0x26d7b80c), + TOBN(0x6b70dbe9, 0xd4292f19), TOBN(0x56f74c26, 0x63f16b18), + TOBN(0xc23db0f7, 0x35fbb42a), TOBN(0xb606bdf6, 0x6ae10040), + TOBN(0x1eb15d4d, 0x044573ac), TOBN(0x7dc3cf86, 0x556b0ba4), + TOBN(0x97af9a33, 0xc60df6f7), TOBN(0x0b1ef85c, 0xa716ce8c), + TOBN(0x2922f884, 0xc96958be), TOBN(0x7c32fa94, 0x35690963), + TOBN(0x2d7f667c, 0xeaa00061), TOBN(0xeaaf7c17, 0x3547365c), + TOBN(0x1eb4de46, 0x87032d58), TOBN(0xc54f3d83, 0x5e2c79e0), + TOBN(0x07818df4, 0x5d04ef23), TOBN(0x55faa9c8, 0x673d41b4), + TOBN(0xced64f6f, 0x89b95355), TOBN(0x4860d2ea, 0xb7415c84), + TOBN(0x5fdb9bd2, 0x050ebad3), TOBN(0xdb53e0cc, 0x6685a5bf), + TOBN(0xb830c031, 0x9feb6593), TOBN(0xdd87f310, 0x6accff17), + TOBN(0x2303ebab, 0x9f555c10), TOBN(0x94603695, 0x287e7065), + TOBN(0xf88311c3, 0x2e83358c), TOBN(0x508dd9b4, 0xeefb0178), + TOBN(0x7ca23706, 0x2dba8652), TOBN(0x62aac5a3, 0x0047abe5), + TOBN(0x9a61d2a0, 0x8b1ea7b3), TOBN(0xd495ab63, 0xae8b1485), + TOBN(0x38740f84, 0x87052f99), TOBN(0x178ebe5b, 0xb2974eea), + TOBN(0x030bbcca, 0x5b36d17f), TOBN(0xb5e4cce3, 0xaaf86eea), + TOBN(0xb51a0220, 0x68f8e9e0), TOBN(0xa4348796, 0x09eb3e75), + TOBN(0xbe592309, 0xeef1a752), TOBN(0x5d7162d7, 0x6f2aa1ed), + TOBN(0xaebfb5ed, 0x0f007dd2), TOBN(0x255e14b2, 0xc89edd22), + TOBN(0xba85e072, 0x0303b697), TOBN(0xc5d17e25, 0xf05720ff), + TOBN(0x02b58d6e, 0x5128ebb6), TOBN(0x2c80242d, 0xd754e113), + TOBN(0x919fca5f, 0xabfae1ca), TOBN(0x937afaac, 0x1a21459b), + TOBN(0x9e0ca91c, 0x1f66a4d2), TOBN(0x194cc7f3, 0x23ec1331), + TOBN(0xad25143a, 0x8aa11690), TOBN(0xbe40ad8d, 0x09b59e08), + TOBN(0x37d60d9b, 0xe750860a), TOBN(0x6c53b008, 0xc6bf434c), + TOBN(0xb572415d, 0x1356eb80), TOBN(0xb8bf9da3, 0x9578ded8), + TOBN(0x22658e36, 0x5e8fb38b), TOBN(0x9b70ce22, 0x5af8cb22), + TOBN(0x7c00018a, 0x829a8180), TOBN(0x84329f93, 0xb81ed295), + TOBN(0x7c343ea2, 0x5f3cea83), TOBN(0x38f8655f, 0x67586536), + TOBN(0xa661a0d0, 0x1d3ec517), TOBN(0x98744652, 0x512321ae), + TOBN(0x084ca591, 0xeca92598), TOBN(0xa9bb9dc9, 0x1dcb3feb), + TOBN(0x14c54355, 0x78b4c240), TOBN(0x5ed62a3b, 0x610cafdc), + TOBN(0x07512f37, 0x1b38846b), TOBN(0x571bb70a, 0xb0e38161), + TOBN(0xb556b95b, 0x2da705d2), TOBN(0x3ef8ada6, 0xb1a08f98), + TOBN(0x85302ca7, 0xddecfbe5), TOBN(0x0e530573, 0x943105cd), + TOBN(0x60554d55, 0x21a9255d), TOBN(0x63a32fa1, 0xf2f3802a), + TOBN(0x35c8c5b0, 0xcd477875), TOBN(0x97f458ea, 0x6ad42da1), + TOBN(0x832d7080, 0xeb6b242d), TOBN(0xd30bd023, 0x3b71e246), + TOBN(0x7027991b, 0xbe31139d), TOBN(0x68797e91, 0x462e4e53), + TOBN(0x423fe20a, 0x6b4e185a), TOBN(0x82f2c67e, 0x42d9b707), + TOBN(0x25c81768, 0x4cf7811b), TOBN(0xbd53005e, 0x045bb95d),} + , + {TOBN(0xe5f649be, 0x9d8e68fd), TOBN(0xdb0f0533, 0x1b044320), + TOBN(0xf6fde9b3, 0xe0c33398), TOBN(0x92f4209b, 0x66c8cfae), + TOBN(0xe9d1afcc, 0x1a739d4b), TOBN(0x09aea75f, 0xa28ab8de), + TOBN(0x14375fb5, 0xeac6f1d0), TOBN(0x6420b560, 0x708f7aa5), + TOBN(0x9eae499c, 0x6254dc41), TOBN(0x7e293924, 0x7a837e7e), + TOBN(0x74aec08c, 0x090524a7), TOBN(0xf82b9219, 0x8d6f55f2), + TOBN(0x493c962e, 0x1402cec5), TOBN(0x9f17ca17, 0xfa2f30e7), + TOBN(0xbcd783e8, 0xe9b879cb), TOBN(0xea3d8c14, 0x5a6f145f), + TOBN(0xdede15e7, 0x5e0dee6e), TOBN(0x74f24872, 0xdc628aa2), + TOBN(0xd3e9c4fe, 0x7861bb93), TOBN(0x56d4822a, 0x6187b2e0), + TOBN(0xb66417cf, 0xc59826f9), TOBN(0xca260969, 0x2408169e), + TOBN(0xedf69d06, 0xc79ef885), TOBN(0x00031f8a, 0xdc7d138f), + TOBN(0x103c46e6, 0x0ebcf726), TOBN(0x4482b831, 0x6231470e), + TOBN(0x6f6dfaca, 0x487c2109), TOBN(0x2e0ace97, 0x62e666ef), + TOBN(0x3246a9d3, 0x1f8d1f42), TOBN(0x1b1e83f1, 0x574944d2), + TOBN(0x13dfa63a, 0xa57f334b), TOBN(0x0cf8daed, 0x9f025d81), + TOBN(0x30d78ea8, 0x00ee11c1), TOBN(0xeb053cd4, 0xb5e3dd75), + TOBN(0x9b65b13e, 0xd58c43c5), TOBN(0xc3ad49bd, 0xbd151663), + TOBN(0x99fd8e41, 0xb6427990), TOBN(0x12cf15bd, 0x707eae1e), + TOBN(0x29ad4f1b, 0x1aabb71e), TOBN(0x5143e74d, 0x07545d0e), + TOBN(0x30266336, 0xc88bdee1), TOBN(0x25f29306, 0x5876767c), + TOBN(0x9c078571, 0xc6731996), TOBN(0xc88690b2, 0xed552951), + TOBN(0x274f2c2d, 0x852705b4), TOBN(0xb0bf8d44, 0x4e09552d), + TOBN(0x7628beeb, 0x986575d1), TOBN(0x407be238, 0x7f864651), + TOBN(0x0e5e3049, 0xa639fc6b), TOBN(0xe75c35d9, 0x86003625), + TOBN(0x0cf35bd8, 0x5dcc1646), TOBN(0x8bcaced2, 0x6c26273a), + TOBN(0xe22ecf1d, 0xb5536742), TOBN(0x013dd897, 0x1a9e068b), + TOBN(0x17f411cb, 0x8a7909c5), TOBN(0x5757ac98, 0x861dd506), + TOBN(0x85de1f0d, 0x1e935abb), TOBN(0xdefd10b4, 0x154de37a), + TOBN(0xb8d9e392, 0x369cebb5), TOBN(0x54d5ef9b, 0x761324be), + TOBN(0x4d6341ba, 0x74f17e26), TOBN(0xc0a0e3c8, 0x78c1dde4), + TOBN(0xa6d77581, 0x87d918fd), TOBN(0x66876015, 0x02ca3a13), + TOBN(0xc7313e9c, 0xf36658f0), TOBN(0xc433ef1c, 0x71f8057e), + TOBN(0x85326246, 0x1b6a835a), TOBN(0xc8f05398, 0x7c86394c), + TOBN(0xff398cdf, 0xe983c4a1), TOBN(0xbf5e8162, 0x03b7b931), + TOBN(0x93193c46, 0xb7b9045b), TOBN(0x1e4ebf5d, 0xa4a6e46b), + TOBN(0xf9942a60, 0x43a24fe7), TOBN(0x29c1191e, 0xffb3492b), + TOBN(0x9f662449, 0x902fde05), TOBN(0xc792a7ac, 0x6713c32d), + TOBN(0x2fd88ad8, 0xb737982c), TOBN(0x7e3a0319, 0xa21e60e3), + TOBN(0x09b0de44, 0x7383591a), TOBN(0x6df141ee, 0x8310a456), + TOBN(0xaec1a039, 0xe6d6f471), TOBN(0x14b2ba0f, 0x1198d12e), + TOBN(0xebc1a160, 0x3aeee5ac), TOBN(0x401f4836, 0xe0b964ce), + TOBN(0x2ee43796, 0x4fd03f66), TOBN(0x3fdb4e49, 0xdd8f3f12), + TOBN(0x6ef267f6, 0x29380f18), TOBN(0x3e8e9670, 0x8da64d16), + TOBN(0xbc19180c, 0x207674f1), TOBN(0x112e09a7, 0x33ae8fdb), + TOBN(0x99667554, 0x6aaeb71e), TOBN(0x79432af1, 0xe101b1c7), + TOBN(0xd5eb558f, 0xde2ddec6), TOBN(0x81392d1f, 0x5357753f), + TOBN(0xa7a76b97, 0x3ae1158a), TOBN(0x416fbbff, 0x4a899991), + TOBN(0x9e65fdfd, 0x0d4a9dcf), TOBN(0x7bc29e48, 0x944ddf12), + TOBN(0xbc1a92d9, 0x3c856866), TOBN(0x273c6905, 0x6e98dfe2), + TOBN(0x69fce418, 0xcdfaa6b8), TOBN(0x606bd823, 0x5061c69f), + TOBN(0x42d495a0, 0x6af75e27), TOBN(0x8ed3d505, 0x6d873a1f), + TOBN(0xaf552841, 0x6ab25b6a), TOBN(0xc6c0ffc7, 0x2b1a4523), + TOBN(0xab18827b, 0x21c99e03), TOBN(0x060e8648, 0x9034691b), + TOBN(0x5207f90f, 0x93c7f398), TOBN(0x9f4a96cb, 0x82f8d10b), + TOBN(0xdd71cd79, 0x3ad0f9e3), TOBN(0x84f435d2, 0xfc3a54f5), + TOBN(0x4b03c55b, 0x8e33787f), TOBN(0xef42f975, 0xa6384673), + TOBN(0xff7304f7, 0x5051b9f0), TOBN(0x18aca1dc, 0x741c87c2), + TOBN(0x56f120a7, 0x2d4bfe80), TOBN(0xfd823b3d, 0x053e732c), + TOBN(0x11bccfe4, 0x7537ca16), TOBN(0xdf6c9c74, 0x1b5a996b), + TOBN(0xee7332c7, 0x904fc3fa), TOBN(0x14a23f45, 0xc7e3636a), + TOBN(0xc38659c3, 0xf091d9aa), TOBN(0x4a995e5d, 0xb12d8540), + TOBN(0x20a53bec, 0xf3a5598a), TOBN(0x56534b17, 0xb1eaa995), + TOBN(0x9ed3dca4, 0xbf04e03c), TOBN(0x716c563a, 0xd8d56268), + TOBN(0x27ba77a4, 0x1d6178e7), TOBN(0xe4c80c40, 0x68a1ff8e), + TOBN(0x75011099, 0x0a13f63d), TOBN(0x7bf33521, 0xa61d46f3), + TOBN(0x0aff218e, 0x10b365bb), TOBN(0x81021804, 0x0fd7ea75), + TOBN(0x05a3fd8a, 0xa4b3a925), TOBN(0xb829e75f, 0x9b3db4e6), + TOBN(0x6bdc75a5, 0x4d53e5fb), TOBN(0x04a5dc02, 0xd52717e3), + TOBN(0x86af502f, 0xe9a42ec2), TOBN(0x8867e8fb, 0x2630e382), + TOBN(0xbf845c6e, 0xbec9889b), TOBN(0x54f491f2, 0xcb47c98d), + TOBN(0xa3091fba, 0x790c2a12), TOBN(0xd7f6fd78, 0xc20f708b), + TOBN(0xa569ac30, 0xacde5e17), TOBN(0xd0f996d0, 0x6852b4d7), + TOBN(0xe51d4bb5, 0x4609ae54), TOBN(0x3fa37d17, 0x0daed061), + TOBN(0x62a88684, 0x34b8fb41), TOBN(0x99a2acbd, 0x9efb64f1), + TOBN(0xb75c1a5e, 0x6448e1f2), TOBN(0xfa99951a, 0x42b5a069), + TOBN(0x6d956e89, 0x2f3b26e7), TOBN(0xf4709860, 0xda875247), + TOBN(0x3ad15179, 0x2482dda3), TOBN(0xd64110e3, 0x017d82f0), + TOBN(0x14928d2c, 0xfad414e4), TOBN(0x2b155f58, 0x2ed02b24), + TOBN(0x481a141b, 0xcb821bf1), TOBN(0x12e3c770, 0x4f81f5da), + TOBN(0xe49c5de5, 0x9fff8381), TOBN(0x11053232, 0x5bbec894), + TOBN(0xa0d051cc, 0x454d88c4), TOBN(0x4f6db89c, 0x1f8e531b), + TOBN(0x34fe3fd6, 0xca563a44), TOBN(0x7f5c2215, 0x58da8ab9), + TOBN(0x8445016d, 0x9474f0a1), TOBN(0x17d34d61, 0xcb7d8a0a), + TOBN(0x8e9d3910, 0x1c474019), TOBN(0xcaff2629, 0xd52ceefb), + TOBN(0xf9cf3e32, 0xc1622c2b), TOBN(0xd4b95e3c, 0xe9071a05), + TOBN(0xfbbca61f, 0x1594438c), TOBN(0x1eb6e6a6, 0x04aadedf), + TOBN(0x853027f4, 0x68e14940), TOBN(0x221d322a, 0xdfabda9c), + TOBN(0xed8ea9f6, 0xb7cb179a), TOBN(0xdc7b764d, 0xb7934dcc), + TOBN(0xfcb13940, 0x5e09180d), TOBN(0x6629a6bf, 0xb47dc2dd), + TOBN(0xbfc55e4e, 0x9f5a915e), TOBN(0xb1db9d37, 0x6204441e), + TOBN(0xf82d68cf, 0x930c5f53), TOBN(0x17d3a142, 0xcbb605b1), + TOBN(0xdd5944ea, 0x308780f2), TOBN(0xdc8de761, 0x3845f5e4), + TOBN(0x6beaba7d, 0x7624d7a3), TOBN(0x1e709afd, 0x304df11e), + TOBN(0x95364376, 0x02170456), TOBN(0xbf204b3a, 0xc8f94b64), + TOBN(0x4e53af7c, 0x5680ca68), TOBN(0x0526074a, 0xe0c67574), + TOBN(0x95d8cef8, 0xecd92af6), TOBN(0xe6b9fa7a, 0x6cd1745a), + TOBN(0x3d546d3d, 0xa325c3e4), TOBN(0x1f57691d, 0x9ae93aae), + TOBN(0xe891f3fe, 0x9d2e1a33), TOBN(0xd430093f, 0xac063d35), + TOBN(0xeda59b12, 0x5513a327), TOBN(0xdc2134f3, 0x5536f18f), + TOBN(0xaa51fe2c, 0x5c210286), TOBN(0x3f68aaee, 0x1cab658c), + TOBN(0x5a23a00b, 0xf9357292), TOBN(0x9a626f39, 0x7efdabed), + TOBN(0xfe2b3bf3, 0x199d78e3), TOBN(0xb7a2af77, 0x71bbc345), + TOBN(0x3d19827a, 0x1e59802c), TOBN(0x823bbc15, 0xb487a51c), + TOBN(0x856139f2, 0x99d0a422), TOBN(0x9ac3df65, 0xf456c6fb), + TOBN(0xaddf65c6, 0x701f8bd6), TOBN(0x149f321e, 0x3758df87), + TOBN(0xb1ecf714, 0x721b7eba), TOBN(0xe17df098, 0x31a3312a), + TOBN(0xdb2fd6ec, 0xd5c4d581), TOBN(0xfd02996f, 0x8fcea1b3), + TOBN(0xe29fa63e, 0x7882f14f), TOBN(0xc9f6dc35, 0x07c6cadc), + TOBN(0x46f22d6f, 0xb882bed0), TOBN(0x1a45755b, 0xd118e52c), + TOBN(0x9f2c7c27, 0x7c4608cf), TOBN(0x7ccbdf32, 0x568012c2), + TOBN(0xfcb0aedd, 0x61729b0e), TOBN(0x7ca2ca9e, 0xf7d75dbf), + TOBN(0xf58fecb1, 0x6f640f62), TOBN(0xe274b92b, 0x39f51946), + TOBN(0x7f4dfc04, 0x6288af44), TOBN(0x0a91f32a, 0xeac329e5), + TOBN(0x43ad274b, 0xd6aaba31), TOBN(0x719a1640, 0x0f6884f9), + TOBN(0x685d29f6, 0xdaf91e20), TOBN(0x5ec1cc33, 0x27e49d52), + TOBN(0x38f4de96, 0x3b54a059), TOBN(0x0e0015e5, 0xefbcfdb3), + TOBN(0x177d23d9, 0x4dbb8da6), TOBN(0x98724aa2, 0x97a617ad), + TOBN(0x30f0885b, 0xfdb6558e), TOBN(0xf9f7a28a, 0xc7899a96), + TOBN(0xd2ae8ac8, 0x872dc112), TOBN(0xfa0642ca, 0x73c3c459), + TOBN(0x15296981, 0xe7dfc8d6), TOBN(0x67cd4450, 0x1fb5b94a), + TOBN(0x0ec71cf1, 0x0eddfd37), TOBN(0xc7e5eeb3, 0x9a8eddc7), + TOBN(0x02ac8e3d, 0x81d95028), TOBN(0x0088f172, 0x70b0e35d), + TOBN(0xec041fab, 0xe1881fe3), TOBN(0x62cf71b8, 0xd99e7faa), + TOBN(0x5043dea7, 0xe0f222c2), TOBN(0x309d42ac, 0x72e65142), + TOBN(0x94fe9ddd, 0x9216cd30), TOBN(0xd6539c7d, 0x0f87feec), + TOBN(0x03c5a57c, 0x432ac7d7), TOBN(0x72692cf0, 0x327fda10), + TOBN(0xec28c85f, 0x280698de), TOBN(0x2331fb46, 0x7ec283b1), + TOBN(0xd34bfa32, 0x2867e633), TOBN(0x78709a82, 0x0a9cc815), + TOBN(0xb7fe6964, 0x875e2fa5), TOBN(0x25cc064f, 0x9e98bfb5), + TOBN(0x9eb0151c, 0x493a65c5), TOBN(0x5fb5d941, 0x53182464), + TOBN(0x69e6f130, 0xf04618e2), TOBN(0xa8ecec22, 0xf89c8ab6), + TOBN(0xcd6ac88b, 0xb96209bd), TOBN(0x65fa8cdb, 0xb3e1c9e0), + TOBN(0xa47d22f5, 0x4a8d8eac), TOBN(0x83895cdf, 0x8d33f963), + TOBN(0xa8adca59, 0xb56cd3d1), TOBN(0x10c8350b, 0xdaf38232), + TOBN(0x2b161fb3, 0xa5080a9f), TOBN(0xbe7f5c64, 0x3af65b3a), + TOBN(0x2c754039, 0x97403a11), TOBN(0x94626cf7, 0x121b96af), + TOBN(0x431de7c4, 0x6a983ec2), TOBN(0x3780dd3a, 0x52cc3df7), + TOBN(0xe28a0e46, 0x2baf8e3b), TOBN(0xabe68aad, 0x51d299ae), + TOBN(0x603eb8f9, 0x647a2408), TOBN(0x14c61ed6, 0x5c750981), + TOBN(0x88b34414, 0xc53352e7), TOBN(0x5a34889c, 0x1337d46e), + TOBN(0x612c1560, 0xf95f2bc8), TOBN(0x8a3f8441, 0xd4807a3a), + TOBN(0x680d9e97, 0x5224da68), TOBN(0x60cd6e88, 0xc3eb00e9), + TOBN(0x3875a98e, 0x9a6bc375), TOBN(0xdc80f924, 0x4fd554c2), + TOBN(0x6c4b3415, 0x6ac77407), TOBN(0xa1e5ea8f, 0x25420681), + TOBN(0x541bfa14, 0x4607a458), TOBN(0x5dbc7e7a, 0x96d7fbf9), + TOBN(0x646a851b, 0x31590a47), TOBN(0x039e85ba, 0x15ee6df8), + TOBN(0xd19fa231, 0xd7b43fc0), TOBN(0x84bc8be8, 0x299a0e04), + TOBN(0x2b9d2936, 0xf20df03a), TOBN(0x24054382, 0x8608d472), + TOBN(0x76b6ba04, 0x9149202a), TOBN(0xb21c3831, 0x3670e7b7), + TOBN(0xddd93059, 0xd6fdee10), TOBN(0x9da47ad3, 0x78488e71), + TOBN(0x99cc1dfd, 0xa0fcfb25), TOBN(0x42abde10, 0x64696954), + TOBN(0x14cc15fc, 0x17eab9fe), TOBN(0xd6e863e4, 0xd3e70972), + TOBN(0x29a7765c, 0x6432112c), TOBN(0x88660001, 0x5b0774d8), + TOBN(0x3729175a, 0x2c088eae), TOBN(0x13afbcae, 0x8230b8d4), + TOBN(0x44768151, 0x915f4379), TOBN(0xf086431a, 0xd8d22812), + TOBN(0x37461955, 0xc298b974), TOBN(0x905fb5f0, 0xf8711e04), + TOBN(0x787abf3a, 0xfe969d18), TOBN(0x392167c2, 0x6f6a494e), + TOBN(0xfc7a0d2d, 0x28c511da), TOBN(0xf127c7dc, 0xb66a262d), + TOBN(0xf9c4bb95, 0xfd63fdf0), TOBN(0x90016589, 0x3913ef46), + TOBN(0x74d2a73c, 0x11aa600d), TOBN(0x2f5379bd, 0x9fb5ab52), + TOBN(0xe49e53a4, 0x7fb70068), TOBN(0x68dd39e5, 0x404aa9a7), + TOBN(0xb9b0cf57, 0x2ecaa9c3), TOBN(0xba0e103b, 0xe824826b), + TOBN(0x60c2198b, 0x4631a3c4), TOBN(0xc5ff84ab, 0xfa8966a2), + TOBN(0x2d6ebe22, 0xac95aff8), TOBN(0x1c9bb6db, 0xb5a46d09), + TOBN(0x419062da, 0x53ee4f8d), TOBN(0x7b9042d0, 0xbb97efef), + TOBN(0x0f87f080, 0x830cf6bd), TOBN(0x4861d19a, 0x6ec8a6c6), + TOBN(0xd3a0daa1, 0x202f01aa), TOBN(0xb0111674, 0xf25afbd5), + TOBN(0x6d00d6cf, 0x1afb20d9), TOBN(0x13695000, 0x40671bc5), + TOBN(0x913ab0dc, 0x2485ea9b), TOBN(0x1f2bed06, 0x9eef61ac), + TOBN(0x850c8217, 0x6d799e20), TOBN(0x93415f37, 0x3271c2de), + TOBN(0x5afb06e9, 0x6c4f5910), TOBN(0x688a52df, 0xc4e9e421), + TOBN(0x30495ba3, 0xe2a9a6db), TOBN(0x4601303d, 0x58f9268b), + TOBN(0xbe3b0dad, 0x7eb0f04f), TOBN(0x4ea47250, 0x4456936d), + TOBN(0x8caf8798, 0xd33fd3e7), TOBN(0x1ccd8a89, 0xeb433708), + TOBN(0x9effe3e8, 0x87fd50ad), TOBN(0xbe240a56, 0x6b29c4df), + TOBN(0xec4ffd98, 0xca0e7ebd), TOBN(0xf586783a, 0xe748616e), + TOBN(0xa5b00d8f, 0xc77baa99), TOBN(0x0acada29, 0xb4f34c9c), + TOBN(0x36dad67d, 0x0fe723ac), TOBN(0x1d8e53a5, 0x39c36c1e), + TOBN(0xe4dd342d, 0x1f4bea41), TOBN(0x64fd5e35, 0xebc9e4e0), + TOBN(0x96f01f90, 0x57908805), TOBN(0xb5b9ea3d, 0x5ed480dd), + TOBN(0x366c5dc2, 0x3efd2dd0), TOBN(0xed2fe305, 0x6e9dfa27), + TOBN(0x4575e892, 0x6e9197e2), TOBN(0x11719c09, 0xab502a5d), + TOBN(0x264c7bec, 0xe81f213f), TOBN(0x741b9241, 0x55f5c457), + TOBN(0x78ac7b68, 0x49a5f4f4), TOBN(0xf91d70a2, 0x9fc45b7d), + TOBN(0x39b05544, 0xb0f5f355), TOBN(0x11f06bce, 0xeef930d9), + TOBN(0xdb84d25d, 0x038d05e1), TOBN(0x04838ee5, 0xbacc1d51), + TOBN(0x9da3ce86, 0x9e8ee00b), TOBN(0xc3412057, 0xc36eda1f), + TOBN(0xae80b913, 0x64d9c2f4), TOBN(0x7468bac3, 0xa010a8ff), + TOBN(0xdfd20037, 0x37359d41), TOBN(0x1a0f5ab8, 0x15efeacc), + TOBN(0x7c25ad2f, 0x659d0ce0), TOBN(0x4011bcbb, 0x6785cff1), + TOBN(0x128b9912, 0x7e2192c7), TOBN(0xa549d8e1, 0x13ccb0e8), + TOBN(0x805588d8, 0xc85438b1), TOBN(0x5680332d, 0xbc25cb27), + TOBN(0xdcd1bc96, 0x1a4bfdf4), TOBN(0x779ff428, 0x706f6566), + TOBN(0x8bbee998, 0xf059987a), TOBN(0xf6ce8cf2, 0xcc686de7), + TOBN(0xf8ad3c4a, 0x953cfdb2), TOBN(0xd1d426d9, 0x2205da36), + TOBN(0xb3c0f13f, 0xc781a241), TOBN(0x3e89360e, 0xd75362a8), + TOBN(0xccd05863, 0xc8a91184), TOBN(0x9bd0c9b7, 0xefa8a7f4), + TOBN(0x97ee4d53, 0x8a912a4b), TOBN(0xde5e15f8, 0xbcf518fd), + TOBN(0x6a055bf8, 0xc467e1e0), TOBN(0x10be4b4b, 0x1587e256), + TOBN(0xd90c14f2, 0x668621c9), TOBN(0xd5518f51, 0xab9c92c1), + TOBN(0x8e6a0100, 0xd6d47b3c), TOBN(0xcbe980dd, 0x66716175), + TOBN(0x500d3f10, 0xddd83683), TOBN(0x3b6cb35d, 0x99cac73c), + TOBN(0x53730c8b, 0x6083d550), TOBN(0xcf159767, 0xdf0a1987), + TOBN(0x84bfcf53, 0x43ad73b3), TOBN(0x1b528c20, 0x4f035a94), + TOBN(0x4294edf7, 0x33eeac69), TOBN(0xb6283e83, 0x817f3240), + TOBN(0xc3fdc959, 0x0a5f25b1), TOBN(0xefaf8aa5, 0x5844ee22), + TOBN(0xde269ba5, 0xdbdde4de), TOBN(0xe3347160, 0xc56133bf), + TOBN(0xc1184219, 0x8d9ea9f8), TOBN(0x090de5db, 0xf3fc1ab5), + TOBN(0x404c37b1, 0x0bf22cda), TOBN(0x7de20ec8, 0xf5618894), + TOBN(0x754c588e, 0xecdaecab), TOBN(0x6ca4b0ed, 0x88342743), + TOBN(0x76f08bdd, 0xf4a938ec), TOBN(0xd182de89, 0x91493ccb), + TOBN(0xd652c53e, 0xc8a4186a), TOBN(0xb3e878db, 0x946d8e33), + TOBN(0x088453c0, 0x5f37663c), TOBN(0x5cd9daaa, 0xb407748b), + TOBN(0xa1f5197f, 0x586d5e72), TOBN(0x47500be8, 0xc443ca59), + TOBN(0x78ef35b2, 0xe2652424), TOBN(0x09c5d26f, 0x6dd7767d), + TOBN(0x7175a79a, 0xa74d3f7b), TOBN(0x0428fd8d, 0xcf5ea459), + TOBN(0x511cb97c, 0xa5d1746d), TOBN(0x36363939, 0xe71d1278), + TOBN(0xcf2df955, 0x10350bf4), TOBN(0xb3817439, 0x60aae782), + TOBN(0xa748c0e4, 0x3e688809), TOBN(0x98021fbf, 0xd7a5a006), + TOBN(0x9076a70c, 0x0e367a98), TOBN(0xbea1bc15, 0x0f62b7c2), + TOBN(0x2645a68c, 0x30fe0343), TOBN(0xacaffa78, 0x699dc14f), + TOBN(0xf4469964, 0x457bf9c4), TOBN(0x0db6407b, 0x0d2ead83), + TOBN(0x68d56cad, 0xb2c6f3eb), TOBN(0x3b512e73, 0xf376356c), + TOBN(0xe43b0e1f, 0xfce10408), TOBN(0x89ddc003, 0x5a5e257d), + TOBN(0xb0ae0d12, 0x0362e5b3), TOBN(0x07f983c7, 0xb0519161), + TOBN(0xc2e94d15, 0x5d5231e7), TOBN(0xcff22aed, 0x0b4f9513), + TOBN(0xb02588dd, 0x6ad0b0b5), TOBN(0xb967d1ac, 0x11d0dcd5), + TOBN(0x8dac6bc6, 0xcf777b6c), TOBN(0x0062bdbd, 0x4c6d1959), + TOBN(0x53da71b5, 0x0ef5cc85), TOBN(0x07012c7d, 0x4006f14f), + TOBN(0x4617f962, 0xac47800d), TOBN(0x53365f2b, 0xc102ed75), + TOBN(0xb422efcb, 0x4ab8c9d3), TOBN(0x195cb26b, 0x34af31c9), + TOBN(0x3a926e29, 0x05f2c4ce), TOBN(0xbd2bdecb, 0x9856966c), + TOBN(0x5d16ab3a, 0x85527015), TOBN(0x9f81609e, 0x4486c231), + TOBN(0xd8b96b2c, 0xda350002), TOBN(0xbd054690, 0xfa1b7d36), + TOBN(0xdc90ebf5, 0xe71d79bc), TOBN(0xf241b6f9, 0x08964e4e), + TOBN(0x7c838643, 0x2fe3cd4c), TOBN(0xe0f33acb, 0xb4bc633c), + TOBN(0xb4a9ecec, 0x3d139f1f), TOBN(0x05ce69cd, 0xdc4a1f49), + TOBN(0xa19d1b16, 0xf5f98aaf), TOBN(0x45bb71d6, 0x6f23e0ef), + TOBN(0x33789fcd, 0x46cdfdd3), TOBN(0x9b8e2978, 0xcee040ca), + TOBN(0x9c69b246, 0xae0a6828), TOBN(0xba533d24, 0x7078d5aa), + TOBN(0x7a2e42c0, 0x7bb4fbdb), TOBN(0xcfb4879a, 0x7035385c), + TOBN(0x8c3dd30b, 0x3281705b), TOBN(0x7e361c6c, 0x404fe081), + TOBN(0x7b21649c, 0x3f604edf), TOBN(0x5dbf6a3f, 0xe52ffe47), + TOBN(0xc41b7c23, 0x4b54d9bf), TOBN(0x1374e681, 0x3511c3d9), + TOBN(0x1863bf16, 0xc1b2b758), TOBN(0x90e78507, 0x1e9e6a96), + TOBN(0xab4bf98d, 0x5d86f174), TOBN(0xd74e0bd3, 0x85e96fe4), + TOBN(0x8afde39f, 0xcac5d344), TOBN(0x90946dbc, 0xbd91b847), + TOBN(0xf5b42358, 0xfe1a838c), TOBN(0x05aae6c5, 0x620ac9d8), + TOBN(0x8e193bd8, 0xa1ce5a0b), TOBN(0x8f710571, 0x4dabfd72), + TOBN(0x8d8fdd48, 0x182caaac), TOBN(0x8c4aeefa, 0x040745cf), + TOBN(0x73c6c30a, 0xf3b93e6d), TOBN(0x991241f3, 0x16f42011), + TOBN(0xa0158eea, 0xe457a477), TOBN(0xd19857db, 0xee6ddc05), + TOBN(0xb3265224, 0x18c41671), TOBN(0x3ffdfc7e, 0x3c2c0d58), + TOBN(0x3a3a5254, 0x26ee7cda), TOBN(0x341b0869, 0xdf02c3a8), + TOBN(0xa023bf42, 0x723bbfc8), TOBN(0x3d15002a, 0x14452691),} + , + {TOBN(0x5ef7324c, 0x85edfa30), TOBN(0x25976554, 0x87d4f3da), + TOBN(0x352f5bc0, 0xdcb50c86), TOBN(0x8f6927b0, 0x4832a96c), + TOBN(0xd08ee1ba, 0x55f2f94c), TOBN(0x6a996f99, 0x344b45fa), + TOBN(0xe133cb8d, 0xa8aa455d), TOBN(0x5d0721ec, 0x758dc1f7), + TOBN(0x6ba7a920, 0x79e5fb67), TOBN(0xe1331feb, 0x70aa725e), + TOBN(0x5080ccf5, 0x7df5d837), TOBN(0xe4cae01d, 0x7ff72e21), + TOBN(0xd9243ee6, 0x0412a77d), TOBN(0x06ff7cac, 0xdf449025), + TOBN(0xbe75f7cd, 0x23ef5a31), TOBN(0xbc957822, 0x0ddef7a8), + TOBN(0x8cf7230c, 0xb0ce1c55), TOBN(0x5b534d05, 0x0bbfb607), + TOBN(0xee1ef113, 0x0e16363b), TOBN(0x27e0aa7a, 0xb4999e82), + TOBN(0xce1dac2d, 0x79362c41), TOBN(0x67920c90, 0x91bb6cb0), + TOBN(0x1e648d63, 0x2223df24), TOBN(0x0f7d9eef, 0xe32e8f28), + TOBN(0x6943f39a, 0xfa833834), TOBN(0x22951722, 0xa6328562), + TOBN(0x81d63dd5, 0x4170fc10), TOBN(0x9f5fa58f, 0xaecc2e6d), + TOBN(0xb66c8725, 0xe77d9a3b), TOBN(0x11235cea, 0x6384ebe0), + TOBN(0x06a8c118, 0x5845e24a), TOBN(0x0137b286, 0xebd093b1), + TOBN(0xc589e1ce, 0x44ace150), TOBN(0xe0f8d3d9, 0x4381e97c), + TOBN(0x59e99b11, 0x62c5a4b8), TOBN(0x90d262f7, 0xfd0ec9f9), + TOBN(0xfbc854c9, 0x283e13c9), TOBN(0x2d04fde7, 0xaedc7085), + TOBN(0x057d7765, 0x47dcbecb), TOBN(0x8dbdf591, 0x9a76fa5f), + TOBN(0xd0150695, 0x0de1e578), TOBN(0x2e1463e7, 0xe9f72bc6), + TOBN(0xffa68441, 0x1b39eca5), TOBN(0x673c8530, 0x7c037f2f), + TOBN(0xd0d6a600, 0x747f91da), TOBN(0xb08d43e1, 0xc9cb78e9), + TOBN(0x0fc0c644, 0x27b5cef5), TOBN(0x5c1d160a, 0xa60a2fd6), + TOBN(0xf98cae53, 0x28c8e13b), TOBN(0x375f10c4, 0xb2eddcd1), + TOBN(0xd4eb8b7f, 0x5cce06ad), TOBN(0xb4669f45, 0x80a2e1ef), + TOBN(0xd593f9d0, 0x5bbd8699), TOBN(0x5528a4c9, 0xe7976d13), + TOBN(0x3923e095, 0x1c7e28d3), TOBN(0xb9293790, 0x3f6bb577), + TOBN(0xdb567d6a, 0xc42bd6d2), TOBN(0x6df86468, 0xbb1f96ae), + TOBN(0x0efe5b1a, 0x4843b28e), TOBN(0x961bbb05, 0x6379b240), + TOBN(0xb6caf5f0, 0x70a6a26b), TOBN(0x70686c0d, 0x328e6e39), + TOBN(0x80da06cf, 0x895fc8d3), TOBN(0x804d8810, 0xb363fdc9), + TOBN(0xbe22877b, 0x207f1670), TOBN(0x9b0dd188, 0x4e615291), + TOBN(0x625ae8dc, 0x97a3c2bf), TOBN(0x08584ef7, 0x439b86e8), + TOBN(0xde7190a5, 0xdcd898ff), TOBN(0x26286c40, 0x2058ee3d), + TOBN(0x3db0b217, 0x5f87b1c1), TOBN(0xcc334771, 0x102a6db5), + TOBN(0xd99de954, 0x2f770fb1), TOBN(0x97c1c620, 0x4cd7535e), + TOBN(0xd3b6c448, 0x3f09cefc), TOBN(0xd725af15, 0x5a63b4f8), + TOBN(0x0c95d24f, 0xc01e20ec), TOBN(0xdfd37494, 0x9ae7121f), + TOBN(0x7d6ddb72, 0xec77b7ec), TOBN(0xfe079d3b, 0x0353a4ae), + TOBN(0x3066e70a, 0x2e6ac8d2), TOBN(0x9c6b5a43, 0x106e5c05), + TOBN(0x52d3c6f5, 0xede59b8c), TOBN(0x30d6a5c3, 0xfccec9ae), + TOBN(0xedec7c22, 0x4fc0a9ef), TOBN(0x190ff083, 0x95c16ced), + TOBN(0xbe12ec8f, 0x94de0fde), TOBN(0x0d131ab8, 0x852d3433), + TOBN(0x42ace07e, 0x85701291), TOBN(0x94793ed9, 0x194061a8), + TOBN(0x30e83ed6, 0xd7f4a485), TOBN(0x9eec7269, 0xf9eeff4d), + TOBN(0x90acba59, 0x0c9d8005), TOBN(0x5feca458, 0x1e79b9d1), + TOBN(0x8fbe5427, 0x1d506a1e), TOBN(0xa32b2c8e, 0x2439cfa7), + TOBN(0x1671c173, 0x73dd0b4e), TOBN(0x37a28214, 0x44a054c6), + TOBN(0x81760a1b, 0x4e8b53f1), TOBN(0xa6c04224, 0xf9f93b9e), + TOBN(0x18784b34, 0xcf671e3c), TOBN(0x81bbecd2, 0xcda9b994), + TOBN(0x38831979, 0xb2ab3848), TOBN(0xef54feb7, 0xf2e03c2d), + TOBN(0xcf197ca7, 0xfb8088fa), TOBN(0x01427247, 0x4ddc96c5), + TOBN(0xa2d2550a, 0x30777176), TOBN(0x53469898, 0x4d0cf71d), + TOBN(0x6ce937b8, 0x3a2aaac6), TOBN(0xe9f91dc3, 0x5af38d9b), + TOBN(0x2598ad83, 0xc8bf2899), TOBN(0x8e706ac9, 0xb5536c16), + TOBN(0x40dc7495, 0xf688dc98), TOBN(0x26490cd7, 0x124c4afc), + TOBN(0xe651ec84, 0x1f18775c), TOBN(0x393ea6c3, 0xb4fdaf4a), + TOBN(0x1e1f3343, 0x7f338e0d), TOBN(0x39fb832b, 0x6053e7b5), + TOBN(0x46e702da, 0x619e14d5), TOBN(0x859cacd1, 0xcdeef6e0), + TOBN(0x63b99ce7, 0x4462007d), TOBN(0xb8ab48a5, 0x4cb5f5b7), + TOBN(0x9ec673d2, 0xf55edde7), TOBN(0xd1567f74, 0x8cfaefda), + TOBN(0x46381b6b, 0x0887bcec), TOBN(0x694497ce, 0xe178f3c2), + TOBN(0x5e6525e3, 0x1e6266cb), TOBN(0x5931de26, 0x697d6413), + TOBN(0x87f8df7c, 0x0e58d493), TOBN(0xb1ae5ed0, 0x58b73f12), + TOBN(0xc368f784, 0xdea0c34d), TOBN(0x9bd0a120, 0x859a91a0), + TOBN(0xb00d88b7, 0xcc863c68), TOBN(0x3a1cc11e, 0x3d1f4d65), + TOBN(0xea38e0e7, 0x0aa85593), TOBN(0x37f13e98, 0x7dc4aee8), + TOBN(0x10d38667, 0xbc947bad), TOBN(0x738e07ce, 0x2a36ee2e), + TOBN(0xc93470cd, 0xc577fcac), TOBN(0xdee1b616, 0x2782470d), + TOBN(0x36a25e67, 0x2e793d12), TOBN(0xd6aa6cae, 0xe0f186da), + TOBN(0x474d0fd9, 0x80e07af7), TOBN(0xf7cdc47d, 0xba8a5cd4), + TOBN(0x28af6d9d, 0xab15247f), TOBN(0x7c789c10, 0x493a537f), + TOBN(0x7ac9b110, 0x23a334e7), TOBN(0x0236ac09, 0x12c9c277), + TOBN(0xa7e5bd25, 0x1d7a5144), TOBN(0x098b9c2a, 0xf13ec4ec), + TOBN(0x3639daca, 0xd3f0abca), TOBN(0x642da81a, 0xa23960f9), + TOBN(0x7d2e5c05, 0x4f7269b1), TOBN(0xfcf30777, 0xe287c385), + TOBN(0x10edc84f, 0xf2a46f21), TOBN(0x35441757, 0x4f43fa36), + TOBN(0xf1327899, 0xfd703431), TOBN(0xa438d7a6, 0x16dd587a), + TOBN(0x65c34c57, 0xe9c8352d), TOBN(0xa728edab, 0x5cc5a24e), + TOBN(0xaed78abc, 0x42531689), TOBN(0x0a51a0e8, 0x010963ef), + TOBN(0x5776fa0a, 0xd717d9b3), TOBN(0xf356c239, 0x7dd3428b), + TOBN(0x29903fff, 0x8d3a3dac), TOBN(0x409597fa, 0x3d94491f), + TOBN(0x4cd7a5ff, 0xbf4a56a4), TOBN(0xe5096474, 0x8adab462), + TOBN(0xa97b5126, 0x5c3427b0), TOBN(0x6401405c, 0xd282c9bd), + TOBN(0x3629f8d7, 0x222c5c45), TOBN(0xb1c02c16, 0xe8d50aed), + TOBN(0xbea2ed75, 0xd9635bc9), TOBN(0x226790c7, 0x6e24552f), + TOBN(0x3c33f2a3, 0x65f1d066), TOBN(0x2a43463e, 0x6dfccc2e), + TOBN(0x8cc3453a, 0xdb483761), TOBN(0xe7cc6085, 0x65d5672b), + TOBN(0x277ed6cb, 0xde3efc87), TOBN(0x19f2f368, 0x69234eaf), + TOBN(0x9aaf4317, 0x5c0b800b), TOBN(0x1f1e7c89, 0x8b6da6e2), + TOBN(0x6cfb4715, 0xb94ec75e), TOBN(0xd590dd5f, 0x453118c2), + TOBN(0x14e49da1, 0x1f17a34c), TOBN(0x5420ab39, 0x235a1456), + TOBN(0xb7637241, 0x2f50363b), TOBN(0x7b15d623, 0xc3fabb6e), + TOBN(0xa0ef40b1, 0xe274e49c), TOBN(0x5cf50744, 0x96b1860a), + TOBN(0xd6583fbf, 0x66afe5a4), TOBN(0x44240510, 0xf47e3e9a), + TOBN(0x99254343, 0x11b2d595), TOBN(0xf1367499, 0xeec8df57), + TOBN(0x3cb12c61, 0x3e73dd05), TOBN(0xd248c033, 0x7dac102a), + TOBN(0xcf154f13, 0xa77739f5), TOBN(0xbf4288cb, 0x23d2af42), + TOBN(0xaa64c9b6, 0x32e4a1cf), TOBN(0xee8c07a8, 0xc8a208f3), + TOBN(0xe10d4999, 0x6fe8393f), TOBN(0x0f809a3f, 0xe91f3a32), + TOBN(0x61096d1c, 0x802f63c8), TOBN(0x289e1462, 0x57750d3d), + TOBN(0xed06167e, 0x9889feea), TOBN(0xd5c9c0e2, 0xe0993909), + TOBN(0x46fca0d8, 0x56508ac6), TOBN(0x91826047, 0x4f1b8e83), + TOBN(0x4f2c877a, 0x9a4a2751), TOBN(0x71bd0072, 0xcae6fead), + TOBN(0x38df8dcc, 0x06aa1941), TOBN(0x5a074b4c, 0x63beeaa8), + TOBN(0xd6d65934, 0xc1cec8ed), TOBN(0xa6ecb49e, 0xaabc03bd), + TOBN(0xaade91c2, 0xde8a8415), TOBN(0xcfb0efdf, 0x691136e0), + TOBN(0x11af45ee, 0x23ab3495), TOBN(0xa132df88, 0x0b77463d), + TOBN(0x8923c15c, 0x815d06f4), TOBN(0xc3ceb3f5, 0x0d61a436), + TOBN(0xaf52291d, 0xe88fb1da), TOBN(0xea057974, 0x1da12179), + TOBN(0xb0d7218c, 0xd2fef720), TOBN(0x6c0899c9, 0x8e1d8845), + TOBN(0x98157504, 0x752ddad7), TOBN(0xd60bd74f, 0xa1a68a97), + TOBN(0x7047a3a9, 0xf658fb99), TOBN(0x1f5d86d6, 0x5f8511e4), + TOBN(0xb8a4bc42, 0x4b5a6d88), TOBN(0x69eb2c33, 0x1abefa7d), + TOBN(0x95bf39e8, 0x13c9c510), TOBN(0xf571960a, 0xd48aab43), + TOBN(0x7e8cfbcf, 0x704e23c6), TOBN(0xc71b7d22, 0x28aaa65b), + TOBN(0xa041b2bd, 0x245e3c83), TOBN(0x69b98834, 0xd21854ff), + TOBN(0x89d227a3, 0x963bfeec), TOBN(0x99947aaa, 0xde7da7cb), + TOBN(0x1d9ee9db, 0xee68a9b1), TOBN(0x0a08f003, 0x698ec368), + TOBN(0xe9ea4094, 0x78ef2487), TOBN(0xc8d2d415, 0x02cfec26), + TOBN(0xc52f9a6e, 0xb7dcf328), TOBN(0x0ed489e3, 0x85b6a937), + TOBN(0x9b94986b, 0xbef3366e), TOBN(0x0de59c70, 0xedddddb8), + TOBN(0xffdb748c, 0xeadddbe2), TOBN(0x9b9784bb, 0x8266ea40), + TOBN(0x142b5502, 0x1a93507a), TOBN(0xb4cd1187, 0x8d3c06cf), + TOBN(0xdf70e76a, 0x91ec3f40), TOBN(0x484e81ad, 0x4e7553c2), + TOBN(0x830f87b5, 0x272e9d6e), TOBN(0xea1c93e5, 0xc6ff514a), + TOBN(0x67cc2adc, 0xc4192a8e), TOBN(0xc77e27e2, 0x42f4535a), + TOBN(0x9cdbab36, 0xd2b713c5), TOBN(0x86274ea0, 0xcf7b0cd3), + TOBN(0x784680f3, 0x09af826b), TOBN(0xbfcc837a, 0x0c72dea3), + TOBN(0xa8bdfe9d, 0xd6529b73), TOBN(0x708aa228, 0x63a88002), + TOBN(0x6c7a9a54, 0xc91d45b9), TOBN(0xdf1a38bb, 0xfd004f56), + TOBN(0x2e8c9a26, 0xb8bad853), TOBN(0x2d52cea3, 0x3723eae7), + TOBN(0x054d6d81, 0x56ca2830), TOBN(0xa3317d14, 0x9a8dc411), + TOBN(0xa08662fe, 0xfd4ddeda), TOBN(0xed2a153a, 0xb55d792b), + TOBN(0x7035c16a, 0xbfc6e944), TOBN(0xb6bc5834, 0x00171cf3), + TOBN(0xe27152b3, 0x83d102b6), TOBN(0xfe695a47, 0x0646b848), + TOBN(0xa5bb09d8, 0x916e6d37), TOBN(0xb4269d64, 0x0d17015e), + TOBN(0x8d8156a1, 0x0a1d2285), TOBN(0xfeef6c51, 0x46d26d72), + TOBN(0x9dac57c8, 0x4c5434a7), TOBN(0x0282e5be, 0x59d39e31), + TOBN(0xedfff181, 0x721c486d), TOBN(0x301baf10, 0xbc58824e), + TOBN(0x8136a6aa, 0x00570031), TOBN(0x55aaf78c, 0x1cddde68), + TOBN(0x26829371, 0x59c63952), TOBN(0x3a3bd274, 0x8bc25baf), + TOBN(0xecdf8657, 0xb7e52dc3), TOBN(0x2dd8c087, 0xfd78e6c8), + TOBN(0x20553274, 0xf5531461), TOBN(0x8b4a1281, 0x5d95499b), + TOBN(0xe2c8763a, 0x1a80f9d2), TOBN(0xd1dbe32b, 0x4ddec758), + TOBN(0xaf12210d, 0x30c34169), TOBN(0xba74a953, 0x78baa533), + TOBN(0x3d133c6e, 0xa438f254), TOBN(0xa431531a, 0x201bef5b), + TOBN(0x15295e22, 0xf669d7ec), TOBN(0xca374f64, 0x357fb515), + TOBN(0x8a8406ff, 0xeaa3fdb3), TOBN(0x106ae448, 0xdf3f2da8), + TOBN(0x8f9b0a90, 0x33c8e9a1), TOBN(0x234645e2, 0x71ad5885), + TOBN(0x3d083224, 0x1c0aed14), TOBN(0xf10a7d3e, 0x7a942d46), + TOBN(0x7c11deee, 0x40d5c9be), TOBN(0xb2bae7ff, 0xba84ed98), + TOBN(0x93e97139, 0xaad58ddd), TOBN(0x3d872796, 0x3f6d1fa3), + TOBN(0x483aca81, 0x8569ff13), TOBN(0x8b89a5fb, 0x9a600f72), + TOBN(0x4cbc27c3, 0xc06f2b86), TOBN(0x22130713, 0x63ad9c0b), + TOBN(0xb5358b1e, 0x48ac2840), TOBN(0x18311294, 0xecba9477), + TOBN(0xda58f990, 0xa6946b43), TOBN(0x3098baf9, 0x9ab41819), + TOBN(0x66c4c158, 0x4198da52), TOBN(0xab4fc17c, 0x146bfd1b), + TOBN(0x2f0a4c3c, 0xbf36a908), TOBN(0x2ae9e34b, 0x58cf7838), + TOBN(0xf411529e, 0x3fa11b1f), TOBN(0x21e43677, 0x974af2b4), + TOBN(0x7c20958e, 0xc230793b), TOBN(0x710ea885, 0x16e840f3), + TOBN(0xfc0b21fc, 0xc5dc67cf), TOBN(0x08d51647, 0x88405718), + TOBN(0xd955c21f, 0xcfe49eb7), TOBN(0x9722a5d5, 0x56dd4a1f), + TOBN(0xc9ef50e2, 0xc861baa5), TOBN(0xc0c21a5d, 0x9505ac3e), + TOBN(0xaf6b9a33, 0x8b7c063f), TOBN(0xc6370339, 0x2f4779c1), + TOBN(0x22df99c7, 0x638167c3), TOBN(0xfe6ffe76, 0x795db30c), + TOBN(0x2b822d33, 0xa4854989), TOBN(0xfef031dd, 0x30563aa5), + TOBN(0x16b09f82, 0xd57c667f), TOBN(0xc70312ce, 0xcc0b76f1), + TOBN(0xbf04a9e6, 0xc9118aec), TOBN(0x82fcb419, 0x3409d133), + TOBN(0x1a8ab385, 0xab45d44d), TOBN(0xfba07222, 0x617b83a3), + TOBN(0xb05f50dd, 0x58e81b52), TOBN(0x1d8db553, 0x21ce5aff), + TOBN(0x3097b8d4, 0xe344a873), TOBN(0x7d8d116d, 0xfe36d53e), + TOBN(0x6db22f58, 0x7875e750), TOBN(0x2dc5e373, 0x43e144ea), + TOBN(0xc05f32e6, 0xe799eb95), TOBN(0xe9e5f4df, 0x6899e6ec), + TOBN(0xbdc3bd68, 0x1fab23d5), TOBN(0xb72b8ab7, 0x73af60e6), + TOBN(0x8db27ae0, 0x2cecc84a), TOBN(0x600016d8, 0x7bdb871c), + TOBN(0x42a44b13, 0xd7c46f58), TOBN(0xb8919727, 0xc3a77d39), + TOBN(0xcfc6bbbd, 0xdafd6088), TOBN(0x1a740146, 0x6bd20d39), + TOBN(0x8c747abd, 0x98c41072), TOBN(0x4c91e765, 0xbdf68ea1), + TOBN(0x7c95e5ca, 0x08819a78), TOBN(0xcf48b729, 0xc9587921), + TOBN(0x091c7c5f, 0xdebbcc7d), TOBN(0x6f287404, 0xf0e05149), + TOBN(0xf83b5ac2, 0x26cd44ec), TOBN(0x88ae32a6, 0xcfea250e), + TOBN(0x6ac5047a, 0x1d06ebc5), TOBN(0xc7e550b4, 0xd434f781), + TOBN(0x61ab1cf2, 0x5c727bd2), TOBN(0x2e4badb1, 0x1cf915b0), + TOBN(0x1b4dadec, 0xf69d3920), TOBN(0xe61b1ca6, 0xf14c1dfe), + TOBN(0x90b479cc, 0xbd6bd51f), TOBN(0x8024e401, 0x8045ec30), + TOBN(0xcab29ca3, 0x25ef0e62), TOBN(0x4f2e9416, 0x49e4ebc0), + TOBN(0x45eb40ec, 0x0ccced58), TOBN(0x25cd4b9c, 0x0da44f98), + TOBN(0x43e06458, 0x871812c6), TOBN(0x99f80d55, 0x16cef651), + TOBN(0x571340c9, 0xce6dc153), TOBN(0x138d5117, 0xd8665521), + TOBN(0xacdb45bc, 0x4e07014d), TOBN(0x2f34bb38, 0x84b60b91), + TOBN(0xf44a4fd2, 0x2ae8921e), TOBN(0xb039288e, 0x892ba1e2), + TOBN(0x9da50174, 0xb1c180b2), TOBN(0x6b70ab66, 0x1693dc87), + TOBN(0x7e9babc9, 0xe7057481), TOBN(0x4581ddef, 0x9c80dc41), + TOBN(0x0c890da9, 0x51294682), TOBN(0x0b5629d3, 0x3f4736e5), + TOBN(0x2340c79e, 0xb06f5b41), TOBN(0xa42e84ce, 0x4e243469), + TOBN(0xf9a20135, 0x045a71a9), TOBN(0xefbfb415, 0xd27b6fb6), + TOBN(0x25ebea23, 0x9d33cd6f), TOBN(0x9caedb88, 0xaa6c0af8), + TOBN(0x53dc7e9a, 0xd9ce6f96), TOBN(0x3897f9fd, 0x51e0b15a), + TOBN(0xf51cb1f8, 0x8e5d788e), TOBN(0x1aec7ba8, 0xe1d490ee), + TOBN(0x265991e0, 0xcc58cb3c), TOBN(0x9f306e8c, 0x9fc3ad31), + TOBN(0x5fed006e, 0x5040a0ac), TOBN(0xca9d5043, 0xfb476f2e), + TOBN(0xa19c06e8, 0xbeea7a23), TOBN(0xd2865801, 0x0edabb63), + TOBN(0xdb92293f, 0x6967469a), TOBN(0x2894d839, 0x8d8a8ed8), + TOBN(0x87c9e406, 0xbbc77122), TOBN(0x8671c6f1, 0x2ea3a26a), + TOBN(0xe42df8d6, 0xd7de9853), TOBN(0x2e3ce346, 0xb1f2bcc7), + TOBN(0xda601dfc, 0x899d50cf), TOBN(0xbfc913de, 0xfb1b598f), + TOBN(0x81c4909f, 0xe61f7908), TOBN(0x192e304f, 0x9bbc7b29), + TOBN(0xc3ed8738, 0xc104b338), TOBN(0xedbe9e47, 0x783f5d61), + TOBN(0x0c06e9be, 0x2db30660), TOBN(0xda3e613f, 0xc0eb7d8e), + TOBN(0xd8fa3e97, 0x322e096e), TOBN(0xfebd91e8, 0xd336e247), + TOBN(0x8f13ccc4, 0xdf655a49), TOBN(0xa9e00dfc, 0x5eb20210), + TOBN(0x84631d0f, 0xc656b6ea), TOBN(0x93a058cd, 0xd8c0d947), + TOBN(0x6846904a, 0x67bd3448), TOBN(0x4a3d4e1a, 0xf394fd5c), + TOBN(0xc102c1a5, 0xdb225f52), TOBN(0xe3455bba, 0xfc4f5e9a), + TOBN(0x6b36985b, 0x4b9ad1ce), TOBN(0xa9818536, 0x5bb7f793), + TOBN(0x6c25e1d0, 0x48b1a416), TOBN(0x1381dd53, 0x3c81bee7), + TOBN(0xd2a30d61, 0x7a4a7620), TOBN(0xc8412926, 0x39b8944c), + TOBN(0x3c1c6fbe, 0x7a97c33a), TOBN(0x941e541d, 0x938664e7), + TOBN(0x417499e8, 0x4a34f239), TOBN(0x15fdb83c, 0xb90402d5), + TOBN(0xb75f46bf, 0x433aa832), TOBN(0xb61e15af, 0x63215db1), + TOBN(0xaabe59d4, 0xa127f89a), TOBN(0x5d541e0c, 0x07e816da), + TOBN(0xaaba0659, 0xa618b692), TOBN(0x55327733, 0x17266026), + TOBN(0xaf53a0fc, 0x95f57552), TOBN(0x32947650, 0x6cacb0c9), + TOBN(0x253ff58d, 0xc821be01), TOBN(0xb0309531, 0xa06f1146), + TOBN(0x59bbbdf5, 0x05c2e54d), TOBN(0x158f27ad, 0x26e8dd22), + TOBN(0xcc5b7ffb, 0x397e1e53), TOBN(0xae03f65b, 0x7fc1e50d), + TOBN(0xa9784ebd, 0x9c95f0f9), TOBN(0x5ed9deb2, 0x24640771), + TOBN(0x31244af7, 0x035561c4), TOBN(0x87332f3a, 0x7ee857de), + TOBN(0x09e16e9e, 0x2b9e0d88), TOBN(0x52d910f4, 0x56a06049), + TOBN(0x507ed477, 0xa9592f48), TOBN(0x85cb917b, 0x2365d678), + TOBN(0xf8511c93, 0x4c8998d1), TOBN(0x2186a3f1, 0x730ea58f), + TOBN(0x50189626, 0xb2029db0), TOBN(0x9137a6d9, 0x02ceb75a), + TOBN(0x2fe17f37, 0x748bc82c), TOBN(0x87c2e931, 0x80469f8c), + TOBN(0x850f71cd, 0xbf891aa2), TOBN(0x0ca1b89b, 0x75ec3d8d), + TOBN(0x516c43aa, 0x5e1cd3cd), TOBN(0x89397808, 0x9a887c28), + TOBN(0x0059c699, 0xddea1f9f), TOBN(0x7737d6fa, 0x8e6868f7), + TOBN(0x6d93746a, 0x60f1524b), TOBN(0x36985e55, 0xba052aa7), + TOBN(0x41b1d322, 0xed923ea5), TOBN(0x3429759f, 0x25852a11), + TOBN(0xbeca6ec3, 0x092e9f41), TOBN(0x3a238c66, 0x62256bbd), + TOBN(0xd82958ea, 0x70ad487d), TOBN(0x4ac8aaf9, 0x65610d93), + TOBN(0x3fa101b1, 0x5e4ccab0), TOBN(0x9bf430f2, 0x9de14bfb), + TOBN(0xa10f5cc6, 0x6531899d), TOBN(0x590005fb, 0xea8ce17d), + TOBN(0xc437912f, 0x24544cb6), TOBN(0x9987b71a, 0xd79ac2e3), + TOBN(0x13e3d9dd, 0xc058a212), TOBN(0x00075aac, 0xd2de9606), + TOBN(0x80ab508b, 0x6cac8369), TOBN(0x87842be7, 0xf54f6c89), + TOBN(0xa7ad663d, 0x6bc532a4), TOBN(0x67813de7, 0x78a91bc8), + TOBN(0x5dcb61ce, 0xc3427239), TOBN(0x5f3c7cf0, 0xc56934d9), + TOBN(0xc079e0fb, 0xe3191591), TOBN(0xe40896bd, 0xb01aada7), + TOBN(0x8d466791, 0x0492d25f), TOBN(0x8aeb30c9, 0xe7408276), + TOBN(0xe9437495, 0x9287aacc), TOBN(0x23d4708d, 0x79fe03d4), + TOBN(0x8cda9cf2, 0xd0c05199), TOBN(0x502fbc22, 0xfae78454), + TOBN(0xc0bda9df, 0xf572a182), TOBN(0x5f9b71b8, 0x6158b372), + TOBN(0xe0f33a59, 0x2b82dd07), TOBN(0x76302735, 0x9523032e), + TOBN(0x7fe1a721, 0xc4505a32), TOBN(0x7b6e3e82, 0xf796409f),} + , + {TOBN(0xe3417bc0, 0x35d0b34a), TOBN(0x440b386b, 0x8327c0a7), + TOBN(0x8fb7262d, 0xac0362d1), TOBN(0x2c41114c, 0xe0cdf943), + TOBN(0x2ba5cef1, 0xad95a0b1), TOBN(0xc09b37a8, 0x67d54362), + TOBN(0x26d6cdd2, 0x01e486c9), TOBN(0x20477abf, 0x42ff9297), + TOBN(0xa004dcb3, 0x292a9287), TOBN(0xddc15cf6, 0x77b092c7), + TOBN(0x083a8464, 0x806c0605), TOBN(0x4a68df70, 0x3db997b0), + TOBN(0x9c134e45, 0x05bf7dd0), TOBN(0xa4e63d39, 0x8ccf7f8c), + TOBN(0xa6e6517f, 0x41b5f8af), TOBN(0xaa8b9342, 0xad7bc1cc), + TOBN(0x126f35b5, 0x1e706ad9), TOBN(0xb99cebb4, 0xc3a9ebdf), + TOBN(0xa75389af, 0xbf608d90), TOBN(0x76113c4f, 0xc6c89858), + TOBN(0x80de8eb0, 0x97e2b5aa), TOBN(0x7e1022cc, 0x63b91304), + TOBN(0x3bdab605, 0x6ccc066c), TOBN(0x33cbb144, 0xb2edf900), + TOBN(0xc4176471, 0x7af715d2), TOBN(0xe2f7f594, 0xd0134a96), + TOBN(0x2c1873ef, 0xa41ec956), TOBN(0xe4e7b4f6, 0x77821304), + TOBN(0xe5c8ff97, 0x88d5374a), TOBN(0x2b915e63, 0x80823d5b), + TOBN(0xea6bc755, 0xb2ee8fe2), TOBN(0x6657624c, 0xe7112651), + TOBN(0x157af101, 0xdace5aca), TOBN(0xc4fdbcf2, 0x11a6a267), + TOBN(0xdaddf340, 0xc49c8609), TOBN(0x97e49f52, 0xe9604a65), + TOBN(0x9be8e790, 0x937e2ad5), TOBN(0x846e2508, 0x326e17f1), + TOBN(0x3f38007a, 0x0bbbc0dc), TOBN(0xcf03603f, 0xb11e16d6), + TOBN(0xd6f800e0, 0x7442f1d5), TOBN(0x475607d1, 0x66e0e3ab), + TOBN(0x82807f16, 0xb7c64047), TOBN(0x8858e1e3, 0xa749883d), + TOBN(0x5859120b, 0x8231ee10), TOBN(0x1b80e7eb, 0x638a1ece), + TOBN(0xcb72525a, 0xc6aa73a4), TOBN(0xa7cdea3d, 0x844423ac), + TOBN(0x5ed0c007, 0xf8ae7c38), TOBN(0x6db07a5c, 0x3d740192), + TOBN(0xbe5e9c2a, 0x5fe36db3), TOBN(0xd5b9d57a, 0x76e95046), + TOBN(0x54ac32e7, 0x8eba20f2), TOBN(0xef11ca8f, 0x71b9a352), + TOBN(0x305e373e, 0xff98a658), TOBN(0xffe5a100, 0x823eb667), + TOBN(0x57477b11, 0xe51732d2), TOBN(0xdfd6eb28, 0x2538fc0e), + TOBN(0x5c43b0cc, 0x3b39eec5), TOBN(0x6af12778, 0xcb36cc57), + TOBN(0x70b0852d, 0x06c425ae), TOBN(0x6df92f8c, 0x5c221b9b), + TOBN(0x6c8d4f9e, 0xce826d9c), TOBN(0xf59aba7b, 0xb49359c3), + TOBN(0x5c8ed8d5, 0xda64309d), TOBN(0x61a6de56, 0x91b30704), + TOBN(0xd6b52f6a, 0x2f9b5808), TOBN(0x0eee4194, 0x98c958a7), + TOBN(0xcddd9aab, 0x771e4caa), TOBN(0x83965dfd, 0x78bc21be), + TOBN(0x02affce3, 0xb3b504f5), TOBN(0x30847a21, 0x561c8291), + TOBN(0xd2eb2cf1, 0x52bfda05), TOBN(0xe0e4c4e9, 0x6197b98c), + TOBN(0x1d35076c, 0xf8a1726f), TOBN(0x6c06085b, 0x2db11e3d), + TOBN(0x15c0c4d7, 0x4463ba14), TOBN(0x9d292f83, 0x0030238c), + TOBN(0x1311ee8b, 0x3727536d), TOBN(0xfeea86ef, 0xbeaedc1e), + TOBN(0xb9d18cd3, 0x66131e2e), TOBN(0xf31d974f, 0x80fe2682), + TOBN(0xb6e49e0f, 0xe4160289), TOBN(0x7c48ec0b, 0x08e92799), + TOBN(0x818111d8, 0xd1989aa7), TOBN(0xb34fa0aa, 0xebf926f9), + TOBN(0xdb5fe2f5, 0xa245474a), TOBN(0xf80a6ebb, 0x3c7ca756), + TOBN(0xa7f96054, 0xafa05dd8), TOBN(0x26dfcf21, 0xfcaf119e), + TOBN(0xe20ef2e3, 0x0564bb59), TOBN(0xef4dca50, 0x61cb02b8), + TOBN(0xcda7838a, 0x65d30672), TOBN(0x8b08d534, 0xfd657e86), + TOBN(0x4c5b4395, 0x46d595c8), TOBN(0x39b58725, 0x425cb836), + TOBN(0x8ea61059, 0x3de9abe3), TOBN(0x40434881, 0x9cdc03be), + TOBN(0x9b261245, 0xcfedce8c), TOBN(0x78c318b4, 0xcf5234a1), + TOBN(0x510bcf16, 0xfde24c99), TOBN(0x2a77cb75, 0xa2c2ff5d), + TOBN(0x9c895c2b, 0x27960fb4), TOBN(0xd30ce975, 0xb0eda42b), + TOBN(0xfda85393, 0x1a62cc26), TOBN(0x23c69b96, 0x50c0e052), + TOBN(0xa227df15, 0xbfc633f3), TOBN(0x2ac78848, 0x1bae7d48), + TOBN(0x487878f9, 0x187d073d), TOBN(0x6c2be919, 0x967f807d), + TOBN(0x765861d8, 0x336e6d8f), TOBN(0x88b8974c, 0xce528a43), + TOBN(0x09521177, 0xff57d051), TOBN(0x2ff38037, 0xfb6a1961), + TOBN(0xfc0aba74, 0xa3d76ad4), TOBN(0x7c764803, 0x25a7ec17), + TOBN(0x7532d75f, 0x48879bc8), TOBN(0xea7eacc0, 0x58ce6bc1), + TOBN(0xc82176b4, 0x8e896c16), TOBN(0x9a30e0b2, 0x2c750fed), + TOBN(0xc37e2c2e, 0x421d3aa4), TOBN(0xf926407c, 0xe84fa840), + TOBN(0x18abc03d, 0x1454e41c), TOBN(0x26605ecd, 0x3f7af644), + TOBN(0x242341a6, 0xd6a5eabf), TOBN(0x1edb84f4, 0x216b668e), + TOBN(0xd836edb8, 0x04010102), TOBN(0x5b337ce7, 0x945e1d8c), + TOBN(0xd2075c77, 0xc055dc14), TOBN(0x2a0ffa25, 0x81d89cdf), + TOBN(0x8ce815ea, 0x6ffdcbaf), TOBN(0xa3428878, 0xfb648867), + TOBN(0x277699cf, 0x884655fb), TOBN(0xfa5b5bd6, 0x364d3e41), + TOBN(0x01f680c6, 0x441e1cb7), TOBN(0x3fd61e66, 0xb70a7d67), + TOBN(0x666ba2dc, 0xcc78cf66), TOBN(0xb3018174, 0x6fdbff77), + TOBN(0x8d4dd0db, 0x168d4668), TOBN(0x259455d0, 0x1dab3a2a), + TOBN(0xf58564c5, 0xcde3acec), TOBN(0x77141925, 0x13adb276), + TOBN(0x527d725d, 0x8a303f65), TOBN(0x55deb6c9, 0xe6f38f7b), + TOBN(0xfd5bb657, 0xb1fa70fb), TOBN(0xfa07f50f, 0xd8073a00), + TOBN(0xf72e3aa7, 0xbca02500), TOBN(0xf68f895d, 0x9975740d), + TOBN(0x30112060, 0x5cae2a6a), TOBN(0x01bd7218, 0x02874842), + TOBN(0x3d423891, 0x7ce47bd3), TOBN(0xa66663c1, 0x789544f6), + TOBN(0x864d05d7, 0x3272d838), TOBN(0xe22924f9, 0xfa6295c5), + TOBN(0x8189593f, 0x6c2fda32), TOBN(0x330d7189, 0xb184b544), + TOBN(0x79efa62c, 0xbde1f714), TOBN(0x35771c94, 0xe5cb1a63), + TOBN(0x2f4826b8, 0x641c8332), TOBN(0x00a894fb, 0xc8cee854), + TOBN(0xb4b9a39b, 0x36194d40), TOBN(0xe857a7c5, 0x77612601), + TOBN(0xf4209dd2, 0x4ecf2f58), TOBN(0x82b9e66d, 0x5a033487), + TOBN(0xc1e36934, 0xe4e8b9dd), TOBN(0xd2372c9d, 0xa42377d7), + TOBN(0x51dc94c7, 0x0e3ae43b), TOBN(0x4c57761e, 0x04474f6f), + TOBN(0xdcdacd0a, 0x1058a318), TOBN(0x369cf3f5, 0x78053a9a), + TOBN(0xc6c3de50, 0x31c68de2), TOBN(0x4653a576, 0x3c4b6d9f), + TOBN(0x1688dd5a, 0xaa4e5c97), TOBN(0x5be80aa1, 0xb7ab3c74), + TOBN(0x70cefe7c, 0xbc65c283), TOBN(0x57f95f13, 0x06867091), + TOBN(0xa39114e2, 0x4415503b), TOBN(0xc08ff7c6, 0x4cbb17e9), + TOBN(0x1eff674d, 0xd7dec966), TOBN(0x6d4690af, 0x53376f63), + TOBN(0xff6fe32e, 0xea74237b), TOBN(0xc436d17e, 0xcd57508e), + TOBN(0x15aa28e1, 0xedcc40fe), TOBN(0x0d769c04, 0x581bbb44), + TOBN(0xc240b6de, 0x34eaacda), TOBN(0xd9e116e8, 0x2ba0f1de), + TOBN(0xcbe45ec7, 0x79438e55), TOBN(0x91787c9d, 0x96f752d7), + TOBN(0x897f532b, 0xf129ac2f), TOBN(0xd307b7c8, 0x5a36e22c), + TOBN(0x91940675, 0x749fb8f3), TOBN(0xd14f95d0, 0x157fdb28), + TOBN(0xfe51d029, 0x6ae55043), TOBN(0x8931e98f, 0x44a87de1), + TOBN(0xe57f1cc6, 0x09e4fee2), TOBN(0x0d063b67, 0x4e072d92), + TOBN(0x70a998b9, 0xed0e4316), TOBN(0xe74a736b, 0x306aca46), + TOBN(0xecf0fbf2, 0x4fda97c7), TOBN(0xa40f65cb, 0x3e178d93), + TOBN(0x16253604, 0x16df4285), TOBN(0xb0c9babb, 0xd0c56ae2), + TOBN(0x73032b19, 0xcfc5cfc3), TOBN(0xe497e5c3, 0x09752056), + TOBN(0x12096bb4, 0x164bda96), TOBN(0x1ee42419, 0xa0b74da1), + TOBN(0x8fc36243, 0x403826ba), TOBN(0x0c8f0069, 0xdc09e660), + TOBN(0x8667e981, 0xc27253c9), TOBN(0x05a6aefb, 0x92b36a45), + TOBN(0xa62c4b36, 0x9cb7bb46), TOBN(0x8394f375, 0x11f7027b), + TOBN(0x747bc79c, 0x5f109d0f), TOBN(0xcad88a76, 0x5b8cc60a), + TOBN(0x80c5a66b, 0x58f09e68), TOBN(0xe753d451, 0xf6127eac), + TOBN(0xc44b74a1, 0x5b0ec6f5), TOBN(0x47989fe4, 0x5289b2b8), + TOBN(0x745f8484, 0x58d6fc73), TOBN(0xec362a6f, 0xf61c70ab), + TOBN(0x070c98a7, 0xb3a8ad41), TOBN(0x73a20fc0, 0x7b63db51), + TOBN(0xed2c2173, 0xf44c35f4), TOBN(0x8a56149d, 0x9acc9dca), + TOBN(0x98f17881, 0x9ac6e0f4), TOBN(0x360fdeaf, 0xa413b5ed), + TOBN(0x0625b8f4, 0xa300b0fd), TOBN(0xf1f4d76a, 0x5b3222d3), + TOBN(0x9d6f5109, 0x587f76b8), TOBN(0x8b4ee08d, 0x2317fdb5), + TOBN(0x88089bb7, 0x8c68b095), TOBN(0x95570e9a, 0x5808d9b9), + TOBN(0xa395c36f, 0x35d33ae7), TOBN(0x200ea123, 0x50bb5a94), + TOBN(0x20c789bd, 0x0bafe84b), TOBN(0x243ef52d, 0x0919276a), + TOBN(0x3934c577, 0xe23ae233), TOBN(0xb93807af, 0xa460d1ec), + TOBN(0xb72a53b1, 0xf8fa76a4), TOBN(0xd8914cb0, 0xc3ca4491), + TOBN(0x2e128494, 0x3fb42622), TOBN(0x3b2700ac, 0x500907d5), + TOBN(0xf370fb09, 0x1a95ec63), TOBN(0xf8f30be2, 0x31b6dfbd), + TOBN(0xf2b2f8d2, 0x69e55f15), TOBN(0x1fead851, 0xcc1323e9), + TOBN(0xfa366010, 0xd9e5eef6), TOBN(0x64d487b0, 0xe316107e), + TOBN(0x4c076b86, 0xd23ddc82), TOBN(0x03fd344c, 0x7e0143f0), + TOBN(0xa95362ff, 0x317af2c5), TOBN(0x0add3db7, 0xe18b7a4f), + TOBN(0x9c673e3f, 0x8260e01b), TOBN(0xfbeb49e5, 0x54a1cc91), + TOBN(0x91351bf2, 0x92f2e433), TOBN(0xc755e7ec, 0x851141eb), + TOBN(0xc9a95139, 0x29607745), TOBN(0x0ca07420, 0xa26f2b28), + TOBN(0xcb2790e7, 0x4bc6f9dd), TOBN(0x345bbb58, 0xadcaffc0), + TOBN(0xc65ea38c, 0xbe0f27a2), TOBN(0x67c24d7c, 0x641fcb56), + TOBN(0x2c25f0a7, 0xa9e2c757), TOBN(0x93f5cdb0, 0x16f16c49), + TOBN(0x2ca5a9d7, 0xc5ee30a1), TOBN(0xd1593635, 0xb909b729), + TOBN(0x804ce9f3, 0xdadeff48), TOBN(0xec464751, 0xb07c30c3), + TOBN(0x89d65ff3, 0x9e49af6a), TOBN(0xf2d6238a, 0x6f3d01bc), + TOBN(0x1095561e, 0x0bced843), TOBN(0x51789e12, 0xc8a13fd8), + TOBN(0xd633f929, 0x763231df), TOBN(0x46df9f7d, 0xe7cbddef), + TOBN(0x01c889c0, 0xcb265da8), TOBN(0xfce1ad10, 0xaf4336d2), + TOBN(0x8d110df6, 0xfc6a0a7e), TOBN(0xdd431b98, 0x6da425dc), + TOBN(0xcdc4aeab, 0x1834aabe), TOBN(0x84deb124, 0x8439b7fc), + TOBN(0x8796f169, 0x3c2a5998), TOBN(0x9b9247b4, 0x7947190d), + TOBN(0x55b9d9a5, 0x11597014), TOBN(0x7e9dd70d, 0x7b1566ee), + TOBN(0x94ad78f7, 0xcbcd5e64), TOBN(0x0359ac17, 0x9bd4c032), + TOBN(0x3b11baaf, 0x7cc222ae), TOBN(0xa6a6e284, 0xba78e812), + TOBN(0x8392053f, 0x24cea1a0), TOBN(0xc97bce4a, 0x33621491), + TOBN(0x7eb1db34, 0x35399ee9), TOBN(0x473f78ef, 0xece81ad1), + TOBN(0x41d72fe0, 0xf63d3d0d), TOBN(0xe620b880, 0xafab62fc), + TOBN(0x92096bc9, 0x93158383), TOBN(0x41a21357, 0x8f896f6c), + TOBN(0x1b5ee2fa, 0xc7dcfcab), TOBN(0x650acfde, 0x9546e007), + TOBN(0xc081b749, 0xb1b02e07), TOBN(0xda9e41a0, 0xf9eca03d), + TOBN(0x013ba727, 0x175a54ab), TOBN(0xca0cd190, 0xea5d8d10), + TOBN(0x85ea52c0, 0x95fd96a9), TOBN(0x2c591b9f, 0xbc5c3940), + TOBN(0x6fb4d4e4, 0x2bad4d5f), TOBN(0xfa4c3590, 0xfef0059b), + TOBN(0x6a10218a, 0xf5122294), TOBN(0x9a78a81a, 0xa85751d1), + TOBN(0x04f20579, 0xa98e84e7), TOBN(0xfe1242c0, 0x4997e5b5), + TOBN(0xe77a273b, 0xca21e1e4), TOBN(0xfcc8b1ef, 0x9411939d), + TOBN(0xe20ea302, 0x92d0487a), TOBN(0x1442dbec, 0x294b91fe), + TOBN(0x1f7a4afe, 0xbb6b0e8f), TOBN(0x1700ef74, 0x6889c318), + TOBN(0xf5bbffc3, 0x70f1fc62), TOBN(0x3b31d4b6, 0x69c79cca), + TOBN(0xe8bc2aab, 0xa7f6340d), TOBN(0xb0b08ab4, 0xa725e10a), + TOBN(0x44f05701, 0xae340050), TOBN(0xba4b3016, 0x1cf0c569), + TOBN(0x5aa29f83, 0xfbe19a51), TOBN(0x1b9ed428, 0xb71d752e), + TOBN(0x1666e54e, 0xeb4819f5), TOBN(0x616cdfed, 0x9e18b75b), + TOBN(0x112ed5be, 0x3ee27b0b), TOBN(0xfbf28319, 0x44c7de4d), + TOBN(0xd685ec85, 0xe0e60d84), TOBN(0x68037e30, 0x1db7ee78), + TOBN(0x5b65bdcd, 0x003c4d6e), TOBN(0x33e7363a, 0x93e29a6a), + TOBN(0x995b3a61, 0x08d0756c), TOBN(0xd727f85c, 0x2faf134b), + TOBN(0xfac6edf7, 0x1d337823), TOBN(0x99b9aa50, 0x0439b8b4), + TOBN(0x722eb104, 0xe2b4e075), TOBN(0x49987295, 0x437c4926), + TOBN(0xb1e4c0e4, 0x46a9b82d), TOBN(0xd0cb3197, 0x57a006f5), + TOBN(0xf3de0f7d, 0xd7808c56), TOBN(0xb5c54d8f, 0x51f89772), + TOBN(0x500a114a, 0xadbd31aa), TOBN(0x9afaaaa6, 0x295f6cab), + TOBN(0x94705e21, 0x04cf667a), TOBN(0xfc2a811b, 0x9d3935d7), + TOBN(0x560b0280, 0x6d09267c), TOBN(0xf19ed119, 0xf780e53b), + TOBN(0xf0227c09, 0x067b6269), TOBN(0x967b8533, 0x5caef599), + TOBN(0x155b9243, 0x68efeebc), TOBN(0xcd6d34f5, 0xc497bae6), + TOBN(0x1dd8d5d3, 0x6cceb370), TOBN(0x2aeac579, 0xa78d7bf9), + TOBN(0x5d65017d, 0x70b67a62), TOBN(0x70c8e44f, 0x17c53f67), + TOBN(0xd1fc0950, 0x86a34d09), TOBN(0xe0fca256, 0xe7134907), + TOBN(0xe24fa29c, 0x80fdd315), TOBN(0x2c4acd03, 0xd87499ad), + TOBN(0xbaaf7517, 0x3b5a9ba6), TOBN(0xb9cbe1f6, 0x12e51a51), + TOBN(0xd88edae3, 0x5e154897), TOBN(0xe4309c3c, 0x77b66ca0), + TOBN(0xf5555805, 0xf67f3746), TOBN(0x85fc37ba, 0xa36401ff), + TOBN(0xdf86e2ca, 0xd9499a53), TOBN(0x6270b2a3, 0xecbc955b), + TOBN(0xafae64f5, 0x974ad33b), TOBN(0x04d85977, 0xfe7b2df1), + TOBN(0x2a3db3ff, 0x4ab03f73), TOBN(0x0b87878a, 0x8702740a), + TOBN(0x6d263f01, 0x5a061732), TOBN(0xc25430ce, 0xa32a1901), + TOBN(0xf7ebab3d, 0xdb155018), TOBN(0x3a86f693, 0x63a9b78e), + TOBN(0x349ae368, 0xda9f3804), TOBN(0x470f07fe, 0xa164349c), + TOBN(0xd52f4cc9, 0x8562baa5), TOBN(0xc74a9e86, 0x2b290df3), + TOBN(0xd3a1aa35, 0x43471a24), TOBN(0x239446be, 0xb8194511), + TOBN(0xbec2dd00, 0x81dcd44d), TOBN(0xca3d7f0f, 0xc42ac82d), + TOBN(0x1f3db085, 0xfdaf4520), TOBN(0xbb6d3e80, 0x4549daf2), + TOBN(0xf5969d8a, 0x19ad5c42), TOBN(0x7052b13d, 0xdbfd1511), + TOBN(0x11890d1b, 0x682b9060), TOBN(0xa71d3883, 0xac34452c), + TOBN(0xa438055b, 0x783805b4), TOBN(0x43241277, 0x4725b23e), + TOBN(0xf20cf96e, 0x4901bbed), TOBN(0x6419c710, 0xf432a2bb), + TOBN(0x57a0fbb9, 0xdfa9cd7d), TOBN(0x589111e4, 0x00daa249), + TOBN(0x19809a33, 0x7b60554e), TOBN(0xea5f8887, 0xede283a4), + TOBN(0x2d713802, 0x503bfd35), TOBN(0x151bb0af, 0x585d2a53), + TOBN(0x40b08f74, 0x43b30ca8), TOBN(0xe10b5bba, 0xd9934583), + TOBN(0xe8a546d6, 0xb51110ad), TOBN(0x1dd50e66, 0x28e0b6c5), + TOBN(0x292e9d54, 0xcff2b821), TOBN(0x3882555d, 0x47281760), + TOBN(0x134838f8, 0x3724d6e3), TOBN(0xf2c679e0, 0x22ddcda1), + TOBN(0x40ee8815, 0x6d2a5768), TOBN(0x7f227bd2, 0x1c1e7e2d), + TOBN(0x487ba134, 0xd04ff443), TOBN(0x76e2ff3d, 0xc614e54b), + TOBN(0x36b88d6f, 0xa3177ec7), TOBN(0xbf731d51, 0x2328fff5), + TOBN(0x758caea2, 0x49ba158e), TOBN(0x5ab8ff4c, 0x02938188), + TOBN(0x33e16056, 0x35edc56d), TOBN(0x5a69d349, 0x7e940d79), + TOBN(0x6c4fd001, 0x03866dcb), TOBN(0x20a38f57, 0x4893cdef), + TOBN(0xfbf3e790, 0xfac3a15b), TOBN(0x6ed7ea2e, 0x7a4f8e6b), + TOBN(0xa663eb4f, 0xbc3aca86), TOBN(0x22061ea5, 0x080d53f7), + TOBN(0x2480dfe6, 0xf546783f), TOBN(0xd38bc6da, 0x5a0a641e), + TOBN(0xfb093cd1, 0x2ede8965), TOBN(0x89654db4, 0xacb455cf), + TOBN(0x413cbf9a, 0x26e1adee), TOBN(0x291f3764, 0x373294d4), + TOBN(0x00797257, 0x648083fe), TOBN(0x25f504d3, 0x208cc341), + TOBN(0x635a8e5e, 0xc3a0ee43), TOBN(0x70aaebca, 0x679898ff), + TOBN(0x9ee9f547, 0x5dc63d56), TOBN(0xce987966, 0xffb34d00), + TOBN(0xf9f86b19, 0x5e26310a), TOBN(0x9e435484, 0x382a8ca8), + TOBN(0x253bcb81, 0xc2352fe4), TOBN(0xa4eac8b0, 0x4474b571), + TOBN(0xc1b97512, 0xc1ad8cf8), TOBN(0x193b4e9e, 0x99e0b697), + TOBN(0x939d2716, 0x01e85df0), TOBN(0x4fb265b3, 0xcd44eafd), + TOBN(0x321e7dcd, 0xe51e1ae2), TOBN(0x8e3a8ca6, 0xe3d8b096), + TOBN(0x8de46cb0, 0x52604998), TOBN(0x91099ad8, 0x39072aa7), + TOBN(0x2617f91c, 0x93aa96b8), TOBN(0x0fc8716b, 0x7fca2e13), + TOBN(0xa7106f5e, 0x95328723), TOBN(0xd1c9c40b, 0x262e6522), + TOBN(0xb9bafe86, 0x42b7c094), TOBN(0x1873439d, 0x1543c021), + TOBN(0xe1baa5de, 0x5cbefd5d), TOBN(0xa363fc5e, 0x521e8aff), + TOBN(0xefe6320d, 0xf862eaac), TOBN(0x14419c63, 0x22c647dc), + TOBN(0x0e06707c, 0x4e46d428), TOBN(0xcb6c834f, 0x4a178f8f), + TOBN(0x0f993a45, 0xd30f917c), TOBN(0xd4c4b049, 0x9879afee), + TOBN(0xb6142a1e, 0x70500063), TOBN(0x7c9b41c3, 0xa5d9d605), + TOBN(0xbc00fc2f, 0x2f8ba2c7), TOBN(0x0966eb2f, 0x7c67aa28), + TOBN(0x13f7b516, 0x5a786972), TOBN(0x3bfb7557, 0x8a2fbba0), + TOBN(0x131c4f23, 0x5a2b9620), TOBN(0xbff3ed27, 0x6faf46be), + TOBN(0x9b4473d1, 0x7e172323), TOBN(0x421e8878, 0x339f6246), + TOBN(0x0fa8587a, 0x25a41632), TOBN(0xc0814124, 0xa35b6c93), + TOBN(0x2b18a9f5, 0x59ebb8db), TOBN(0x264e3357, 0x76edb29c), + TOBN(0xaf245ccd, 0xc87c51e2), TOBN(0x16b3015b, 0x501e6214), + TOBN(0xbb31c560, 0x0a3882ce), TOBN(0x6961bb94, 0xfec11e04), + TOBN(0x3b825b8d, 0xeff7a3a0), TOBN(0xbec33738, 0xb1df7326), + TOBN(0x68ad747c, 0x99604a1f), TOBN(0xd154c934, 0x9a3bd499), + TOBN(0xac33506f, 0x1cc7a906), TOBN(0x73bb5392, 0x6c560e8f), + TOBN(0x6428fcbe, 0x263e3944), TOBN(0xc11828d5, 0x1c387434), + TOBN(0x3cd04be1, 0x3e4b12ff), TOBN(0xc3aad9f9, 0x2d88667c), + TOBN(0xc52ddcf8, 0x248120cf), TOBN(0x985a892e, 0x2a389532), + TOBN(0xfbb4b21b, 0x3bb85fa0), TOBN(0xf95375e0, 0x8dfc6269), + TOBN(0xfb4fb06c, 0x7ee2acea), TOBN(0x6785426e, 0x309c4d1f), + TOBN(0x659b17c8, 0xd8ceb147), TOBN(0x9b649eee, 0xb70a5554), + TOBN(0x6b7fa0b5, 0xac6bc634), TOBN(0xd99fe2c7, 0x1d6e732f), + TOBN(0x30e6e762, 0x8d3abba2), TOBN(0x18fee6e7, 0xa797b799), + TOBN(0x5c9d360d, 0xc696464d), TOBN(0xe3baeb48, 0x27bfde12), + TOBN(0x2bf5db47, 0xf23206d5), TOBN(0x2f6d3420, 0x1d260152), + TOBN(0x17b87653, 0x3f8ff89a), TOBN(0x5157c30c, 0x378fa458), + TOBN(0x7517c5c5, 0x2d4fb936), TOBN(0xef22f7ac, 0xe6518cdc), + TOBN(0xdeb483e6, 0xbf847a64), TOBN(0xf5084558, 0x92e0fa89),} + , + {TOBN(0xab9659d8, 0xdf7304d4), TOBN(0xb71bcf1b, 0xff210e8e), + TOBN(0xa9a2438b, 0xd73fbd60), TOBN(0x4595cd1f, 0x5d11b4de), + TOBN(0x9c0d329a, 0x4835859d), TOBN(0x4a0f0d2d, 0x7dbb6e56), + TOBN(0xc6038e5e, 0xdf928a4e), TOBN(0xc9429621, 0x8f5ad154), + TOBN(0x91213462, 0xf23f2d92), TOBN(0x6cab71bd, 0x60b94078), + TOBN(0x6bdd0a63, 0x176cde20), TOBN(0x54c9b20c, 0xee4d54bc), + TOBN(0x3cd2d8aa, 0x9f2ac02f), TOBN(0x03f8e617, 0x206eedb0), + TOBN(0xc7f68e16, 0x93086434), TOBN(0x831469c5, 0x92dd3db9), + TOBN(0x8521df24, 0x8f981354), TOBN(0x587e23ec, 0x3588a259), + TOBN(0xcbedf281, 0xd7a0992c), TOBN(0x06930a55, 0x38961407), + TOBN(0x09320deb, 0xbe5bbe21), TOBN(0xa7ffa5b5, 0x2491817f), + TOBN(0xe6c8b4d9, 0x09065160), TOBN(0xac4f3992, 0xfff6d2a9), + TOBN(0x7aa7a158, 0x3ae9c1bd), TOBN(0xe0af6d98, 0xe37ce240), + TOBN(0xe54342d9, 0x28ab38b4), TOBN(0xe8b75007, 0x0a1c98ca), + TOBN(0xefce86af, 0xe02358f2), TOBN(0x31b8b856, 0xea921228), + TOBN(0x052a1912, 0x0a1c67fc), TOBN(0xb4069ea4, 0xe3aead59), + TOBN(0x3232d6e2, 0x7fa03cb3), TOBN(0xdb938e5b, 0x0fdd7d88), + TOBN(0x04c1d2cd, 0x2ccbfc5d), TOBN(0xd2f45c12, 0xaf3a580f), + TOBN(0x592620b5, 0x7883e614), TOBN(0x5fd27e68, 0xbe7c5f26), + TOBN(0x139e45a9, 0x1567e1e3), TOBN(0x2cc71d2d, 0x44d8aaaf), + TOBN(0x4a9090cd, 0xe36d0757), TOBN(0xf722d7b1, 0xd9a29382), + TOBN(0xfb7fb04c, 0x04b48ddf), TOBN(0x628ad2a7, 0xebe16f43), + TOBN(0xcd3fbfb5, 0x20226040), TOBN(0x6c34ecb1, 0x5104b6c4), + TOBN(0x30c0754e, 0xc903c188), TOBN(0xec336b08, 0x2d23cab0), + TOBN(0x473d62a2, 0x1e206ee5), TOBN(0xf1e27480, 0x8c49a633), + TOBN(0x87ab956c, 0xe9f6b2c3), TOBN(0x61830b48, 0x62b606ea), + TOBN(0x67cd6846, 0xe78e815f), TOBN(0xfe40139f, 0x4c02082a), + TOBN(0x52bbbfcb, 0x952ec365), TOBN(0x74c11642, 0x6b9836ab), + TOBN(0x9f51439e, 0x558df019), TOBN(0x230da4ba, 0xac712b27), + TOBN(0x518919e3, 0x55185a24), TOBN(0x4dcefcdd, 0x84b78f50), + TOBN(0xa7d90fb2, 0xa47d4c5a), TOBN(0x55ac9abf, 0xb30e009e), + TOBN(0xfd2fc359, 0x74eed273), TOBN(0xb72d824c, 0xdbea8faf), + TOBN(0xce721a74, 0x4513e2ca), TOBN(0x0b418612, 0x38240b2c), + TOBN(0x05199968, 0xd5baa450), TOBN(0xeb1757ed, 0x2b0e8c25), + TOBN(0x6ebc3e28, 0x3dfac6d5), TOBN(0xb2431e2e, 0x48a237f5), + TOBN(0x2acb5e23, 0x52f61499), TOBN(0x5558a2a7, 0xe06c936b), + TOBN(0xd213f923, 0xcbb13d1b), TOBN(0x98799f42, 0x5bfb9bfe), + TOBN(0x1ae8ddc9, 0x701144a9), TOBN(0x0b8b3bb6, 0x4c5595ee), + TOBN(0x0ea9ef2e, 0x3ecebb21), TOBN(0x17cb6c4b, 0x3671f9a7), + TOBN(0x47ef464f, 0x726f1d1f), TOBN(0x171b9484, 0x6943a276), + TOBN(0x51a4ae2d, 0x7ef0329c), TOBN(0x08509222, 0x91c4402a), + TOBN(0x64a61d35, 0xafd45bbc), TOBN(0x38f096fe, 0x3035a851), + TOBN(0xc7468b74, 0xa1dec027), TOBN(0xe8cf10e7, 0x4fc7dcba), + TOBN(0xea35ff40, 0xf4a06353), TOBN(0x0b4c0dfa, 0x8b77dd66), + TOBN(0x779b8552, 0xde7e5c19), TOBN(0xfab28609, 0xc1c0256c), + TOBN(0x64f58eee, 0xabd4743d), TOBN(0x4e8ef838, 0x7b6cc93b), + TOBN(0xee650d26, 0x4cb1bf3d), TOBN(0x4c1f9d09, 0x73dedf61), + TOBN(0xaef7c9d7, 0xbfb70ced), TOBN(0x1ec0507e, 0x1641de1e), + TOBN(0xcd7e5cc7, 0xcde45079), TOBN(0xde173c9a, 0x516ac9e4), + TOBN(0x517a8494, 0xc170315c), TOBN(0x438fd905, 0x91d8e8fb), + TOBN(0x5145c506, 0xc7d9630b), TOBN(0x6457a87b, 0xf47d4d75), + TOBN(0xd31646bf, 0x0d9a80e8), TOBN(0x453add2b, 0xcef3aabe), + TOBN(0xc9941109, 0xa607419d), TOBN(0xfaa71e62, 0xbb6bca80), + TOBN(0x34158c13, 0x07c431f3), TOBN(0x594abebc, 0x992bc47a), + TOBN(0x6dfea691, 0xeb78399f), TOBN(0x48aafb35, 0x3f42cba4), + TOBN(0xedcd65af, 0x077c04f0), TOBN(0x1a29a366, 0xe884491a), + TOBN(0x023a40e5, 0x1c21f2bf), TOBN(0xf99a513c, 0xa5057aee), + TOBN(0xa3fe7e25, 0xbcab072e), TOBN(0x8568d2e1, 0x40e32bcf), + TOBN(0x904594eb, 0xd3f69d9f), TOBN(0x181a9733, 0x07affab1), + TOBN(0xe4d68d76, 0xb6e330f4), TOBN(0x87a6dafb, 0xc75a7fc1), + TOBN(0x549db2b5, 0xef7d9289), TOBN(0x2480d4a8, 0x197f015a), + TOBN(0x61d5590b, 0xc40493b6), TOBN(0x3a55b52e, 0x6f780331), + TOBN(0x40eb8115, 0x309eadb0), TOBN(0xdea7de5a, 0x92e5c625), + TOBN(0x64d631f0, 0xcc6a3d5a), TOBN(0x9d5e9d7c, 0x93e8dd61), + TOBN(0xf297bef5, 0x206d3ffc), TOBN(0x23d5e033, 0x7d808bd4), + TOBN(0x4a4f6912, 0xd24cf5ba), TOBN(0xe4d8163b, 0x09cdaa8a), + TOBN(0x0e0de9ef, 0xd3082e8e), TOBN(0x4fe1246c, 0x0192f360), + TOBN(0x1f900150, 0x4b8eee0a), TOBN(0x5219da81, 0xf1da391b), + TOBN(0x7bf6a5c1, 0xf7ea25aa), TOBN(0xd165e6bf, 0xfbb07d5f), + TOBN(0xe3539361, 0x89e78671), TOBN(0xa3fcac89, 0x2bac4219), + TOBN(0xdfab6fd4, 0xf0baa8ab), TOBN(0x5a4adac1, 0xe2c1c2e5), + TOBN(0x6cd75e31, 0x40d85849), TOBN(0xce263fea, 0x19b39181), + TOBN(0xcb6803d3, 0x07032c72), TOBN(0x7f40d5ce, 0x790968c8), + TOBN(0xa6de86bd, 0xdce978f0), TOBN(0x25547c4f, 0x368f751c), + TOBN(0xb1e685fd, 0x65fb2a9e), TOBN(0xce69336f, 0x1eb9179c), + TOBN(0xb15d1c27, 0x12504442), TOBN(0xb7df465c, 0xb911a06b), + TOBN(0xb8d804a3, 0x315980cd), TOBN(0x693bc492, 0xfa3bebf7), + TOBN(0x3578aeee, 0x2253c504), TOBN(0x158de498, 0xcd2474a2), + TOBN(0x1331f5c7, 0xcfda8368), TOBN(0xd2d7bbb3, 0x78d7177e), + TOBN(0xdf61133a, 0xf3c1e46e), TOBN(0x5836ce7d, 0xd30e7be8), + TOBN(0x83084f19, 0x94f834cb), TOBN(0xd35653d4, 0x429ed782), + TOBN(0xa542f16f, 0x59e58243), TOBN(0xc2b52f65, 0x0470a22d), + TOBN(0xe3b6221b, 0x18f23d96), TOBN(0xcb05abac, 0x3f5252b4), + TOBN(0xca00938b, 0x87d61402), TOBN(0x2f186cdd, 0x411933e4), + TOBN(0xe042ece5, 0x9a29a5c5), TOBN(0xb19b3c07, 0x3b6c8402), + TOBN(0xc97667c7, 0x19d92684), TOBN(0xb5624622, 0xebc66372), + TOBN(0x0cb96e65, 0x3c04fa02), TOBN(0x83a7176c, 0x8eaa39aa), + TOBN(0x2033561d, 0xeaa1633f), TOBN(0x45a9d086, 0x4533df73), + TOBN(0xe0542c1d, 0x3dc090bc), TOBN(0x82c996ef, 0xaa59c167), + TOBN(0xe3f735e8, 0x0ee7fc4d), TOBN(0x7b179393, 0x7c35db79), + TOBN(0xb6419e25, 0xf8c5dbfd), TOBN(0x4d9d7a1e, 0x1f327b04), + TOBN(0x979f6f9b, 0x298dfca8), TOBN(0xc7c5dff1, 0x8de9366a), + TOBN(0x1b7a588d, 0x04c82bdd), TOBN(0x68005534, 0xf8319dfd), + TOBN(0xde8a55b5, 0xd8eb9580), TOBN(0x5ea886da, 0x8d5bca81), + TOBN(0xe8530a01, 0x252a0b4d), TOBN(0x1bffb4fe, 0x35eaa0a1), + TOBN(0x2ad828b1, 0xd8e99563), TOBN(0x7de96ef5, 0x95f9cd87), + TOBN(0x4abb2d0c, 0xd77d970c), TOBN(0x03cfb933, 0xd33ef9cb), + TOBN(0xb0547c01, 0x8b211fe9), TOBN(0x2fe64809, 0xa56ed1c6), + TOBN(0xcb7d5624, 0xc2ac98cc), TOBN(0x2a1372c0, 0x1a393e33), + TOBN(0xc8d1ec1c, 0x29660521), TOBN(0xf3d31b04, 0xb37ac3e9), + TOBN(0xa29ae9df, 0x5ece6e7c), TOBN(0x0603ac8f, 0x0facfb55), + TOBN(0xcfe85b7a, 0xdda233a5), TOBN(0xe618919f, 0xbd75f0b8), + TOBN(0xf555a3d2, 0x99bf1603), TOBN(0x1f43afc9, 0xf184255a), + TOBN(0xdcdaf341, 0x319a3e02), TOBN(0xd3b117ef, 0x03903a39), + TOBN(0xe095da13, 0x65d1d131), TOBN(0x86f16367, 0xc37ad03e), + TOBN(0x5f37389e, 0x462cd8dd), TOBN(0xc103fa04, 0xd67a60e6), + TOBN(0x57c34344, 0xf4b478f0), TOBN(0xce91edd8, 0xe117c98d), + TOBN(0x001777b0, 0x231fc12e), TOBN(0x11ae47f2, 0xb207bccb), + TOBN(0xd983cf8d, 0x20f8a242), TOBN(0x7aff5b1d, 0xf22e1ad8), + TOBN(0x68fd11d0, 0x7fc4feb3), TOBN(0x5d53ae90, 0xb0f1c3e1), + TOBN(0x50fb7905, 0xec041803), TOBN(0x85e3c977, 0x14404888), + TOBN(0x0e67faed, 0xac628d8f), TOBN(0x2e865150, 0x6668532c), + TOBN(0x15acaaa4, 0x6a67a6b0), TOBN(0xf4cdee25, 0xb25cec41), + TOBN(0x49ee565a, 0xe4c6701e), TOBN(0x2a04ca66, 0xfc7d63d8), + TOBN(0xeb105018, 0xef0543fb), TOBN(0xf709a4f5, 0xd1b0d81d), + TOBN(0x5b906ee6, 0x2915d333), TOBN(0xf4a87412, 0x96f1f0ab), + TOBN(0xb6b82fa7, 0x4d82f4c2), TOBN(0x90725a60, 0x6804efb3), + TOBN(0xbc82ec46, 0xadc3425e), TOBN(0xb7b80581, 0x2787843e), + TOBN(0xdf46d91c, 0xdd1fc74c), TOBN(0xdc1c62cb, 0xe783a6c4), + TOBN(0x59d1b9f3, 0x1a04cbba), TOBN(0xd87f6f72, 0x95e40764), + TOBN(0x02b4cfc1, 0x317f4a76), TOBN(0x8d2703eb, 0x91036bce), + TOBN(0x98206cc6, 0xa5e72a56), TOBN(0x57be9ed1, 0xcf53fb0f), + TOBN(0x09374571, 0xef0b17ac), TOBN(0x74b2655e, 0xd9181b38), + TOBN(0xc8f80ea8, 0x89935d0e), TOBN(0xc0d9e942, 0x91529936), + TOBN(0x19686041, 0x1e84e0e5), TOBN(0xa5db84d3, 0xaea34c93), + TOBN(0xf9d5bb19, 0x7073a732), TOBN(0xb8d2fe56, 0x6bcfd7c0), + TOBN(0x45775f36, 0xf3eb82fa), TOBN(0x8cb20ccc, 0xfdff8b58), + TOBN(0x1659b65f, 0x8374c110), TOBN(0xb8b4a422, 0x330c789a), + TOBN(0x75e3c3ea, 0x6fe8208b), TOBN(0xbd74b9e4, 0x286e78fe), + TOBN(0x0be2e81b, 0xd7d93a1a), TOBN(0x7ed06e27, 0xdd0a5aae), + TOBN(0x721f5a58, 0x6be8b800), TOBN(0x428299d1, 0xd846db28), + TOBN(0x95cb8e6b, 0x5be88ed3), TOBN(0xc3186b23, 0x1c034e11), + TOBN(0xa6312c9e, 0x8977d99b), TOBN(0xbe944331, 0x83f531e7), + TOBN(0x8232c0c2, 0x18d3b1d4), TOBN(0x617aae8b, 0xe1247b73), + TOBN(0x40153fc4, 0x282aec3b), TOBN(0xc6063d2f, 0xf7b8f823), + TOBN(0x68f10e58, 0x3304f94c), TOBN(0x31efae74, 0xee676346), + TOBN(0xbadb6c6d, 0x40a9b97c), TOBN(0x14702c63, 0x4f666256), + TOBN(0xdeb954f1, 0x5184b2e3), TOBN(0x5184a526, 0x94b6ca40), + TOBN(0xfff05337, 0x003c32ea), TOBN(0x5aa374dd, 0x205974c7), + TOBN(0x9a763854, 0x4b0dd71a), TOBN(0x459cd27f, 0xdeb947ec), + TOBN(0xa6e28161, 0x459c2b92), TOBN(0x2f020fa8, 0x75ee8ef5), + TOBN(0xb132ec2d, 0x30b06310), TOBN(0xc3e15899, 0xbc6a4530), + TOBN(0xdc5f53fe, 0xaa3f451a), TOBN(0x3a3c7f23, 0xc2d9acac), + TOBN(0x2ec2f892, 0x6b27e58b), TOBN(0x68466ee7, 0xd742799f), + TOBN(0x98324dd4, 0x1fa26613), TOBN(0xa2dc6dab, 0xbdc29d63), + TOBN(0xf9675faa, 0xd712d657), TOBN(0x813994be, 0x21fd8d15), + TOBN(0x5ccbb722, 0xfd4f7553), TOBN(0x5135ff8b, 0xf3a36b20), + TOBN(0x44be28af, 0x69559df5), TOBN(0x40b65bed, 0x9d41bf30), + TOBN(0xd98bf2a4, 0x3734e520), TOBN(0x5e3abbe3, 0x209bdcba), + TOBN(0x77c76553, 0xbc945b35), TOBN(0x5331c093, 0xc6ef14aa), + TOBN(0x518ffe29, 0x76b60c80), TOBN(0x2285593b, 0x7ace16f8), + TOBN(0xab1f64cc, 0xbe2b9784), TOBN(0xe8f2c0d9, 0xab2421b6), + TOBN(0x617d7174, 0xc1df065c), TOBN(0xafeeb5ab, 0x5f6578fa), + TOBN(0x16ff1329, 0x263b54a8), TOBN(0x45c55808, 0xc990dce3), + TOBN(0x42eab6c0, 0xecc8c177), TOBN(0x799ea9b5, 0x5982ecaa), + TOBN(0xf65da244, 0xb607ef8e), TOBN(0x8ab226ce, 0x32a3fc2c), + TOBN(0x745741e5, 0x7ea973dc), TOBN(0x5c00ca70, 0x20888f2e), + TOBN(0x7cdce3cf, 0x45fd9cf1), TOBN(0x8a741ef1, 0x5507f872), + TOBN(0x47c51c2f, 0x196b4cec), TOBN(0x70d08e43, 0xc97ea618), + TOBN(0x930da15c, 0x15b18a2b), TOBN(0x33b6c678, 0x2f610514), + TOBN(0xc662e4f8, 0x07ac9794), TOBN(0x1eccf050, 0xba06cb79), + TOBN(0x1ff08623, 0xe7d954e5), TOBN(0x6ef2c5fb, 0x24cf71c3), + TOBN(0xb2c063d2, 0x67978453), TOBN(0xa0cf3796, 0x1d654af8), + TOBN(0x7cb242ea, 0x7ebdaa37), TOBN(0x206e0b10, 0xb86747e0), + TOBN(0x481dae5f, 0xd5ecfefc), TOBN(0x07084fd8, 0xc2bff8fc), + TOBN(0x8040a01a, 0xea324596), TOBN(0x4c646980, 0xd4de4036), + TOBN(0x9eb8ab4e, 0xd65abfc3), TOBN(0xe01cb91f, 0x13541ec7), + TOBN(0x8f029adb, 0xfd695012), TOBN(0x9ae28483, 0x3c7569ec), + TOBN(0xa5614c9e, 0xa66d80a1), TOBN(0x680a3e44, 0x75f5f911), + TOBN(0x0c07b14d, 0xceba4fc1), TOBN(0x891c285b, 0xa13071c1), + TOBN(0xcac67ceb, 0x799ece3c), TOBN(0x29b910a9, 0x41e07e27), + TOBN(0x66bdb409, 0xf2e43123), TOBN(0x06f8b137, 0x7ac9ecbe), + TOBN(0x5981fafd, 0x38547090), TOBN(0x19ab8b9f, 0x85e3415d), + TOBN(0xfc28c194, 0xc7e31b27), TOBN(0x843be0aa, 0x6fbcbb42), + TOBN(0xf3b1ed43, 0xa6db836c), TOBN(0x2a1330e4, 0x01a45c05), + TOBN(0x4f19f3c5, 0x95c1a377), TOBN(0xa85f39d0, 0x44b5ee33), + TOBN(0x3da18e6d, 0x4ae52834), TOBN(0x5a403b39, 0x7423dcb0), + TOBN(0xbb555e0a, 0xf2374aef), TOBN(0x2ad599c4, 0x1e8ca111), + TOBN(0x1b3a2fb9, 0x014b3bf8), TOBN(0x73092684, 0xf66d5007), + TOBN(0x079f1426, 0xc4340102), TOBN(0x1827cf81, 0x8fddf4de), + TOBN(0xc83605f6, 0xf10ff927), TOBN(0xd3871451, 0x23739fc6), + TOBN(0x6d163450, 0xcac1c2cc), TOBN(0x6b521296, 0xa2ec1ac5), + TOBN(0x0606c4f9, 0x6e3cb4a5), TOBN(0xe47d3f41, 0x778abff7), + TOBN(0x425a8d5e, 0xbe8e3a45), TOBN(0x53ea9e97, 0xa6102160), + TOBN(0x477a106e, 0x39cbb688), TOBN(0x532401d2, 0xf3386d32), + TOBN(0x8e564f64, 0xb1b9b421), TOBN(0xca9b8388, 0x81dad33f), + TOBN(0xb1422b4e, 0x2093913e), TOBN(0x533d2f92, 0x69bc8112), + TOBN(0x3fa017be, 0xebe7b2c7), TOBN(0xb2767c4a, 0xcaf197c6), + TOBN(0xc925ff87, 0xaedbae9f), TOBN(0x7daf0eb9, 0x36880a54), + TOBN(0x9284ddf5, 0x9c4d0e71), TOBN(0x1581cf93, 0x316f8cf5), + TOBN(0x3eeca887, 0x3ac1f452), TOBN(0xb417fce9, 0xfb6aeffe), + TOBN(0xa5918046, 0xeefb8dc3), TOBN(0x73d318ac, 0x02209400), + TOBN(0xe800400f, 0x728693e5), TOBN(0xe87d814b, 0x339927ed), + TOBN(0x93e94d3b, 0x57ea9910), TOBN(0xff8a35b6, 0x2245fb69), + TOBN(0x043853d7, 0x7f200d34), TOBN(0x470f1e68, 0x0f653ce1), + TOBN(0x81ac05bd, 0x59a06379), TOBN(0xa14052c2, 0x03930c29), + TOBN(0x6b72fab5, 0x26bc2797), TOBN(0x13670d16, 0x99f16771), + TOBN(0x00170052, 0x1e3e48d1), TOBN(0x978fe401, 0xb7adf678), + TOBN(0x55ecfb92, 0xd41c5dd4), TOBN(0x5ff8e247, 0xc7b27da5), + TOBN(0xe7518272, 0x013fb606), TOBN(0x5768d7e5, 0x2f547a3c), + TOBN(0xbb24eaa3, 0x60017a5f), TOBN(0x6b18e6e4, 0x9c64ce9b), + TOBN(0xc225c655, 0x103dde07), TOBN(0xfc3672ae, 0x7592f7ea), + TOBN(0x9606ad77, 0xd06283a1), TOBN(0x542fc650, 0xe4d59d99), + TOBN(0xabb57c49, 0x2a40e7c2), TOBN(0xac948f13, 0xa8db9f55), + TOBN(0x6d4c9682, 0xb04465c3), TOBN(0xe3d062fa, 0x6468bd15), + TOBN(0xa51729ac, 0x5f318d7e), TOBN(0x1fc87df6, 0x9eb6fc95), + TOBN(0x63d146a8, 0x0591f652), TOBN(0xa861b8f7, 0x589621aa), + TOBN(0x59f5f15a, 0xce31348c), TOBN(0x8f663391, 0x440da6da), + TOBN(0xcfa778ac, 0xb591ffa3), TOBN(0x027ca9c5, 0x4cdfebce), + TOBN(0xbe8e05a5, 0x444ea6b3), TOBN(0x8aab4e69, 0xa78d8254), + TOBN(0x2437f04f, 0xb474d6b8), TOBN(0x6597ffd4, 0x045b3855), + TOBN(0xbb0aea4e, 0xca47ecaa), TOBN(0x568aae83, 0x85c7ebfc), + TOBN(0x0e966e64, 0xc73b2383), TOBN(0x49eb3447, 0xd17d8762), + TOBN(0xde107821, 0x8da05dab), TOBN(0x443d8baa, 0x016b7236), + TOBN(0x163b63a5, 0xea7610d6), TOBN(0xe47e4185, 0xce1ca979), + TOBN(0xae648b65, 0x80baa132), TOBN(0xebf53de2, 0x0e0d5b64), + TOBN(0x8d3bfcb4, 0xd3c8c1ca), TOBN(0x0d914ef3, 0x5d04b309), + TOBN(0x55ef6415, 0x3de7d395), TOBN(0xbde1666f, 0x26b850e8), + TOBN(0xdbe1ca6e, 0xd449ab19), TOBN(0x8902b322, 0xe89a2672), + TOBN(0xb1674b7e, 0xdacb7a53), TOBN(0x8e9faf6e, 0xf52523ff), + TOBN(0x6ba535da, 0x9a85788b), TOBN(0xd21f03ae, 0xbd0626d4), + TOBN(0x099f8c47, 0xe873dc64), TOBN(0xcda8564d, 0x018ec97e), + TOBN(0x3e8d7a5c, 0xde92c68c), TOBN(0x78e035a1, 0x73323cc4), + TOBN(0x3ef26275, 0xf880ff7c), TOBN(0xa4ee3dff, 0x273eedaa), + TOBN(0x58823507, 0xaf4e18f8), TOBN(0x967ec9b5, 0x0672f328), + TOBN(0x9ded19d9, 0x559d3186), TOBN(0x5e2ab3de, 0x6cdce39c), + TOBN(0xabad6e4d, 0x11c226df), TOBN(0xf9783f43, 0x87723014), + TOBN(0x9a49a0cf, 0x1a885719), TOBN(0xfc0c1a5a, 0x90da9dbf), + TOBN(0x8bbaec49, 0x571d92ac), TOBN(0x569e85fe, 0x4692517f), + TOBN(0x8333b014, 0xa14ea4af), TOBN(0x32f2a62f, 0x12e5c5ad), + TOBN(0x98c2ce3a, 0x06d89b85), TOBN(0xb90741aa, 0x2ff77a08), + TOBN(0x2530defc, 0x01f795a2), TOBN(0xd6e5ba0b, 0x84b3c199), + TOBN(0x7d8e8451, 0x12e4c936), TOBN(0xae419f7d, 0xbd0be17b), + TOBN(0xa583fc8c, 0x22262bc9), TOBN(0x6b842ac7, 0x91bfe2bd), + TOBN(0x33cef4e9, 0x440d6827), TOBN(0x5f69f4de, 0xef81fb14), + TOBN(0xf16cf6f6, 0x234fbb92), TOBN(0x76ae3fc3, 0xd9e7e158), + TOBN(0x4e89f6c2, 0xe9740b33), TOBN(0x677bc85d, 0x4962d6a1), + TOBN(0x6c6d8a7f, 0x68d10d15), TOBN(0x5f9a7224, 0x0257b1cd), + TOBN(0x7096b916, 0x4ad85961), TOBN(0x5f8c47f7, 0xe657ab4a), + TOBN(0xde57d7d0, 0xf7461d7e), TOBN(0x7eb6094d, 0x80ce5ee2), + TOBN(0x0b1e1dfd, 0x34190547), TOBN(0x8a394f43, 0xf05dd150), + TOBN(0x0a9eb24d, 0x97df44e6), TOBN(0x78ca06bf, 0x87675719), + TOBN(0x6f0b3462, 0x6ffeec22), TOBN(0x9d91bcea, 0x36cdd8fb), + TOBN(0xac83363c, 0xa105be47), TOBN(0x81ba76c1, 0x069710e3), + TOBN(0x3d1b24cb, 0x28c682c6), TOBN(0x27f25228, 0x8612575b), + TOBN(0xb587c779, 0xe8e66e98), TOBN(0x7b0c03e9, 0x405eb1fe), + TOBN(0xfdf0d030, 0x15b548e7), TOBN(0xa8be76e0, 0x38b36af7), + TOBN(0x4cdab04a, 0x4f310c40), TOBN(0x6287223e, 0xf47ecaec), + TOBN(0x678e6055, 0x8b399320), TOBN(0x61fe3fa6, 0xc01e4646), + TOBN(0xc482866b, 0x03261a5e), TOBN(0xdfcf45b8, 0x5c2f244a), + TOBN(0x8fab9a51, 0x2f684b43), TOBN(0xf796c654, 0xc7220a66), + TOBN(0x1d90707e, 0xf5afa58f), TOBN(0x2c421d97, 0x4fdbe0de), + TOBN(0xc4f4cda3, 0xaf2ebc2f), TOBN(0xa0af843d, 0xcb4efe24), + TOBN(0x53b857c1, 0x9ccd10b1), TOBN(0xddc9d1eb, 0x914d3e04), + TOBN(0x7bdec8bb, 0x62771deb), TOBN(0x829277aa, 0x91c5aa81), + TOBN(0x7af18dd6, 0x832391ae), TOBN(0x1740f316, 0xc71a84ca),} + , + {TOBN(0x8928e99a, 0xeeaf8c49), TOBN(0xee7aa73d, 0x6e24d728), + TOBN(0x4c5007c2, 0xe72b156c), TOBN(0x5fcf57c5, 0xed408a1d), + TOBN(0x9f719e39, 0xb6057604), TOBN(0x7d343c01, 0xc2868bbf), + TOBN(0x2cca254b, 0x7e103e2d), TOBN(0xe6eb38a9, 0xf131bea2), + TOBN(0xb33e624f, 0x8be762b4), TOBN(0x2a9ee4d1, 0x058e3413), + TOBN(0x968e6369, 0x67d805fa), TOBN(0x9848949b, 0x7db8bfd7), + TOBN(0x5308d7e5, 0xd23a8417), TOBN(0x892f3b1d, 0xf3e29da5), + TOBN(0xc95c139e, 0x3dee471f), TOBN(0x8631594d, 0xd757e089), + TOBN(0xe0c82a3c, 0xde918dcc), TOBN(0x2e7b5994, 0x26fdcf4b), + TOBN(0x82c50249, 0x32cb1b2d), TOBN(0xea613a9d, 0x7657ae07), + TOBN(0xc2eb5f6c, 0xf1fdc9f7), TOBN(0xb6eae8b8, 0x879fe682), + TOBN(0x253dfee0, 0x591cbc7f), TOBN(0x000da713, 0x3e1290e6), + TOBN(0x1083e2ea, 0x1f095615), TOBN(0x0a28ad77, 0x14e68c33), + TOBN(0x6bfc0252, 0x3d8818be), TOBN(0xb585113a, 0xf35850cd), + TOBN(0x7d935f0b, 0x30df8aa1), TOBN(0xaddda07c, 0x4ab7e3ac), + TOBN(0x92c34299, 0x552f00cb), TOBN(0xc33ed1de, 0x2909df6c), + TOBN(0x22c2195d, 0x80e87766), TOBN(0x9e99e6d8, 0x9ddf4ac0), + TOBN(0x09642e4e, 0x65e74934), TOBN(0x2610ffa2, 0xff1ff241), + TOBN(0x4d1d47d4, 0x751c8159), TOBN(0x697b4985, 0xaf3a9363), + TOBN(0x0318ca46, 0x87477c33), TOBN(0xa90cb565, 0x9441eff3), + TOBN(0x58bb3848, 0x36f024cb), TOBN(0x85be1f77, 0x36016168), + TOBN(0x6c59587c, 0xdc7e07f1), TOBN(0x191be071, 0xaf1d8f02), + TOBN(0xbf169fa5, 0xcca5e55c), TOBN(0x3864ba3c, 0xf7d04eac), + TOBN(0x915e367f, 0x8d7d05db), TOBN(0xb48a876d, 0xa6549e5d), + TOBN(0xef89c656, 0x580e40a2), TOBN(0xf194ed8c, 0x728068bc), + TOBN(0x74528045, 0xa47990c9), TOBN(0xf53fc7d7, 0x5e1a4649), + TOBN(0xbec5ae9b, 0x78593e7d), TOBN(0x2cac4ee3, 0x41db65d7), + TOBN(0xa8c1eb24, 0x04a3d39b), TOBN(0x53b7d634, 0x03f8f3ef), + TOBN(0x2dc40d48, 0x3e07113c), TOBN(0x6e4a5d39, 0x7d8b63ae), + TOBN(0x5582a94b, 0x79684c2b), TOBN(0x932b33d4, 0x622da26c), + TOBN(0xf534f651, 0x0dbbf08d), TOBN(0x211d07c9, 0x64c23a52), + TOBN(0x0eeece0f, 0xee5bdc9b), TOBN(0xdf178168, 0xf7015558), + TOBN(0xd4294635, 0x0a712229), TOBN(0x93cbe448, 0x09273f8c), + TOBN(0x00b095ef, 0x8f13bc83), TOBN(0xbb741972, 0x8798978c), + TOBN(0x9d7309a2, 0x56dbe6e7), TOBN(0xe578ec56, 0x5a5d39ec), + TOBN(0x3961151b, 0x851f9a31), TOBN(0x2da7715d, 0xe5709eb4), + TOBN(0x867f3017, 0x53dfabf0), TOBN(0x728d2078, 0xb8e39259), + TOBN(0x5c75a0cd, 0x815d9958), TOBN(0xf84867a6, 0x16603be1), + TOBN(0xc865b13d, 0x70e35b1c), TOBN(0x02414468, 0x19b03e2c), + TOBN(0xe46041da, 0xac1f3121), TOBN(0x7c9017ad, 0x6f028a7c), + TOBN(0xabc96de9, 0x0a482873), TOBN(0x4265d6b1, 0xb77e54d4), + TOBN(0x68c38e79, 0xa57d88e7), TOBN(0xd461d766, 0x9ce82de3), + TOBN(0x817a9ec5, 0x64a7e489), TOBN(0xcc5675cd, 0xa0def5f2), + TOBN(0x9a00e785, 0x985d494e), TOBN(0xc626833f, 0x1b03514a), + TOBN(0xabe7905a, 0x83cdd60e), TOBN(0x50602fb5, 0xa1170184), + TOBN(0x689886cd, 0xb023642a), TOBN(0xd568d090, 0xa6e1fb00), + TOBN(0x5b1922c7, 0x0259217f), TOBN(0x93831cd9, 0xc43141e4), + TOBN(0xdfca3587, 0x0c95f86e), TOBN(0xdec2057a, 0x568ae828), + TOBN(0xc44ea599, 0xf98a759a), TOBN(0x55a0a7a2, 0xf7c23c1d), + TOBN(0xd5ffb6e6, 0x94c4f687), TOBN(0x3563cce2, 0x12848478), + TOBN(0x812b3517, 0xe7b1fbe1), TOBN(0x8a7dc979, 0x4f7338e0), + TOBN(0x211ecee9, 0x52d048db), TOBN(0x2eea4056, 0xc86ea3b8), + TOBN(0xd8cb68a7, 0xba772b34), TOBN(0xe16ed341, 0x5f4e2541), + TOBN(0x9b32f6a6, 0x0fec14db), TOBN(0xeee376f7, 0x391698be), + TOBN(0xe9a7aa17, 0x83674c02), TOBN(0x65832f97, 0x5843022a), + TOBN(0x29f3a8da, 0x5ba4990f), TOBN(0x79a59c3a, 0xfb8e3216), + TOBN(0x9cdc4d2e, 0xbd19bb16), TOBN(0xc6c7cfd0, 0xb3262d86), + TOBN(0xd4ce14d0, 0x969c0b47), TOBN(0x1fa352b7, 0x13e56128), + TOBN(0x383d55b8, 0x973db6d3), TOBN(0x71836850, 0xe8e5b7bf), + TOBN(0xc7714596, 0xe6bb571f), TOBN(0x259df31f, 0x2d5b2dd2), + TOBN(0x568f8925, 0x913cc16d), TOBN(0x18bc5b6d, 0xe1a26f5a), + TOBN(0xdfa413be, 0xf5f499ae), TOBN(0xf8835dec, 0xc3f0ae84), + TOBN(0xb6e60bd8, 0x65a40ab0), TOBN(0x65596439, 0x194b377e), + TOBN(0xbcd85625, 0x92084a69), TOBN(0x5ce433b9, 0x4f23ede0), + TOBN(0xe8e8f04f, 0x6ad65143), TOBN(0x11511827, 0xd6e14af6), + TOBN(0x3d390a10, 0x8295c0c7), TOBN(0x71e29ee4, 0x621eba16), + TOBN(0xa588fc09, 0x63717b46), TOBN(0x02be02fe, 0xe06ad4a2), + TOBN(0x931558c6, 0x04c22b22), TOBN(0xbb4d4bd6, 0x12f3c849), + TOBN(0x54a4f496, 0x20efd662), TOBN(0x92ba6d20, 0xc5952d14), + TOBN(0x2db8ea1e, 0xcc9784c2), TOBN(0x81cc10ca, 0x4b353644), + TOBN(0x40b570ad, 0x4b4d7f6c), TOBN(0x5c9f1d96, 0x84a1dcd2), + TOBN(0x01379f81, 0x3147e797), TOBN(0xe5c6097b, 0x2bd499f5), + TOBN(0x40dcafa6, 0x328e5e20), TOBN(0xf7b5244a, 0x54815550), + TOBN(0xb9a4f118, 0x47bfc978), TOBN(0x0ea0e79f, 0xd25825b1), + TOBN(0xa50f96eb, 0x646c7ecf), TOBN(0xeb811493, 0x446dea9d), + TOBN(0x2af04677, 0xdfabcf69), TOBN(0xbe3a068f, 0xc713f6e8), + TOBN(0x860d523d, 0x42e06189), TOBN(0xbf077941, 0x4e3aff13), + TOBN(0x0b616dca, 0xc1b20650), TOBN(0xe66dd6d1, 0x2131300d), + TOBN(0xd4a0fd67, 0xff99abde), TOBN(0xc9903550, 0xc7aac50d), + TOBN(0x022ecf8b, 0x7c46b2d7), TOBN(0x3333b1e8, 0x3abf92af), + TOBN(0x11cc113c, 0x6c491c14), TOBN(0x05976688, 0x80dd3f88), + TOBN(0xf5b4d9e7, 0x29d932ed), TOBN(0xe982aad8, 0xa2c38b6d), + TOBN(0x6f925347, 0x8be0dcf0), TOBN(0x700080ae, 0x65ca53f2), + TOBN(0xd8131156, 0x443ca77f), TOBN(0xe92d6942, 0xec51f984), + TOBN(0xd2a08af8, 0x85dfe9ae), TOBN(0xd825d9a5, 0x4d2a86ca), + TOBN(0x2c53988d, 0x39dff020), TOBN(0xf38b135a, 0x430cdc40), + TOBN(0x0c918ae0, 0x62a7150b), TOBN(0xf31fd8de, 0x0c340e9b), + TOBN(0xafa0e7ae, 0x4dbbf02e), TOBN(0x5847fb2a, 0x5eba6239), + TOBN(0x6b1647dc, 0xdccbac8b), TOBN(0xb642aa78, 0x06f485c8), + TOBN(0x873f3765, 0x7038ecdf), TOBN(0x2ce5e865, 0xfa49d3fe), + TOBN(0xea223788, 0xc98c4400), TOBN(0x8104a8cd, 0xf1fa5279), + TOBN(0xbcf7cc7a, 0x06becfd7), TOBN(0x49424316, 0xc8f974ae), + TOBN(0xc0da65e7, 0x84d6365d), TOBN(0xbcb7443f, 0x8f759fb8), + TOBN(0x35c712b1, 0x7ae81930), TOBN(0x80428dff, 0x4c6e08ab), + TOBN(0xf19dafef, 0xa4faf843), TOBN(0xced8538d, 0xffa9855f), + TOBN(0x20ac409c, 0xbe3ac7ce), TOBN(0x358c1fb6, 0x882da71e), + TOBN(0xafa9c0e5, 0xfd349961), TOBN(0x2b2cfa51, 0x8421c2fc), + TOBN(0x2a80db17, 0xf3a28d38), TOBN(0xa8aba539, 0x5d138e7e), + TOBN(0x52012d1d, 0x6e96eb8d), TOBN(0x65d8dea0, 0xcbaf9622), + TOBN(0x57735447, 0xb264f56c), TOBN(0xbeebef3f, 0x1b6c8da2), + TOBN(0xfc346d98, 0xce785254), TOBN(0xd50e8d72, 0xbb64a161), + TOBN(0xc03567c7, 0x49794add), TOBN(0x15a76065, 0x752c7ef6), + TOBN(0x59f3a222, 0x961f23d6), TOBN(0x378e4438, 0x73ecc0b0), + TOBN(0xc74be434, 0x5a82fde4), TOBN(0xae509af2, 0xd8b9cf34), + TOBN(0x4a61ee46, 0x577f44a1), TOBN(0xe09b748c, 0xb611deeb), + TOBN(0xc0481b2c, 0xf5f7b884), TOBN(0x35626678, 0x61acfa6b), + TOBN(0x37f4c518, 0xbf8d21e6), TOBN(0x22d96531, 0xb205a76d), + TOBN(0x37fb85e1, 0x954073c0), TOBN(0xbceafe4f, 0x65b3a567), + TOBN(0xefecdef7, 0xbe42a582), TOBN(0xd3fc6080, 0x65046be6), + TOBN(0xc9af13c8, 0x09e8dba9), TOBN(0x1e6c9847, 0x641491ff), + TOBN(0x3b574925, 0xd30c31f7), TOBN(0xb7eb72ba, 0xac2a2122), + TOBN(0x776a0dac, 0xef0859e7), TOBN(0x06fec314, 0x21900942), + TOBN(0x2464bc10, 0xf8c22049), TOBN(0x9bfbcce7, 0x875ebf69), + TOBN(0xd7a88e2a, 0x4336326b), TOBN(0xda05261c, 0x5bc2acfa), + TOBN(0xc29f5bdc, 0xeba7efc8), TOBN(0x471237ca, 0x25dbbf2e), + TOBN(0xa72773f2, 0x2975f127), TOBN(0xdc744e8e, 0x04d0b326), + TOBN(0x38a7ed16, 0xa56edb73), TOBN(0x64357e37, 0x2c007e70), + TOBN(0xa167d15b, 0x5080b400), TOBN(0x07b41164, 0x23de4be1), + TOBN(0xb2d91e32, 0x74c89883), TOBN(0x3c162821, 0x2882e7ed), + TOBN(0xad6b36ba, 0x7503e482), TOBN(0x48434e8e, 0x0ea34331), + TOBN(0x79f4f24f, 0x2c7ae0b9), TOBN(0xc46fbf81, 0x1939b44a), + TOBN(0x76fefae8, 0x56595eb1), TOBN(0x417b66ab, 0xcd5f29c7), + TOBN(0x5f2332b2, 0xc5ceec20), TOBN(0xd69661ff, 0xe1a1cae2), + TOBN(0x5ede7e52, 0x9b0286e6), TOBN(0x9d062529, 0xe276b993), + TOBN(0x324794b0, 0x7e50122b), TOBN(0xdd744f8b, 0x4af07ca5), + TOBN(0x30a12f08, 0xd63fc97b), TOBN(0x39650f1a, 0x76626d9d), + TOBN(0x101b47f7, 0x1fa38477), TOBN(0x3d815f19, 0xd4dc124f), + TOBN(0x1569ae95, 0xb26eb58a), TOBN(0xc3cde188, 0x95fb1887), + TOBN(0x54e9f37b, 0xf9539a48), TOBN(0xb0100e06, 0x7408c1a5), + TOBN(0x821d9811, 0xea580cbb), TOBN(0x8af52d35, 0x86e50c56), + TOBN(0xdfbd9d47, 0xdbbf698b), TOBN(0x2961a1ea, 0x03dc1c73), + TOBN(0x203d38f8, 0xe76a5df8), TOBN(0x08a53a68, 0x6def707a), + TOBN(0x26eefb48, 0x1bee45d4), TOBN(0xb3cee346, 0x3c688036), + TOBN(0x463c5315, 0xc42f2469), TOBN(0x19d84d2e, 0x81378162), + TOBN(0x22d7c3c5, 0x1c4d349f), TOBN(0x65965844, 0x163d59c5), + TOBN(0xcf198c56, 0xb8abceae), TOBN(0x6fb1fb1b, 0x628559d5), + TOBN(0x8bbffd06, 0x07bf8fe3), TOBN(0x46259c58, 0x3467734b), + TOBN(0xd8953cea, 0x35f7f0d3), TOBN(0x1f0bece2, 0xd65b0ff1), + TOBN(0xf7d5b4b3, 0xf3c72914), TOBN(0x29e8ea95, 0x3cb53389), + TOBN(0x4a365626, 0x836b6d46), TOBN(0xe849f910, 0xea174fde), + TOBN(0x7ec62fbb, 0xf4737f21), TOBN(0xd8dba5ab, 0x6209f5ac), + TOBN(0x24b5d7a9, 0xa5f9adbe), TOBN(0x707d28f7, 0xa61dc768), + TOBN(0x7711460b, 0xcaa999ea), TOBN(0xba7b174d, 0x1c92e4cc), + TOBN(0x3c4bab66, 0x18d4bf2d), TOBN(0xb8f0c980, 0xeb8bd279), + TOBN(0x024bea9a, 0x324b4737), TOBN(0xfba9e423, 0x32a83bca), + TOBN(0x6e635643, 0xa232dced), TOBN(0x99619367, 0x2571c8ba), + TOBN(0xe8c9f357, 0x54b7032b), TOBN(0xf936b3ba, 0x2442d54a), + TOBN(0x2263f0f0, 0x8290c65a), TOBN(0x48989780, 0xee2c7fdb), + TOBN(0xadc5d55a, 0x13d4f95e), TOBN(0x737cff85, 0xad9b8500), + TOBN(0x271c557b, 0x8a73f43d), TOBN(0xbed617a4, 0xe18bc476), + TOBN(0x66245401, 0x7dfd8ab2), TOBN(0xae7b89ae, 0x3a2870aa), + TOBN(0x1b555f53, 0x23a7e545), TOBN(0x6791e247, 0xbe057e4c), + TOBN(0x860136ad, 0x324fa34d), TOBN(0xea111447, 0x4cbeae28), + TOBN(0x023a4270, 0xbedd3299), TOBN(0x3d5c3a7f, 0xc1c35c34), + TOBN(0xb0f6db67, 0x8d0412d2), TOBN(0xd92625e2, 0xfcdc6b9a), + TOBN(0x92ae5ccc, 0x4e28a982), TOBN(0xea251c36, 0x47a3ce7e), + TOBN(0x9d658932, 0x790691bf), TOBN(0xed610589, 0x06b736ae), + TOBN(0x712c2f04, 0xc0d63b6e), TOBN(0x5cf06fd5, 0xc63d488f), + TOBN(0x97363fac, 0xd9588e41), TOBN(0x1f9bf762, 0x2b93257e), + TOBN(0xa9d1ffc4, 0x667acace), TOBN(0x1cf4a1aa, 0x0a061ecf), + TOBN(0x40e48a49, 0xdc1818d0), TOBN(0x0643ff39, 0xa3621ab0), + TOBN(0x5768640c, 0xe39ef639), TOBN(0x1fc099ea, 0x04d86854), + TOBN(0x9130b9c3, 0xeccd28fd), TOBN(0xd743cbd2, 0x7eec54ab), + TOBN(0x052b146f, 0xe5b475b6), TOBN(0x058d9a82, 0x900a7d1f), + TOBN(0x65e02292, 0x91262b72), TOBN(0x96f924f9, 0xbb0edf03), + TOBN(0x5cfa59c8, 0xfe206842), TOBN(0xf6037004, 0x5eafa720), + TOBN(0x5f30699e, 0x18d7dd96), TOBN(0x381e8782, 0xcbab2495), + TOBN(0x91669b46, 0xdd8be949), TOBN(0xb40606f5, 0x26aae8ef), + TOBN(0x2812b839, 0xfc6751a4), TOBN(0x16196214, 0xfba800ef), + TOBN(0x4398d5ca, 0x4c1a2875), TOBN(0x720c00ee, 0x653d8349), + TOBN(0xc2699eb0, 0xd820007c), TOBN(0x880ee660, 0xa39b5825), + TOBN(0x70694694, 0x471f6984), TOBN(0xf7d16ea8, 0xe3dda99a), + TOBN(0x28d675b2, 0xc0519a23), TOBN(0x9ebf94fe, 0x4f6952e3), + TOBN(0xf28bb767, 0xa2294a8a), TOBN(0x85512b4d, 0xfe0af3f5), + TOBN(0x18958ba8, 0x99b16a0d), TOBN(0x95c2430c, 0xba7548a7), + TOBN(0xb30d1b10, 0xa16be615), TOBN(0xe3ebbb97, 0x85bfb74c), + TOBN(0xa3273cfe, 0x18549fdb), TOBN(0xf6e200bf, 0x4fcdb792), + TOBN(0x54a76e18, 0x83aba56c), TOBN(0x73ec66f6, 0x89ef6aa2), + TOBN(0x8d17add7, 0xd1b9a305), TOBN(0xa959c5b9, 0xb7ae1b9d), + TOBN(0x88643522, 0x6bcc094a), TOBN(0xcc5616c4, 0xd7d429b9), + TOBN(0xa6dada01, 0xe6a33f7c), TOBN(0xc6217a07, 0x9d4e70ad), + TOBN(0xd619a818, 0x09c15b7c), TOBN(0xea06b329, 0x0e80c854), + TOBN(0x174811ce, 0xa5f5e7b9), TOBN(0x66dfc310, 0x787c65f4), + TOBN(0x4ea7bd69, 0x3316ab54), TOBN(0xc12c4acb, 0x1dcc0f70), + TOBN(0xe4308d1a, 0x1e407dd9), TOBN(0xe8a3587c, 0x91afa997), + TOBN(0xea296c12, 0xab77b7a5), TOBN(0xb5ad49e4, 0x673c0d52), + TOBN(0x40f9b2b2, 0x7006085a), TOBN(0xa88ff340, 0x87bf6ec2), + TOBN(0x978603b1, 0x4e3066a6), TOBN(0xb3f99fc2, 0xb5e486e2), + TOBN(0x07b53f5e, 0xb2e63645), TOBN(0xbe57e547, 0x84c84232), + TOBN(0xd779c216, 0x7214d5cf), TOBN(0x617969cd, 0x029a3aca), + TOBN(0xd17668cd, 0x8a7017a0), TOBN(0x77b4d19a, 0xbe9b7ee8), + TOBN(0x58fd0e93, 0x9c161776), TOBN(0xa8c4f4ef, 0xd5968a72), + TOBN(0x296071cc, 0x67b3de77), TOBN(0xae3c0b8e, 0x634f7905), + TOBN(0x67e440c2, 0x8a7100c9), TOBN(0xbb8c3c1b, 0xeb4b9b42), + TOBN(0x6d71e8ea, 0xc51b3583), TOBN(0x7591f5af, 0x9525e642), + TOBN(0xf73a2f7b, 0x13f509f3), TOBN(0x618487aa, 0x5619ac9b), + TOBN(0x3a72e5f7, 0x9d61718a), TOBN(0x00413bcc, 0x7592d28c), + TOBN(0x7d9b11d3, 0x963c35cf), TOBN(0x77623bcf, 0xb90a46ed), + TOBN(0xdeef273b, 0xdcdd2a50), TOBN(0x4a741f9b, 0x0601846e), + TOBN(0x33b89e51, 0x0ec6e929), TOBN(0xcb02319f, 0x8b7f22cd), + TOBN(0xbbe1500d, 0x084bae24), TOBN(0x2f0ae8d7, 0x343d2693), + TOBN(0xacffb5f2, 0x7cdef811), TOBN(0xaa0c030a, 0x263fb94f), + TOBN(0x6eef0d61, 0xa0f442de), TOBN(0xf92e1817, 0x27b139d3), + TOBN(0x1ae6deb7, 0x0ad8bc28), TOBN(0xa89e38dc, 0xc0514130), + TOBN(0x81eeb865, 0xd2fdca23), TOBN(0x5a15ee08, 0xcc8ef895), + TOBN(0x768fa10a, 0x01905614), TOBN(0xeff5b8ef, 0x880ee19b), + TOBN(0xf0c0cabb, 0xcb1c8a0e), TOBN(0x2e1ee9cd, 0xb8c838f9), + TOBN(0x0587d8b8, 0x8a4a14c0), TOBN(0xf6f27896, 0x2ff698e5), + TOBN(0xed38ef1c, 0x89ee6256), TOBN(0xf44ee1fe, 0x6b353b45), + TOBN(0x9115c0c7, 0x70e903b3), TOBN(0xc78ec0a1, 0x818f31df), + TOBN(0x6c003324, 0xb7dccbc6), TOBN(0xd96dd1f3, 0x163bbc25), + TOBN(0x33aa82dd, 0x5cedd805), TOBN(0x123aae4f, 0x7f7eb2f1), + TOBN(0x1723fcf5, 0xa26262cd), TOBN(0x1f7f4d5d, 0x0060ebd5), + TOBN(0xf19c5c01, 0xb2eaa3af), TOBN(0x2ccb9b14, 0x9790accf), + TOBN(0x1f9c1cad, 0x52324aa6), TOBN(0x63200526, 0x7247df54), + TOBN(0x5732fe42, 0xbac96f82), TOBN(0x52fe771f, 0x01a1c384), + TOBN(0x546ca13d, 0xb1001684), TOBN(0xb56b4eee, 0xa1709f75), + TOBN(0x266545a9, 0xd5db8672), TOBN(0xed971c90, 0x1e8f3cfb), + TOBN(0x4e7d8691, 0xe3a07b29), TOBN(0x7570d9ec, 0xe4b696b9), + TOBN(0xdc5fa067, 0x7bc7e9ae), TOBN(0x68b44caf, 0xc82c4844), + TOBN(0x519d34b3, 0xbf44da80), TOBN(0x283834f9, 0x5ab32e66), + TOBN(0x6e608797, 0x6278a000), TOBN(0x1e62960e, 0x627312f6), + TOBN(0x9b87b27b, 0xe6901c55), TOBN(0x80e78538, 0x24fdbc1f), + TOBN(0xbbbc0951, 0x2facc27d), TOBN(0x06394239, 0xac143b5a), + TOBN(0x35bb4a40, 0x376c1944), TOBN(0x7cb62694, 0x63da1511), + TOBN(0xafd29161, 0xb7148a3b), TOBN(0xa6f9d9ed, 0x4e2ea2ee), + TOBN(0x15dc2ca2, 0x880dd212), TOBN(0x903c3813, 0xa61139a9), + TOBN(0x2aa7b46d, 0x6c0f8785), TOBN(0x36ce2871, 0x901c60ff), + TOBN(0xc683b028, 0xe10d9c12), TOBN(0x7573baa2, 0x032f33d3), + TOBN(0x87a9b1f6, 0x67a31b58), TOBN(0xfd3ed11a, 0xf4ffae12), + TOBN(0x83dcaa9a, 0x0cb2748e), TOBN(0x8239f018, 0x5d6fdf16), + TOBN(0xba67b49c, 0x72753941), TOBN(0x2beec455, 0xc321cb36), + TOBN(0x88015606, 0x3f8b84ce), TOBN(0x76417083, 0x8d38c86f), + TOBN(0x054f1ca7, 0x598953dd), TOBN(0xc939e110, 0x4e8e7429), + TOBN(0x9b1ac2b3, 0x5a914f2f), TOBN(0x39e35ed3, 0xe74b8f9c), + TOBN(0xd0debdb2, 0x781b2fb0), TOBN(0x1585638f, 0x2d997ba2), + TOBN(0x9c4b646e, 0x9e2fce99), TOBN(0x68a21081, 0x1e80857f), + TOBN(0x06d54e44, 0x3643b52a), TOBN(0xde8d6d63, 0x0d8eb843), + TOBN(0x70321563, 0x42146a0a), TOBN(0x8ba826f2, 0x5eaa3622), + TOBN(0x227a58bd, 0x86138787), TOBN(0x43b6c03c, 0x10281d37), + TOBN(0x6326afbb, 0xb54dde39), TOBN(0x744e5e8a, 0xdb6f2d5f), + TOBN(0x48b2a99a, 0xcff158e1), TOBN(0xa93c8fa0, 0xef87918f), + TOBN(0x2182f956, 0xde058c5c), TOBN(0x216235d2, 0x936f9e7a), + TOBN(0xace0c0db, 0xd2e31e67), TOBN(0xc96449bf, 0xf23ac3e7), + TOBN(0x7e9a2874, 0x170693bd), TOBN(0xa28e14fd, 0xa45e6335), + TOBN(0x5757f6b3, 0x56427344), TOBN(0x822e4556, 0xacf8edf9), + TOBN(0x2b7a6ee2, 0xe6a285cd), TOBN(0x5866f211, 0xa9df3af0), + TOBN(0x40dde2dd, 0xf845b844), TOBN(0x986c3726, 0x110e5e49), + TOBN(0x73680c2a, 0xf7172277), TOBN(0x57b94f0f, 0x0cccb244), + TOBN(0xbdff7267, 0x2d438ca7), TOBN(0xbad1ce11, 0xcf4663fd), + TOBN(0x9813ed9d, 0xd8f71cae), TOBN(0xf43272a6, 0x961fdaa6), + TOBN(0xbeff0119, 0xbd6d1637), TOBN(0xfebc4f91, 0x30361978), + TOBN(0x02b37a95, 0x2f41deff), TOBN(0x0e44a59a, 0xe63b89b7), + TOBN(0x673257dc, 0x143ff951), TOBN(0x19c02205, 0xd752baf4), + TOBN(0x46c23069, 0xc4b7d692), TOBN(0x2e6392c3, 0xfd1502ac), + TOBN(0x6057b1a2, 0x1b220846), TOBN(0xe51ff946, 0x0c1b5b63),} + , + {TOBN(0x6e85cb51, 0x566c5c43), TOBN(0xcff9c919, 0x3597f046), + TOBN(0x9354e90c, 0x4994d94a), TOBN(0xe0a39332, 0x2147927d), + TOBN(0x8427fac1, 0x0dc1eb2b), TOBN(0x88cfd8c2, 0x2ff319fa), + TOBN(0xe2d4e684, 0x01965274), TOBN(0xfa2e067d, 0x67aaa746), + TOBN(0xb6d92a7f, 0x3e5f9f11), TOBN(0x9afe153a, 0xd6cb3b8e), + TOBN(0x4d1a6dd7, 0xddf800bd), TOBN(0xf6c13cc0, 0xcaf17e19), + TOBN(0x15f6c58e, 0x325fc3ee), TOBN(0x71095400, 0xa31dc3b2), + TOBN(0x168e7c07, 0xafa3d3e7), TOBN(0x3f8417a1, 0x94c7ae2d), + TOBN(0xec234772, 0x813b230d), TOBN(0x634d0f5f, 0x17344427), + TOBN(0x11548ab1, 0xd77fc56a), TOBN(0x7fab1750, 0xce06af77), + TOBN(0xb62c10a7, 0x4f7c4f83), TOBN(0xa7d2edc4, 0x220a67d9), + TOBN(0x1c404170, 0x921209a0), TOBN(0x0b9815a0, 0xface59f0), + TOBN(0x2842589b, 0x319540c3), TOBN(0x18490f59, 0xa283d6f8), + TOBN(0xa2731f84, 0xdaae9fcb), TOBN(0x3db6d960, 0xc3683ba0), + TOBN(0xc85c63bb, 0x14611069), TOBN(0xb19436af, 0x0788bf05), + TOBN(0x905459df, 0x347460d2), TOBN(0x73f6e094, 0xe11a7db1), + TOBN(0xdc7f938e, 0xb6357f37), TOBN(0xc5d00f79, 0x2bd8aa62), + TOBN(0xc878dcb9, 0x2ca979fc), TOBN(0x37e83ed9, 0xeb023a99), + TOBN(0x6b23e273, 0x1560bf3d), TOBN(0x1086e459, 0x1d0fae61), + TOBN(0x78248316, 0x9a9414bd), TOBN(0x1b956bc0, 0xf0ea9ea1), + TOBN(0x7b85bb91, 0xc31b9c38), TOBN(0x0c5aa90b, 0x48ef57b5), + TOBN(0xdedeb169, 0xaf3bab6f), TOBN(0xe610ad73, 0x2d373685), + TOBN(0xf13870df, 0x02ba8e15), TOBN(0x0337edb6, 0x8ca7f771), + TOBN(0xe4acf747, 0xb62c036c), TOBN(0xd921d576, 0xb6b94e81), + TOBN(0xdbc86439, 0x2c422f7a), TOBN(0xfb635362, 0xed348898), + TOBN(0x83084668, 0xc45bfcd1), TOBN(0xc357c9e3, 0x2b315e11), + TOBN(0xb173b540, 0x5b2e5b8c), TOBN(0x7e946931, 0xe102b9a4), + TOBN(0x17c890eb, 0x7b0fb199), TOBN(0xec225a83, 0xd61b662b), + TOBN(0xf306a3c8, 0xee3c76cb), TOBN(0x3cf11623, 0xd32a1f6e), + TOBN(0xe6d5ab64, 0x6863e956), TOBN(0x3b8a4cbe, 0x5c005c26), + TOBN(0xdcd529a5, 0x9ce6bb27), TOBN(0xc4afaa52, 0x04d4b16f), + TOBN(0xb0624a26, 0x7923798d), TOBN(0x85e56df6, 0x6b307fab), + TOBN(0x0281893c, 0x2bf29698), TOBN(0x91fc19a4, 0xd7ce7603), + TOBN(0x75a5dca3, 0xad9a558f), TOBN(0x40ceb3fa, 0x4d50bf77), + TOBN(0x1baf6060, 0xbc9ba369), TOBN(0x927e1037, 0x597888c2), + TOBN(0xd936bf19, 0x86a34c07), TOBN(0xd4cf10c1, 0xc34ae980), + TOBN(0x3a3e5334, 0x859dd614), TOBN(0x9c475b5b, 0x18d0c8ee), + TOBN(0x63080d1f, 0x07cd51d5), TOBN(0xc9c0d0a6, 0xb88b4326), + TOBN(0x1ac98691, 0xc234296f), TOBN(0x2a0a83a4, 0x94887fb6), + TOBN(0x56511427, 0x0cea9cf2), TOBN(0x5230a6e8, 0xa24802f5), + TOBN(0xf7a2bf0f, 0x72e3d5c1), TOBN(0x37717446, 0x4f21439e), + TOBN(0xfedcbf25, 0x9ce30334), TOBN(0xe0030a78, 0x7ce202f9), + TOBN(0x6f2d9ebf, 0x1202e9ca), TOBN(0xe79dde6c, 0x75e6e591), + TOBN(0xf52072af, 0xf1dac4f8), TOBN(0x6c8d087e, 0xbb9b404d), + TOBN(0xad0fc73d, 0xbce913af), TOBN(0x909e587b, 0x458a07cb), + TOBN(0x1300da84, 0xd4f00c8a), TOBN(0x425cd048, 0xb54466ac), + TOBN(0xb59cb9be, 0x90e9d8bf), TOBN(0x991616db, 0x3e431b0e), + TOBN(0xd3aa117a, 0x531aecff), TOBN(0x91af92d3, 0x59f4dc3b), + TOBN(0x9b1ec292, 0xe93fda29), TOBN(0x76bb6c17, 0xe97d91bc), + TOBN(0x7509d95f, 0xaface1e6), TOBN(0x3653fe47, 0xbe855ae3), + TOBN(0x73180b28, 0x0f680e75), TOBN(0x75eefd1b, 0xeeb6c26c), + TOBN(0xa4cdf29f, 0xb66d4236), TOBN(0x2d70a997, 0x6b5821d8), + TOBN(0x7a3ee207, 0x20445c36), TOBN(0x71d1ac82, 0x59877174), + TOBN(0x0fc539f7, 0x949f73e9), TOBN(0xd05cf3d7, 0x982e3081), + TOBN(0x8758e20b, 0x7b1c7129), TOBN(0xffadcc20, 0x569e61f2), + TOBN(0xb05d3a2f, 0x59544c2d), TOBN(0xbe16f5c1, 0x9fff5e53), + TOBN(0x73cf65b8, 0xaad58135), TOBN(0x622c2119, 0x037aa5be), + TOBN(0x79373b3f, 0x646fd6a0), TOBN(0x0e029db5, 0x0d3978cf), + TOBN(0x8bdfc437, 0x94fba037), TOBN(0xaefbd687, 0x620797a6), + TOBN(0x3fa5382b, 0xbd30d38e), TOBN(0x7627cfbf, 0x585d7464), + TOBN(0xb2330fef, 0x4e4ca463), TOBN(0xbcef7287, 0x3566cc63), + TOBN(0xd161d2ca, 0xcf780900), TOBN(0x135dc539, 0x5b54827d), + TOBN(0x638f052e, 0x27bf1bc6), TOBN(0x10a224f0, 0x07dfa06c), + TOBN(0xe973586d, 0x6d3321da), TOBN(0x8b0c5738, 0x26152c8f), + TOBN(0x07ef4f2a, 0x34606074), TOBN(0x80fe7fe8, 0xa0f7047a), + TOBN(0x3d1a8152, 0xe1a0e306), TOBN(0x32cf43d8, 0x88da5222), + TOBN(0xbf89a95f, 0x5f02ffe6), TOBN(0x3d9eb9a4, 0x806ad3ea), + TOBN(0x012c17bb, 0x79c8e55e), TOBN(0xfdcd1a74, 0x99c81dac), + TOBN(0x7043178b, 0xb9556098), TOBN(0x4090a1df, 0x801c3886), + TOBN(0x759800ff, 0x9b67b912), TOBN(0x3e5c0304, 0x232620c8), + TOBN(0x4b9d3c4b, 0x70dceeca), TOBN(0xbb2d3c15, 0x181f648e), + TOBN(0xf981d837, 0x6e33345c), TOBN(0xb626289b, 0x0cf2297a), + TOBN(0x766ac659, 0x8baebdcf), TOBN(0x1a28ae09, 0x75df01e5), + TOBN(0xb71283da, 0x375876d8), TOBN(0x4865a96d, 0x607b9800), + TOBN(0x25dd1bcd, 0x237936b2), TOBN(0x332f4f4b, 0x60417494), + TOBN(0xd0923d68, 0x370a2147), TOBN(0x497f5dfb, 0xdc842203), + TOBN(0x9dc74cbd, 0x32be5e0f), TOBN(0x7475bcb7, 0x17a01375), + TOBN(0x438477c9, 0x50d872b1), TOBN(0xcec67879, 0xffe1d63d), + TOBN(0x9b006014, 0xd8578c70), TOBN(0xc9ad99a8, 0x78bb6b8b), + TOBN(0x6799008e, 0x11fb3806), TOBN(0xcfe81435, 0xcd44cab3), + TOBN(0xa2ee1582, 0x2f4fb344), TOBN(0xb8823450, 0x483fa6eb), + TOBN(0x622d323d, 0x652c7749), TOBN(0xd8474a98, 0xbeb0a15b), + TOBN(0xe43c154d, 0x5d1c00d0), TOBN(0x7fd581d9, 0x0e3e7aac), + TOBN(0x2b44c619, 0x2525ddf8), TOBN(0x67a033eb, 0xb8ae9739), + TOBN(0x113ffec1, 0x9ef2d2e4), TOBN(0x1bf6767e, 0xd5a0ea7f), + TOBN(0x57fff75e, 0x03714c0a), TOBN(0xa23c422e, 0x0a23e9ee), + TOBN(0xdd5f6b2d, 0x540f83af), TOBN(0xc2c2c27e, 0x55ea46a7), + TOBN(0xeb6b4246, 0x672a1208), TOBN(0xd13599f7, 0xae634f7a), + TOBN(0xcf914b5c, 0xd7b32c6e), TOBN(0x61a5a640, 0xeaf61814), + TOBN(0x8dc3df8b, 0x208a1bbb), TOBN(0xef627fd6, 0xb6d79aa5), + TOBN(0x44232ffc, 0xc4c86bc8), TOBN(0xe6f9231b, 0x061539fe), + TOBN(0x1d04f25a, 0x958b9533), TOBN(0x180cf934, 0x49e8c885), + TOBN(0x89689595, 0x9884aaf7), TOBN(0xb1959be3, 0x07b348a6), + TOBN(0x96250e57, 0x3c147c87), TOBN(0xae0efb3a, 0xdd0c61f8), + TOBN(0xed00745e, 0xca8c325e), TOBN(0x3c911696, 0xecff3f70), + TOBN(0x73acbc65, 0x319ad41d), TOBN(0x7b01a020, 0xf0b1c7ef), + TOBN(0xea32b293, 0x63a1483f), TOBN(0x89eabe71, 0x7a248f96), + TOBN(0x9c6231d3, 0x343157e5), TOBN(0x93a375e5, 0xdf3c546d), + TOBN(0xe76e9343, 0x6a2afe69), TOBN(0xc4f89100, 0xe166c88e), + TOBN(0x248efd0d, 0x4f872093), TOBN(0xae0eb3ea, 0x8fe0ea61), + TOBN(0xaf89790d, 0x9d79046e), TOBN(0x4d650f2d, 0x6cee0976), + TOBN(0xa3935d9a, 0x43071eca), TOBN(0x66fcd2c9, 0x283b0bfe), + TOBN(0x0e665eb5, 0x696605f1), TOBN(0xe77e5d07, 0xa54cd38d), + TOBN(0x90ee050a, 0x43d950cf), TOBN(0x86ddebda, 0xd32e69b5), + TOBN(0x6ad94a3d, 0xfddf7415), TOBN(0xf7fa1309, 0x3f6e8d5a), + TOBN(0xc4831d1d, 0xe9957f75), TOBN(0x7de28501, 0xd5817447), + TOBN(0x6f1d7078, 0x9e2aeb6b), TOBN(0xba2b9ff4, 0xf67a53c2), + TOBN(0x36963767, 0xdf9defc3), TOBN(0x479deed3, 0x0d38022c), + TOBN(0xd2edb89b, 0x3a8631e8), TOBN(0x8de855de, 0x7a213746), + TOBN(0xb2056cb7, 0xb00c5f11), TOBN(0xdeaefbd0, 0x2c9b85e4), + TOBN(0x03f39a8d, 0xd150892d), TOBN(0x37b84686, 0x218b7985), + TOBN(0x36296dd8, 0xb7375f1a), TOBN(0x472cd4b1, 0xb78e898e), + TOBN(0x15dff651, 0xe9f05de9), TOBN(0xd4045069, 0x2ce98ba9), + TOBN(0x8466a7ae, 0x9b38024c), TOBN(0xb910e700, 0xe5a6b5ef), + TOBN(0xae1c56ea, 0xb3aa8f0d), TOBN(0xbab2a507, 0x7eee74a6), + TOBN(0x0dca11e2, 0x4b4c4620), TOBN(0xfd896e2e, 0x4c47d1f4), + TOBN(0xeb45ae53, 0x308fbd93), TOBN(0x46cd5a2e, 0x02c36fda), + TOBN(0x6a3d4e90, 0xbaa48385), TOBN(0xdd55e62e, 0x9dbe9960), + TOBN(0xa1406aa0, 0x2a81ede7), TOBN(0x6860dd14, 0xf9274ea7), + TOBN(0xcfdcb0c2, 0x80414f86), TOBN(0xff410b10, 0x22f94327), + TOBN(0x5a33cc38, 0x49ad467b), TOBN(0xefb48b6c, 0x0a7335f1), + TOBN(0x14fb54a4, 0xb153a360), TOBN(0x604aa9d2, 0xb52469cc), + TOBN(0x5e9dc486, 0x754e48e9), TOBN(0x693cb455, 0x37471e8e), + TOBN(0xfb2fd7cd, 0x8d3b37b6), TOBN(0x63345e16, 0xcf09ff07), + TOBN(0x9910ba6b, 0x23a5d896), TOBN(0x1fe19e35, 0x7fe4364e), + TOBN(0x6e1da8c3, 0x9a33c677), TOBN(0x15b4488b, 0x29fd9fd0), + TOBN(0x1f439254, 0x1a1f22bf), TOBN(0x920a8a70, 0xab8163e8), + TOBN(0x3fd1b249, 0x07e5658e), TOBN(0xf2c4f79c, 0xb6ec839b), + TOBN(0x1abbc3d0, 0x4aa38d1b), TOBN(0x3b0db35c, 0xb5d9510e), + TOBN(0x1754ac78, 0x3e60dec0), TOBN(0x53272fd7, 0xea099b33), + TOBN(0x5fb0494f, 0x07a8e107), TOBN(0x4a89e137, 0x6a8191fa), + TOBN(0xa113b7f6, 0x3c4ad544), TOBN(0x88a2e909, 0x6cb9897b), + TOBN(0x17d55de3, 0xb44a3f84), TOBN(0xacb2f344, 0x17c6c690), + TOBN(0x32088168, 0x10232390), TOBN(0xf2e8a61f, 0x6c733bf7), + TOBN(0xa774aab6, 0x9c2d7652), TOBN(0xfb5307e3, 0xed95c5bc), + TOBN(0xa05c73c2, 0x4981f110), TOBN(0x1baae31c, 0xa39458c9), + TOBN(0x1def185b, 0xcbea62e7), TOBN(0xe8ac9eae, 0xeaf63059), + TOBN(0x098a8cfd, 0x9921851c), TOBN(0xd959c3f1, 0x3abe2f5b), + TOBN(0xa4f19525, 0x20e40ae5), TOBN(0x320789e3, 0x07a24aa1), + TOBN(0x259e6927, 0x7392b2bc), TOBN(0x58f6c667, 0x1918668b), + TOBN(0xce1db2bb, 0xc55d2d8b), TOBN(0x41d58bb7, 0xf4f6ca56), + TOBN(0x7650b680, 0x8f877614), TOBN(0x905e16ba, 0xf4c349ed), + TOBN(0xed415140, 0xf661acac), TOBN(0x3b8784f0, 0xcb2270af), + TOBN(0x3bc280ac, 0x8a402cba), TOBN(0xd53f7146, 0x0937921a), + TOBN(0xc03c8ee5, 0xe5681e83), TOBN(0x62126105, 0xf6ac9e4a), + TOBN(0x9503a53f, 0x936b1a38), TOBN(0x3d45e2d4, 0x782fecbd), + TOBN(0x69a5c439, 0x76e8ae98), TOBN(0xb53b2eeb, 0xbfb4b00e), + TOBN(0xf1674712, 0x72386c89), TOBN(0x30ca34a2, 0x4268bce4), + TOBN(0x7f1ed86c, 0x78341730), TOBN(0x8ef5beb8, 0xb525e248), + TOBN(0xbbc489fd, 0xb74fbf38), TOBN(0x38a92a0e, 0x91a0b382), + TOBN(0x7a77ba3f, 0x22433ccf), TOBN(0xde8362d6, 0xa29f05a9), + TOBN(0x7f6a30ea, 0x61189afc), TOBN(0x693b5505, 0x59ef114f), + TOBN(0x50266bc0, 0xcd1797a1), TOBN(0xea17b47e, 0xf4b7af2d), + TOBN(0xd6c4025c, 0x3df9483e), TOBN(0x8cbb9d9f, 0xa37b18c9), + TOBN(0x91cbfd9c, 0x4d8424cf), TOBN(0xdb7048f1, 0xab1c3506), + TOBN(0x9eaf641f, 0x028206a3), TOBN(0xf986f3f9, 0x25bdf6ce), + TOBN(0x262143b5, 0x224c08dc), TOBN(0x2bbb09b4, 0x81b50c91), + TOBN(0xc16ed709, 0xaca8c84f), TOBN(0xa6210d9d, 0xb2850ca8), + TOBN(0x6d8df67a, 0x09cb54d6), TOBN(0x91eef6e0, 0x500919a4), + TOBN(0x90f61381, 0x0f132857), TOBN(0x9acede47, 0xf8d5028b), + TOBN(0x844d1b71, 0x90b771c3), TOBN(0x563b71e4, 0xba6426be), + TOBN(0x2efa2e83, 0xbdb802ff), TOBN(0x3410cbab, 0xab5b4a41), + TOBN(0x555b2d26, 0x30da84dd), TOBN(0xd0711ae9, 0xee1cc29a), + TOBN(0xcf3e8c60, 0x2f547792), TOBN(0x03d7d5de, 0xdc678b35), + TOBN(0x071a2fa8, 0xced806b8), TOBN(0x222e6134, 0x697f1478), + TOBN(0xdc16fd5d, 0xabfcdbbf), TOBN(0x44912ebf, 0x121b53b8), + TOBN(0xac943674, 0x2496c27c), TOBN(0x8ea3176c, 0x1ffc26b0), + TOBN(0xb6e224ac, 0x13debf2c), TOBN(0x524cc235, 0xf372a832), + TOBN(0xd706e1d8, 0x9f6f1b18), TOBN(0x2552f005, 0x44cce35b), + TOBN(0x8c8326c2, 0xa88e31fc), TOBN(0xb5468b2c, 0xf9552047), + TOBN(0xce683e88, 0x3ff90f2b), TOBN(0x77947bdf, 0x2f0a5423), + TOBN(0xd0a1b28b, 0xed56e328), TOBN(0xaee35253, 0xc20134ac), + TOBN(0x7e98367d, 0x3567962f), TOBN(0x379ed61f, 0x8188bffb), + TOBN(0x73bba348, 0xfaf130a1), TOBN(0x6c1f75e1, 0x904ed734), + TOBN(0x18956642, 0x3b4a79fc), TOBN(0xf20bc83d, 0x54ef4493), + TOBN(0x836d425d, 0x9111eca1), TOBN(0xe5b5c318, 0x009a8dcf), + TOBN(0x3360b25d, 0x13221bc5), TOBN(0x707baad2, 0x6b3eeaf7), + TOBN(0xd7279ed8, 0x743a95a1), TOBN(0x7450a875, 0x969e809f), + TOBN(0x32b6bd53, 0xe5d0338f), TOBN(0x1e77f7af, 0x2b883bbc), + TOBN(0x90da12cc, 0x1063ecd0), TOBN(0xe2697b58, 0xc315be47), + TOBN(0x2771a5bd, 0xda85d534), TOBN(0x53e78c1f, 0xff980eea), + TOBN(0xadf1cf84, 0x900385e7), TOBN(0x7d3b14f6, 0xc9387b62), + TOBN(0x170e74b0, 0xcb8f2bd2), TOBN(0x2d50b486, 0x827fa993), + TOBN(0xcdbe8c9a, 0xf6f32bab), TOBN(0x55e906b0, 0xc3b93ab8), + TOBN(0x747f22fc, 0x8fe280d1), TOBN(0xcd8e0de5, 0xb2e114ab), + TOBN(0x5ab7dbeb, 0xe10b68b0), TOBN(0x9dc63a9c, 0xa480d4b2), + TOBN(0x78d4bc3b, 0x4be1495f), TOBN(0x25eb3db8, 0x9359122d), + TOBN(0x3f8ac05b, 0x0809cbdc), TOBN(0xbf4187bb, 0xd37c702f), + TOBN(0x84cea069, 0x1416a6a5), TOBN(0x8f860c79, 0x43ef881c), + TOBN(0x41311f8a, 0x38038a5d), TOBN(0xe78c2ec0, 0xfc612067), + TOBN(0x494d2e81, 0x5ad73581), TOBN(0xb4cc9e00, 0x59604097), + TOBN(0xff558aec, 0xf3612cba), TOBN(0x35beef7a, 0x9e36c39e), + TOBN(0x1845c7cf, 0xdbcf41b9), TOBN(0x5703662a, 0xaea997c0), + TOBN(0x8b925afe, 0xe402f6d8), TOBN(0xd0a1b1ae, 0x4dd72162), + TOBN(0x9f47b375, 0x03c41c4b), TOBN(0xa023829b, 0x0391d042), + TOBN(0x5f5045c3, 0x503b8b0a), TOBN(0x123c2688, 0x98c010e5), + TOBN(0x324ec0cc, 0x36ba06ee), TOBN(0xface3115, 0x3dd2cc0c), + TOBN(0xb364f3be, 0xf333e91f), TOBN(0xef8aff73, 0x28e832b0), + TOBN(0x1e9bad04, 0x2d05841b), TOBN(0x42f0e3df, 0x356a21e2), + TOBN(0xa3270bcb, 0x4add627e), TOBN(0xb09a8158, 0xd322e711), + TOBN(0x86e326a1, 0x0fee104a), TOBN(0xad7788f8, 0x3703f65d), + TOBN(0x7e765430, 0x47bc4833), TOBN(0x6cee582b, 0x2b9b893a), + TOBN(0x9cd2a167, 0xe8f55a7b), TOBN(0xefbee3c6, 0xd9e4190d), + TOBN(0x33ee7185, 0xd40c2e9d), TOBN(0x844cc9c5, 0xa380b548), + TOBN(0x323f8ecd, 0x66926e04), TOBN(0x0001e38f, 0x8110c1ba), + TOBN(0x8dbcac12, 0xfc6a7f07), TOBN(0xd65e1d58, 0x0cec0827), + TOBN(0xd2cd4141, 0xbe76ca2d), TOBN(0x7895cf5c, 0xe892f33a), + TOBN(0x956d230d, 0x367139d2), TOBN(0xa91abd3e, 0xd012c4c1), + TOBN(0x34fa4883, 0x87eb36bf), TOBN(0xc5f07102, 0x914b8fb4), + TOBN(0x90f0e579, 0xadb9c95f), TOBN(0xfe6ea8cb, 0x28888195), + TOBN(0x7b9b5065, 0xedfa9284), TOBN(0x6c510bd2, 0x2b8c8d65), + TOBN(0xd7b8ebef, 0xcbe8aafd), TOBN(0xedb3af98, 0x96b1da07), + TOBN(0x28ff779d, 0x6295d426), TOBN(0x0c4f6ac7, 0x3fa3ad7b), + TOBN(0xec44d054, 0x8b8e2604), TOBN(0x9b32a66d, 0x8b0050e1), + TOBN(0x1f943366, 0xf0476ce2), TOBN(0x7554d953, 0xa602c7b4), + TOBN(0xbe35aca6, 0x524f2809), TOBN(0xb6881229, 0xfd4edbea), + TOBN(0xe8cd0c8f, 0x508efb63), TOBN(0x9eb5b5c8, 0x6abcefc7), + TOBN(0xf5621f5f, 0xb441ab4f), TOBN(0x79e6c046, 0xb76a2b22), + TOBN(0x74a4792c, 0xe37a1f69), TOBN(0xcbd252cb, 0x03542b60), + TOBN(0x785f65d5, 0xb3c20bd3), TOBN(0x8dea6143, 0x4fabc60c), + TOBN(0x45e21446, 0xde673629), TOBN(0x57f7aa1e, 0x703c2d21), + TOBN(0xa0e99b7f, 0x98c868c7), TOBN(0x4e42f66d, 0x8b641676), + TOBN(0x602884dc, 0x91077896), TOBN(0xa0d690cf, 0xc2c9885b), + TOBN(0xfeb4da33, 0x3b9a5187), TOBN(0x5f789598, 0x153c87ee), + TOBN(0x2192dd47, 0x52b16dba), TOBN(0xdeefc0e6, 0x3524c1b1), + TOBN(0x465ea76e, 0xe4383693), TOBN(0x79401711, 0x361b8d98), + TOBN(0xa5f9ace9, 0xf21a15cb), TOBN(0x73d26163, 0xefee9aeb), + TOBN(0xcca844b3, 0xe677016c), TOBN(0x6c122b07, 0x57eaee06), + TOBN(0xb782dce7, 0x15f09690), TOBN(0x508b9b12, 0x2dfc0fc9), + TOBN(0x9015ab4b, 0x65d89fc6), TOBN(0x5e79dab7, 0xd6d5bb0f), + TOBN(0x64f021f0, 0x6c775aa2), TOBN(0xdf09d8cc, 0x37c7eca1), + TOBN(0x9a761367, 0xef2fa506), TOBN(0xed4ca476, 0x5b81eec6), + TOBN(0x262ede36, 0x10bbb8b5), TOBN(0x0737ce83, 0x0641ada3), + TOBN(0x4c94288a, 0xe9831ccc), TOBN(0x487fc1ce, 0x8065e635), + TOBN(0xb13d7ab3, 0xb8bb3659), TOBN(0xdea5df3e, 0x855e4120), + TOBN(0xb9a18573, 0x85eb0244), TOBN(0x1a1b8ea3, 0xa7cfe0a3), + TOBN(0x3b837119, 0x67b0867c), TOBN(0x8d5e0d08, 0x9d364520), + TOBN(0x52dccc1e, 0xd930f0e3), TOBN(0xefbbcec7, 0xbf20bbaf), + TOBN(0x99cffcab, 0x0263ad10), TOBN(0xd8199e6d, 0xfcd18f8a), + TOBN(0x64e2773f, 0xe9f10617), TOBN(0x0079e8e1, 0x08704848), + TOBN(0x1169989f, 0x8a342283), TOBN(0x8097799c, 0xa83012e6), + TOBN(0xece966cb, 0x8a6a9001), TOBN(0x93b3afef, 0x072ac7fc), + TOBN(0xe6893a2a, 0x2db3d5ba), TOBN(0x263dc462, 0x89bf4fdc), + TOBN(0x8852dfc9, 0xe0396673), TOBN(0x7ac70895, 0x3af362b6), + TOBN(0xbb9cce4d, 0x5c2f342b), TOBN(0xbf80907a, 0xb52d7aae), + TOBN(0x97f3d3cd, 0x2161bcd0), TOBN(0xb25b0834, 0x0962744d), + TOBN(0xc5b18ea5, 0x6c3a1dda), TOBN(0xfe4ec7eb, 0x06c92317), + TOBN(0xb787b890, 0xad1c4afe), TOBN(0xdccd9a92, 0x0ede801a), + TOBN(0x9ac6ddda, 0xdb58da1f), TOBN(0x22bbc12f, 0xb8cae6ee), + TOBN(0xc6f8bced, 0x815c4a43), TOBN(0x8105a92c, 0xf96480c7), + TOBN(0x0dc3dbf3, 0x7a859d51), TOBN(0xe3ec7ce6, 0x3041196b), + TOBN(0xd9f64b25, 0x0d1067c9), TOBN(0xf2321321, 0x3d1f8dd8), + TOBN(0x8b5c619c, 0x76497ee8), TOBN(0x5d2b0ac6, 0xc717370e), + TOBN(0x98204cb6, 0x4fcf68e1), TOBN(0x0bdec211, 0x62bc6792), + TOBN(0x6973ccef, 0xa63b1011), TOBN(0xf9e3fa97, 0xe0de1ac5), + TOBN(0x5efb693e, 0x3d0e0c8b), TOBN(0x037248e9, 0xd2d4fcb4),} + , + {TOBN(0x80802dc9, 0x1ec34f9e), TOBN(0xd8772d35, 0x33810603), + TOBN(0x3f06d66c, 0x530cb4f3), TOBN(0x7be5ed0d, 0xc475c129), + TOBN(0xcb9e3c19, 0x31e82b10), TOBN(0xc63d2857, 0xc9ff6b4c), + TOBN(0xb92118c6, 0x92a1b45e), TOBN(0x0aec4414, 0x7285bbca), + TOBN(0xfc189ae7, 0x1e29a3ef), TOBN(0xcbe906f0, 0x4c93302e), + TOBN(0xd0107914, 0xceaae10e), TOBN(0xb7a23f34, 0xb68e19f8), + TOBN(0xe9d875c2, 0xefd2119d), TOBN(0x03198c6e, 0xfcadc9c8), + TOBN(0x65591bf6, 0x4da17113), TOBN(0x3cf0bbf8, 0x3d443038), + TOBN(0xae485bb7, 0x2b724759), TOBN(0x945353e1, 0xb2d4c63a), + TOBN(0x82159d07, 0xde7d6f2c), TOBN(0x389caef3, 0x4ec5b109), + TOBN(0x4a8ebb53, 0xdb65ef14), TOBN(0x2dc2cb7e, 0xdd99de43), + TOBN(0x816fa3ed, 0x83f2405f), TOBN(0x73429bb9, 0xc14208a3), + TOBN(0xb618d590, 0xb01e6e27), TOBN(0x047e2ccd, 0xe180b2dc), + TOBN(0xd1b299b5, 0x04aea4a9), TOBN(0x412c9e1e, 0x9fa403a4), + TOBN(0x88d28a36, 0x79407552), TOBN(0x49c50136, 0xf332b8e3), + TOBN(0x3a1b6fcc, 0xe668de19), TOBN(0x178851bc, 0x75122b97), + TOBN(0xb1e13752, 0xfb85fa4c), TOBN(0xd61257ce, 0x383c8ce9), + TOBN(0xd43da670, 0xd2f74dae), TOBN(0xa35aa23f, 0xbf846bbb), + TOBN(0x5e74235d, 0x4421fc83), TOBN(0xf6df8ee0, 0xc363473b), + TOBN(0x34d7f52a, 0x3c4aa158), TOBN(0x50d05aab, 0x9bc6d22e), + TOBN(0x8c56e735, 0xa64785f4), TOBN(0xbc56637b, 0x5f29cd07), + TOBN(0x53b2bb80, 0x3ee35067), TOBN(0x50235a0f, 0xdc919270), + TOBN(0x191ab6d8, 0xf2c4aa65), TOBN(0xc3475831, 0x8396023b), + TOBN(0x80400ba5, 0xf0f805ba), TOBN(0x8881065b, 0x5ec0f80f), + TOBN(0xc370e522, 0xcc1b5e83), TOBN(0xde2d4ad1, 0x860b8bfb), + TOBN(0xad364df0, 0x67b256df), TOBN(0x8f12502e, 0xe0138997), + TOBN(0x503fa0dc, 0x7783920a), TOBN(0xe80014ad, 0xc0bc866a), + TOBN(0x3f89b744, 0xd3064ba6), TOBN(0x03511dcd, 0xcba5dba5), + TOBN(0x197dd46d, 0x95a7b1a2), TOBN(0x9c4e7ad6, 0x3c6341fb), + TOBN(0x426eca29, 0x484c2ece), TOBN(0x9211e489, 0xde7f4f8a), + TOBN(0x14997f6e, 0xc78ef1f4), TOBN(0x2b2c0910, 0x06574586), + TOBN(0x17286a6e, 0x1c3eede8), TOBN(0x25f92e47, 0x0f60e018), + TOBN(0x805c5646, 0x31890a36), TOBN(0x703ef600, 0x57feea5b), + TOBN(0x389f747c, 0xaf3c3030), TOBN(0xe0e5daeb, 0x54dd3739), + TOBN(0xfe24a4c3, 0xc9c9f155), TOBN(0x7e4bf176, 0xb5393962), + TOBN(0x37183de2, 0xaf20bf29), TOBN(0x4a1bd7b5, 0xf95a8c3b), + TOBN(0xa83b9699, 0x46191d3d), TOBN(0x281fc8dd, 0x7b87f257), + TOBN(0xb18e2c13, 0x54107588), TOBN(0x6372def7, 0x9b2bafe8), + TOBN(0xdaf4bb48, 0x0d8972ca), TOBN(0x3f2dd4b7, 0x56167a3f), + TOBN(0x1eace32d, 0x84310cf4), TOBN(0xe3bcefaf, 0xe42700aa), + TOBN(0x5fe5691e, 0xd785e73d), TOBN(0xa5db5ab6, 0x2ea60467), + TOBN(0x02e23d41, 0xdfc6514a), TOBN(0x35e8048e, 0xe03c3665), + TOBN(0x3f8b118f, 0x1adaa0f8), TOBN(0x28ec3b45, 0x84ce1a5a), + TOBN(0xe8cacc6e, 0x2c6646b8), TOBN(0x1343d185, 0xdbd0e40f), + TOBN(0xe5d7f844, 0xcaaa358c), TOBN(0x1a1db7e4, 0x9924182a), + TOBN(0xd64cd42d, 0x9c875d9a), TOBN(0xb37b515f, 0x042eeec8), + TOBN(0x4d4dd409, 0x7b165fbe), TOBN(0xfc322ed9, 0xe206eff3), + TOBN(0x7dee4102, 0x59b7e17e), TOBN(0x55a481c0, 0x8236ca00), + TOBN(0x8c885312, 0xc23fc975), TOBN(0x15715806, 0x05d6297b), + TOBN(0xa078868e, 0xf78edd39), TOBN(0x956b31e0, 0x03c45e52), + TOBN(0x470275d5, 0xff7b33a6), TOBN(0xc8d5dc3a, 0x0c7e673f), + TOBN(0x419227b4, 0x7e2f2598), TOBN(0x8b37b634, 0x4c14a975), + TOBN(0xd0667ed6, 0x8b11888c), TOBN(0x5e0e8c3e, 0x803e25dc), + TOBN(0x34e5d0dc, 0xb987a24a), TOBN(0x9f40ac3b, 0xae920323), + TOBN(0x5463de95, 0x34e0f63a), TOBN(0xa128bf92, 0x6b6328f9), + TOBN(0x491ccd7c, 0xda64f1b7), TOBN(0x7ef1ec27, 0xc47bde35), + TOBN(0xa857240f, 0xa36a2737), TOBN(0x35dc1366, 0x63621bc1), + TOBN(0x7a3a6453, 0xd4fb6897), TOBN(0x80f1a439, 0xc929319d), + TOBN(0xfc18274b, 0xf8cb0ba0), TOBN(0xb0b53766, 0x8078c5eb), + TOBN(0xfb0d4924, 0x1e01d0ef), TOBN(0x50d7c67d, 0x372ab09c), + TOBN(0xb4e370af, 0x3aeac968), TOBN(0xe4f7fee9, 0xc4b63266), + TOBN(0xb4acd4c2, 0xe3ac5664), TOBN(0xf8910bd2, 0xceb38cbf), + TOBN(0x1c3ae50c, 0xc9c0726e), TOBN(0x15309569, 0xd97b40bf), + TOBN(0x70884b7f, 0xfd5a5a1b), TOBN(0x3890896a, 0xef8314cd), + TOBN(0x58e1515c, 0xa5618c93), TOBN(0xe665432b, 0x77d942d1), + TOBN(0xb32181bf, 0xb6f767a8), TOBN(0x753794e8, 0x3a604110), + TOBN(0x09afeb7c, 0xe8c0dbcc), TOBN(0x31e02613, 0x598673a3), + TOBN(0x5d98e557, 0x7d46db00), TOBN(0xfc21fb8c, 0x9d985b28), + TOBN(0xc9040116, 0xb0843e0b), TOBN(0x53b1b3a8, 0x69b04531), + TOBN(0xdd1649f0, 0x85d7d830), TOBN(0xbb3bcc87, 0xcb7427e8), + TOBN(0x77261100, 0xc93dce83), TOBN(0x7e79da61, 0xa1922a2a), + TOBN(0x587a2b02, 0xf3149ce8), TOBN(0x147e1384, 0xde92ec83), + TOBN(0x484c83d3, 0xaf077f30), TOBN(0xea78f844, 0x0658b53a), + TOBN(0x912076c2, 0x027aec53), TOBN(0xf34714e3, 0x93c8177d), + TOBN(0x37ef5d15, 0xc2376c84), TOBN(0x8315b659, 0x3d1aa783), + TOBN(0x3a75c484, 0xef852a90), TOBN(0x0ba0c58a, 0x16086bd4), + TOBN(0x29688d7a, 0x529a6d48), TOBN(0x9c7f250d, 0xc2f19203), + TOBN(0x123042fb, 0x682e2df9), TOBN(0x2b7587e7, 0xad8121bc), + TOBN(0x30fc0233, 0xe0182a65), TOBN(0xb82ecf87, 0xe3e1128a), + TOBN(0x71682861, 0x93fb098f), TOBN(0x043e21ae, 0x85e9e6a7), + TOBN(0xab5b49d6, 0x66c834ea), TOBN(0x3be43e18, 0x47414287), + TOBN(0xf40fb859, 0x219a2a47), TOBN(0x0e6559e9, 0xcc58df3c), + TOBN(0xfe1dfe8e, 0x0c6615b4), TOBN(0x14abc8fd, 0x56459d70), + TOBN(0x7be0fa8e, 0x05de0386), TOBN(0x8e63ef68, 0xe9035c7c), + TOBN(0x116401b4, 0x53b31e91), TOBN(0x0cba7ad4, 0x4436b4d8), + TOBN(0x9151f9a0, 0x107afd66), TOBN(0xafaca8d0, 0x1f0ee4c4), + TOBN(0x75fe5c1d, 0x9ee9761c), TOBN(0x3497a16b, 0xf0c0588f), + TOBN(0x3ee2bebd, 0x0304804c), TOBN(0xa8fb9a60, 0xc2c990b9), + TOBN(0xd14d32fe, 0x39251114), TOBN(0x36bf25bc, 0xcac73366), + TOBN(0xc9562c66, 0xdba7495c), TOBN(0x324d301b, 0x46ad348b), + TOBN(0x9f46620c, 0xd670407e), TOBN(0x0ea8d4f1, 0xe3733a01), + TOBN(0xd396d532, 0xb0c324e0), TOBN(0x5b211a0e, 0x03c317cd), + TOBN(0x090d7d20, 0x5ffe7b37), TOBN(0x3b7f3efb, 0x1747d2da), + TOBN(0xa2cb525f, 0xb54fc519), TOBN(0x6e220932, 0xf66a971e), + TOBN(0xddc160df, 0xb486d440), TOBN(0x7fcfec46, 0x3fe13465), + TOBN(0x83da7e4e, 0x76e4c151), TOBN(0xd6fa48a1, 0xd8d302b5), + TOBN(0xc6304f26, 0x5872cd88), TOBN(0x806c1d3c, 0x278b90a1), + TOBN(0x3553e725, 0xcaf0bc1c), TOBN(0xff59e603, 0xbb9d8d5c), + TOBN(0xa4550f32, 0x7a0b85dd), TOBN(0xdec5720a, 0x93ecc217), + TOBN(0x0b88b741, 0x69d62213), TOBN(0x7212f245, 0x5b365955), + TOBN(0x20764111, 0xb5cae787), TOBN(0x13cb7f58, 0x1dfd3124), + TOBN(0x2dca77da, 0x1175aefb), TOBN(0xeb75466b, 0xffaae775), + TOBN(0x74d76f3b, 0xdb6cff32), TOBN(0x7440f37a, 0x61fcda9a), + TOBN(0x1bb3ac92, 0xb525028b), TOBN(0x20fbf8f7, 0xa1975f29), + TOBN(0x982692e1, 0xdf83097f), TOBN(0x28738f6c, 0x554b0800), + TOBN(0xdc703717, 0xa2ce2f2f), TOBN(0x7913b93c, 0x40814194), + TOBN(0x04924593, 0x1fe89636), TOBN(0x7b98443f, 0xf78834a6), + TOBN(0x11c6ab01, 0x5114a5a1), TOBN(0x60deb383, 0xffba5f4c), + TOBN(0x4caa54c6, 0x01a982e6), TOBN(0x1dd35e11, 0x3491cd26), + TOBN(0x973c315f, 0x7cbd6b05), TOBN(0xcab00775, 0x52494724), + TOBN(0x04659b1f, 0x6565e15a), TOBN(0xbf30f529, 0x8c8fb026), + TOBN(0xfc21641b, 0xa8a0de37), TOBN(0xe9c7a366, 0xfa5e5114), + TOBN(0xdb849ca5, 0x52f03ad8), TOBN(0xc7e8dbe9, 0x024e35c0), + TOBN(0xa1a2bbac, 0xcfc3c789), TOBN(0xbf733e7d, 0x9c26f262), + TOBN(0x882ffbf5, 0xb8444823), TOBN(0xb7224e88, 0x6bf8483b), + TOBN(0x53023b8b, 0x65bef640), TOBN(0xaabfec91, 0xd4d5f8cd), + TOBN(0xa40e1510, 0x079ea1bd), TOBN(0x1ad9addc, 0xd05d5d26), + TOBN(0xdb3f2eab, 0x13e68d4f), TOBN(0x1cff1ae2, 0x640f803f), + TOBN(0xe0e7b749, 0xd4cee117), TOBN(0x8e9f275b, 0x4036d909), + TOBN(0xce34e31d, 0x8f4d4c38), TOBN(0x22b37f69, 0xd75130fc), + TOBN(0x83e0f1fd, 0xb4014604), TOBN(0xa8ce9919, 0x89415078), + TOBN(0x82375b75, 0x41792efe), TOBN(0x4f59bf5c, 0x97d4515b), + TOBN(0xac4f324f, 0x923a277d), TOBN(0xd9bc9b7d, 0x650f3406), + TOBN(0xc6fa87d1, 0x8a39bc51), TOBN(0x82588530, 0x5ccc108f), + TOBN(0x5ced3c9f, 0x82e4c634), TOBN(0x8efb8314, 0x3a4464f8), + TOBN(0xe706381b, 0x7a1dca25), TOBN(0x6cd15a3c, 0x5a2a412b), + TOBN(0x9347a8fd, 0xbfcd8fb5), TOBN(0x31db2eef, 0x6e54cd22), + TOBN(0xc4aeb11e, 0xf8d8932f), TOBN(0x11e7c1ed, 0x344411af), + TOBN(0x2653050c, 0xdc9a151e), TOBN(0x9edbfc08, 0x3bb0a859), + TOBN(0x926c81c7, 0xfd5691e7), TOBN(0x9c1b2342, 0x6f39019a), + TOBN(0x64a81c8b, 0x7f8474b9), TOBN(0x90657c07, 0x01761819), + TOBN(0x390b3331, 0x55e0375a), TOBN(0xc676c626, 0xb6ebc47d), + TOBN(0x51623247, 0xb7d6dee8), TOBN(0x0948d927, 0x79659313), + TOBN(0x99700161, 0xe9ab35ed), TOBN(0x06cc32b4, 0x8ddde408), + TOBN(0x6f2fd664, 0x061ef338), TOBN(0x1606fa02, 0xc202e9ed), + TOBN(0x55388bc1, 0x929ba99b), TOBN(0xc4428c5e, 0x1e81df69), + TOBN(0xce2028ae, 0xf91b0b2a), TOBN(0xce870a23, 0xf03dfd3f), + TOBN(0x66ec2c87, 0x0affe8ed), TOBN(0xb205fb46, 0x284d0c00), + TOBN(0xbf5dffe7, 0x44cefa48), TOBN(0xb6fc37a8, 0xa19876d7), + TOBN(0xbecfa84c, 0x08b72863), TOBN(0xd7205ff5, 0x2576374f), + TOBN(0x80330d32, 0x8887de41), TOBN(0x5de0df0c, 0x869ea534), + TOBN(0x13f42753, 0x3c56ea17), TOBN(0xeb1f6069, 0x452b1a78), + TOBN(0x50474396, 0xe30ea15c), TOBN(0x575816a1, 0xc1494125), + TOBN(0xbe1ce55b, 0xfe6bb38f), TOBN(0xb901a948, 0x96ae30f7), + TOBN(0xe5af0f08, 0xd8fc3548), TOBN(0x5010b5d0, 0xd73bfd08), + TOBN(0x993d2880, 0x53fe655a), TOBN(0x99f2630b, 0x1c1309fd), + TOBN(0xd8677baf, 0xb4e3b76f), TOBN(0x14e51ddc, 0xb840784b), + TOBN(0x326c750c, 0xbf0092ce), TOBN(0xc83d306b, 0xf528320f), + TOBN(0xc4456715, 0x77d4715c), TOBN(0xd30019f9, 0x6b703235), + TOBN(0x207ccb2e, 0xd669e986), TOBN(0x57c824af, 0xf6dbfc28), + TOBN(0xf0eb532f, 0xd8f92a23), TOBN(0x4a557fd4, 0x9bb98fd2), + TOBN(0xa57acea7, 0xc1e6199a), TOBN(0x0c663820, 0x8b94b1ed), + TOBN(0x9b42be8f, 0xf83a9266), TOBN(0xc7741c97, 0x0101bd45), + TOBN(0x95770c11, 0x07bd9ceb), TOBN(0x1f50250a, 0x8b2e0744), + TOBN(0xf762eec8, 0x1477b654), TOBN(0xc65b900e, 0x15efe59a), + TOBN(0x88c96148, 0x9546a897), TOBN(0x7e8025b3, 0xc30b4d7c), + TOBN(0xae4065ef, 0x12045cf9), TOBN(0x6fcb2caf, 0x9ccce8bd), + TOBN(0x1fa0ba4e, 0xf2cf6525), TOBN(0xf683125d, 0xcb72c312), + TOBN(0xa01da4ea, 0xe312410e), TOBN(0x67e28677, 0x6cd8e830), + TOBN(0xabd95752, 0x98fb3f07), TOBN(0x05f11e11, 0xeef649a5), + TOBN(0xba47faef, 0x9d3472c2), TOBN(0x3adff697, 0xc77d1345), + TOBN(0x4761fa04, 0xdd15afee), TOBN(0x64f1f61a, 0xb9e69462), + TOBN(0xfa691fab, 0x9bfb9093), TOBN(0x3df8ae8f, 0xa1133dfe), + TOBN(0xcd5f8967, 0x58cc710d), TOBN(0xfbb88d50, 0x16c7fe79), + TOBN(0x8e011b4c, 0xe88c50d1), TOBN(0x7532e807, 0xa8771c4f), + TOBN(0x64c78a48, 0xe2278ee4), TOBN(0x0b283e83, 0x3845072a), + TOBN(0x98a6f291, 0x49e69274), TOBN(0xb96e9668, 0x1868b21c), + TOBN(0x38f0adc2, 0xb1a8908e), TOBN(0x90afcff7, 0x1feb829d), + TOBN(0x9915a383, 0x210b0856), TOBN(0xa5a80602, 0xdef04889), + TOBN(0x800e9af9, 0x7c64d509), TOBN(0x81382d0b, 0xb8996f6f), + TOBN(0x490eba53, 0x81927e27), TOBN(0x46c63b32, 0x4af50182), + TOBN(0x784c5fd9, 0xd3ad62ce), TOBN(0xe4fa1870, 0xf8ae8736), + TOBN(0x4ec9d0bc, 0xd7466b25), TOBN(0x84ddbe1a, 0xdb235c65), + TOBN(0x5e2645ee, 0x163c1688), TOBN(0x570bd00e, 0x00eba747), + TOBN(0xfa51b629, 0x128bfa0f), TOBN(0x92fce1bd, 0x6c1d3b68), + TOBN(0x3e7361dc, 0xb66778b1), TOBN(0x9c7d249d, 0x5561d2bb), + TOBN(0xa40b28bf, 0x0bbc6229), TOBN(0x1c83c05e, 0xdfd91497), + TOBN(0x5f9f5154, 0xf083df05), TOBN(0xbac38b3c, 0xeee66c9d), + TOBN(0xf71db7e3, 0xec0dfcfd), TOBN(0xf2ecda8e, 0x8b0a8416), + TOBN(0x52fddd86, 0x7812aa66), TOBN(0x2896ef10, 0x4e6f4272), + TOBN(0xff27186a, 0x0fe9a745), TOBN(0x08249fcd, 0x49ca70db), + TOBN(0x7425a2e6, 0x441cac49), TOBN(0xf4a0885a, 0xece5ff57), + TOBN(0x6e2cb731, 0x7d7ead58), TOBN(0xf96cf7d6, 0x1898d104), + TOBN(0xafe67c9d, 0x4f2c9a89), TOBN(0x89895a50, 0x1c7bf5bc), + TOBN(0xdc7cb8e5, 0x573cecfa), TOBN(0x66497eae, 0xd15f03e6), + TOBN(0x6bc0de69, 0x3f084420), TOBN(0x323b9b36, 0xacd532b0), + TOBN(0xcfed390a, 0x0115a3c1), TOBN(0x9414c40b, 0x2d65ca0e), + TOBN(0x641406bd, 0x2f530c78), TOBN(0x29369a44, 0x833438f2), + TOBN(0x996884f5, 0x903fa271), TOBN(0xe6da0fd2, 0xb9da921e), + TOBN(0xa6f2f269, 0x5db01e54), TOBN(0x1ee3e9bd, 0x6876214e), + TOBN(0xa26e181c, 0xe27a9497), TOBN(0x36d254e4, 0x8e215e04), + TOBN(0x42f32a6c, 0x252cabca), TOBN(0x99481487, 0x80b57614), + TOBN(0x4c4dfe69, 0x40d9cae1), TOBN(0x05869580, 0x11a10f09), + TOBN(0xca287b57, 0x3491b64b), TOBN(0x77862d5d, 0x3fd4a53b), + TOBN(0xbf94856e, 0x50349126), TOBN(0x2be30bd1, 0x71c5268f), + TOBN(0x10393f19, 0xcbb650a6), TOBN(0x639531fe, 0x778cf9fd), + TOBN(0x02556a11, 0xb2935359), TOBN(0xda38aa96, 0xaf8c126e), + TOBN(0x47dbe6c2, 0x0960167f), TOBN(0x37bbabb6, 0x501901cd), + TOBN(0xb6e979e0, 0x2c947778), TOBN(0xd69a5175, 0x7a1a1dc6), + TOBN(0xc3ed5095, 0x9d9faf0c), TOBN(0x4dd9c096, 0x1d5fa5f0), + TOBN(0xa0c4304d, 0x64f16ea8), TOBN(0x8b1cac16, 0x7e718623), + TOBN(0x0b576546, 0x7c67f03e), TOBN(0x559cf5ad, 0xcbd88c01), + TOBN(0x074877bb, 0x0e2af19a), TOBN(0x1f717ec1, 0xa1228c92), + TOBN(0x70bcb800, 0x326e8920), TOBN(0xec6e2c5c, 0x4f312804), + TOBN(0x426aea7d, 0x3fca4752), TOBN(0xf12c0949, 0x2211f62a), + TOBN(0x24beecd8, 0x7be7b6b5), TOBN(0xb77eaf4c, 0x36d7a27d), + TOBN(0x154c2781, 0xfda78fd3), TOBN(0x848a83b0, 0x264eeabe), + TOBN(0x81287ef0, 0x4ffe2bc4), TOBN(0x7b6d88c6, 0xb6b6fc2a), + TOBN(0x805fb947, 0xce417d99), TOBN(0x4b93dcc3, 0x8b916cc4), + TOBN(0x72e65bb3, 0x21273323), TOBN(0xbcc1badd, 0x6ea9886e), + TOBN(0x0e223011, 0x4bc5ee85), TOBN(0xa561be74, 0xc18ee1e4), + TOBN(0x762fd2d4, 0xa6bcf1f1), TOBN(0x50e6a5a4, 0x95231489), + TOBN(0xca96001f, 0xa00b500b), TOBN(0x5c098cfc, 0x5d7dcdf5), + TOBN(0xa64e2d2e, 0x8c446a85), TOBN(0xbae9bcf1, 0x971f3c62), + TOBN(0x4ec22683, 0x8435a2c5), TOBN(0x8ceaed6c, 0x4bad4643), + TOBN(0xe9f8fb47, 0xccccf4e3), TOBN(0xbd4f3fa4, 0x1ce3b21e), + TOBN(0xd79fb110, 0xa3db3292), TOBN(0xe28a37da, 0xb536c66a), + TOBN(0x279ce87b, 0x8e49e6a9), TOBN(0x70ccfe8d, 0xfdcec8e3), + TOBN(0x2193e4e0, 0x3ba464b2), TOBN(0x0f39d60e, 0xaca9a398), + TOBN(0x7d7932af, 0xf82c12ab), TOBN(0xd8ff50ed, 0x91e7e0f7), + TOBN(0xea961058, 0xfa28a7e0), TOBN(0xc726cf25, 0x0bf5ec74), + TOBN(0xe74d55c8, 0xdb229666), TOBN(0x0bd9abbf, 0xa57f5799), + TOBN(0x7479ef07, 0x4dfc47b3), TOBN(0xd9c65fc3, 0x0c52f91d), + TOBN(0x8e0283fe, 0x36a8bde2), TOBN(0xa32a8b5e, 0x7d4b7280), + TOBN(0x6a677c61, 0x12e83233), TOBN(0x0fbb3512, 0xdcc9bf28), + TOBN(0x562e8ea5, 0x0d780f61), TOBN(0x0db8b22b, 0x1dc4e89c), + TOBN(0x0a6fd1fb, 0x89be0144), TOBN(0x8c77d246, 0xca57113b), + TOBN(0x4639075d, 0xff09c91c), TOBN(0x5b47b17f, 0x5060824c), + TOBN(0x58aea2b0, 0x16287b52), TOBN(0xa1343520, 0xd0cd8eb0), + TOBN(0x6148b4d0, 0xc5d58573), TOBN(0xdd2b6170, 0x291c68ae), + TOBN(0xa61b3929, 0x1da3b3b7), TOBN(0x5f946d79, 0x08c4ac10), + TOBN(0x4105d4a5, 0x7217d583), TOBN(0x5061da3d, 0x25e6de5e), + TOBN(0x3113940d, 0xec1b4991), TOBN(0xf12195e1, 0x36f485ae), + TOBN(0xa7507fb2, 0x731a2ee0), TOBN(0x95057a8e, 0x6e9e196e), + TOBN(0xa3c2c911, 0x2e130136), TOBN(0x97dfbb36, 0x33c60d15), + TOBN(0xcaf3c581, 0xb300ee2b), TOBN(0x77f25d90, 0xf4bac8b8), + TOBN(0xdb1c4f98, 0x6d840cd6), TOBN(0x471d62c0, 0xe634288c), + TOBN(0x8ec2f85e, 0xcec8a161), TOBN(0x41f37cbc, 0xfa6f4ae2), + TOBN(0x6793a20f, 0x4b709985), TOBN(0x7a7bd33b, 0xefa8985b), + TOBN(0x2c6a3fbd, 0x938e6446), TOBN(0x19042619, 0x2a8d47c1), + TOBN(0x16848667, 0xcc36975f), TOBN(0x02acf168, 0x9d5f1dfb), + TOBN(0x62d41ad4, 0x613baa94), TOBN(0xb56fbb92, 0x9f684670), + TOBN(0xce610d0d, 0xe9e40569), TOBN(0x7b99c65f, 0x35489fef), + TOBN(0x0c88ad1b, 0x3df18b97), TOBN(0x81b7d9be, 0x5d0e9edb), + TOBN(0xd85218c0, 0xc716cc0a), TOBN(0xf4b5ff90, 0x85691c49), + TOBN(0xa4fd666b, 0xce356ac6), TOBN(0x17c72895, 0x4b327a7a), + TOBN(0xf93d5085, 0xda6be7de), TOBN(0xff71530e, 0x3301d34e), + TOBN(0x4cd96442, 0xd8f448e8), TOBN(0x9283d331, 0x2ed18ffa), + TOBN(0x4d33dd99, 0x2a849870), TOBN(0xa716964b, 0x41576335), + TOBN(0xff5e3a9b, 0x179be0e5), TOBN(0x5b9d6b1b, 0x83b13632), + TOBN(0x3b8bd7d4, 0xa52f313b), TOBN(0xc9dd95a0, 0x637a4660), + TOBN(0x30035962, 0x0b3e218f), TOBN(0xce1481a3, 0xc7b28a3c), + TOBN(0xab41b43a, 0x43228d83), TOBN(0x24ae1c30, 0x4ad63f99), + TOBN(0x8e525f1a, 0x46a51229), TOBN(0x14af860f, 0xcd26d2b4), + TOBN(0xd6baef61, 0x3f714aa1), TOBN(0xf51865ad, 0xeb78795e), + TOBN(0xd3e21fce, 0xe6a9d694), TOBN(0x82ceb1dd, 0x8a37b527)} +}; diff --git a/crypto/ecdh/Makefile b/crypto/ecdh/Makefile index f0766356a132..1b31ba1f0b3f 100644 --- a/crypto/ecdh/Makefile +++ b/crypto/ecdh/Makefile @@ -17,9 +17,9 @@ TEST=ecdhtest.c APPS= LIB=$(TOP)/libcrypto.a -LIBSRC= ech_lib.c ech_ossl.c ech_key.c ech_err.c +LIBSRC= ech_lib.c ech_ossl.c ech_key.c ech_err.c ech_kdf.c -LIBOBJ= ech_lib.o ech_ossl.o ech_key.o ech_err.o +LIBOBJ= ech_lib.o ech_ossl.o ech_key.o ech_err.o ech_kdf.o SRC= $(LIBSRC) @@ -85,6 +85,14 @@ ech_err.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h ech_err.o: ../../include/openssl/ossl_typ.h ../../include/openssl/safestack.h ech_err.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h ech_err.o: ech_err.c +ech_kdf.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h +ech_kdf.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h +ech_kdf.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h +ech_kdf.o: ../../include/openssl/evp.h ../../include/openssl/obj_mac.h +ech_kdf.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h +ech_kdf.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h +ech_kdf.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h +ech_kdf.o: ../../include/openssl/symhacks.h ech_kdf.c ech_key.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h ech_key.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h ech_key.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h diff --git a/crypto/ecdh/ecdh.h b/crypto/ecdh/ecdh.h index a9b811abd050..25348b30fe7c 100644 --- a/crypto/ecdh/ecdh.h +++ b/crypto/ecdh/ecdh.h @@ -85,6 +85,8 @@ extern "C" { #endif +# define EC_FLAG_COFACTOR_ECDH 0x1000 + const ECDH_METHOD *ECDH_OpenSSL(void); void ECDH_set_default_method(const ECDH_METHOD *); @@ -101,6 +103,11 @@ int ECDH_get_ex_new_index(long argl, void *argp, CRYPTO_EX_new int ECDH_set_ex_data(EC_KEY *d, int idx, void *arg); void *ECDH_get_ex_data(EC_KEY *d, int idx); +int ECDH_KDF_X9_62(unsigned char *out, size_t outlen, + const unsigned char *Z, size_t Zlen, + const unsigned char *sinfo, size_t sinfolen, + const EVP_MD *md); + /* BEGIN ERROR CODES */ /* * The following lines are auto generated by the script mkerr.pl. Any changes diff --git a/crypto/ecdh/ecdhtest.c b/crypto/ecdh/ecdhtest.c index 996321d1ee43..2fe2c66443d0 100644 --- a/crypto/ecdh/ecdhtest.c +++ b/crypto/ecdh/ecdhtest.c @@ -312,6 +312,170 @@ static int test_ecdh_curve(int nid, const char *text, BN_CTX *ctx, BIO *out) return (ret); } +/* Keys and shared secrets from RFC 7027 */ + +static const unsigned char bp256_da[] = { + 0x81, 0xDB, 0x1E, 0xE1, 0x00, 0x15, 0x0F, 0xF2, 0xEA, 0x33, 0x8D, 0x70, + 0x82, 0x71, 0xBE, 0x38, 0x30, 0x0C, 0xB5, 0x42, 0x41, 0xD7, 0x99, 0x50, + 0xF7, 0x7B, 0x06, 0x30, 0x39, 0x80, 0x4F, 0x1D +}; + +static const unsigned char bp256_db[] = { + 0x55, 0xE4, 0x0B, 0xC4, 0x1E, 0x37, 0xE3, 0xE2, 0xAD, 0x25, 0xC3, 0xC6, + 0x65, 0x45, 0x11, 0xFF, 0xA8, 0x47, 0x4A, 0x91, 0xA0, 0x03, 0x20, 0x87, + 0x59, 0x38, 0x52, 0xD3, 0xE7, 0xD7, 0x6B, 0xD3 +}; + +static const unsigned char bp256_Z[] = { + 0x89, 0xAF, 0xC3, 0x9D, 0x41, 0xD3, 0xB3, 0x27, 0x81, 0x4B, 0x80, 0x94, + 0x0B, 0x04, 0x25, 0x90, 0xF9, 0x65, 0x56, 0xEC, 0x91, 0xE6, 0xAE, 0x79, + 0x39, 0xBC, 0xE3, 0x1F, 0x3A, 0x18, 0xBF, 0x2B +}; + +static const unsigned char bp384_da[] = { + 0x1E, 0x20, 0xF5, 0xE0, 0x48, 0xA5, 0x88, 0x6F, 0x1F, 0x15, 0x7C, 0x74, + 0xE9, 0x1B, 0xDE, 0x2B, 0x98, 0xC8, 0xB5, 0x2D, 0x58, 0xE5, 0x00, 0x3D, + 0x57, 0x05, 0x3F, 0xC4, 0xB0, 0xBD, 0x65, 0xD6, 0xF1, 0x5E, 0xB5, 0xD1, + 0xEE, 0x16, 0x10, 0xDF, 0x87, 0x07, 0x95, 0x14, 0x36, 0x27, 0xD0, 0x42 +}; + +static const unsigned char bp384_db[] = { + 0x03, 0x26, 0x40, 0xBC, 0x60, 0x03, 0xC5, 0x92, 0x60, 0xF7, 0x25, 0x0C, + 0x3D, 0xB5, 0x8C, 0xE6, 0x47, 0xF9, 0x8E, 0x12, 0x60, 0xAC, 0xCE, 0x4A, + 0xCD, 0xA3, 0xDD, 0x86, 0x9F, 0x74, 0xE0, 0x1F, 0x8B, 0xA5, 0xE0, 0x32, + 0x43, 0x09, 0xDB, 0x6A, 0x98, 0x31, 0x49, 0x7A, 0xBA, 0xC9, 0x66, 0x70 +}; + +static const unsigned char bp384_Z[] = { + 0x0B, 0xD9, 0xD3, 0xA7, 0xEA, 0x0B, 0x3D, 0x51, 0x9D, 0x09, 0xD8, 0xE4, + 0x8D, 0x07, 0x85, 0xFB, 0x74, 0x4A, 0x6B, 0x35, 0x5E, 0x63, 0x04, 0xBC, + 0x51, 0xC2, 0x29, 0xFB, 0xBC, 0xE2, 0x39, 0xBB, 0xAD, 0xF6, 0x40, 0x37, + 0x15, 0xC3, 0x5D, 0x4F, 0xB2, 0xA5, 0x44, 0x4F, 0x57, 0x5D, 0x4F, 0x42 +}; + +static const unsigned char bp512_da[] = { + 0x16, 0x30, 0x2F, 0xF0, 0xDB, 0xBB, 0x5A, 0x8D, 0x73, 0x3D, 0xAB, 0x71, + 0x41, 0xC1, 0xB4, 0x5A, 0xCB, 0xC8, 0x71, 0x59, 0x39, 0x67, 0x7F, 0x6A, + 0x56, 0x85, 0x0A, 0x38, 0xBD, 0x87, 0xBD, 0x59, 0xB0, 0x9E, 0x80, 0x27, + 0x96, 0x09, 0xFF, 0x33, 0x3E, 0xB9, 0xD4, 0xC0, 0x61, 0x23, 0x1F, 0xB2, + 0x6F, 0x92, 0xEE, 0xB0, 0x49, 0x82, 0xA5, 0xF1, 0xD1, 0x76, 0x4C, 0xAD, + 0x57, 0x66, 0x54, 0x22 +}; + +static const unsigned char bp512_db[] = { + 0x23, 0x0E, 0x18, 0xE1, 0xBC, 0xC8, 0x8A, 0x36, 0x2F, 0xA5, 0x4E, 0x4E, + 0xA3, 0x90, 0x20, 0x09, 0x29, 0x2F, 0x7F, 0x80, 0x33, 0x62, 0x4F, 0xD4, + 0x71, 0xB5, 0xD8, 0xAC, 0xE4, 0x9D, 0x12, 0xCF, 0xAB, 0xBC, 0x19, 0x96, + 0x3D, 0xAB, 0x8E, 0x2F, 0x1E, 0xBA, 0x00, 0xBF, 0xFB, 0x29, 0xE4, 0xD7, + 0x2D, 0x13, 0xF2, 0x22, 0x45, 0x62, 0xF4, 0x05, 0xCB, 0x80, 0x50, 0x36, + 0x66, 0xB2, 0x54, 0x29 +}; + +static const unsigned char bp512_Z[] = { + 0xA7, 0x92, 0x70, 0x98, 0x65, 0x5F, 0x1F, 0x99, 0x76, 0xFA, 0x50, 0xA9, + 0xD5, 0x66, 0x86, 0x5D, 0xC5, 0x30, 0x33, 0x18, 0x46, 0x38, 0x1C, 0x87, + 0x25, 0x6B, 0xAF, 0x32, 0x26, 0x24, 0x4B, 0x76, 0xD3, 0x64, 0x03, 0xC0, + 0x24, 0xD7, 0xBB, 0xF0, 0xAA, 0x08, 0x03, 0xEA, 0xFF, 0x40, 0x5D, 0x3D, + 0x24, 0xF1, 0x1A, 0x9B, 0x5C, 0x0B, 0xEF, 0x67, 0x9F, 0xE1, 0x45, 0x4B, + 0x21, 0xC4, 0xCD, 0x1F +}; + +/* Given private value and NID, create EC_KEY structure */ + +static EC_KEY *mk_eckey(int nid, const unsigned char *p, size_t plen) +{ + int ok = 0; + EC_KEY *k = NULL; + BIGNUM *priv = NULL; + EC_POINT *pub = NULL; + const EC_GROUP *grp; + k = EC_KEY_new_by_curve_name(nid); + if (!k) + goto err; + priv = BN_bin2bn(p, plen, NULL); + if (!priv) + goto err; + if (!EC_KEY_set_private_key(k, priv)) + goto err; + grp = EC_KEY_get0_group(k); + pub = EC_POINT_new(grp); + if (!pub) + goto err; + if (!EC_POINT_mul(grp, pub, priv, NULL, NULL, NULL)) + goto err; + if (!EC_KEY_set_public_key(k, pub)) + goto err; + ok = 1; + err: + if (priv) + BN_clear_free(priv); + if (pub) + EC_POINT_free(pub); + if (ok) + return k; + else if (k) + EC_KEY_free(k); + return NULL; +} + +/* + * Known answer test: compute shared secret and check it matches expected + * value. + */ + +static int ecdh_kat(BIO *out, const char *cname, int nid, + const unsigned char *k1, size_t k1_len, + const unsigned char *k2, size_t k2_len, + const unsigned char *Z, size_t Zlen) +{ + int rv = 0; + EC_KEY *key1 = NULL, *key2 = NULL; + unsigned char *Ztmp = NULL; + size_t Ztmplen; + BIO_puts(out, "Testing ECDH shared secret with "); + BIO_puts(out, cname); + key1 = mk_eckey(nid, k1, k1_len); + key2 = mk_eckey(nid, k2, k2_len); + if (!key1 || !key2) + goto err; + Ztmplen = (EC_GROUP_get_degree(EC_KEY_get0_group(key1)) + 7) / 8; + if (Ztmplen != Zlen) + goto err; + Ztmp = OPENSSL_malloc(Ztmplen); + if (!ECDH_compute_key(Ztmp, Ztmplen, + EC_KEY_get0_public_key(key2), key1, 0)) + goto err; + if (memcmp(Ztmp, Z, Zlen)) + goto err; + memset(Ztmp, 0, Zlen); + if (!ECDH_compute_key(Ztmp, Ztmplen, + EC_KEY_get0_public_key(key1), key2, 0)) + goto err; + if (memcmp(Ztmp, Z, Zlen)) + goto err; + rv = 1; + err: + if (key1) + EC_KEY_free(key1); + if (key2) + EC_KEY_free(key2); + if (Ztmp) + OPENSSL_free(Ztmp); + if (rv) + BIO_puts(out, " ok\n"); + else { + fprintf(stderr, "Error in ECDH routines\n"); + ERR_print_errors_fp(stderr); + } + return rv; +} + +# define test_ecdh_kat(bio, curve, bits) \ + ecdh_kat(bio, curve, NID_brainpoolP##bits##r1, \ + bp##bits##_da, sizeof(bp##bits##_da), \ + bp##bits##_db, sizeof(bp##bits##_db), \ + bp##bits##_Z, sizeof(bp##bits##_Z)) + int main(int argc, char *argv[]) { BN_CTX *ctx = NULL; @@ -372,6 +536,12 @@ int main(int argc, char *argv[]) if (!test_ecdh_curve(NID_sect571r1, "NIST Binary-Curve B-571", ctx, out)) goto err; # endif + if (!test_ecdh_kat(out, "Brainpool Prime-Curve brainpoolP256r1", 256)) + goto err; + if (!test_ecdh_kat(out, "Brainpool Prime-Curve brainpoolP384r1", 384)) + goto err; + if (!test_ecdh_kat(out, "Brainpool Prime-Curve brainpoolP512r1", 512)) + goto err; ret = 0; diff --git a/crypto/ecdh/ech_kdf.c b/crypto/ecdh/ech_kdf.c new file mode 100644 index 000000000000..ac722ac9ee69 --- /dev/null +++ b/crypto/ecdh/ech_kdf.c @@ -0,0 +1,111 @@ +/* crypto/ecdh/ec_kdf.c */ +/* + * Written by Stephen Henson for the OpenSSL project. + */ +/* ==================================================================== + * Copyright (c) 2013 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + */ + +#define OPENSSL_FIPSAPI + +#include +#include +#include + +/* Key derivation function from X9.62/SECG */ +/* Way more than we will ever need */ +#define ECDH_KDF_MAX (1 << 30) + +int ECDH_KDF_X9_62(unsigned char *out, size_t outlen, + const unsigned char *Z, size_t Zlen, + const unsigned char *sinfo, size_t sinfolen, + const EVP_MD *md) +{ + EVP_MD_CTX mctx; + int rv = 0; + unsigned int i; + size_t mdlen; + unsigned char ctr[4]; + if (sinfolen > ECDH_KDF_MAX || outlen > ECDH_KDF_MAX + || Zlen > ECDH_KDF_MAX) + return 0; + mdlen = EVP_MD_size(md); + EVP_MD_CTX_init(&mctx); + for (i = 1;; i++) { + unsigned char mtmp[EVP_MAX_MD_SIZE]; + EVP_DigestInit_ex(&mctx, md, NULL); + ctr[3] = i & 0xFF; + ctr[2] = (i >> 8) & 0xFF; + ctr[1] = (i >> 16) & 0xFF; + ctr[0] = (i >> 24) & 0xFF; + if (!EVP_DigestUpdate(&mctx, Z, Zlen)) + goto err; + if (!EVP_DigestUpdate(&mctx, ctr, sizeof(ctr))) + goto err; + if (!EVP_DigestUpdate(&mctx, sinfo, sinfolen)) + goto err; + if (outlen >= mdlen) { + if (!EVP_DigestFinal(&mctx, out, NULL)) + goto err; + outlen -= mdlen; + if (outlen == 0) + break; + out += mdlen; + } else { + if (!EVP_DigestFinal(&mctx, mtmp, NULL)) + goto err; + memcpy(out, mtmp, outlen); + OPENSSL_cleanse(mtmp, mdlen); + break; + } + } + rv = 1; + err: + EVP_MD_CTX_cleanup(&mctx); + return rv; +} diff --git a/crypto/ecdh/ech_ossl.c b/crypto/ecdh/ech_ossl.c index d448b19a528f..df115cc262e5 100644 --- a/crypto/ecdh/ech_ossl.c +++ b/crypto/ecdh/ech_ossl.c @@ -138,6 +138,16 @@ static int ecdh_compute_key(void *out, size_t outlen, const EC_POINT *pub_key, } group = EC_KEY_get0_group(ecdh); + + if (EC_KEY_get_flags(ecdh) & EC_FLAG_COFACTOR_ECDH) { + if (!EC_GROUP_get_cofactor(group, x, ctx) || + !BN_mul(x, x, priv_key, ctx)) { + ECDHerr(ECDH_F_ECDH_COMPUTE_KEY, ERR_R_MALLOC_FAILURE); + goto err; + } + priv_key = x; + } + if ((tmp = EC_POINT_new(group)) == NULL) { ECDHerr(ECDH_F_ECDH_COMPUTE_KEY, ERR_R_MALLOC_FAILURE); goto err; diff --git a/crypto/ecdsa/ecdsa.h b/crypto/ecdsa/ecdsa.h index faf76b1104fc..c4016ac3e19b 100644 --- a/crypto/ecdsa/ecdsa.h +++ b/crypto/ecdsa/ecdsa.h @@ -228,6 +228,80 @@ int ECDSA_get_ex_new_index(long argl, void *argp, CRYPTO_EX_new int ECDSA_set_ex_data(EC_KEY *d, int idx, void *arg); void *ECDSA_get_ex_data(EC_KEY *d, int idx); +/** Allocates and initialize a ECDSA_METHOD structure + * \param ecdsa_method pointer to ECDSA_METHOD to copy. (May be NULL) + * \return pointer to a ECDSA_METHOD structure or NULL if an error occurred + */ + +ECDSA_METHOD *ECDSA_METHOD_new(ECDSA_METHOD *ecdsa_method); + +/** frees a ECDSA_METHOD structure + * \param ecdsa_method pointer to the ECDSA_METHOD structure + */ +void ECDSA_METHOD_free(ECDSA_METHOD *ecdsa_method); + +/** Sets application specific data in the ECDSA_METHOD + * \param ecdsa_method pointer to existing ECDSA_METHOD + * \param app application specific data to set + */ + +void ECDSA_METHOD_set_app_data(ECDSA_METHOD *ecdsa_method, void *app); + +/** Returns application specific data from a ECDSA_METHOD structure + * \param ecdsa_method pointer to ECDSA_METHOD structure + * \return pointer to application specific data. + */ + +void *ECDSA_METHOD_get_app_data(ECDSA_METHOD *ecdsa_method); + +/** Set the ECDSA_do_sign function in the ECDSA_METHOD + * \param ecdsa_method pointer to existing ECDSA_METHOD + * \param ecdsa_do_sign a funtion of type ECDSA_do_sign + */ + +void ECDSA_METHOD_set_sign(ECDSA_METHOD *ecdsa_method, + ECDSA_SIG *(*ecdsa_do_sign) (const unsigned char + *dgst, int dgst_len, + const BIGNUM *inv, + const BIGNUM *rp, + EC_KEY *eckey)); + +/** Set the ECDSA_sign_setup function in the ECDSA_METHOD + * \param ecdsa_method pointer to existing ECDSA_METHOD + * \param ecdsa_sign_setup a funtion of type ECDSA_sign_setup + */ + +void ECDSA_METHOD_set_sign_setup(ECDSA_METHOD *ecdsa_method, + int (*ecdsa_sign_setup) (EC_KEY *eckey, + BN_CTX *ctx, + BIGNUM **kinv, + BIGNUM **r)); + +/** Set the ECDSA_do_verify function in the ECDSA_METHOD + * \param ecdsa_method pointer to existing ECDSA_METHOD + * \param ecdsa_do_verify a funtion of type ECDSA_do_verify + */ + +void ECDSA_METHOD_set_verify(ECDSA_METHOD *ecdsa_method, + int (*ecdsa_do_verify) (const unsigned char + *dgst, int dgst_len, + const ECDSA_SIG *sig, + EC_KEY *eckey)); + +void ECDSA_METHOD_set_flags(ECDSA_METHOD *ecdsa_method, int flags); + +/** Set the flags field in the ECDSA_METHOD + * \param ecdsa_method pointer to existing ECDSA_METHOD + * \param flags flags value to set + */ + +void ECDSA_METHOD_set_name(ECDSA_METHOD *ecdsa_method, char *name); + +/** Set the name field in the ECDSA_METHOD + * \param ecdsa_method pointer to existing ECDSA_METHOD + * \param name name to set + */ + /* BEGIN ERROR CODES */ /* * The following lines are auto generated by the script mkerr.pl. Any changes @@ -242,6 +316,7 @@ void ERR_load_ECDSA_strings(void); # define ECDSA_F_ECDSA_DATA_NEW_METHOD 100 # define ECDSA_F_ECDSA_DO_SIGN 101 # define ECDSA_F_ECDSA_DO_VERIFY 102 +# define ECDSA_F_ECDSA_METHOD_NEW 105 # define ECDSA_F_ECDSA_SIGN_SETUP 103 /* Reason codes. */ diff --git a/crypto/ecdsa/ecs_err.c b/crypto/ecdsa/ecs_err.c index 6fc64a0077ac..f1fa7b55f977 100644 --- a/crypto/ecdsa/ecs_err.c +++ b/crypto/ecdsa/ecs_err.c @@ -74,6 +74,7 @@ static ERR_STRING_DATA ECDSA_str_functs[] = { {ERR_FUNC(ECDSA_F_ECDSA_DATA_NEW_METHOD), "ECDSA_DATA_NEW_METHOD"}, {ERR_FUNC(ECDSA_F_ECDSA_DO_SIGN), "ECDSA_do_sign"}, {ERR_FUNC(ECDSA_F_ECDSA_DO_VERIFY), "ECDSA_do_verify"}, + {ERR_FUNC(ECDSA_F_ECDSA_METHOD_NEW), "ECDSA_METHOD_new"}, {ERR_FUNC(ECDSA_F_ECDSA_SIGN_SETUP), "ECDSA_sign_setup"}, {0, NULL} }; diff --git a/crypto/ecdsa/ecs_lib.c b/crypto/ecdsa/ecs_lib.c index 0f2d3432f666..1c0231031850 100644 --- a/crypto/ecdsa/ecs_lib.c +++ b/crypto/ecdsa/ecs_lib.c @@ -275,3 +275,80 @@ void *ECDSA_get_ex_data(EC_KEY *d, int idx) return NULL; return (CRYPTO_get_ex_data(&ecdsa->ex_data, idx)); } + +ECDSA_METHOD *ECDSA_METHOD_new(ECDSA_METHOD *ecdsa_meth) +{ + ECDSA_METHOD *ret; + + ret = OPENSSL_malloc(sizeof(ECDSA_METHOD)); + if (ret == NULL) { + ECDSAerr(ECDSA_F_ECDSA_METHOD_NEW, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (ecdsa_meth) + *ret = *ecdsa_meth; + else { + ret->ecdsa_sign_setup = 0; + ret->ecdsa_do_sign = 0; + ret->ecdsa_do_verify = 0; + ret->name = NULL; + ret->flags = 0; + } + ret->flags |= ECDSA_METHOD_FLAG_ALLOCATED; + return ret; +} + +void ECDSA_METHOD_set_sign(ECDSA_METHOD *ecdsa_method, + ECDSA_SIG *(*ecdsa_do_sign) (const unsigned char + *dgst, int dgst_len, + const BIGNUM *inv, + const BIGNUM *rp, + EC_KEY *eckey)) +{ + ecdsa_method->ecdsa_do_sign = ecdsa_do_sign; +} + +void ECDSA_METHOD_set_sign_setup(ECDSA_METHOD *ecdsa_method, + int (*ecdsa_sign_setup) (EC_KEY *eckey, + BN_CTX *ctx, + BIGNUM **kinv, + BIGNUM **r)) +{ + ecdsa_method->ecdsa_sign_setup = ecdsa_sign_setup; +} + +void ECDSA_METHOD_set_verify(ECDSA_METHOD *ecdsa_method, + int (*ecdsa_do_verify) (const unsigned char + *dgst, int dgst_len, + const ECDSA_SIG *sig, + EC_KEY *eckey)) +{ + ecdsa_method->ecdsa_do_verify = ecdsa_do_verify; +} + +void ECDSA_METHOD_set_flags(ECDSA_METHOD *ecdsa_method, int flags) +{ + ecdsa_method->flags = flags | ECDSA_METHOD_FLAG_ALLOCATED; +} + +void ECDSA_METHOD_set_name(ECDSA_METHOD *ecdsa_method, char *name) +{ + ecdsa_method->name = name; +} + +void ECDSA_METHOD_free(ECDSA_METHOD *ecdsa_method) +{ + if (ecdsa_method->flags & ECDSA_METHOD_FLAG_ALLOCATED) + OPENSSL_free(ecdsa_method); +} + +void ECDSA_METHOD_set_app_data(ECDSA_METHOD *ecdsa_method, void *app) +{ + ecdsa_method->app_data = app; +} + +void *ECDSA_METHOD_get_app_data(ECDSA_METHOD *ecdsa_method) +{ + return ecdsa_method->app_data; +} diff --git a/crypto/ecdsa/ecs_locl.h b/crypto/ecdsa/ecs_locl.h index 76b2caf1f4a9..d3a5efc54738 100644 --- a/crypto/ecdsa/ecs_locl.h +++ b/crypto/ecdsa/ecs_locl.h @@ -79,9 +79,13 @@ struct ecdsa_method { int (*finish) (EC_KEY *eckey); # endif int flags; - char *app_data; + void *app_data; }; +/* The ECDSA_METHOD was allocated and can be freed */ + +# define ECDSA_METHOD_FLAG_ALLOCATED 0x2 + /* * If this flag is set the ECDSA method is FIPS compliant and can be used in * FIPS mode. This is set in the validated module method. If an application diff --git a/crypto/ecdsa/ecs_ossl.c b/crypto/ecdsa/ecs_ossl.c index 4c5fa6b926e4..dd769609be4c 100644 --- a/crypto/ecdsa/ecs_ossl.c +++ b/crypto/ecdsa/ecs_ossl.c @@ -179,10 +179,32 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp, while (BN_is_zero(r)); /* compute the inverse of k */ - if (!BN_mod_inverse(k, k, order, ctx)) { - ECDSAerr(ECDSA_F_ECDSA_SIGN_SETUP, ERR_R_BN_LIB); - goto err; + if (EC_GROUP_get_mont_data(group) != NULL) { + /* + * We want inverse in constant time, therefore we utilize the fact + * order must be prime and use Fermats Little Theorem instead. + */ + if (!BN_set_word(X, 2)) { + ECDSAerr(ECDSA_F_ECDSA_SIGN_SETUP, ERR_R_BN_LIB); + goto err; + } + if (!BN_mod_sub(X, order, X, order, ctx)) { + ECDSAerr(ECDSA_F_ECDSA_SIGN_SETUP, ERR_R_BN_LIB); + goto err; + } + BN_set_flags(X, BN_FLG_CONSTTIME); + if (!BN_mod_exp_mont_consttime + (k, k, X, order, ctx, EC_GROUP_get_mont_data(group))) { + ECDSAerr(ECDSA_F_ECDSA_SIGN_SETUP, ERR_R_BN_LIB); + goto err; + } + } else { + if (!BN_mod_inverse(k, k, order, ctx)) { + ECDSAerr(ECDSA_F_ECDSA_SIGN_SETUP, ERR_R_BN_LIB); + goto err; + } } + /* clear old values if necessary */ if (*rp != NULL) BN_clear_free(*rp); diff --git a/crypto/engine/Makefile b/crypto/engine/Makefile index 8ceb747fabdc..426388e9b174 100644 --- a/crypto/engine/Makefile +++ b/crypto/engine/Makefile @@ -22,13 +22,13 @@ LIBSRC= eng_err.c eng_lib.c eng_list.c eng_init.c eng_ctrl.c \ tb_rsa.c tb_dsa.c tb_ecdsa.c tb_dh.c tb_ecdh.c tb_rand.c tb_store.c \ tb_cipher.c tb_digest.c tb_pkmeth.c tb_asnmth.c \ eng_openssl.c eng_cnf.c eng_dyn.c eng_cryptodev.c \ - eng_rsax.c eng_rdrand.c + eng_rdrand.c LIBOBJ= eng_err.o eng_lib.o eng_list.o eng_init.o eng_ctrl.o \ eng_table.o eng_pkey.o eng_fat.o eng_all.o \ tb_rsa.o tb_dsa.o tb_ecdsa.o tb_dh.o tb_ecdh.o tb_rand.o tb_store.o \ tb_cipher.o tb_digest.o tb_pkmeth.o tb_asnmth.o \ eng_openssl.o eng_cnf.o eng_dyn.o eng_cryptodev.o \ - eng_rsax.o eng_rdrand.o + eng_rdrand.o SRC= $(LIBSRC) @@ -267,20 +267,6 @@ eng_rdrand.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h eng_rdrand.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h eng_rdrand.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h eng_rdrand.o: eng_rdrand.c -eng_rsax.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h -eng_rsax.o: ../../include/openssl/bn.h ../../include/openssl/buffer.h -eng_rsax.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h -eng_rsax.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h -eng_rsax.o: ../../include/openssl/ecdsa.h ../../include/openssl/engine.h -eng_rsax.o: ../../include/openssl/err.h ../../include/openssl/evp.h -eng_rsax.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h -eng_rsax.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h -eng_rsax.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h -eng_rsax.o: ../../include/openssl/pkcs7.h ../../include/openssl/rsa.h -eng_rsax.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h -eng_rsax.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h -eng_rsax.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h -eng_rsax.o: eng_rsax.c eng_table.o: ../../e_os.h ../../include/openssl/asn1.h eng_table.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h eng_table.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h diff --git a/crypto/engine/eng_all.c b/crypto/engine/eng_all.c index 7edf12e7e174..195a3a95542a 100644 --- a/crypto/engine/eng_all.c +++ b/crypto/engine/eng_all.c @@ -76,9 +76,6 @@ void ENGINE_load_builtin_engines(void) #if !defined(OPENSSL_NO_HW) && (defined(__OpenBSD__) || defined(__FreeBSD__) || defined(HAVE_CRYPTODEV)) ENGINE_load_cryptodev(); #endif -#ifndef OPENSSL_NO_RSAX - ENGINE_load_rsax(); -#endif #ifndef OPENSSL_NO_RDRAND ENGINE_load_rdrand(); #endif diff --git a/crypto/engine/eng_cryptodev.c b/crypto/engine/eng_cryptodev.c index bcb936dfa754..926d95c0d7fc 100644 --- a/crypto/engine/eng_cryptodev.c +++ b/crypto/engine/eng_cryptodev.c @@ -54,10 +54,10 @@ void ENGINE_load_cryptodev(void) # include # include -# include -# include -# include -# include +# include +# include +# include +# include # include # include # include @@ -160,6 +160,17 @@ static struct { { CRYPTO_AES_CBC, NID_aes_256_cbc, 16, 32, }, +# ifdef CRYPTO_AES_CTR + { + CRYPTO_AES_CTR, NID_aes_128_ctr, 14, 16, + }, + { + CRYPTO_AES_CTR, NID_aes_192_ctr, 14, 24, + }, + { + CRYPTO_AES_CTR, NID_aes_256_ctr, 14, 32, + }, +# endif { CRYPTO_BLF_CBC, NID_bf_cbc, 8, 16, }, @@ -630,6 +641,46 @@ const EVP_CIPHER cryptodev_aes_256_cbc = { NULL }; +# ifdef CRYPTO_AES_CTR +const EVP_CIPHER cryptodev_aes_ctr = { + NID_aes_128_ctr, + 16, 16, 14, + EVP_CIPH_CTR_MODE, + cryptodev_init_key, + cryptodev_cipher, + cryptodev_cleanup, + sizeof(struct dev_crypto_state), + EVP_CIPHER_set_asn1_iv, + EVP_CIPHER_get_asn1_iv, + NULL +}; + +const EVP_CIPHER cryptodev_aes_ctr_192 = { + NID_aes_192_ctr, + 16, 24, 14, + EVP_CIPH_CTR_MODE, + cryptodev_init_key, + cryptodev_cipher, + cryptodev_cleanup, + sizeof(struct dev_crypto_state), + EVP_CIPHER_set_asn1_iv, + EVP_CIPHER_get_asn1_iv, + NULL +}; + +const EVP_CIPHER cryptodev_aes_ctr_256 = { + NID_aes_256_ctr, + 16, 32, 14, + EVP_CIPH_CTR_MODE, + cryptodev_init_key, + cryptodev_cipher, + cryptodev_cleanup, + sizeof(struct dev_crypto_state), + EVP_CIPHER_set_asn1_iv, + EVP_CIPHER_get_asn1_iv, + NULL +}; +# endif /* * Registered by the ENGINE when used to find out how to deal with * a particular NID in the ENGINE. this says what we'll do at the @@ -667,6 +718,17 @@ cryptodev_engine_ciphers(ENGINE *e, const EVP_CIPHER **cipher, case NID_aes_256_cbc: *cipher = &cryptodev_aes_256_cbc; break; +# ifdef CRYPTO_AES_CTR + case NID_aes_128_ctr: + *cipher = &cryptodev_aes_ctr; + break; + case NID_aes_192_ctr: + *cipher = &cryptodev_aes_ctr_192; + break; + case NID_aes_256_ctr: + *cipher = &cryptodev_aes_ctr_256; + break; +# endif default: *cipher = NULL; break; diff --git a/crypto/engine/eng_rsax.c b/crypto/engine/eng_rsax.c deleted file mode 100644 index 86ee9d893995..000000000000 --- a/crypto/engine/eng_rsax.c +++ /dev/null @@ -1,701 +0,0 @@ -/* crypto/engine/eng_rsax.c */ -/* Copyright (c) 2010-2010 Intel Corp. - * Author: Vinodh.Gopal@intel.com - * Jim Guilford - * Erdinc.Ozturk@intel.com - * Maxim.Perminov@intel.com - * Ying.Huang@intel.com - * - * More information about algorithm used can be found at: - * http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf - */ -/* ==================================================================== - * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * licensing@OpenSSL.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - */ - -#include - -#include -#include -#include -#include -#include -#ifndef OPENSSL_NO_RSA -# include -#endif -#include -#include - -/* RSAX is available **ONLY* on x86_64 CPUs */ -#undef COMPILE_RSAX - -#if (defined(__x86_64) || defined(__x86_64__) || \ - defined(_M_AMD64) || defined (_M_X64)) && !defined(OPENSSL_NO_ASM) -# define COMPILE_RSAX -static ENGINE *ENGINE_rsax(void); -#endif - -void ENGINE_load_rsax(void) -{ -/* On non-x86 CPUs it just returns. */ -#ifdef COMPILE_RSAX - ENGINE *toadd = ENGINE_rsax(); - if (!toadd) - return; - ENGINE_add(toadd); - ENGINE_free(toadd); - ERR_clear_error(); -#endif -} - -#ifdef COMPILE_RSAX -# define E_RSAX_LIB_NAME "rsax engine" - -static int e_rsax_destroy(ENGINE *e); -static int e_rsax_init(ENGINE *e); -static int e_rsax_finish(ENGINE *e); -static int e_rsax_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void)); - -# ifndef OPENSSL_NO_RSA -/* RSA stuff */ -static int e_rsax_rsa_mod_exp(BIGNUM *r, const BIGNUM *I, RSA *rsa, - BN_CTX *ctx); -static int e_rsax_rsa_finish(RSA *r); -# endif - -static const ENGINE_CMD_DEFN e_rsax_cmd_defns[] = { - {0, NULL, NULL, 0} -}; - -# ifndef OPENSSL_NO_RSA -/* Our internal RSA_METHOD that we provide pointers to */ -static RSA_METHOD e_rsax_rsa = { - "Intel RSA-X method", - NULL, - NULL, - NULL, - NULL, - e_rsax_rsa_mod_exp, - NULL, - NULL, - e_rsax_rsa_finish, - RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE, - NULL, - NULL, - NULL -}; -# endif - -/* Constants used when creating the ENGINE */ -static const char *engine_e_rsax_id = "rsax"; -static const char *engine_e_rsax_name = "RSAX engine support"; - -/* This internal function is used by ENGINE_rsax() */ -static int bind_helper(ENGINE *e) -{ -# ifndef OPENSSL_NO_RSA - const RSA_METHOD *meth1; -# endif - if (!ENGINE_set_id(e, engine_e_rsax_id) || - !ENGINE_set_name(e, engine_e_rsax_name) || -# ifndef OPENSSL_NO_RSA - !ENGINE_set_RSA(e, &e_rsax_rsa) || -# endif - !ENGINE_set_destroy_function(e, e_rsax_destroy) || - !ENGINE_set_init_function(e, e_rsax_init) || - !ENGINE_set_finish_function(e, e_rsax_finish) || - !ENGINE_set_ctrl_function(e, e_rsax_ctrl) || - !ENGINE_set_cmd_defns(e, e_rsax_cmd_defns)) - return 0; - -# ifndef OPENSSL_NO_RSA - meth1 = RSA_PKCS1_SSLeay(); - e_rsax_rsa.rsa_pub_enc = meth1->rsa_pub_enc; - e_rsax_rsa.rsa_pub_dec = meth1->rsa_pub_dec; - e_rsax_rsa.rsa_priv_enc = meth1->rsa_priv_enc; - e_rsax_rsa.rsa_priv_dec = meth1->rsa_priv_dec; - e_rsax_rsa.bn_mod_exp = meth1->bn_mod_exp; -# endif - return 1; -} - -static ENGINE *ENGINE_rsax(void) -{ - ENGINE *ret = ENGINE_new(); - if (!ret) - return NULL; - if (!bind_helper(ret)) { - ENGINE_free(ret); - return NULL; - } - return ret; -} - -# ifndef OPENSSL_NO_RSA -/* Used to attach our own key-data to an RSA structure */ -static int rsax_ex_data_idx = -1; -# endif - -static int e_rsax_destroy(ENGINE *e) -{ - return 1; -} - -/* (de)initialisation functions. */ -static int e_rsax_init(ENGINE *e) -{ -# ifndef OPENSSL_NO_RSA - if (rsax_ex_data_idx == -1) - rsax_ex_data_idx = RSA_get_ex_new_index(0, NULL, NULL, NULL, NULL); -# endif - if (rsax_ex_data_idx == -1) - return 0; - return 1; -} - -static int e_rsax_finish(ENGINE *e) -{ - return 1; -} - -static int e_rsax_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void)) -{ - int to_return = 1; - - switch (cmd) { - /* The command isn't understood by this engine */ - default: - to_return = 0; - break; - } - - return to_return; -} - -# ifndef OPENSSL_NO_RSA - -# ifdef _WIN32 -typedef unsigned __int64 UINT64; -# else -typedef unsigned long long UINT64; -# endif -typedef unsigned short UINT16; - -/* - * Table t is interleaved in the following manner: The order in memory is - * t[0][0], t[0][1], ..., t[0][7], t[1][0], ... A particular 512-bit value is - * stored in t[][index] rather than the more normal t[index][]; i.e. the - * qwords of a particular entry in t are not adjacent in memory - */ - -/* Init BIGNUM b from the interleaved UINT64 array */ -static int interleaved_array_to_bn_512(BIGNUM *b, UINT64 *array); - -/* - * Extract array elements from BIGNUM b To set the whole array from b, call - * with n=8 - */ -static int bn_extract_to_array_512(const BIGNUM *b, unsigned int n, - UINT64 *array); - -struct mod_ctx_512 { - UINT64 t[8][8]; - UINT64 m[8]; - UINT64 m1[8]; /* 2^278 % m */ - UINT64 m2[8]; /* 2^640 % m */ - UINT64 k1[2]; /* (- 1/m) % 2^128 */ -}; - -static int mod_exp_pre_compute_data_512(UINT64 *m, struct mod_ctx_512 *data); - -void mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */ - UINT64 *g, /* 512 bits, 8 qwords */ - UINT64 *exp, /* 512 bits, 8 qwords */ - struct mod_ctx_512 *data); - -typedef struct st_e_rsax_mod_ctx { - UINT64 type; - union { - struct mod_ctx_512 b512; - } ctx; - -} E_RSAX_MOD_CTX; - -static E_RSAX_MOD_CTX *e_rsax_get_ctx(RSA *rsa, int idx, BIGNUM *m) -{ - E_RSAX_MOD_CTX *hptr; - - if (idx < 0 || idx > 2) - return NULL; - - hptr = RSA_get_ex_data(rsa, rsax_ex_data_idx); - if (!hptr) { - hptr = OPENSSL_malloc(3 * sizeof(E_RSAX_MOD_CTX)); - if (!hptr) - return NULL; - hptr[2].type = hptr[1].type = hptr[0].type = 0; - RSA_set_ex_data(rsa, rsax_ex_data_idx, hptr); - } - - if (hptr[idx].type == (UINT64)BN_num_bits(m)) - return hptr + idx; - - if (BN_num_bits(m) == 512) { - UINT64 _m[8]; - bn_extract_to_array_512(m, 8, _m); - memset(&hptr[idx].ctx.b512, 0, sizeof(struct mod_ctx_512)); - mod_exp_pre_compute_data_512(_m, &hptr[idx].ctx.b512); - } - - hptr[idx].type = BN_num_bits(m); - return hptr + idx; -} - -static int e_rsax_rsa_finish(RSA *rsa) -{ - E_RSAX_MOD_CTX *hptr = RSA_get_ex_data(rsa, rsax_ex_data_idx); - if (hptr) { - OPENSSL_free(hptr); - RSA_set_ex_data(rsa, rsax_ex_data_idx, NULL); - } - if (rsa->_method_mod_n) - BN_MONT_CTX_free(rsa->_method_mod_n); - if (rsa->_method_mod_p) - BN_MONT_CTX_free(rsa->_method_mod_p); - if (rsa->_method_mod_q) - BN_MONT_CTX_free(rsa->_method_mod_q); - return 1; -} - -static int e_rsax_bn_mod_exp(BIGNUM *r, const BIGNUM *g, const BIGNUM *e, - const BIGNUM *m, BN_CTX *ctx, - BN_MONT_CTX *in_mont, - E_RSAX_MOD_CTX *rsax_mod_ctx) -{ - if (rsax_mod_ctx && BN_get_flags(e, BN_FLG_CONSTTIME) != 0) { - if (BN_num_bits(m) == 512) { - UINT64 _r[8]; - UINT64 _g[8]; - UINT64 _e[8]; - - /* Init the arrays from the BIGNUMs */ - bn_extract_to_array_512(g, 8, _g); - bn_extract_to_array_512(e, 8, _e); - - mod_exp_512(_r, _g, _e, &rsax_mod_ctx->ctx.b512); - /* Return the result in the BIGNUM */ - interleaved_array_to_bn_512(r, _r); - return 1; - } - } - - return BN_mod_exp_mont(r, g, e, m, ctx, in_mont); -} - -/* - * Declares for the Intel CIAP 512-bit / CRT / 1024 bit RSA modular - * exponentiation routine precalculations and a structure to hold the - * necessary values. These files are meant to live in crypto/rsa/ in the - * target openssl. - */ - -/* - * Local method: extracts a piece from a BIGNUM, to fit it into - * an array. Call with n=8 to extract an entire 512-bit BIGNUM - */ -static int bn_extract_to_array_512(const BIGNUM *b, unsigned int n, - UINT64 *array) -{ - int i; - UINT64 tmp; - unsigned char bn_buff[64]; - memset(bn_buff, 0, 64); - if (BN_num_bytes(b) > 64) { - printf("Can't support this byte size\n"); - return 0; - } - if (BN_num_bytes(b) != 0) { - if (!BN_bn2bin(b, bn_buff + (64 - BN_num_bytes(b)))) { - printf("Error's in bn2bin\n"); - /* We have to error, here */ - return 0; - } - } - while (n-- > 0) { - array[n] = 0; - for (i = 7; i >= 0; i--) { - tmp = bn_buff[63 - (n * 8 + i)]; - array[n] |= tmp << (8 * i); - } - } - return 1; -} - -/* Init a 512-bit BIGNUM from the UINT64*_ (8 * 64) interleaved array */ -static int interleaved_array_to_bn_512(BIGNUM *b, UINT64 *array) -{ - unsigned char tmp[64]; - int n = 8; - int i; - while (n-- > 0) { - for (i = 7; i >= 0; i--) { - tmp[63 - (n * 8 + i)] = (unsigned char)(array[n] >> (8 * i)); - }} - BN_bin2bn(tmp, 64, b); - return 0; -} - -/* The main 512bit precompute call */ -static int mod_exp_pre_compute_data_512(UINT64 *m, struct mod_ctx_512 *data) -{ - BIGNUM two_768, two_640, two_128, two_512, tmp, _m, tmp2; - - /* We need a BN_CTX for the modulo functions */ - BN_CTX *ctx; - /* Some tmps */ - UINT64 _t[8]; - int i, j, ret = 0; - - /* Init _m with m */ - BN_init(&_m); - interleaved_array_to_bn_512(&_m, m); - memset(_t, 0, 64); - - /* Inits */ - BN_init(&two_768); - BN_init(&two_640); - BN_init(&two_128); - BN_init(&two_512); - BN_init(&tmp); - BN_init(&tmp2); - - /* Create our context */ - if ((ctx = BN_CTX_new()) == NULL) { - goto err; - } - BN_CTX_start(ctx); - - /* - * For production, if you care, these only need to be set once, - * and may be made constants. - */ - BN_lshift(&two_768, BN_value_one(), 768); - BN_lshift(&two_640, BN_value_one(), 640); - BN_lshift(&two_128, BN_value_one(), 128); - BN_lshift(&two_512, BN_value_one(), 512); - - if (0 == (m[7] & 0x8000000000000000)) { - goto err; - } - if (0 == (m[0] & 0x1)) { /* Odd modulus required for Mont */ - goto err; - } - - /* Precompute m1 */ - BN_mod(&tmp, &two_768, &_m, ctx); - if (!bn_extract_to_array_512(&tmp, 8, &data->m1[0])) { - goto err; - } - - /* Precompute m2 */ - BN_mod(&tmp, &two_640, &_m, ctx); - if (!bn_extract_to_array_512(&tmp, 8, &data->m2[0])) { - goto err; - } - - /* - * Precompute k1, a 128b number = ((-1)* m-1 ) mod 2128; k1 should - * be non-negative. - */ - BN_mod_inverse(&tmp, &_m, &two_128, ctx); - if (!BN_is_zero(&tmp)) { - BN_sub(&tmp, &two_128, &tmp); - } - if (!bn_extract_to_array_512(&tmp, 2, &data->k1[0])) { - goto err; - } - - /* Precompute t */ - for (i = 0; i < 8; i++) { - BN_zero(&tmp); - if (i & 1) { - BN_add(&tmp, &two_512, &tmp); - } - if (i & 2) { - BN_add(&tmp, &two_512, &tmp); - } - if (i & 4) { - BN_add(&tmp, &two_640, &tmp); - } - - BN_nnmod(&tmp2, &tmp, &_m, ctx); - if (!bn_extract_to_array_512(&tmp2, 8, _t)) { - goto err; - } - for (j = 0; j < 8; j++) - data->t[j][i] = _t[j]; - } - - /* Precompute m */ - for (i = 0; i < 8; i++) { - data->m[i] = m[i]; - } - - ret = 1; - - err: - /* Cleanup */ - if (ctx != NULL) { - BN_CTX_end(ctx); - BN_CTX_free(ctx); - } - BN_free(&two_768); - BN_free(&two_640); - BN_free(&two_128); - BN_free(&two_512); - BN_free(&tmp); - BN_free(&tmp2); - BN_free(&_m); - - return ret; -} - -static int e_rsax_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, - BN_CTX *ctx) -{ - BIGNUM *r1, *m1, *vrfy; - BIGNUM local_dmp1, local_dmq1, local_c, local_r1; - BIGNUM *dmp1, *dmq1, *c, *pr1; - int ret = 0; - - BN_CTX_start(ctx); - r1 = BN_CTX_get(ctx); - m1 = BN_CTX_get(ctx); - vrfy = BN_CTX_get(ctx); - - { - BIGNUM local_p, local_q; - BIGNUM *p = NULL, *q = NULL; - int error = 0; - - /* - * Make sure BN_mod_inverse in Montgomery intialization uses the - * BN_FLG_CONSTTIME flag (unless RSA_FLAG_NO_CONSTTIME is set) - */ - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - BN_init(&local_p); - p = &local_p; - BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME); - - BN_init(&local_q); - q = &local_q; - BN_with_flags(q, rsa->q, BN_FLG_CONSTTIME); - } else { - p = rsa->p; - q = rsa->q; - } - - if (rsa->flags & RSA_FLAG_CACHE_PRIVATE) { - if (!BN_MONT_CTX_set_locked - (&rsa->_method_mod_p, CRYPTO_LOCK_RSA, p, ctx)) - error = 1; - if (!BN_MONT_CTX_set_locked - (&rsa->_method_mod_q, CRYPTO_LOCK_RSA, q, ctx)) - error = 1; - } - - /* clean up */ - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - BN_free(&local_p); - BN_free(&local_q); - } - if (error) - goto err; - } - - if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) - if (!BN_MONT_CTX_set_locked - (&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx)) - goto err; - - /* compute I mod q */ - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - c = &local_c; - BN_with_flags(c, I, BN_FLG_CONSTTIME); - if (!BN_mod(r1, c, rsa->q, ctx)) - goto err; - } else { - if (!BN_mod(r1, I, rsa->q, ctx)) - goto err; - } - - /* compute r1^dmq1 mod q */ - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - dmq1 = &local_dmq1; - BN_with_flags(dmq1, rsa->dmq1, BN_FLG_CONSTTIME); - } else - dmq1 = rsa->dmq1; - - if (!e_rsax_bn_mod_exp(m1, r1, dmq1, rsa->q, ctx, - rsa->_method_mod_q, e_rsax_get_ctx(rsa, 0, - rsa->q))) - goto err; - - /* compute I mod p */ - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - c = &local_c; - BN_with_flags(c, I, BN_FLG_CONSTTIME); - if (!BN_mod(r1, c, rsa->p, ctx)) - goto err; - } else { - if (!BN_mod(r1, I, rsa->p, ctx)) - goto err; - } - - /* compute r1^dmp1 mod p */ - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - dmp1 = &local_dmp1; - BN_with_flags(dmp1, rsa->dmp1, BN_FLG_CONSTTIME); - } else - dmp1 = rsa->dmp1; - - if (!e_rsax_bn_mod_exp(r0, r1, dmp1, rsa->p, ctx, - rsa->_method_mod_p, e_rsax_get_ctx(rsa, 1, - rsa->p))) - goto err; - - if (!BN_sub(r0, r0, m1)) - goto err; - /* - * This will help stop the size of r0 increasing, which does affect the - * multiply if it optimised for a power of 2 size - */ - if (BN_is_negative(r0)) - if (!BN_add(r0, r0, rsa->p)) - goto err; - - if (!BN_mul(r1, r0, rsa->iqmp, ctx)) - goto err; - - /* Turn BN_FLG_CONSTTIME flag on before division operation */ - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - pr1 = &local_r1; - BN_with_flags(pr1, r1, BN_FLG_CONSTTIME); - } else - pr1 = r1; - if (!BN_mod(r0, pr1, rsa->p, ctx)) - goto err; - - /* - * If p < q it is occasionally possible for the correction of adding 'p' - * if r0 is negative above to leave the result still negative. This can - * break the private key operations: the following second correction - * should *always* correct this rare occurrence. This will *never* happen - * with OpenSSL generated keys because they ensure p > q [steve] - */ - if (BN_is_negative(r0)) - if (!BN_add(r0, r0, rsa->p)) - goto err; - if (!BN_mul(r1, r0, rsa->q, ctx)) - goto err; - if (!BN_add(r0, r1, m1)) - goto err; - - if (rsa->e && rsa->n) { - if (!e_rsax_bn_mod_exp - (vrfy, r0, rsa->e, rsa->n, ctx, rsa->_method_mod_n, - e_rsax_get_ctx(rsa, 2, rsa->n))) - goto err; - - /* - * If 'I' was greater than (or equal to) rsa->n, the operation will - * be equivalent to using 'I mod n'. However, the result of the - * verify will *always* be less than 'n' so we don't check for - * absolute equality, just congruency. - */ - if (!BN_sub(vrfy, vrfy, I)) - goto err; - if (!BN_mod(vrfy, vrfy, rsa->n, ctx)) - goto err; - if (BN_is_negative(vrfy)) - if (!BN_add(vrfy, vrfy, rsa->n)) - goto err; - if (!BN_is_zero(vrfy)) { - /* - * 'I' and 'vrfy' aren't congruent mod n. Don't leak - * miscalculated CRT output, just do a raw (slower) mod_exp and - * return that instead. - */ - - BIGNUM local_d; - BIGNUM *d = NULL; - - if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { - d = &local_d; - BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME); - } else - d = rsa->d; - if (!e_rsax_bn_mod_exp(r0, I, d, rsa->n, ctx, - rsa->_method_mod_n, e_rsax_get_ctx(rsa, 2, - rsa->n))) - goto err; - } - } - ret = 1; - - err: - BN_CTX_end(ctx); - - return ret; -} -# endif /* !OPENSSL_NO_RSA */ -#endif /* !COMPILE_RSAX */ diff --git a/crypto/engine/engine.h b/crypto/engine/engine.h index e81096ae02a6..bd7b591447dd 100644 --- a/crypto/engine/engine.h +++ b/crypto/engine/engine.h @@ -413,7 +413,6 @@ void ENGINE_load_gost(void); # endif # endif void ENGINE_load_cryptodev(void); -void ENGINE_load_rsax(void); void ENGINE_load_rdrand(void); void ENGINE_load_builtin_engines(void); diff --git a/crypto/evp/Makefile b/crypto/evp/Makefile index ed1502d3a694..aaaad986e0e8 100644 --- a/crypto/evp/Makefile +++ b/crypto/evp/Makefile @@ -28,8 +28,8 @@ LIBSRC= encode.c digest.c evp_enc.c evp_key.c evp_acnf.c evp_cnf.c \ bio_md.c bio_b64.c bio_enc.c evp_err.c e_null.c \ c_all.c c_allc.c c_alld.c evp_lib.c bio_ok.c \ evp_pkey.c evp_pbe.c p5_crpt.c p5_crpt2.c \ - e_old.c pmeth_lib.c pmeth_fn.c pmeth_gn.c m_sigver.c evp_fips.c \ - e_aes_cbc_hmac_sha1.c e_rc4_hmac_md5.c + e_old.c pmeth_lib.c pmeth_fn.c pmeth_gn.c m_sigver.c \ + e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ e_des.o e_bf.o e_idea.o e_des3.o e_camellia.o\ @@ -41,8 +41,8 @@ LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ bio_md.o bio_b64.o bio_enc.o evp_err.o e_null.o \ c_all.o c_allc.o c_alld.o evp_lib.o bio_ok.o \ evp_pkey.o evp_pbe.o p5_crpt.o p5_crpt2.o \ - e_old.o pmeth_lib.o pmeth_fn.o pmeth_gn.o m_sigver.o evp_fips.o \ - e_aes_cbc_hmac_sha1.o e_rc4_hmac_md5.o + e_old.o pmeth_lib.o pmeth_fn.o pmeth_gn.o m_sigver.o \ + e_aes_cbc_hmac_sha1.o e_aes_cbc_hmac_sha256.o e_rc4_hmac_md5.o SRC= $(LIBSRC) @@ -204,16 +204,36 @@ e_aes_cbc_hmac_sha1.o: ../../include/openssl/bio.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/crypto.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/e_os2.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/evp.h +e_aes_cbc_hmac_sha1.o: ../../include/openssl/modes.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/obj_mac.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/objects.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/opensslconf.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/opensslv.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/ossl_typ.h +e_aes_cbc_hmac_sha1.o: ../../include/openssl/rand.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/safestack.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/sha.h e_aes_cbc_hmac_sha1.o: ../../include/openssl/stack.h -e_aes_cbc_hmac_sha1.o: ../../include/openssl/symhacks.h e_aes_cbc_hmac_sha1.c -e_aes_cbc_hmac_sha1.o: evp_locl.h +e_aes_cbc_hmac_sha1.o: ../../include/openssl/symhacks.h ../modes/modes_lcl.h +e_aes_cbc_hmac_sha1.o: e_aes_cbc_hmac_sha1.c +e_aes_cbc_hmac_sha256.o: ../../include/openssl/aes.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/asn1.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/bio.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/crypto.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/e_os2.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/evp.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/modes.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/obj_mac.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/objects.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/opensslconf.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/opensslv.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/ossl_typ.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/rand.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/safestack.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/sha.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/stack.h +e_aes_cbc_hmac_sha256.o: ../../include/openssl/symhacks.h ../modes/modes_lcl.h +e_aes_cbc_hmac_sha256.o: e_aes_cbc_hmac_sha256.c e_bf.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h e_bf.o: ../../include/openssl/blowfish.h ../../include/openssl/buffer.h e_bf.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h @@ -227,11 +247,13 @@ e_camellia.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h e_camellia.o: ../../include/openssl/camellia.h ../../include/openssl/crypto.h e_camellia.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h e_camellia.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h -e_camellia.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h +e_camellia.o: ../../include/openssl/modes.h ../../include/openssl/obj_mac.h +e_camellia.o: ../../include/openssl/objects.h e_camellia.o: ../../include/openssl/opensslconf.h e_camellia.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h e_camellia.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h -e_camellia.o: ../../include/openssl/symhacks.h e_camellia.c evp_locl.h +e_camellia.o: ../../include/openssl/symhacks.h ../modes/modes_lcl.h +e_camellia.o: e_camellia.c evp_locl.h e_cast.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h e_cast.o: ../../include/openssl/buffer.h ../../include/openssl/cast.h e_cast.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h @@ -260,9 +282,10 @@ e_des3.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h e_des3.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h e_des3.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h e_des3.o: ../../include/openssl/ossl_typ.h ../../include/openssl/rand.h -e_des3.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h -e_des3.o: ../../include/openssl/symhacks.h ../../include/openssl/ui.h -e_des3.o: ../../include/openssl/ui_compat.h ../cryptlib.h e_des3.c evp_locl.h +e_des3.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h +e_des3.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h +e_des3.o: ../../include/openssl/ui.h ../../include/openssl/ui_compat.h +e_des3.o: ../cryptlib.h e_des3.c evp_locl.h e_idea.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h e_idea.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h e_idea.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h @@ -394,13 +417,6 @@ evp_err.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h evp_err.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h evp_err.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h evp_err.o: ../../include/openssl/symhacks.h evp_err.c -evp_fips.o: ../../include/openssl/asn1.h ../../include/openssl/bio.h -evp_fips.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h -evp_fips.o: ../../include/openssl/evp.h ../../include/openssl/obj_mac.h -evp_fips.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h -evp_fips.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h -evp_fips.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h -evp_fips.o: ../../include/openssl/symhacks.h evp_fips.c evp_key.o: ../../e_os.h ../../include/openssl/asn1.h evp_key.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h evp_key.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h diff --git a/crypto/evp/c_allc.c b/crypto/evp/c_allc.c index 3097c2112efc..280e58408f2b 100644 --- a/crypto/evp/c_allc.c +++ b/crypto/evp/c_allc.c @@ -93,6 +93,7 @@ void OpenSSL_add_all_ciphers(void) EVP_add_cipher(EVP_des_ecb()); EVP_add_cipher(EVP_des_ede()); EVP_add_cipher(EVP_des_ede3()); + EVP_add_cipher(EVP_des_ede3_wrap()); #endif #ifndef OPENSSL_NO_RC4 @@ -172,6 +173,8 @@ void OpenSSL_add_all_ciphers(void) EVP_add_cipher(EVP_aes_128_ctr()); EVP_add_cipher(EVP_aes_128_gcm()); EVP_add_cipher(EVP_aes_128_xts()); + EVP_add_cipher(EVP_aes_128_ccm()); + EVP_add_cipher(EVP_aes_128_wrap()); EVP_add_cipher_alias(SN_aes_128_cbc, "AES128"); EVP_add_cipher_alias(SN_aes_128_cbc, "aes128"); EVP_add_cipher(EVP_aes_192_ecb()); @@ -182,6 +185,8 @@ void OpenSSL_add_all_ciphers(void) EVP_add_cipher(EVP_aes_192_ofb()); EVP_add_cipher(EVP_aes_192_ctr()); EVP_add_cipher(EVP_aes_192_gcm()); + EVP_add_cipher(EVP_aes_192_ccm()); + EVP_add_cipher(EVP_aes_192_wrap()); EVP_add_cipher_alias(SN_aes_192_cbc, "AES192"); EVP_add_cipher_alias(SN_aes_192_cbc, "aes192"); EVP_add_cipher(EVP_aes_256_ecb()); @@ -193,12 +198,18 @@ void OpenSSL_add_all_ciphers(void) EVP_add_cipher(EVP_aes_256_ctr()); EVP_add_cipher(EVP_aes_256_gcm()); EVP_add_cipher(EVP_aes_256_xts()); + EVP_add_cipher(EVP_aes_256_ccm()); + EVP_add_cipher(EVP_aes_256_wrap()); EVP_add_cipher_alias(SN_aes_256_cbc, "AES256"); EVP_add_cipher_alias(SN_aes_256_cbc, "aes256"); # if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA1) EVP_add_cipher(EVP_aes_128_cbc_hmac_sha1()); EVP_add_cipher(EVP_aes_256_cbc_hmac_sha1()); # endif +# if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA256) + EVP_add_cipher(EVP_aes_128_cbc_hmac_sha256()); + EVP_add_cipher(EVP_aes_256_cbc_hmac_sha256()); +# endif #endif #ifndef OPENSSL_NO_CAMELLIA diff --git a/crypto/evp/digest.c b/crypto/evp/digest.c index 2e202c8fe031..f2643f32486a 100644 --- a/crypto/evp/digest.c +++ b/crypto/evp/digest.c @@ -119,6 +119,7 @@ #ifdef OPENSSL_FIPS # include +# include "evp_locl.h" #endif void EVP_MD_CTX_init(EVP_MD_CTX *ctx) @@ -145,6 +146,17 @@ int EVP_DigestInit(EVP_MD_CTX *ctx, const EVP_MD *type) int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl) { EVP_MD_CTX_clear_flags(ctx, EVP_MD_CTX_FLAG_CLEANED); +#ifdef OPENSSL_FIPS + /* If FIPS mode switch to approved implementation if possible */ + if (FIPS_mode()) { + const EVP_MD *fipsmd; + if (type) { + fipsmd = evp_get_fips_md(type); + if (fipsmd) + type = fipsmd; + } + } +#endif #ifndef OPENSSL_NO_ENGINE /* * Whether it's nice or not, "Inits" can be used on "Final"'d contexts so diff --git a/crypto/evp/e_aes.c b/crypto/evp/e_aes.c index a4327fcb0554..1734a823c1e5 100644 --- a/crypto/evp/e_aes.c +++ b/crypto/evp/e_aes.c @@ -57,12 +57,17 @@ # include # include # include "evp_locl.h" -# ifndef OPENSSL_FIPS -# include "modes_lcl.h" -# include +# include "modes_lcl.h" +# include + +# undef EVP_CIPH_FLAG_FIPS +# define EVP_CIPH_FLAG_FIPS 0 typedef struct { - AES_KEY ks; + union { + double align; + AES_KEY ks; + } ks; block128_f block; union { cbc128_f cbc; @@ -71,7 +76,10 @@ typedef struct { } EVP_AES_KEY; typedef struct { - AES_KEY ks; /* AES key schedule to use */ + union { + double align; + AES_KEY ks; + } ks; /* AES key schedule to use */ int key_set; /* Set if key initialised */ int iv_set; /* Set if an iv is set */ GCM128_CONTEXT gcm; @@ -84,7 +92,10 @@ typedef struct { } EVP_AES_GCM_CTX; typedef struct { - AES_KEY ks1, ks2; /* AES key schedules to use */ + union { + double align; + AES_KEY ks; + } ks1, ks2; /* AES key schedules to use */ XTS128_CONTEXT xts; void (*stream) (const unsigned char *in, unsigned char *out, size_t length, @@ -93,7 +104,10 @@ typedef struct { } EVP_AES_XTS_CTX; typedef struct { - AES_KEY ks; /* AES key schedule to use */ + union { + double align; + AES_KEY ks; + } ks; /* AES key schedule to use */ int key_set; /* Set if key initialised */ int iv_set; /* Set if an iv is set */ int tag_set; /* Set if tag is valid */ @@ -103,9 +117,9 @@ typedef struct { ccm128_f str; } EVP_AES_CCM_CTX; -# define MAXBITCHUNK ((size_t)1<<(sizeof(size_t)*8-4)) +# define MAXBITCHUNK ((size_t)1<<(sizeof(size_t)*8-4)) -# ifdef VPAES_ASM +# ifdef VPAES_ASM int vpaes_set_encrypt_key(const unsigned char *userKey, int bits, AES_KEY *key); int vpaes_set_decrypt_key(const unsigned char *userKey, int bits, @@ -120,8 +134,8 @@ void vpaes_cbc_encrypt(const unsigned char *in, unsigned char *out, size_t length, const AES_KEY *key, unsigned char *ivec, int enc); -# endif -# ifdef BSAES_ASM +# endif +# ifdef BSAES_ASM void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out, size_t length, const AES_KEY *key, unsigned char ivec[16], int enc); @@ -134,40 +148,54 @@ void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out, void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len, const AES_KEY *key1, const AES_KEY *key2, const unsigned char iv[16]); -# endif -# ifdef AES_CTR_ASM +# endif +# ifdef AES_CTR_ASM void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out, size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]); -# endif -# ifdef AES_XTS_ASM +# endif +# ifdef AES_XTS_ASM void AES_xts_encrypt(const char *inp, char *out, size_t len, const AES_KEY *key1, const AES_KEY *key2, const unsigned char iv[16]); void AES_xts_decrypt(const char *inp, char *out, size_t len, const AES_KEY *key1, const AES_KEY *key2, const unsigned char iv[16]); +# endif + +# if defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC)) +# include "ppc_arch.h" +# ifdef VPAES_ASM +# define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC) # endif +# define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207) +# define HWAES_set_encrypt_key aes_p8_set_encrypt_key +# define HWAES_set_decrypt_key aes_p8_set_decrypt_key +# define HWAES_encrypt aes_p8_encrypt +# define HWAES_decrypt aes_p8_decrypt +# define HWAES_cbc_encrypt aes_p8_cbc_encrypt +# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks +# endif -# if defined(AES_ASM) && !defined(I386_ONLY) && ( \ +# if defined(AES_ASM) && !defined(I386_ONLY) && ( \ ((defined(__i386) || defined(__i386__) || \ defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \ defined(__x86_64) || defined(__x86_64__) || \ defined(_M_AMD64) || defined(_M_X64) || \ defined(__INTEL__) ) -extern unsigned int OPENSSL_ia32cap_P[2]; +extern unsigned int OPENSSL_ia32cap_P[]; -# ifdef VPAES_ASM -# define VPAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32))) -# endif -# ifdef BSAES_ASM -# define BSAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32))) -# endif +# ifdef VPAES_ASM +# define VPAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32))) +# endif +# ifdef BSAES_ASM +# define BSAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32))) +# endif /* * AES-NI section */ -# define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32))) +# define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32))) int aesni_set_encrypt_key(const unsigned char *userKey, int bits, AES_KEY *key); @@ -218,6 +246,26 @@ void aesni_ccm64_decrypt_blocks(const unsigned char *in, const unsigned char ivec[16], unsigned char cmac[16]); +# if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64) +size_t aesni_gcm_encrypt(const unsigned char *in, + unsigned char *out, + size_t len, + const void *key, unsigned char ivec[16], u64 *Xi); +# define AES_gcm_encrypt aesni_gcm_encrypt +size_t aesni_gcm_decrypt(const unsigned char *in, + unsigned char *out, + size_t len, + const void *key, unsigned char ivec[16], u64 *Xi); +# define AES_gcm_decrypt aesni_gcm_decrypt +void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in, + size_t len); +# define AES_GCM_ASM(gctx) (gctx->ctr==aesni_ctr32_encrypt_blocks && \ + gctx->gcm.ghash==gcm_ghash_avx) +# define AES_GCM_ASM2(gctx) (gctx->gcm.block==(block128_f)aesni_encrypt && \ + gctx->gcm.ghash==gcm_ghash_avx) +# undef AES_GCM_ASM2 /* minor size optimization */ +# endif + static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { @@ -271,23 +319,23 @@ static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return 1; } -# define aesni_ofb_cipher aes_ofb_cipher +# define aesni_ofb_cipher aes_ofb_cipher static int aesni_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); -# define aesni_cfb_cipher aes_cfb_cipher +# define aesni_cfb_cipher aes_cfb_cipher static int aesni_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); -# define aesni_cfb8_cipher aes_cfb8_cipher +# define aesni_cfb8_cipher aes_cfb8_cipher static int aesni_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); -# define aesni_cfb1_cipher aes_cfb1_cipher +# define aesni_cfb1_cipher aes_cfb1_cipher static int aesni_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); -# define aesni_ctr_cipher aes_ctr_cipher +# define aesni_ctr_cipher aes_ctr_cipher static int aesni_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); @@ -298,7 +346,7 @@ static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, if (!iv && !key) return 1; if (key) { - aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks); + aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) aesni_encrypt); gctx->ctr = (ctr128_f) aesni_ctr32_encrypt_blocks; /* @@ -323,7 +371,7 @@ static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, return 1; } -# define aesni_gcm_cipher aes_gcm_cipher +# define aesni_gcm_cipher aes_gcm_cipher static int aesni_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); @@ -337,17 +385,17 @@ static int aesni_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, if (key) { /* key_len is two AES keys */ if (enc) { - aesni_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1); + aesni_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks); xctx->xts.block1 = (block128_f) aesni_encrypt; xctx->stream = aesni_xts_encrypt; } else { - aesni_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1); + aesni_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks); xctx->xts.block1 = (block128_f) aesni_decrypt; xctx->stream = aesni_xts_decrypt; } aesni_set_encrypt_key(key + ctx->key_len / 2, - ctx->key_len * 4, &xctx->ks2); + ctx->key_len * 4, &xctx->ks2.ks); xctx->xts.block2 = (block128_f) aesni_encrypt; xctx->xts.key1 = &xctx->ks1; @@ -361,7 +409,7 @@ static int aesni_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, return 1; } -# define aesni_xts_cipher aes_xts_cipher +# define aesni_xts_cipher aes_xts_cipher static int aesni_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); @@ -372,7 +420,7 @@ static int aesni_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, if (!iv && !key) return 1; if (key) { - aesni_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks); + aesni_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) aesni_encrypt); cctx->str = enc ? (ccm128_f) aesni_ccm64_encrypt_blocks : @@ -386,11 +434,11 @@ static int aesni_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, return 1; } -# define aesni_ccm_cipher aes_ccm_cipher +# define aesni_ccm_cipher aes_ccm_cipher static int aesni_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len); -# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ +# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ static const EVP_CIPHER aesni_##keylen##_##mode = { \ nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ flags|EVP_CIPH_##MODE##_MODE, \ @@ -411,7 +459,7 @@ static const EVP_CIPHER aes_##keylen##_##mode = { \ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ { return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; } -# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \ +# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \ static const EVP_CIPHER aesni_##keylen##_##mode = { \ nid##_##keylen##_##mode,blocksize, \ (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \ @@ -433,9 +481,378 @@ static const EVP_CIPHER aes_##keylen##_##mode = { \ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ { return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; } +# elif defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) + +# include "sparc_arch.h" + +extern unsigned int OPENSSL_sparcv9cap_P[]; + +# define SPARC_AES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_AES) + +void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks); +void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks); +void aes_t4_encrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void aes_t4_decrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +/* + * Key-length specific subroutines were chosen for following reason. + * Each SPARC T4 core can execute up to 8 threads which share core's + * resources. Loading as much key material to registers allows to + * minimize references to shared memory interface, as well as amount + * of instructions in inner loops [much needed on T4]. But then having + * non-key-length specific routines would require conditional branches + * either in inner loops or on subroutines' entries. Former is hardly + * acceptable, while latter means code size increase to size occupied + * by multiple key-length specfic subroutines, so why fight? + */ +void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + unsigned char *ivec); +void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + unsigned char *ivec); +void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + unsigned char *ivec); +void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + unsigned char *ivec); +void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + unsigned char *ivec); +void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + unsigned char *ivec); +void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, + unsigned char *ivec); +void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, + unsigned char *ivec); +void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, + unsigned char *ivec); +void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char *ivec); +void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char *ivec); +void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char *ivec); +void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char *ivec); + +static int aes_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + int ret, mode, bits; + EVP_AES_KEY *dat = (EVP_AES_KEY *) ctx->cipher_data; + + mode = ctx->cipher->flags & EVP_CIPH_MODE; + bits = ctx->key_len * 8; + if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) + && !enc) { + ret = 0; + aes_t4_set_decrypt_key(key, bits, ctx->cipher_data); + dat->block = (block128_f) aes_t4_decrypt; + switch (bits) { + case 128: + dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? + (cbc128_f) aes128_t4_cbc_decrypt : NULL; + break; + case 192: + dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? + (cbc128_f) aes192_t4_cbc_decrypt : NULL; + break; + case 256: + dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? + (cbc128_f) aes256_t4_cbc_decrypt : NULL; + break; + default: + ret = -1; + } + } else { + ret = 0; + aes_t4_set_encrypt_key(key, bits, ctx->cipher_data); + dat->block = (block128_f) aes_t4_encrypt; + switch (bits) { + case 128: + if (mode == EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f) aes128_t4_cbc_encrypt; + else if (mode == EVP_CIPH_CTR_MODE) + dat->stream.ctr = (ctr128_f) aes128_t4_ctr32_encrypt; + else + dat->stream.cbc = NULL; + break; + case 192: + if (mode == EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f) aes192_t4_cbc_encrypt; + else if (mode == EVP_CIPH_CTR_MODE) + dat->stream.ctr = (ctr128_f) aes192_t4_ctr32_encrypt; + else + dat->stream.cbc = NULL; + break; + case 256: + if (mode == EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f) aes256_t4_cbc_encrypt; + else if (mode == EVP_CIPH_CTR_MODE) + dat->stream.ctr = (ctr128_f) aes256_t4_ctr32_encrypt; + else + dat->stream.cbc = NULL; + break; + default: + ret = -1; + } + } + + if (ret < 0) { + EVPerr(EVP_F_AES_T4_INIT_KEY, EVP_R_AES_KEY_SETUP_FAILED); + return 0; + } + + return 1; +} + +# define aes_t4_cbc_cipher aes_cbc_cipher +static int aes_t4_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define aes_t4_ecb_cipher aes_ecb_cipher +static int aes_t4_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define aes_t4_ofb_cipher aes_ofb_cipher +static int aes_t4_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define aes_t4_cfb_cipher aes_cfb_cipher +static int aes_t4_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define aes_t4_cfb8_cipher aes_cfb8_cipher +static int aes_t4_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define aes_t4_cfb1_cipher aes_cfb1_cipher +static int aes_t4_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define aes_t4_ctr_cipher aes_ctr_cipher +static int aes_t4_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +static int aes_t4_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + EVP_AES_GCM_CTX *gctx = ctx->cipher_data; + if (!iv && !key) + return 1; + if (key) { + int bits = ctx->key_len * 8; + aes_t4_set_encrypt_key(key, bits, &gctx->ks.ks); + CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, + (block128_f) aes_t4_encrypt); + switch (bits) { + case 128: + gctx->ctr = (ctr128_f) aes128_t4_ctr32_encrypt; + break; + case 192: + gctx->ctr = (ctr128_f) aes192_t4_ctr32_encrypt; + break; + case 256: + gctx->ctr = (ctr128_f) aes256_t4_ctr32_encrypt; + break; + default: + return 0; + } + /* + * If we have an iv can set it directly, otherwise use saved IV. + */ + if (iv == NULL && gctx->iv_set) + iv = gctx->iv; + if (iv) { + CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen); + gctx->iv_set = 1; + } + gctx->key_set = 1; + } else { + /* If key set use IV, otherwise copy */ + if (gctx->key_set) + CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen); + else + memcpy(gctx->iv, iv, gctx->ivlen); + gctx->iv_set = 1; + gctx->iv_gen = 0; + } + return 1; +} + +# define aes_t4_gcm_cipher aes_gcm_cipher +static int aes_t4_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + EVP_AES_XTS_CTX *xctx = ctx->cipher_data; + if (!iv && !key) + return 1; + + if (key) { + int bits = ctx->key_len * 4; + xctx->stream = NULL; + /* key_len is two AES keys */ + if (enc) { + aes_t4_set_encrypt_key(key, bits, &xctx->ks1.ks); + xctx->xts.block1 = (block128_f) aes_t4_encrypt; + switch (bits) { + case 128: + xctx->stream = aes128_t4_xts_encrypt; + break; +# if 0 /* not yet */ + case 192: + xctx->stream = aes192_t4_xts_encrypt; + break; +# endif + case 256: + xctx->stream = aes256_t4_xts_encrypt; + break; + default: + return 0; + } + } else { + aes_t4_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks); + xctx->xts.block1 = (block128_f) aes_t4_decrypt; + switch (bits) { + case 128: + xctx->stream = aes128_t4_xts_decrypt; + break; +# if 0 /* not yet */ + case 192: + xctx->stream = aes192_t4_xts_decrypt; + break; +# endif + case 256: + xctx->stream = aes256_t4_xts_decrypt; + break; + default: + return 0; + } + } + + aes_t4_set_encrypt_key(key + ctx->key_len / 2, + ctx->key_len * 4, &xctx->ks2.ks); + xctx->xts.block2 = (block128_f) aes_t4_encrypt; + + xctx->xts.key1 = &xctx->ks1; + } + + if (iv) { + xctx->xts.key2 = &xctx->ks2; + memcpy(ctx->iv, iv, 16); + } + + return 1; +} + +# define aes_t4_xts_cipher aes_xts_cipher +static int aes_t4_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +static int aes_t4_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + EVP_AES_CCM_CTX *cctx = ctx->cipher_data; + if (!iv && !key) + return 1; + if (key) { + int bits = ctx->key_len * 8; + aes_t4_set_encrypt_key(key, bits, &cctx->ks.ks); + CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, + &cctx->ks, (block128_f) aes_t4_encrypt); +# if 0 /* not yet */ + switch (bits) { + case 128: + cctx->str = enc ? (ccm128_f) aes128_t4_ccm64_encrypt : + (ccm128_f) ae128_t4_ccm64_decrypt; + break; + case 192: + cctx->str = enc ? (ccm128_f) aes192_t4_ccm64_encrypt : + (ccm128_f) ae192_t4_ccm64_decrypt; + break; + case 256: + cctx->str = enc ? (ccm128_f) aes256_t4_ccm64_encrypt : + (ccm128_f) ae256_t4_ccm64_decrypt; + break; + default: + return 0; + } # else + cctx->str = NULL; +# endif + cctx->key_set = 1; + } + if (iv) { + memcpy(ctx->iv, iv, 15 - cctx->L); + cctx->iv_set = 1; + } + return 1; +} -# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ +# define aes_t4_ccm_cipher aes_ccm_cipher +static int aes_t4_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ +static const EVP_CIPHER aes_t4_##keylen##_##mode = { \ + nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + aes_t4_init_key, \ + aes_t4_##mode##_cipher, \ + NULL, \ + sizeof(EVP_AES_KEY), \ + NULL,NULL,NULL,NULL }; \ +static const EVP_CIPHER aes_##keylen##_##mode = { \ + nid##_##keylen##_##nmode,blocksize, \ + keylen/8,ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + aes_init_key, \ + aes_##mode##_cipher, \ + NULL, \ + sizeof(EVP_AES_KEY), \ + NULL,NULL,NULL,NULL }; \ +const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ +{ return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; } + +# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \ +static const EVP_CIPHER aes_t4_##keylen##_##mode = { \ + nid##_##keylen##_##mode,blocksize, \ + (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + aes_t4_##mode##_init_key, \ + aes_t4_##mode##_cipher, \ + aes_##mode##_cleanup, \ + sizeof(EVP_AES_##MODE##_CTX), \ + NULL,NULL,aes_##mode##_ctrl,NULL }; \ +static const EVP_CIPHER aes_##keylen##_##mode = { \ + nid##_##keylen##_##mode,blocksize, \ + (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + aes_##mode##_init_key, \ + aes_##mode##_cipher, \ + aes_##mode##_cleanup, \ + sizeof(EVP_AES_##MODE##_CTX), \ + NULL,NULL,aes_##mode##_ctrl,NULL }; \ +const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ +{ return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; } + +# else + +# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ static const EVP_CIPHER aes_##keylen##_##mode = { \ nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ flags|EVP_CIPH_##MODE##_MODE, \ @@ -447,7 +864,7 @@ static const EVP_CIPHER aes_##keylen##_##mode = { \ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ { return &aes_##keylen##_##mode; } -# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \ +# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \ static const EVP_CIPHER aes_##keylen##_##mode = { \ nid##_##keylen##_##mode,blocksize, \ (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \ @@ -459,9 +876,42 @@ static const EVP_CIPHER aes_##keylen##_##mode = { \ NULL,NULL,aes_##mode##_ctrl,NULL }; \ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ { return &aes_##keylen##_##mode; } +# endif + +# if defined(OPENSSL_CPUID_OBJ) && (defined(__arm__) || defined(__arm) || defined(__aarch64__)) +# include "arm_arch.h" +# if __ARM_MAX_ARCH__>=7 +# if defined(BSAES_ASM) +# define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) +# endif +# define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES) +# define HWAES_set_encrypt_key aes_v8_set_encrypt_key +# define HWAES_set_decrypt_key aes_v8_set_decrypt_key +# define HWAES_encrypt aes_v8_encrypt +# define HWAES_decrypt aes_v8_decrypt +# define HWAES_cbc_encrypt aes_v8_cbc_encrypt +# define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks # endif +# endif + +# if defined(HWAES_CAPABLE) +int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits, + AES_KEY *key); +int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits, + AES_KEY *key); +void HWAES_encrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void HWAES_decrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t length, const AES_KEY *key, + unsigned char *ivec, const int enc); +void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + const unsigned char ivec[16]); +# endif -# define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \ +# define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \ BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ BLOCK_CIPHER_generic(nid,keylen,1,16,ofb128,ofb,OFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ @@ -479,51 +929,80 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, mode = ctx->cipher->flags & EVP_CIPH_MODE; if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) -# ifdef BSAES_CAPABLE +# ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) { + ret = HWAES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); + dat->block = (block128_f) HWAES_decrypt; + dat->stream.cbc = NULL; +# ifdef HWAES_cbc_encrypt + if (mode == EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f) HWAES_cbc_encrypt; +# endif + } else +# endif +# ifdef BSAES_CAPABLE if (BSAES_CAPABLE && mode == EVP_CIPH_CBC_MODE) { - ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks); + ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = (block128_f) AES_decrypt; dat->stream.cbc = (cbc128_f) bsaes_cbc_encrypt; } else -# endif -# ifdef VPAES_CAPABLE +# endif +# ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - ret = vpaes_set_decrypt_key(key, ctx->key_len * 8, &dat->ks); + ret = vpaes_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = (block128_f) vpaes_decrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) vpaes_cbc_encrypt : NULL; } else -# endif +# endif { - ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks); + ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = (block128_f) AES_decrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) AES_cbc_encrypt : NULL; } else -# ifdef BSAES_CAPABLE +# ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) { + ret = HWAES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); + dat->block = (block128_f) HWAES_encrypt; + dat->stream.cbc = NULL; +# ifdef HWAES_cbc_encrypt + if (mode == EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f) HWAES_cbc_encrypt; + else +# endif +# ifdef HWAES_ctr32_encrypt_blocks + if (mode == EVP_CIPH_CTR_MODE) + dat->stream.ctr = (ctr128_f) HWAES_ctr32_encrypt_blocks; + else +# endif + (void)0; /* terminate potentially open 'else' */ + } else +# endif +# ifdef BSAES_CAPABLE if (BSAES_CAPABLE && mode == EVP_CIPH_CTR_MODE) { - ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks); + ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = (block128_f) AES_encrypt; dat->stream.ctr = (ctr128_f) bsaes_ctr32_encrypt_blocks; } else -# endif -# ifdef VPAES_CAPABLE +# endif +# ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - ret = vpaes_set_encrypt_key(key, ctx->key_len * 8, &dat->ks); + ret = vpaes_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = (block128_f) vpaes_encrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) vpaes_cbc_encrypt : NULL; } else -# endif +# endif { - ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks); + ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = (block128_f) AES_encrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) AES_cbc_encrypt : NULL; -# ifdef AES_CTR_ASM +# ifdef AES_CTR_ASM if (mode == EVP_CIPH_CTR_MODE) dat->stream.ctr = (ctr128_f) AES_ctr32_encrypt; -# endif +# endif } if (ret < 0) { @@ -544,7 +1023,7 @@ static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, else if (ctx->encrypt) CRYPTO_cbc128_encrypt(in, out, len, &dat->ks, ctx->iv, dat->block); else - CRYPTO_cbc128_encrypt(in, out, len, &dat->ks, ctx->iv, dat->block); + CRYPTO_cbc128_decrypt(in, out, len, &dat->ks, ctx->iv, dat->block); return 1; } @@ -680,11 +1159,6 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) case EVP_CTRL_GCM_SET_IVLEN: if (arg <= 0) return 0; -# ifdef OPENSSL_FIPS - if (FIPS_module_mode() && !(c->flags & EVP_CIPH_FLAG_NON_FIPS_ALLOW) - && arg < 12) - return 0; -# endif /* Allocate memory for IV if needed */ if ((arg > EVP_MAX_IV_LENGTH) && (arg > gctx->ivlen)) { if (gctx->iv != c->iv) @@ -805,34 +1279,47 @@ static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, return 1; if (key) { do { -# ifdef BSAES_CAPABLE +# ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) { + HWAES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); + CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, + (block128_f) HWAES_encrypt); +# ifdef HWAES_ctr32_encrypt_blocks + gctx->ctr = (ctr128_f) HWAES_ctr32_encrypt_blocks; +# else + gctx->ctr = NULL; +# endif + break; + } else +# endif +# ifdef BSAES_CAPABLE if (BSAES_CAPABLE) { - AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks); + AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) AES_encrypt); gctx->ctr = (ctr128_f) bsaes_ctr32_encrypt_blocks; break; } else -# endif -# ifdef VPAES_CAPABLE +# endif +# ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - vpaes_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks); + vpaes_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) vpaes_encrypt); gctx->ctr = NULL; break; } else -# endif +# endif (void)0; /* terminate potentially open 'else' */ - AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks); + AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) AES_encrypt); -# ifdef AES_CTR_ASM +# ifdef AES_CTR_ASM gctx->ctr = (ctr128_f) AES_ctr32_encrypt; -# else +# else gctx->ctr = NULL; -# endif +# endif } while (0); /* @@ -891,11 +1378,38 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, if (ctx->encrypt) { /* Encrypt payload */ if (gctx->ctr) { + size_t bulk = 0; +# if defined(AES_GCM_ASM) + if (len >= 32 && AES_GCM_ASM(gctx)) { + if (CRYPTO_gcm128_encrypt(&gctx->gcm, NULL, NULL, 0)) + return -1; + + bulk = AES_gcm_encrypt(in, out, len, + gctx->gcm.key, + gctx->gcm.Yi.c, gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + } +# endif if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm, - in, out, len, gctx->ctr)) + in + bulk, + out + bulk, + len - bulk, gctx->ctr)) goto err; } else { - if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, len)) + size_t bulk = 0; +# if defined(AES_GCM_ASM2) + if (len >= 32 && AES_GCM_ASM2(gctx)) { + if (CRYPTO_gcm128_encrypt(&gctx->gcm, NULL, NULL, 0)) + return -1; + + bulk = AES_gcm_encrypt(in, out, len, + gctx->gcm.key, + gctx->gcm.Yi.c, gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + } +# endif + if (CRYPTO_gcm128_encrypt(&gctx->gcm, + in + bulk, out + bulk, len - bulk)) goto err; } out += len; @@ -905,11 +1419,38 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } else { /* Decrypt */ if (gctx->ctr) { + size_t bulk = 0; +# if defined(AES_GCM_ASM) + if (len >= 16 && AES_GCM_ASM(gctx)) { + if (CRYPTO_gcm128_decrypt(&gctx->gcm, NULL, NULL, 0)) + return -1; + + bulk = AES_gcm_decrypt(in, out, len, + gctx->gcm.key, + gctx->gcm.Yi.c, gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + } +# endif if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm, - in, out, len, gctx->ctr)) + in + bulk, + out + bulk, + len - bulk, gctx->ctr)) goto err; } else { - if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, len)) + size_t bulk = 0; +# if defined(AES_GCM_ASM2) + if (len >= 16 && AES_GCM_ASM2(gctx)) { + if (CRYPTO_gcm128_decrypt(&gctx->gcm, NULL, NULL, 0)) + return -1; + + bulk = AES_gcm_decrypt(in, out, len, + gctx->gcm.key, + gctx->gcm.Yi.c, gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + } +# endif + if (CRYPTO_gcm128_decrypt(&gctx->gcm, + in + bulk, out + bulk, len - bulk)) goto err; } /* Retrieve tag */ @@ -947,20 +1488,90 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return -1; } else if (ctx->encrypt) { if (gctx->ctr) { + size_t bulk = 0; +# if defined(AES_GCM_ASM) + if (len >= 32 && AES_GCM_ASM(gctx)) { + size_t res = (16 - gctx->gcm.mres) % 16; + + if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, res)) + return -1; + + bulk = AES_gcm_encrypt(in + res, + out + res, len - res, + gctx->gcm.key, gctx->gcm.Yi.c, + gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + bulk += res; + } +# endif if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm, - in, out, len, gctx->ctr)) + in + bulk, + out + bulk, + len - bulk, gctx->ctr)) return -1; } else { - if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, len)) + size_t bulk = 0; +# if defined(AES_GCM_ASM2) + if (len >= 32 && AES_GCM_ASM2(gctx)) { + size_t res = (16 - gctx->gcm.mres) % 16; + + if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, res)) + return -1; + + bulk = AES_gcm_encrypt(in + res, + out + res, len - res, + gctx->gcm.key, gctx->gcm.Yi.c, + gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + bulk += res; + } +# endif + if (CRYPTO_gcm128_encrypt(&gctx->gcm, + in + bulk, out + bulk, len - bulk)) return -1; } } else { if (gctx->ctr) { + size_t bulk = 0; +# if defined(AES_GCM_ASM) + if (len >= 16 && AES_GCM_ASM(gctx)) { + size_t res = (16 - gctx->gcm.mres) % 16; + + if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, res)) + return -1; + + bulk = AES_gcm_decrypt(in + res, + out + res, len - res, + gctx->gcm.key, + gctx->gcm.Yi.c, gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + bulk += res; + } +# endif if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm, - in, out, len, gctx->ctr)) + in + bulk, + out + bulk, + len - bulk, gctx->ctr)) return -1; } else { - if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, len)) + size_t bulk = 0; +# if defined(AES_GCM_ASM2) + if (len >= 16 && AES_GCM_ASM2(gctx)) { + size_t res = (16 - gctx->gcm.mres) % 16; + + if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, res)) + return -1; + + bulk = AES_gcm_decrypt(in + res, + out + res, len - res, + gctx->gcm.key, + gctx->gcm.Yi.c, gctx->gcm.Xi.u); + gctx->gcm.len.u[1] += bulk; + bulk += res; + } +# endif + if (CRYPTO_gcm128_decrypt(&gctx->gcm, + in + bulk, out + bulk, len - bulk)) return -1; } } @@ -983,7 +1594,7 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } -# define CUSTOM_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 \ +# define CUSTOM_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 \ | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \ | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \ | EVP_CIPH_CUSTOM_COPY) @@ -1032,47 +1643,69 @@ static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, if (key) do { -# ifdef AES_XTS_ASM +# ifdef AES_XTS_ASM xctx->stream = enc ? AES_xts_encrypt : AES_xts_decrypt; -# else +# else xctx->stream = NULL; -# endif +# endif /* key_len is two AES keys */ -# ifdef BSAES_CAPABLE +# ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) { + if (enc) { + HWAES_set_encrypt_key(key, ctx->key_len * 4, + &xctx->ks1.ks); + xctx->xts.block1 = (block128_f) HWAES_encrypt; + } else { + HWAES_set_decrypt_key(key, ctx->key_len * 4, + &xctx->ks1.ks); + xctx->xts.block1 = (block128_f) HWAES_decrypt; + } + + HWAES_set_encrypt_key(key + ctx->key_len / 2, + ctx->key_len * 4, &xctx->ks2.ks); + xctx->xts.block2 = (block128_f) HWAES_encrypt; + + xctx->xts.key1 = &xctx->ks1; + break; + } else +# endif +# ifdef BSAES_CAPABLE if (BSAES_CAPABLE) xctx->stream = enc ? bsaes_xts_encrypt : bsaes_xts_decrypt; else -# endif -# ifdef VPAES_CAPABLE +# endif +# ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { if (enc) { - vpaes_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1); + vpaes_set_encrypt_key(key, ctx->key_len * 4, + &xctx->ks1.ks); xctx->xts.block1 = (block128_f) vpaes_encrypt; } else { - vpaes_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1); + vpaes_set_decrypt_key(key, ctx->key_len * 4, + &xctx->ks1.ks); xctx->xts.block1 = (block128_f) vpaes_decrypt; } vpaes_set_encrypt_key(key + ctx->key_len / 2, - ctx->key_len * 4, &xctx->ks2); + ctx->key_len * 4, &xctx->ks2.ks); xctx->xts.block2 = (block128_f) vpaes_encrypt; xctx->xts.key1 = &xctx->ks1; break; } else -# endif +# endif (void)0; /* terminate potentially open 'else' */ if (enc) { - AES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1); + AES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks); xctx->xts.block1 = (block128_f) AES_encrypt; } else { - AES_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1); + AES_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks); xctx->xts.block1 = (block128_f) AES_decrypt; } AES_set_encrypt_key(key + ctx->key_len / 2, - ctx->key_len * 4, &xctx->ks2); + ctx->key_len * 4, &xctx->ks2.ks); xctx->xts.block2 = (block128_f) AES_encrypt; xctx->xts.key1 = &xctx->ks1; @@ -1094,14 +1727,6 @@ static int aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return 0; if (!out || !in || len < AES_BLOCK_SIZE) return 0; -# ifdef OPENSSL_FIPS - /* Requirement of SP800-38E */ - if (FIPS_module_mode() && !(ctx->flags & EVP_CIPH_FLAG_NON_FIPS_ALLOW) && - (len > (1UL << 20) * 16)) { - EVPerr(EVP_F_AES_XTS_CIPHER, EVP_R_TOO_LARGE); - return 0; - } -# endif if (xctx->stream) (*xctx->stream) (in, out, len, xctx->xts.key1, xctx->xts.key2, ctx->iv); @@ -1111,9 +1736,9 @@ static int aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return 1; } -# define aes_xts_cleanup NULL +# define aes_xts_cleanup NULL -# define XTS_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 | EVP_CIPH_CUSTOM_IV \ +# define XTS_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 | EVP_CIPH_CUSTOM_IV \ | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \ | EVP_CIPH_CUSTOM_COPY) @@ -1191,17 +1816,28 @@ static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, return 1; if (key) do { -# ifdef VPAES_CAPABLE +# ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) { + HWAES_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks); + + CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, + &cctx->ks, (block128_f) HWAES_encrypt); + cctx->str = NULL; + cctx->key_set = 1; + break; + } else +# endif +# ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - vpaes_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks); + vpaes_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) vpaes_encrypt); cctx->str = NULL; cctx->key_set = 1; break; } -# endif - AES_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks); +# endif + AES_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) AES_encrypt); cctx->str = NULL; @@ -1274,7 +1910,7 @@ static int aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } -# define aes_ccm_cleanup NULL +# define aes_ccm_cleanup NULL BLOCK_CIPHER_custom(NID_aes, 128, 1, 12, ccm, CCM, EVP_CIPH_FLAG_FIPS | CUSTOM_FLAGS) @@ -1282,5 +1918,107 @@ BLOCK_CIPHER_custom(NID_aes, 128, 1, 12, ccm, CCM, EVP_CIPH_FLAG_FIPS | CUSTOM_FLAGS) BLOCK_CIPHER_custom(NID_aes, 256, 1, 12, ccm, CCM, EVP_CIPH_FLAG_FIPS | CUSTOM_FLAGS) -# endif #endif +typedef struct { + union { + double align; + AES_KEY ks; + } ks; + /* Indicates if IV has been set */ + unsigned char *iv; +} EVP_AES_WRAP_CTX; + +static int aes_wrap_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + EVP_AES_WRAP_CTX *wctx = ctx->cipher_data; + if (!iv && !key) + return 1; + if (key) { + if (ctx->encrypt) + AES_set_encrypt_key(key, ctx->key_len * 8, &wctx->ks.ks); + else + AES_set_decrypt_key(key, ctx->key_len * 8, &wctx->ks.ks); + if (!iv) + wctx->iv = NULL; + } + if (iv) { + memcpy(ctx->iv, iv, 8); + wctx->iv = ctx->iv; + } + return 1; +} + +static int aes_wrap_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t inlen) +{ + EVP_AES_WRAP_CTX *wctx = ctx->cipher_data; + size_t rv; + if (!in) + return 0; + if (inlen % 8) + return -1; + if (ctx->encrypt && inlen < 8) + return -1; + if (!ctx->encrypt && inlen < 16) + return -1; + if (!out) { + if (ctx->encrypt) + return inlen + 8; + else + return inlen - 8; + } + if (ctx->encrypt) + rv = CRYPTO_128_wrap(&wctx->ks.ks, wctx->iv, out, in, inlen, + (block128_f) AES_encrypt); + else + rv = CRYPTO_128_unwrap(&wctx->ks.ks, wctx->iv, out, in, inlen, + (block128_f) AES_decrypt); + return rv ? (int)rv : -1; +} + +#define WRAP_FLAGS (EVP_CIPH_WRAP_MODE \ + | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \ + | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_FLAG_DEFAULT_ASN1) + +static const EVP_CIPHER aes_128_wrap = { + NID_id_aes128_wrap, + 8, 16, 8, WRAP_FLAGS, + aes_wrap_init_key, aes_wrap_cipher, + NULL, + sizeof(EVP_AES_WRAP_CTX), + NULL, NULL, NULL, NULL +}; + +const EVP_CIPHER *EVP_aes_128_wrap(void) +{ + return &aes_128_wrap; +} + +static const EVP_CIPHER aes_192_wrap = { + NID_id_aes192_wrap, + 8, 24, 8, WRAP_FLAGS, + aes_wrap_init_key, aes_wrap_cipher, + NULL, + sizeof(EVP_AES_WRAP_CTX), + NULL, NULL, NULL, NULL +}; + +const EVP_CIPHER *EVP_aes_192_wrap(void) +{ + return &aes_192_wrap; +} + +static const EVP_CIPHER aes_256_wrap = { + NID_id_aes256_wrap, + 8, 32, 8, WRAP_FLAGS, + aes_wrap_init_key, aes_wrap_cipher, + NULL, + sizeof(EVP_AES_WRAP_CTX), + NULL, NULL, NULL, NULL +}; + +const EVP_CIPHER *EVP_aes_256_wrap(void) +{ + return &aes_256_wrap; +} diff --git a/crypto/evp/e_aes_cbc_hmac_sha1.c b/crypto/evp/e_aes_cbc_hmac_sha1.c index d1f5928f628a..8330964ee16b 100644 --- a/crypto/evp/e_aes_cbc_hmac_sha1.c +++ b/crypto/evp/e_aes_cbc_hmac_sha1.c @@ -58,7 +58,8 @@ # include # include # include -# include "evp_locl.h" +# include +# include "modes_lcl.h" # ifndef EVP_CIPH_FLAG_AEAD_CIPHER # define EVP_CIPH_FLAG_AEAD_CIPHER 0x200000 @@ -70,6 +71,10 @@ # define EVP_CIPH_FLAG_DEFAULT_ASN1 0 # endif +# if !defined(EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) +# define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0 +# endif + # define TLS1_1_VERSION 0x0302 typedef struct { @@ -89,11 +94,7 @@ typedef struct { defined(_M_AMD64) || defined(_M_X64) || \ defined(__INTEL__) ) -# if defined(__GNUC__) && __GNUC__>=2 && !defined(PEDANTIC) -# define BSWAP(x) ({ unsigned int r=(x); asm ("bswapl %0":"=r"(r):"0"(r)); r; }) -# endif - -extern unsigned int OPENSSL_ia32cap_P[2]; +extern unsigned int OPENSSL_ia32cap_P[]; # define AESNI_CAPABLE (1<<(57-32)) int aesni_set_encrypt_key(const unsigned char *userKey, int bits, @@ -110,6 +111,10 @@ void aesni_cbc_sha1_enc(const void *inp, void *out, size_t blocks, const AES_KEY *key, unsigned char iv[16], SHA_CTX *ctx, const void *in0); +void aesni256_cbc_sha1_dec(const void *inp, void *out, size_t blocks, + const AES_KEY *key, unsigned char iv[16], + SHA_CTX *ctx, const void *in0); + # define data(ctx) ((EVP_AES_HMAC_SHA1 *)(ctx)->cipher_data) static int aesni_cbc_hmac_sha1_init_key(EVP_CIPHER_CTX *ctx, @@ -134,6 +139,7 @@ static int aesni_cbc_hmac_sha1_init_key(EVP_CIPHER_CTX *ctx, } # define STITCHED_CALL +# undef STITCHED_DECRYPT_CALL # if !defined(STITCHED_CALL) # define aes_off 0 @@ -177,6 +183,275 @@ static void sha1_update(SHA_CTX *c, const void *data, size_t len) # endif # define SHA1_Update sha1_update +# if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + +typedef struct { + unsigned int A[8], B[8], C[8], D[8], E[8]; +} SHA1_MB_CTX; +typedef struct { + const unsigned char *ptr; + int blocks; +} HASH_DESC; + +void sha1_multi_block(SHA1_MB_CTX *, const HASH_DESC *, int); + +typedef struct { + const unsigned char *inp; + unsigned char *out; + int blocks; + u64 iv[2]; +} CIPH_DESC; + +void aesni_multi_cbc_encrypt(CIPH_DESC *, void *, int); + +static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA1 *key, + unsigned char *out, + const unsigned char *inp, + size_t inp_len, int n4x) +{ /* n4x is 1 or 2 */ + HASH_DESC hash_d[8], edges[8]; + CIPH_DESC ciph_d[8]; + unsigned char storage[sizeof(SHA1_MB_CTX) + 32]; + union { + u64 q[16]; + u32 d[32]; + u8 c[128]; + } blocks[8]; + SHA1_MB_CTX *ctx; + unsigned int frag, last, packlen, i, x4 = 4 * n4x, minblocks, processed = + 0; + size_t ret = 0; + u8 *IVs; +# if defined(BSWAP8) + u64 seqnum; +# endif + + /* ask for IVs in bulk */ + if (RAND_bytes((IVs = blocks[0].c), 16 * x4) <= 0) + return 0; + + ctx = (SHA1_MB_CTX *) (storage + 32 - ((size_t)storage % 32)); /* align */ + + frag = (unsigned int)inp_len >> (1 + n4x); + last = (unsigned int)inp_len + frag - (frag << (1 + n4x)); + if (last > frag && ((last + 13 + 9) % 64) < (x4 - 1)) { + frag++; + last -= x4 - 1; + } + + packlen = 5 + 16 + ((frag + 20 + 16) & -16); + + /* populate descriptors with pointers and IVs */ + hash_d[0].ptr = inp; + ciph_d[0].inp = inp; + /* 5+16 is place for header and explicit IV */ + ciph_d[0].out = out + 5 + 16; + memcpy(ciph_d[0].out - 16, IVs, 16); + memcpy(ciph_d[0].iv, IVs, 16); + IVs += 16; + + for (i = 1; i < x4; i++) { + ciph_d[i].inp = hash_d[i].ptr = hash_d[i - 1].ptr + frag; + ciph_d[i].out = ciph_d[i - 1].out + packlen; + memcpy(ciph_d[i].out - 16, IVs, 16); + memcpy(ciph_d[i].iv, IVs, 16); + IVs += 16; + } + +# if defined(BSWAP8) + memcpy(blocks[0].c, key->md.data, 8); + seqnum = BSWAP8(blocks[0].q[0]); +# endif + for (i = 0; i < x4; i++) { + unsigned int len = (i == (x4 - 1) ? last : frag); +# if !defined(BSWAP8) + unsigned int carry, j; +# endif + + ctx->A[i] = key->md.h0; + ctx->B[i] = key->md.h1; + ctx->C[i] = key->md.h2; + ctx->D[i] = key->md.h3; + ctx->E[i] = key->md.h4; + + /* fix seqnum */ +# if defined(BSWAP8) + blocks[i].q[0] = BSWAP8(seqnum + i); +# else + for (carry = i, j = 8; j--;) { + blocks[i].c[j] = ((u8 *)key->md.data)[j] + carry; + carry = (blocks[i].c[j] - carry) >> (sizeof(carry) * 8 - 1); + } +# endif + blocks[i].c[8] = ((u8 *)key->md.data)[8]; + blocks[i].c[9] = ((u8 *)key->md.data)[9]; + blocks[i].c[10] = ((u8 *)key->md.data)[10]; + /* fix length */ + blocks[i].c[11] = (u8)(len >> 8); + blocks[i].c[12] = (u8)(len); + + memcpy(blocks[i].c + 13, hash_d[i].ptr, 64 - 13); + hash_d[i].ptr += 64 - 13; + hash_d[i].blocks = (len - (64 - 13)) / 64; + + edges[i].ptr = blocks[i].c; + edges[i].blocks = 1; + } + + /* hash 13-byte headers and first 64-13 bytes of inputs */ + sha1_multi_block(ctx, edges, n4x); + /* hash bulk inputs */ +# define MAXCHUNKSIZE 2048 +# if MAXCHUNKSIZE%64 +# error "MAXCHUNKSIZE is not divisible by 64" +# elif MAXCHUNKSIZE + /* + * goal is to minimize pressure on L1 cache by moving in shorter steps, + * so that hashed data is still in the cache by the time we encrypt it + */ + minblocks = ((frag <= last ? frag : last) - (64 - 13)) / 64; + if (minblocks > MAXCHUNKSIZE / 64) { + for (i = 0; i < x4; i++) { + edges[i].ptr = hash_d[i].ptr; + edges[i].blocks = MAXCHUNKSIZE / 64; + ciph_d[i].blocks = MAXCHUNKSIZE / 16; + } + do { + sha1_multi_block(ctx, edges, n4x); + aesni_multi_cbc_encrypt(ciph_d, &key->ks, n4x); + + for (i = 0; i < x4; i++) { + edges[i].ptr = hash_d[i].ptr += MAXCHUNKSIZE; + hash_d[i].blocks -= MAXCHUNKSIZE / 64; + edges[i].blocks = MAXCHUNKSIZE / 64; + ciph_d[i].inp += MAXCHUNKSIZE; + ciph_d[i].out += MAXCHUNKSIZE; + ciph_d[i].blocks = MAXCHUNKSIZE / 16; + memcpy(ciph_d[i].iv, ciph_d[i].out - 16, 16); + } + processed += MAXCHUNKSIZE; + minblocks -= MAXCHUNKSIZE / 64; + } while (minblocks > MAXCHUNKSIZE / 64); + } +# endif +# undef MAXCHUNKSIZE + sha1_multi_block(ctx, hash_d, n4x); + + memset(blocks, 0, sizeof(blocks)); + for (i = 0; i < x4; i++) { + unsigned int len = (i == (x4 - 1) ? last : frag), + off = hash_d[i].blocks * 64; + const unsigned char *ptr = hash_d[i].ptr + off; + + off = (len - processed) - (64 - 13) - off; /* remainder actually */ + memcpy(blocks[i].c, ptr, off); + blocks[i].c[off] = 0x80; + len += 64 + 13; /* 64 is HMAC header */ + len *= 8; /* convert to bits */ + if (off < (64 - 8)) { +# ifdef BSWAP4 + blocks[i].d[15] = BSWAP4(len); +# else + PUTU32(blocks[i].c + 60, len); +# endif + edges[i].blocks = 1; + } else { +# ifdef BSWAP4 + blocks[i].d[31] = BSWAP4(len); +# else + PUTU32(blocks[i].c + 124, len); +# endif + edges[i].blocks = 2; + } + edges[i].ptr = blocks[i].c; + } + + /* hash input tails and finalize */ + sha1_multi_block(ctx, edges, n4x); + + memset(blocks, 0, sizeof(blocks)); + for (i = 0; i < x4; i++) { +# ifdef BSWAP4 + blocks[i].d[0] = BSWAP4(ctx->A[i]); + ctx->A[i] = key->tail.h0; + blocks[i].d[1] = BSWAP4(ctx->B[i]); + ctx->B[i] = key->tail.h1; + blocks[i].d[2] = BSWAP4(ctx->C[i]); + ctx->C[i] = key->tail.h2; + blocks[i].d[3] = BSWAP4(ctx->D[i]); + ctx->D[i] = key->tail.h3; + blocks[i].d[4] = BSWAP4(ctx->E[i]); + ctx->E[i] = key->tail.h4; + blocks[i].c[20] = 0x80; + blocks[i].d[15] = BSWAP4((64 + 20) * 8); +# else + PUTU32(blocks[i].c + 0, ctx->A[i]); + ctx->A[i] = key->tail.h0; + PUTU32(blocks[i].c + 4, ctx->B[i]); + ctx->B[i] = key->tail.h1; + PUTU32(blocks[i].c + 8, ctx->C[i]); + ctx->C[i] = key->tail.h2; + PUTU32(blocks[i].c + 12, ctx->D[i]); + ctx->D[i] = key->tail.h3; + PUTU32(blocks[i].c + 16, ctx->E[i]); + ctx->E[i] = key->tail.h4; + blocks[i].c[20] = 0x80; + PUTU32(blocks[i].c + 60, (64 + 20) * 8); +# endif + edges[i].ptr = blocks[i].c; + edges[i].blocks = 1; + } + + /* finalize MACs */ + sha1_multi_block(ctx, edges, n4x); + + for (i = 0; i < x4; i++) { + unsigned int len = (i == (x4 - 1) ? last : frag), pad, j; + unsigned char *out0 = out; + + memcpy(ciph_d[i].out, ciph_d[i].inp, len - processed); + ciph_d[i].inp = ciph_d[i].out; + + out += 5 + 16 + len; + + /* write MAC */ + PUTU32(out + 0, ctx->A[i]); + PUTU32(out + 4, ctx->B[i]); + PUTU32(out + 8, ctx->C[i]); + PUTU32(out + 12, ctx->D[i]); + PUTU32(out + 16, ctx->E[i]); + out += 20; + len += 20; + + /* pad */ + pad = 15 - len % 16; + for (j = 0; j <= pad; j++) + *(out++) = pad; + len += pad + 1; + + ciph_d[i].blocks = (len - processed) / 16; + len += 16; /* account for explicit iv */ + + /* arrange header */ + out0[0] = ((u8 *)key->md.data)[8]; + out0[1] = ((u8 *)key->md.data)[9]; + out0[2] = ((u8 *)key->md.data)[10]; + out0[3] = (u8)(len >> 8); + out0[4] = (u8)(len); + + ret += len + 5; + inp += frag; + } + + aesni_multi_cbc_encrypt(ciph_d, &key->ks, n4x); + + OPENSSL_cleanse(blocks, sizeof(blocks)); + OPENSSL_cleanse(ctx, sizeof(*ctx)); + + return ret; +} +# endif + static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len) { @@ -257,10 +532,7 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, /* arrange cache line alignment */ pmac = (void *)(((size_t)mac.c + 31) & ((size_t)0 - 32)); - /* decrypt HMAC|padding at once */ - aesni_cbc_encrypt(in, out, len, &key->ks, ctx->iv, 0); - - if (plen) { /* "TLS" mode of operation */ + if (plen != NO_PAYLOAD_LENGTH) { /* "TLS" mode of operation */ size_t inp_len, mask, j, i; unsigned int res, maxpad, pad, bitlen; int ret = 1; @@ -268,17 +540,37 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, unsigned int u[SHA_LBLOCK]; unsigned char c[SHA_CBLOCK]; } *data = (void *)key->md.data; +# if defined(STITCHED_DECRYPT_CALL) + unsigned char tail_iv[AES_BLOCK_SIZE]; + int stitch = 0; +# endif if ((key->aux.tls_aad[plen - 4] << 8 | key->aux.tls_aad[plen - 3]) - >= TLS1_1_VERSION) - iv = AES_BLOCK_SIZE; - - if (len < (iv + SHA_DIGEST_LENGTH + 1)) + >= TLS1_1_VERSION) { + if (len < (AES_BLOCK_SIZE + SHA_DIGEST_LENGTH + 1)) + return 0; + + /* omit explicit iv */ + memcpy(ctx->iv, in, AES_BLOCK_SIZE); + in += AES_BLOCK_SIZE; + out += AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } else if (len < (SHA_DIGEST_LENGTH + 1)) return 0; - /* omit explicit iv */ - out += iv; - len -= iv; +# if defined(STITCHED_DECRYPT_CALL) + if (len >= 1024 && ctx->key_len == 32) { + /* decrypt last block */ + memcpy(tail_iv, in + len - 2 * AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + aesni_cbc_encrypt(in + len - AES_BLOCK_SIZE, + out + len - AES_BLOCK_SIZE, AES_BLOCK_SIZE, + &key->ks, tail_iv, 0); + stitch = 1; + } else +# endif + /* decrypt HMAC|padding at once */ + aesni_cbc_encrypt(in, out, len, &key->ks, ctx->iv, 0); /* figure out payload length */ pad = out[len - 1]; @@ -298,6 +590,29 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, key->md = key->head; SHA1_Update(&key->md, key->aux.tls_aad, plen); +# if defined(STITCHED_DECRYPT_CALL) + if (stitch) { + blocks = (len - (256 + 32 + SHA_CBLOCK)) / SHA_CBLOCK; + aes_off = len - AES_BLOCK_SIZE - blocks * SHA_CBLOCK; + sha_off = SHA_CBLOCK - plen; + + aesni_cbc_encrypt(in, out, aes_off, &key->ks, ctx->iv, 0); + + SHA1_Update(&key->md, out, sha_off); + aesni256_cbc_sha1_dec(in + aes_off, + out + aes_off, blocks, &key->ks, + ctx->iv, &key->md, out + sha_off); + + sha_off += blocks *= SHA_CBLOCK; + out += sha_off; + len -= sha_off; + inp_len -= sha_off; + + key->md.Nl += (blocks << 3); /* at most 18 bits */ + memcpy(ctx->iv, tail_iv, AES_BLOCK_SIZE); + } +# endif + # if 1 len -= SHA_DIGEST_LENGTH; /* amend mac */ if (len >= (256 + SHA_CBLOCK)) { @@ -311,8 +626,8 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, /* but pretend as if we hashed padded payload */ bitlen = key->md.Nl + (inp_len << 3); /* at most 18 bits */ -# ifdef BSWAP - bitlen = BSWAP(bitlen); +# ifdef BSWAP4 + bitlen = BSWAP4(bitlen); # else mac.c[0] = 0; mac.c[1] = (unsigned char)(bitlen >> 16); @@ -376,12 +691,12 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, pmac->u[3] |= key->md.h3 & mask; pmac->u[4] |= key->md.h4 & mask; -# ifdef BSWAP - pmac->u[0] = BSWAP(pmac->u[0]); - pmac->u[1] = BSWAP(pmac->u[1]); - pmac->u[2] = BSWAP(pmac->u[2]); - pmac->u[3] = BSWAP(pmac->u[3]); - pmac->u[4] = BSWAP(pmac->u[4]); +# ifdef BSWAP4 + pmac->u[0] = BSWAP4(pmac->u[0]); + pmac->u[1] = BSWAP4(pmac->u[1]); + pmac->u[2] = BSWAP4(pmac->u[2]); + pmac->u[3] = BSWAP4(pmac->u[3]); + pmac->u[4] = BSWAP4(pmac->u[4]); # else for (i = 0; i < 5; i++) { res = pmac->u[i]; @@ -458,6 +773,33 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, # endif return ret; } else { +# if defined(STITCHED_DECRYPT_CALL) + if (len >= 1024 && ctx->key_len == 32) { + if (sha_off %= SHA_CBLOCK) + blocks = (len - 3 * SHA_CBLOCK) / SHA_CBLOCK; + else + blocks = (len - 2 * SHA_CBLOCK) / SHA_CBLOCK; + aes_off = len - blocks * SHA_CBLOCK; + + aesni_cbc_encrypt(in, out, aes_off, &key->ks, ctx->iv, 0); + SHA1_Update(&key->md, out, sha_off); + aesni256_cbc_sha1_dec(in + aes_off, + out + aes_off, blocks, &key->ks, + ctx->iv, &key->md, out + sha_off); + + sha_off += blocks *= SHA_CBLOCK; + out += sha_off; + len -= sha_off; + + key->md.Nh += blocks >> 29; + key->md.Nl += blocks <<= 3; + if (key->md.Nl < (unsigned int)blocks) + key->md.Nh++; + } else +# endif + /* decrypt HMAC|padding at once */ + aesni_cbc_encrypt(in, out, len, &key->ks, ctx->iv, 0); + SHA1_Update(&key->md, out, len); } } @@ -531,6 +873,70 @@ static int aesni_cbc_hmac_sha1_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, return SHA_DIGEST_LENGTH; } } +# if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + case EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE: + return (int)(5 + 16 + ((arg + 20 + 16) & -16)); + case EVP_CTRL_TLS1_1_MULTIBLOCK_AAD: + { + EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *param = + (EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *) ptr; + unsigned int n4x = 1, x4; + unsigned int frag, last, packlen, inp_len; + + if (arg < (int)sizeof(EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM)) + return -1; + + inp_len = param->inp[11] << 8 | param->inp[12]; + + if (ctx->encrypt) { + if ((param->inp[9] << 8 | param->inp[10]) < TLS1_1_VERSION) + return -1; + + if (inp_len) { + if (inp_len < 4096) + return 0; /* too short */ + + if (inp_len >= 8192 && OPENSSL_ia32cap_P[2] & (1 << 5)) + n4x = 2; /* AVX2 */ + } else if ((n4x = param->interleave / 4) && n4x <= 2) + inp_len = param->len; + else + return -1; + + key->md = key->head; + SHA1_Update(&key->md, param->inp, 13); + + x4 = 4 * n4x; + n4x += 1; + + frag = inp_len >> n4x; + last = inp_len + frag - (frag << n4x); + if (last > frag && ((last + 13 + 9) % 64 < (x4 - 1))) { + frag++; + last -= x4 - 1; + } + + packlen = 5 + 16 + ((frag + 20 + 16) & -16); + packlen = (packlen << n4x) - packlen; + packlen += 5 + 16 + ((last + 20 + 16) & -16); + + param->interleave = x4; + + return (int)packlen; + } else + return -1; /* not yet */ + } + case EVP_CTRL_TLS1_1_MULTIBLOCK_ENCRYPT: + { + EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *param = + (EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *) ptr; + + return (int)tls1_1_multi_block_encrypt(key, param->out, + param->inp, param->len, + param->interleave / 4); + } + case EVP_CTRL_TLS1_1_MULTIBLOCK_DECRYPT: +# endif default: return -1; } @@ -544,7 +950,7 @@ static EVP_CIPHER aesni_128_cbc_hmac_sha1_cipher = { # endif 16, 16, 16, EVP_CIPH_CBC_MODE | EVP_CIPH_FLAG_DEFAULT_ASN1 | - EVP_CIPH_FLAG_AEAD_CIPHER, + EVP_CIPH_FLAG_AEAD_CIPHER | EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK, aesni_cbc_hmac_sha1_init_key, aesni_cbc_hmac_sha1_cipher, NULL, @@ -563,7 +969,7 @@ static EVP_CIPHER aesni_256_cbc_hmac_sha1_cipher = { # endif 16, 32, 16, EVP_CIPH_CBC_MODE | EVP_CIPH_FLAG_DEFAULT_ASN1 | - EVP_CIPH_FLAG_AEAD_CIPHER, + EVP_CIPH_FLAG_AEAD_CIPHER | EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK, aesni_cbc_hmac_sha1_init_key, aesni_cbc_hmac_sha1_cipher, NULL, diff --git a/crypto/evp/e_aes_cbc_hmac_sha256.c b/crypto/evp/e_aes_cbc_hmac_sha256.c new file mode 100644 index 000000000000..b1c586e6fd96 --- /dev/null +++ b/crypto/evp/e_aes_cbc_hmac_sha256.c @@ -0,0 +1,973 @@ +/* ==================================================================== + * Copyright (c) 2011-2013 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + */ + +#include + +#include +#include + +#if !defined(OPENSSL_NO_AES) && !defined(OPENSSL_NO_SHA256) + +# include +# include +# include +# include +# include +# include "modes_lcl.h" + +# ifndef EVP_CIPH_FLAG_AEAD_CIPHER +# define EVP_CIPH_FLAG_AEAD_CIPHER 0x200000 +# define EVP_CTRL_AEAD_TLS1_AAD 0x16 +# define EVP_CTRL_AEAD_SET_MAC_KEY 0x17 +# endif + +# if !defined(EVP_CIPH_FLAG_DEFAULT_ASN1) +# define EVP_CIPH_FLAG_DEFAULT_ASN1 0 +# endif + +# if !defined(EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) +# define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0 +# endif + +# define TLS1_1_VERSION 0x0302 + +typedef struct { + AES_KEY ks; + SHA256_CTX head, tail, md; + size_t payload_length; /* AAD length in decrypt case */ + union { + unsigned int tls_ver; + unsigned char tls_aad[16]; /* 13 used */ + } aux; +} EVP_AES_HMAC_SHA256; + +# define NO_PAYLOAD_LENGTH ((size_t)-1) + +# if defined(AES_ASM) && ( \ + defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_AMD64) || defined(_M_X64) || \ + defined(__INTEL__) ) + +extern unsigned int OPENSSL_ia32cap_P[]; +# define AESNI_CAPABLE (1<<(57-32)) + +int aesni_set_encrypt_key(const unsigned char *userKey, int bits, + AES_KEY *key); +int aesni_set_decrypt_key(const unsigned char *userKey, int bits, + AES_KEY *key); + +void aesni_cbc_encrypt(const unsigned char *in, + unsigned char *out, + size_t length, + const AES_KEY *key, unsigned char *ivec, int enc); + +int aesni_cbc_sha256_enc(const void *inp, void *out, size_t blocks, + const AES_KEY *key, unsigned char iv[16], + SHA256_CTX *ctx, const void *in0); + +# define data(ctx) ((EVP_AES_HMAC_SHA256 *)(ctx)->cipher_data) + +static int aesni_cbc_hmac_sha256_init_key(EVP_CIPHER_CTX *ctx, + const unsigned char *inkey, + const unsigned char *iv, int enc) +{ + EVP_AES_HMAC_SHA256 *key = data(ctx); + int ret; + + if (enc) + memset(&key->ks, 0, sizeof(key->ks.rd_key)), + ret = aesni_set_encrypt_key(inkey, ctx->key_len * 8, &key->ks); + else + ret = aesni_set_decrypt_key(inkey, ctx->key_len * 8, &key->ks); + + SHA256_Init(&key->head); /* handy when benchmarking */ + key->tail = key->head; + key->md = key->head; + + key->payload_length = NO_PAYLOAD_LENGTH; + + return ret < 0 ? 0 : 1; +} + +# define STITCHED_CALL + +# if !defined(STITCHED_CALL) +# define aes_off 0 +# endif + +void sha256_block_data_order(void *c, const void *p, size_t len); + +static void sha256_update(SHA256_CTX *c, const void *data, size_t len) +{ + const unsigned char *ptr = data; + size_t res; + + if ((res = c->num)) { + res = SHA256_CBLOCK - res; + if (len < res) + res = len; + SHA256_Update(c, ptr, res); + ptr += res; + len -= res; + } + + res = len % SHA256_CBLOCK; + len -= res; + + if (len) { + sha256_block_data_order(c, ptr, len / SHA256_CBLOCK); + + ptr += len; + c->Nh += len >> 29; + c->Nl += len <<= 3; + if (c->Nl < (unsigned int)len) + c->Nh++; + } + + if (res) + SHA256_Update(c, ptr, res); +} + +# ifdef SHA256_Update +# undef SHA256_Update +# endif +# define SHA256_Update sha256_update + +# if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + +typedef struct { + unsigned int A[8], B[8], C[8], D[8], E[8], F[8], G[8], H[8]; +} SHA256_MB_CTX; +typedef struct { + const unsigned char *ptr; + int blocks; +} HASH_DESC; + +void sha256_multi_block(SHA256_MB_CTX *, const HASH_DESC *, int); + +typedef struct { + const unsigned char *inp; + unsigned char *out; + int blocks; + u64 iv[2]; +} CIPH_DESC; + +void aesni_multi_cbc_encrypt(CIPH_DESC *, void *, int); + +static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key, + unsigned char *out, + const unsigned char *inp, + size_t inp_len, int n4x) +{ /* n4x is 1 or 2 */ + HASH_DESC hash_d[8], edges[8]; + CIPH_DESC ciph_d[8]; + unsigned char storage[sizeof(SHA256_MB_CTX) + 32]; + union { + u64 q[16]; + u32 d[32]; + u8 c[128]; + } blocks[8]; + SHA256_MB_CTX *ctx; + unsigned int frag, last, packlen, i, x4 = 4 * n4x, minblocks, processed = + 0; + size_t ret = 0; + u8 *IVs; +# if defined(BSWAP8) + u64 seqnum; +# endif + + /* ask for IVs in bulk */ + if (RAND_bytes((IVs = blocks[0].c), 16 * x4) <= 0) + return 0; + + /* align */ + ctx = (SHA256_MB_CTX *) (storage + 32 - ((size_t)storage % 32)); + + frag = (unsigned int)inp_len >> (1 + n4x); + last = (unsigned int)inp_len + frag - (frag << (1 + n4x)); + if (last > frag && ((last + 13 + 9) % 64) < (x4 - 1)) { + frag++; + last -= x4 - 1; + } + + packlen = 5 + 16 + ((frag + 32 + 16) & -16); + + /* populate descriptors with pointers and IVs */ + hash_d[0].ptr = inp; + ciph_d[0].inp = inp; + /* 5+16 is place for header and explicit IV */ + ciph_d[0].out = out + 5 + 16; + memcpy(ciph_d[0].out - 16, IVs, 16); + memcpy(ciph_d[0].iv, IVs, 16); + IVs += 16; + + for (i = 1; i < x4; i++) { + ciph_d[i].inp = hash_d[i].ptr = hash_d[i - 1].ptr + frag; + ciph_d[i].out = ciph_d[i - 1].out + packlen; + memcpy(ciph_d[i].out - 16, IVs, 16); + memcpy(ciph_d[i].iv, IVs, 16); + IVs += 16; + } + +# if defined(BSWAP8) + memcpy(blocks[0].c, key->md.data, 8); + seqnum = BSWAP8(blocks[0].q[0]); +# endif + for (i = 0; i < x4; i++) { + unsigned int len = (i == (x4 - 1) ? last : frag); +# if !defined(BSWAP8) + unsigned int carry, j; +# endif + + ctx->A[i] = key->md.h[0]; + ctx->B[i] = key->md.h[1]; + ctx->C[i] = key->md.h[2]; + ctx->D[i] = key->md.h[3]; + ctx->E[i] = key->md.h[4]; + ctx->F[i] = key->md.h[5]; + ctx->G[i] = key->md.h[6]; + ctx->H[i] = key->md.h[7]; + + /* fix seqnum */ +# if defined(BSWAP8) + blocks[i].q[0] = BSWAP8(seqnum + i); +# else + for (carry = i, j = 8; j--;) { + blocks[i].c[j] = ((u8 *)key->md.data)[j] + carry; + carry = (blocks[i].c[j] - carry) >> (sizeof(carry) * 8 - 1); + } +# endif + blocks[i].c[8] = ((u8 *)key->md.data)[8]; + blocks[i].c[9] = ((u8 *)key->md.data)[9]; + blocks[i].c[10] = ((u8 *)key->md.data)[10]; + /* fix length */ + blocks[i].c[11] = (u8)(len >> 8); + blocks[i].c[12] = (u8)(len); + + memcpy(blocks[i].c + 13, hash_d[i].ptr, 64 - 13); + hash_d[i].ptr += 64 - 13; + hash_d[i].blocks = (len - (64 - 13)) / 64; + + edges[i].ptr = blocks[i].c; + edges[i].blocks = 1; + } + + /* hash 13-byte headers and first 64-13 bytes of inputs */ + sha256_multi_block(ctx, edges, n4x); + /* hash bulk inputs */ +# define MAXCHUNKSIZE 2048 +# if MAXCHUNKSIZE%64 +# error "MAXCHUNKSIZE is not divisible by 64" +# elif MAXCHUNKSIZE + /* + * goal is to minimize pressure on L1 cache by moving in shorter steps, + * so that hashed data is still in the cache by the time we encrypt it + */ + minblocks = ((frag <= last ? frag : last) - (64 - 13)) / 64; + if (minblocks > MAXCHUNKSIZE / 64) { + for (i = 0; i < x4; i++) { + edges[i].ptr = hash_d[i].ptr; + edges[i].blocks = MAXCHUNKSIZE / 64; + ciph_d[i].blocks = MAXCHUNKSIZE / 16; + } + do { + sha256_multi_block(ctx, edges, n4x); + aesni_multi_cbc_encrypt(ciph_d, &key->ks, n4x); + + for (i = 0; i < x4; i++) { + edges[i].ptr = hash_d[i].ptr += MAXCHUNKSIZE; + hash_d[i].blocks -= MAXCHUNKSIZE / 64; + edges[i].blocks = MAXCHUNKSIZE / 64; + ciph_d[i].inp += MAXCHUNKSIZE; + ciph_d[i].out += MAXCHUNKSIZE; + ciph_d[i].blocks = MAXCHUNKSIZE / 16; + memcpy(ciph_d[i].iv, ciph_d[i].out - 16, 16); + } + processed += MAXCHUNKSIZE; + minblocks -= MAXCHUNKSIZE / 64; + } while (minblocks > MAXCHUNKSIZE / 64); + } +# endif +# undef MAXCHUNKSIZE + sha256_multi_block(ctx, hash_d, n4x); + + memset(blocks, 0, sizeof(blocks)); + for (i = 0; i < x4; i++) { + unsigned int len = (i == (x4 - 1) ? last : frag), + off = hash_d[i].blocks * 64; + const unsigned char *ptr = hash_d[i].ptr + off; + + off = (len - processed) - (64 - 13) - off; /* remainder actually */ + memcpy(blocks[i].c, ptr, off); + blocks[i].c[off] = 0x80; + len += 64 + 13; /* 64 is HMAC header */ + len *= 8; /* convert to bits */ + if (off < (64 - 8)) { +# ifdef BSWAP4 + blocks[i].d[15] = BSWAP4(len); +# else + PUTU32(blocks[i].c + 60, len); +# endif + edges[i].blocks = 1; + } else { +# ifdef BSWAP4 + blocks[i].d[31] = BSWAP4(len); +# else + PUTU32(blocks[i].c + 124, len); +# endif + edges[i].blocks = 2; + } + edges[i].ptr = blocks[i].c; + } + + /* hash input tails and finalize */ + sha256_multi_block(ctx, edges, n4x); + + memset(blocks, 0, sizeof(blocks)); + for (i = 0; i < x4; i++) { +# ifdef BSWAP4 + blocks[i].d[0] = BSWAP4(ctx->A[i]); + ctx->A[i] = key->tail.h[0]; + blocks[i].d[1] = BSWAP4(ctx->B[i]); + ctx->B[i] = key->tail.h[1]; + blocks[i].d[2] = BSWAP4(ctx->C[i]); + ctx->C[i] = key->tail.h[2]; + blocks[i].d[3] = BSWAP4(ctx->D[i]); + ctx->D[i] = key->tail.h[3]; + blocks[i].d[4] = BSWAP4(ctx->E[i]); + ctx->E[i] = key->tail.h[4]; + blocks[i].d[5] = BSWAP4(ctx->F[i]); + ctx->F[i] = key->tail.h[5]; + blocks[i].d[6] = BSWAP4(ctx->G[i]); + ctx->G[i] = key->tail.h[6]; + blocks[i].d[7] = BSWAP4(ctx->H[i]); + ctx->H[i] = key->tail.h[7]; + blocks[i].c[32] = 0x80; + blocks[i].d[15] = BSWAP4((64 + 32) * 8); +# else + PUTU32(blocks[i].c + 0, ctx->A[i]); + ctx->A[i] = key->tail.h[0]; + PUTU32(blocks[i].c + 4, ctx->B[i]); + ctx->B[i] = key->tail.h[1]; + PUTU32(blocks[i].c + 8, ctx->C[i]); + ctx->C[i] = key->tail.h[2]; + PUTU32(blocks[i].c + 12, ctx->D[i]); + ctx->D[i] = key->tail.h[3]; + PUTU32(blocks[i].c + 16, ctx->E[i]); + ctx->E[i] = key->tail.h[4]; + PUTU32(blocks[i].c + 20, ctx->F[i]); + ctx->F[i] = key->tail.h[5]; + PUTU32(blocks[i].c + 24, ctx->G[i]); + ctx->G[i] = key->tail.h[6]; + PUTU32(blocks[i].c + 28, ctx->H[i]); + ctx->H[i] = key->tail.h[7]; + blocks[i].c[32] = 0x80; + PUTU32(blocks[i].c + 60, (64 + 32) * 8); +# endif + edges[i].ptr = blocks[i].c; + edges[i].blocks = 1; + } + + /* finalize MACs */ + sha256_multi_block(ctx, edges, n4x); + + for (i = 0; i < x4; i++) { + unsigned int len = (i == (x4 - 1) ? last : frag), pad, j; + unsigned char *out0 = out; + + memcpy(ciph_d[i].out, ciph_d[i].inp, len - processed); + ciph_d[i].inp = ciph_d[i].out; + + out += 5 + 16 + len; + + /* write MAC */ + PUTU32(out + 0, ctx->A[i]); + PUTU32(out + 4, ctx->B[i]); + PUTU32(out + 8, ctx->C[i]); + PUTU32(out + 12, ctx->D[i]); + PUTU32(out + 16, ctx->E[i]); + PUTU32(out + 20, ctx->F[i]); + PUTU32(out + 24, ctx->G[i]); + PUTU32(out + 28, ctx->H[i]); + out += 32; + len += 32; + + /* pad */ + pad = 15 - len % 16; + for (j = 0; j <= pad; j++) + *(out++) = pad; + len += pad + 1; + + ciph_d[i].blocks = (len - processed) / 16; + len += 16; /* account for explicit iv */ + + /* arrange header */ + out0[0] = ((u8 *)key->md.data)[8]; + out0[1] = ((u8 *)key->md.data)[9]; + out0[2] = ((u8 *)key->md.data)[10]; + out0[3] = (u8)(len >> 8); + out0[4] = (u8)(len); + + ret += len + 5; + inp += frag; + } + + aesni_multi_cbc_encrypt(ciph_d, &key->ks, n4x); + + OPENSSL_cleanse(blocks, sizeof(blocks)); + OPENSSL_cleanse(ctx, sizeof(*ctx)); + + return ret; +} +# endif + +static int aesni_cbc_hmac_sha256_cipher(EVP_CIPHER_CTX *ctx, + unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_AES_HMAC_SHA256 *key = data(ctx); + unsigned int l; + size_t plen = key->payload_length, iv = 0, /* explicit IV in TLS 1.1 and + * later */ + sha_off = 0; +# if defined(STITCHED_CALL) + size_t aes_off = 0, blocks; + + sha_off = SHA256_CBLOCK - key->md.num; +# endif + + key->payload_length = NO_PAYLOAD_LENGTH; + + if (len % AES_BLOCK_SIZE) + return 0; + + if (ctx->encrypt) { + if (plen == NO_PAYLOAD_LENGTH) + plen = len; + else if (len != + ((plen + SHA256_DIGEST_LENGTH + + AES_BLOCK_SIZE) & -AES_BLOCK_SIZE)) + return 0; + else if (key->aux.tls_ver >= TLS1_1_VERSION) + iv = AES_BLOCK_SIZE; + +# if defined(STITCHED_CALL) + if (OPENSSL_ia32cap_P[1] & (1 << (60 - 32)) && /* AVX? */ + plen > (sha_off + iv) && + (blocks = (plen - (sha_off + iv)) / SHA256_CBLOCK)) { + SHA256_Update(&key->md, in + iv, sha_off); + + (void)aesni_cbc_sha256_enc(in, out, blocks, &key->ks, + ctx->iv, &key->md, in + iv + sha_off); + blocks *= SHA256_CBLOCK; + aes_off += blocks; + sha_off += blocks; + key->md.Nh += blocks >> 29; + key->md.Nl += blocks <<= 3; + if (key->md.Nl < (unsigned int)blocks) + key->md.Nh++; + } else { + sha_off = 0; + } +# endif + sha_off += iv; + SHA256_Update(&key->md, in + sha_off, plen - sha_off); + + if (plen != len) { /* "TLS" mode of operation */ + if (in != out) + memcpy(out + aes_off, in + aes_off, plen - aes_off); + + /* calculate HMAC and append it to payload */ + SHA256_Final(out + plen, &key->md); + key->md = key->tail; + SHA256_Update(&key->md, out + plen, SHA256_DIGEST_LENGTH); + SHA256_Final(out + plen, &key->md); + + /* pad the payload|hmac */ + plen += SHA256_DIGEST_LENGTH; + for (l = len - plen - 1; plen < len; plen++) + out[plen] = l; + /* encrypt HMAC|padding at once */ + aesni_cbc_encrypt(out + aes_off, out + aes_off, len - aes_off, + &key->ks, ctx->iv, 1); + } else { + aesni_cbc_encrypt(in + aes_off, out + aes_off, len - aes_off, + &key->ks, ctx->iv, 1); + } + } else { + union { + unsigned int u[SHA256_DIGEST_LENGTH / sizeof(unsigned int)]; + unsigned char c[64 + SHA256_DIGEST_LENGTH]; + } mac, *pmac; + + /* arrange cache line alignment */ + pmac = (void *)(((size_t)mac.c + 63) & ((size_t)0 - 64)); + + /* decrypt HMAC|padding at once */ + aesni_cbc_encrypt(in, out, len, &key->ks, ctx->iv, 0); + + if (plen != NO_PAYLOAD_LENGTH) { /* "TLS" mode of operation */ + size_t inp_len, mask, j, i; + unsigned int res, maxpad, pad, bitlen; + int ret = 1; + union { + unsigned int u[SHA_LBLOCK]; + unsigned char c[SHA256_CBLOCK]; + } *data = (void *)key->md.data; + + if ((key->aux.tls_aad[plen - 4] << 8 | key->aux.tls_aad[plen - 3]) + >= TLS1_1_VERSION) + iv = AES_BLOCK_SIZE; + + if (len < (iv + SHA256_DIGEST_LENGTH + 1)) + return 0; + + /* omit explicit iv */ + out += iv; + len -= iv; + + /* figure out payload length */ + pad = out[len - 1]; + maxpad = len - (SHA256_DIGEST_LENGTH + 1); + maxpad |= (255 - maxpad) >> (sizeof(maxpad) * 8 - 8); + maxpad &= 255; + + inp_len = len - (SHA256_DIGEST_LENGTH + pad + 1); + mask = (0 - ((inp_len - len) >> (sizeof(inp_len) * 8 - 1))); + inp_len &= mask; + ret &= (int)mask; + + key->aux.tls_aad[plen - 2] = inp_len >> 8; + key->aux.tls_aad[plen - 1] = inp_len; + + /* calculate HMAC */ + key->md = key->head; + SHA256_Update(&key->md, key->aux.tls_aad, plen); + +# if 1 + len -= SHA256_DIGEST_LENGTH; /* amend mac */ + if (len >= (256 + SHA256_CBLOCK)) { + j = (len - (256 + SHA256_CBLOCK)) & (0 - SHA256_CBLOCK); + j += SHA256_CBLOCK - key->md.num; + SHA256_Update(&key->md, out, j); + out += j; + len -= j; + inp_len -= j; + } + + /* but pretend as if we hashed padded payload */ + bitlen = key->md.Nl + (inp_len << 3); /* at most 18 bits */ +# ifdef BSWAP4 + bitlen = BSWAP4(bitlen); +# else + mac.c[0] = 0; + mac.c[1] = (unsigned char)(bitlen >> 16); + mac.c[2] = (unsigned char)(bitlen >> 8); + mac.c[3] = (unsigned char)bitlen; + bitlen = mac.u[0]; +# endif + + pmac->u[0] = 0; + pmac->u[1] = 0; + pmac->u[2] = 0; + pmac->u[3] = 0; + pmac->u[4] = 0; + pmac->u[5] = 0; + pmac->u[6] = 0; + pmac->u[7] = 0; + + for (res = key->md.num, j = 0; j < len; j++) { + size_t c = out[j]; + mask = (j - inp_len) >> (sizeof(j) * 8 - 8); + c &= mask; + c |= 0x80 & ~mask & ~((inp_len - j) >> (sizeof(j) * 8 - 8)); + data->c[res++] = (unsigned char)c; + + if (res != SHA256_CBLOCK) + continue; + + /* j is not incremented yet */ + mask = 0 - ((inp_len + 7 - j) >> (sizeof(j) * 8 - 1)); + data->u[SHA_LBLOCK - 1] |= bitlen & mask; + sha256_block_data_order(&key->md, data, 1); + mask &= 0 - ((j - inp_len - 72) >> (sizeof(j) * 8 - 1)); + pmac->u[0] |= key->md.h[0] & mask; + pmac->u[1] |= key->md.h[1] & mask; + pmac->u[2] |= key->md.h[2] & mask; + pmac->u[3] |= key->md.h[3] & mask; + pmac->u[4] |= key->md.h[4] & mask; + pmac->u[5] |= key->md.h[5] & mask; + pmac->u[6] |= key->md.h[6] & mask; + pmac->u[7] |= key->md.h[7] & mask; + res = 0; + } + + for (i = res; i < SHA256_CBLOCK; i++, j++) + data->c[i] = 0; + + if (res > SHA256_CBLOCK - 8) { + mask = 0 - ((inp_len + 8 - j) >> (sizeof(j) * 8 - 1)); + data->u[SHA_LBLOCK - 1] |= bitlen & mask; + sha256_block_data_order(&key->md, data, 1); + mask &= 0 - ((j - inp_len - 73) >> (sizeof(j) * 8 - 1)); + pmac->u[0] |= key->md.h[0] & mask; + pmac->u[1] |= key->md.h[1] & mask; + pmac->u[2] |= key->md.h[2] & mask; + pmac->u[3] |= key->md.h[3] & mask; + pmac->u[4] |= key->md.h[4] & mask; + pmac->u[5] |= key->md.h[5] & mask; + pmac->u[6] |= key->md.h[6] & mask; + pmac->u[7] |= key->md.h[7] & mask; + + memset(data, 0, SHA256_CBLOCK); + j += 64; + } + data->u[SHA_LBLOCK - 1] = bitlen; + sha256_block_data_order(&key->md, data, 1); + mask = 0 - ((j - inp_len - 73) >> (sizeof(j) * 8 - 1)); + pmac->u[0] |= key->md.h[0] & mask; + pmac->u[1] |= key->md.h[1] & mask; + pmac->u[2] |= key->md.h[2] & mask; + pmac->u[3] |= key->md.h[3] & mask; + pmac->u[4] |= key->md.h[4] & mask; + pmac->u[5] |= key->md.h[5] & mask; + pmac->u[6] |= key->md.h[6] & mask; + pmac->u[7] |= key->md.h[7] & mask; + +# ifdef BSWAP4 + pmac->u[0] = BSWAP4(pmac->u[0]); + pmac->u[1] = BSWAP4(pmac->u[1]); + pmac->u[2] = BSWAP4(pmac->u[2]); + pmac->u[3] = BSWAP4(pmac->u[3]); + pmac->u[4] = BSWAP4(pmac->u[4]); + pmac->u[5] = BSWAP4(pmac->u[5]); + pmac->u[6] = BSWAP4(pmac->u[6]); + pmac->u[7] = BSWAP4(pmac->u[7]); +# else + for (i = 0; i < 8; i++) { + res = pmac->u[i]; + pmac->c[4 * i + 0] = (unsigned char)(res >> 24); + pmac->c[4 * i + 1] = (unsigned char)(res >> 16); + pmac->c[4 * i + 2] = (unsigned char)(res >> 8); + pmac->c[4 * i + 3] = (unsigned char)res; + } +# endif + len += SHA256_DIGEST_LENGTH; +# else + SHA256_Update(&key->md, out, inp_len); + res = key->md.num; + SHA256_Final(pmac->c, &key->md); + + { + unsigned int inp_blocks, pad_blocks; + + /* but pretend as if we hashed padded payload */ + inp_blocks = + 1 + ((SHA256_CBLOCK - 9 - res) >> (sizeof(res) * 8 - 1)); + res += (unsigned int)(len - inp_len); + pad_blocks = res / SHA256_CBLOCK; + res %= SHA256_CBLOCK; + pad_blocks += + 1 + ((SHA256_CBLOCK - 9 - res) >> (sizeof(res) * 8 - 1)); + for (; inp_blocks < pad_blocks; inp_blocks++) + sha1_block_data_order(&key->md, data, 1); + } +# endif + key->md = key->tail; + SHA256_Update(&key->md, pmac->c, SHA256_DIGEST_LENGTH); + SHA256_Final(pmac->c, &key->md); + + /* verify HMAC */ + out += inp_len; + len -= inp_len; +# if 1 + { + unsigned char *p = + out + len - 1 - maxpad - SHA256_DIGEST_LENGTH; + size_t off = out - p; + unsigned int c, cmask; + + maxpad += SHA256_DIGEST_LENGTH; + for (res = 0, i = 0, j = 0; j < maxpad; j++) { + c = p[j]; + cmask = + ((int)(j - off - SHA256_DIGEST_LENGTH)) >> + (sizeof(int) * 8 - 1); + res |= (c ^ pad) & ~cmask; /* ... and padding */ + cmask &= ((int)(off - 1 - j)) >> (sizeof(int) * 8 - 1); + res |= (c ^ pmac->c[i]) & cmask; + i += 1 & cmask; + } + maxpad -= SHA256_DIGEST_LENGTH; + + res = 0 - ((0 - res) >> (sizeof(res) * 8 - 1)); + ret &= (int)~res; + } +# else + for (res = 0, i = 0; i < SHA256_DIGEST_LENGTH; i++) + res |= out[i] ^ pmac->c[i]; + res = 0 - ((0 - res) >> (sizeof(res) * 8 - 1)); + ret &= (int)~res; + + /* verify padding */ + pad = (pad & ~res) | (maxpad & res); + out = out + len - 1 - pad; + for (res = 0, i = 0; i < pad; i++) + res |= out[i] ^ pad; + + res = (0 - res) >> (sizeof(res) * 8 - 1); + ret &= (int)~res; +# endif + return ret; + } else { + SHA256_Update(&key->md, out, len); + } + } + + return 1; +} + +static int aesni_cbc_hmac_sha256_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, + void *ptr) +{ + EVP_AES_HMAC_SHA256 *key = data(ctx); + + switch (type) { + case EVP_CTRL_AEAD_SET_MAC_KEY: + { + unsigned int i; + unsigned char hmac_key[64]; + + memset(hmac_key, 0, sizeof(hmac_key)); + + if (arg > (int)sizeof(hmac_key)) { + SHA256_Init(&key->head); + SHA256_Update(&key->head, ptr, arg); + SHA256_Final(hmac_key, &key->head); + } else { + memcpy(hmac_key, ptr, arg); + } + + for (i = 0; i < sizeof(hmac_key); i++) + hmac_key[i] ^= 0x36; /* ipad */ + SHA256_Init(&key->head); + SHA256_Update(&key->head, hmac_key, sizeof(hmac_key)); + + for (i = 0; i < sizeof(hmac_key); i++) + hmac_key[i] ^= 0x36 ^ 0x5c; /* opad */ + SHA256_Init(&key->tail); + SHA256_Update(&key->tail, hmac_key, sizeof(hmac_key)); + + OPENSSL_cleanse(hmac_key, sizeof(hmac_key)); + + return 1; + } + case EVP_CTRL_AEAD_TLS1_AAD: + { + unsigned char *p = ptr; + unsigned int len = p[arg - 2] << 8 | p[arg - 1]; + + if (arg != EVP_AEAD_TLS1_AAD_LEN) + return -1; + + len = p[arg - 2] << 8 | p[arg - 1]; + + if (ctx->encrypt) { + key->payload_length = len; + if ((key->aux.tls_ver = + p[arg - 4] << 8 | p[arg - 3]) >= TLS1_1_VERSION) { + len -= AES_BLOCK_SIZE; + p[arg - 2] = len >> 8; + p[arg - 1] = len; + } + key->md = key->head; + SHA256_Update(&key->md, p, arg); + + return (int)(((len + SHA256_DIGEST_LENGTH + + AES_BLOCK_SIZE) & -AES_BLOCK_SIZE) + - len); + } else { + memcpy(key->aux.tls_aad, ptr, arg); + key->payload_length = arg; + + return SHA256_DIGEST_LENGTH; + } + } +# if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + case EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE: + return (int)(5 + 16 + ((arg + 32 + 16) & -16)); + case EVP_CTRL_TLS1_1_MULTIBLOCK_AAD: + { + EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *param = + (EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *) ptr; + unsigned int n4x = 1, x4; + unsigned int frag, last, packlen, inp_len; + + if (arg < (int)sizeof(EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM)) + return -1; + + inp_len = param->inp[11] << 8 | param->inp[12]; + + if (ctx->encrypt) { + if ((param->inp[9] << 8 | param->inp[10]) < TLS1_1_VERSION) + return -1; + + if (inp_len) { + if (inp_len < 4096) + return 0; /* too short */ + + if (inp_len >= 8192 && OPENSSL_ia32cap_P[2] & (1 << 5)) + n4x = 2; /* AVX2 */ + } else if ((n4x = param->interleave / 4) && n4x <= 2) + inp_len = param->len; + else + return -1; + + key->md = key->head; + SHA256_Update(&key->md, param->inp, 13); + + x4 = 4 * n4x; + n4x += 1; + + frag = inp_len >> n4x; + last = inp_len + frag - (frag << n4x); + if (last > frag && ((last + 13 + 9) % 64 < (x4 - 1))) { + frag++; + last -= x4 - 1; + } + + packlen = 5 + 16 + ((frag + 32 + 16) & -16); + packlen = (packlen << n4x) - packlen; + packlen += 5 + 16 + ((last + 32 + 16) & -16); + + param->interleave = x4; + + return (int)packlen; + } else + return -1; /* not yet */ + } + case EVP_CTRL_TLS1_1_MULTIBLOCK_ENCRYPT: + { + EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *param = + (EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *) ptr; + + return (int)tls1_1_multi_block_encrypt(key, param->out, + param->inp, param->len, + param->interleave / 4); + } + case EVP_CTRL_TLS1_1_MULTIBLOCK_DECRYPT: +# endif + default: + return -1; + } +} + +static EVP_CIPHER aesni_128_cbc_hmac_sha256_cipher = { +# ifdef NID_aes_128_cbc_hmac_sha256 + NID_aes_128_cbc_hmac_sha256, +# else + NID_undef, +# endif + 16, 16, 16, + EVP_CIPH_CBC_MODE | EVP_CIPH_FLAG_DEFAULT_ASN1 | + EVP_CIPH_FLAG_AEAD_CIPHER | EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK, + aesni_cbc_hmac_sha256_init_key, + aesni_cbc_hmac_sha256_cipher, + NULL, + sizeof(EVP_AES_HMAC_SHA256), + EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : EVP_CIPHER_set_asn1_iv, + EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : EVP_CIPHER_get_asn1_iv, + aesni_cbc_hmac_sha256_ctrl, + NULL +}; + +static EVP_CIPHER aesni_256_cbc_hmac_sha256_cipher = { +# ifdef NID_aes_256_cbc_hmac_sha256 + NID_aes_256_cbc_hmac_sha256, +# else + NID_undef, +# endif + 16, 32, 16, + EVP_CIPH_CBC_MODE | EVP_CIPH_FLAG_DEFAULT_ASN1 | + EVP_CIPH_FLAG_AEAD_CIPHER | EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK, + aesni_cbc_hmac_sha256_init_key, + aesni_cbc_hmac_sha256_cipher, + NULL, + sizeof(EVP_AES_HMAC_SHA256), + EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : EVP_CIPHER_set_asn1_iv, + EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : EVP_CIPHER_get_asn1_iv, + aesni_cbc_hmac_sha256_ctrl, + NULL +}; + +const EVP_CIPHER *EVP_aes_128_cbc_hmac_sha256(void) +{ + return ((OPENSSL_ia32cap_P[1] & AESNI_CAPABLE) && + aesni_cbc_sha256_enc(NULL, NULL, 0, NULL, NULL, NULL, NULL) ? + &aesni_128_cbc_hmac_sha256_cipher : NULL); +} + +const EVP_CIPHER *EVP_aes_256_cbc_hmac_sha256(void) +{ + return ((OPENSSL_ia32cap_P[1] & AESNI_CAPABLE) && + aesni_cbc_sha256_enc(NULL, NULL, 0, NULL, NULL, NULL, NULL) ? + &aesni_256_cbc_hmac_sha256_cipher : NULL); +} +# else +const EVP_CIPHER *EVP_aes_128_cbc_hmac_sha256(void) +{ + return NULL; +} + +const EVP_CIPHER *EVP_aes_256_cbc_hmac_sha256(void) +{ + return NULL; +} +# endif +#endif diff --git a/crypto/evp/e_camellia.c b/crypto/evp/e_camellia.c index 27bc4892fff8..f9c84013675d 100644 --- a/crypto/evp/e_camellia.c +++ b/crypto/evp/e_camellia.c @@ -61,6 +61,7 @@ # include # include # include "evp_locl.h" +# include "modes_lcl.h" static int camellia_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc); @@ -68,48 +69,322 @@ static int camellia_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, /* Camellia subkey Structure */ typedef struct { CAMELLIA_KEY ks; + block128_f block; + union { + cbc128_f cbc; + ctr128_f ctr; + } stream; } EVP_CAMELLIA_KEY; +# define MAXBITCHUNK ((size_t)1<<(sizeof(size_t)*8-4)) + /* Attribute operation for Camellia */ # define data(ctx) EVP_C_DATA(EVP_CAMELLIA_KEY,ctx) -IMPLEMENT_BLOCK_CIPHER(camellia_128, ks, Camellia, EVP_CAMELLIA_KEY, - NID_camellia_128, 16, 16, 16, 128, - 0, camellia_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, NULL) - IMPLEMENT_BLOCK_CIPHER(camellia_192, ks, Camellia, EVP_CAMELLIA_KEY, - NID_camellia_192, 16, 24, 16, 128, - 0, camellia_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, NULL) - IMPLEMENT_BLOCK_CIPHER(camellia_256, ks, Camellia, EVP_CAMELLIA_KEY, - NID_camellia_256, 16, 32, 16, 128, - 0, camellia_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, NULL) -# define IMPLEMENT_CAMELLIA_CFBR(ksize,cbits) IMPLEMENT_CFBR(camellia,Camellia,EVP_CAMELLIA_KEY,ks,ksize,cbits,16) - IMPLEMENT_CAMELLIA_CFBR(128, 1) - IMPLEMENT_CAMELLIA_CFBR(192, 1) - IMPLEMENT_CAMELLIA_CFBR(256, 1) - - IMPLEMENT_CAMELLIA_CFBR(128, 8) - IMPLEMENT_CAMELLIA_CFBR(192, 8) - IMPLEMENT_CAMELLIA_CFBR(256, 8) +# if defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) +/* ---------^^^ this is not a typo, just a way to detect that + * assembler support was in general requested... */ +# include "sparc_arch.h" + +extern unsigned int OPENSSL_sparcv9cap_P[]; + +# define SPARC_CMLL_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_CAMELLIA) + +void cmll_t4_set_key(const unsigned char *key, int bits, CAMELLIA_KEY *ks); +void cmll_t4_encrypt(const unsigned char *in, unsigned char *out, + const CAMELLIA_KEY *key); +void cmll_t4_decrypt(const unsigned char *in, unsigned char *out, + const CAMELLIA_KEY *key); + +void cmll128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t len, const CAMELLIA_KEY *key, + unsigned char *ivec); +void cmll128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const CAMELLIA_KEY *key, + unsigned char *ivec); +void cmll256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t len, const CAMELLIA_KEY *key, + unsigned char *ivec); +void cmll256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, + size_t len, const CAMELLIA_KEY *key, + unsigned char *ivec); +void cmll128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const CAMELLIA_KEY *key, + unsigned char *ivec); +void cmll256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const CAMELLIA_KEY *key, + unsigned char *ivec); + +static int cmll_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + int ret, mode, bits; + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + mode = ctx->cipher->flags & EVP_CIPH_MODE; + bits = ctx->key_len * 8; + + cmll_t4_set_key(key, bits, &dat->ks); + + if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) + && !enc) { + ret = 0; + dat->block = (block128_f) cmll_t4_decrypt; + switch (bits) { + case 128: + dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? + (cbc128_f) cmll128_t4_cbc_decrypt : NULL; + break; + case 192: + case 256: + dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? + (cbc128_f) cmll256_t4_cbc_decrypt : NULL; + break; + default: + ret = -1; + } + } else { + ret = 0; + dat->block = (block128_f) cmll_t4_encrypt; + switch (bits) { + case 128: + if (mode == EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f) cmll128_t4_cbc_encrypt; + else if (mode == EVP_CIPH_CTR_MODE) + dat->stream.ctr = (ctr128_f) cmll128_t4_ctr32_encrypt; + else + dat->stream.cbc = NULL; + break; + case 192: + case 256: + if (mode == EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f) cmll256_t4_cbc_encrypt; + else if (mode == EVP_CIPH_CTR_MODE) + dat->stream.ctr = (ctr128_f) cmll256_t4_ctr32_encrypt; + else + dat->stream.cbc = NULL; + break; + default: + ret = -1; + } + } + + if (ret < 0) { + EVPerr(EVP_F_CMLL_T4_INIT_KEY, EVP_R_CAMELLIA_KEY_SETUP_FAILED); + return 0; + } + + return 1; +} + +# define cmll_t4_cbc_cipher camellia_cbc_cipher +static int cmll_t4_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define cmll_t4_ecb_cipher camellia_ecb_cipher +static int cmll_t4_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); +# define cmll_t4_ofb_cipher camellia_ofb_cipher +static int cmll_t4_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define cmll_t4_cfb_cipher camellia_cfb_cipher +static int cmll_t4_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define cmll_t4_cfb8_cipher camellia_cfb8_cipher +static int cmll_t4_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define cmll_t4_cfb1_cipher camellia_cfb1_cipher +static int cmll_t4_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define cmll_t4_ctr_cipher camellia_ctr_cipher +static int cmll_t4_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); + +# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ +static const EVP_CIPHER cmll_t4_##keylen##_##mode = { \ + nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + cmll_t4_init_key, \ + cmll_t4_##mode##_cipher, \ + NULL, \ + sizeof(EVP_CAMELLIA_KEY), \ + NULL,NULL,NULL,NULL }; \ +static const EVP_CIPHER camellia_##keylen##_##mode = { \ + nid##_##keylen##_##nmode,blocksize, \ + keylen/8,ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + camellia_init_key, \ + camellia_##mode##_cipher, \ + NULL, \ + sizeof(EVP_CAMELLIA_KEY), \ + NULL,NULL,NULL,NULL }; \ +const EVP_CIPHER *EVP_camellia_##keylen##_##mode(void) \ +{ return SPARC_CMLL_CAPABLE?&cmll_t4_##keylen##_##mode:&camellia_##keylen##_##mode; } + +# else + +# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ +static const EVP_CIPHER camellia_##keylen##_##mode = { \ + nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + camellia_init_key, \ + camellia_##mode##_cipher, \ + NULL, \ + sizeof(EVP_CAMELLIA_KEY), \ + NULL,NULL,NULL,NULL }; \ +const EVP_CIPHER *EVP_camellia_##keylen##_##mode(void) \ +{ return &camellia_##keylen##_##mode; } + +# endif + +# define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \ + BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ + BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ + BLOCK_CIPHER_generic(nid,keylen,1,16,ofb128,ofb,OFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ + BLOCK_CIPHER_generic(nid,keylen,1,16,cfb128,cfb,CFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ + BLOCK_CIPHER_generic(nid,keylen,1,16,cfb1,cfb1,CFB,flags) \ + BLOCK_CIPHER_generic(nid,keylen,1,16,cfb8,cfb8,CFB,flags) +# if 0 /* not yet, missing NID */ +BLOCK_CIPHER_generic(nid, keylen, 1, 16, ctr, ctr, CTR, flags) +# endif /* The subkey for Camellia is generated. */ static int camellia_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { - int ret; - - ret = Camellia_set_key(key, ctx->key_len * 8, ctx->cipher_data); + int ret, mode; + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + ret = Camellia_set_key(key, ctx->key_len * 8, &dat->ks); if (ret < 0) { EVPerr(EVP_F_CAMELLIA_INIT_KEY, EVP_R_CAMELLIA_KEY_SETUP_FAILED); return 0; } + mode = ctx->cipher->flags & EVP_CIPH_MODE; + if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) + && !enc) { + dat->block = (block128_f) Camellia_decrypt; + dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? + (cbc128_f) Camellia_cbc_encrypt : NULL; + } else { + dat->block = (block128_f) Camellia_encrypt; + dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? + (cbc128_f) Camellia_cbc_encrypt : NULL; + } + return 1; } +static int camellia_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + if (dat->stream.cbc) + (*dat->stream.cbc) (in, out, len, &dat->ks, ctx->iv, ctx->encrypt); + else if (ctx->encrypt) + CRYPTO_cbc128_encrypt(in, out, len, &dat->ks, ctx->iv, dat->block); + else + CRYPTO_cbc128_decrypt(in, out, len, &dat->ks, ctx->iv, dat->block); + + return 1; +} + +static int camellia_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + size_t bl = ctx->cipher->block_size; + size_t i; + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + if (len < bl) + return 1; + + for (i = 0, len -= bl; i <= len; i += bl) + (*dat->block) (in + i, out + i, &dat->ks); + + return 1; +} + +static int camellia_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + CRYPTO_ofb128_encrypt(in, out, len, &dat->ks, + ctx->iv, &ctx->num, dat->block); + return 1; +} + +static int camellia_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + CRYPTO_cfb128_encrypt(in, out, len, &dat->ks, + ctx->iv, &ctx->num, ctx->encrypt, dat->block); + return 1; +} + +static int camellia_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + CRYPTO_cfb128_8_encrypt(in, out, len, &dat->ks, + ctx->iv, &ctx->num, ctx->encrypt, dat->block); + return 1; +} + +static int camellia_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + if (ctx->flags & EVP_CIPH_FLAG_LENGTH_BITS) { + CRYPTO_cfb128_1_encrypt(in, out, len, &dat->ks, + ctx->iv, &ctx->num, ctx->encrypt, dat->block); + return 1; + } + + while (len >= MAXBITCHUNK) { + CRYPTO_cfb128_1_encrypt(in, out, MAXBITCHUNK * 8, &dat->ks, + ctx->iv, &ctx->num, ctx->encrypt, dat->block); + len -= MAXBITCHUNK; + } + if (len) + CRYPTO_cfb128_1_encrypt(in, out, len * 8, &dat->ks, + ctx->iv, &ctx->num, ctx->encrypt, dat->block); + + return 1; +} + +# if 0 /* not yet, missing NID */ +static int camellia_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + unsigned int num = ctx->num; + EVP_CAMELLIA_KEY *dat = (EVP_CAMELLIA_KEY *) ctx->cipher_data; + + if (dat->stream.ctr) + CRYPTO_ctr128_encrypt_ctr32(in, out, len, &dat->ks, + ctx->iv, ctx->buf, &num, dat->stream.ctr); + else + CRYPTO_ctr128_encrypt(in, out, len, &dat->ks, + ctx->iv, ctx->buf, &num, dat->block); + ctx->num = (size_t)num; + return 1; +} +# endif + +BLOCK_CIPHER_generic_pack(NID_camellia, 128, 0) + BLOCK_CIPHER_generic_pack(NID_camellia, 192, 0) + BLOCK_CIPHER_generic_pack(NID_camellia, 256, 0) #else # ifdef PEDANTIC diff --git a/crypto/evp/e_des.c b/crypto/evp/e_des.c index ea1a4c422879..aae13a675694 100644 --- a/crypto/evp/e_des.c +++ b/crypto/evp/e_des.c @@ -65,6 +65,32 @@ # include # include +typedef struct { + union { + double align; + DES_key_schedule ks; + } ks; + union { + void (*cbc) (const void *, void *, size_t, const void *, void *); + } stream; +} EVP_DES_KEY; + +# if defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) +/* ---------^^^ this is not a typo, just a way to detect that + * assembler support was in general requested... */ +# include "sparc_arch.h" + +extern unsigned int OPENSSL_sparcv9cap_P[]; + +# define SPARC_DES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_DES) + +void des_t4_key_expand(const void *key, DES_key_schedule *ks); +void des_t4_cbc_encrypt(const void *inp, void *out, size_t len, + DES_key_schedule *ks, unsigned char iv[8]); +void des_t4_cbc_decrypt(const void *inp, void *out, size_t len, + DES_key_schedule *ks, unsigned char iv[8]); +# endif + static int des_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc); static int des_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr); @@ -102,6 +128,12 @@ static int des_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, static int des_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) { + EVP_DES_KEY *dat = (EVP_DES_KEY *) ctx->cipher_data; + + if (dat->stream.cbc) { + (*dat->stream.cbc) (in, out, inl, &dat->ks.ks, ctx->iv); + return 1; + } while (inl >= EVP_MAXCHUNK) { DES_ncbc_encrypt(in, out, (long)EVP_MAXCHUNK, ctx->cipher_data, (DES_cblock *)ctx->iv, ctx->encrypt); @@ -179,16 +211,15 @@ static int des_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return 1; } -BLOCK_CIPHER_defs(des, DES_key_schedule, NID_des, 8, 8, 8, 64, +BLOCK_CIPHER_defs(des, EVP_DES_KEY, NID_des, 8, 8, 8, 64, EVP_CIPH_RAND_KEY, des_init_key, NULL, EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, des_ctrl) + BLOCK_CIPHER_def_cfb(des, EVP_DES_KEY, NID_des, 8, 8, 1, + EVP_CIPH_RAND_KEY, des_init_key, NULL, + EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, des_ctrl) -BLOCK_CIPHER_def_cfb(des, DES_key_schedule, NID_des, 8, 8, 1, - EVP_CIPH_RAND_KEY, des_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, des_ctrl) - -BLOCK_CIPHER_def_cfb(des, DES_key_schedule, NID_des, 8, 8, 8, + BLOCK_CIPHER_def_cfb(des, EVP_DES_KEY, NID_des, 8, 8, 8, EVP_CIPH_RAND_KEY, des_init_key, NULL, EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, des_ctrl) @@ -196,8 +227,22 @@ static int des_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { DES_cblock *deskey = (DES_cblock *)key; + EVP_DES_KEY *dat = (EVP_DES_KEY *) ctx->cipher_data; + + dat->stream.cbc = NULL; +# if defined(SPARC_DES_CAPABLE) + if (SPARC_DES_CAPABLE) { + int mode = ctx->cipher->flags & EVP_CIPH_MODE; + + if (mode == EVP_CIPH_CBC_MODE) { + des_t4_key_expand(key, &dat->ks.ks); + dat->stream.cbc = enc ? des_t4_cbc_encrypt : des_t4_cbc_decrypt; + return 1; + } + } +# endif # ifdef EVP_CHECK_DES_KEY - if (DES_set_key_checked(deskey, ctx->cipher_data) != 0) + if (DES_set_key_checked(deskey, dat->ks.ks) != 0) return 0; # else DES_set_key_unchecked(deskey, ctx->cipher_data); diff --git a/crypto/evp/e_des3.c b/crypto/evp/e_des3.c index 07a5aca606de..96f272eb8046 100644 --- a/crypto/evp/e_des3.c +++ b/crypto/evp/e_des3.c @@ -65,7 +65,38 @@ # include # include -# ifndef OPENSSL_FIPS +/* Block use of implementations in FIPS mode */ +# undef EVP_CIPH_FLAG_FIPS +# define EVP_CIPH_FLAG_FIPS 0 + +typedef struct { + union { + double align; + DES_key_schedule ks[3]; + } ks; + union { + void (*cbc) (const void *, void *, size_t, const void *, void *); + } stream; +} DES_EDE_KEY; +# define ks1 ks.ks[0] +# define ks2 ks.ks[1] +# define ks3 ks.ks[2] + +# if defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) +/* ---------^^^ this is not a typo, just a way to detect that + * assembler support was in general requested... */ +# include "sparc_arch.h" + +extern unsigned int OPENSSL_sparcv9cap_P[]; + +# define SPARC_DES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_DES) + +void des_t4_key_expand(const void *key, DES_key_schedule *ks); +void des_t4_ede3_cbc_encrypt(const void *inp, void *out, size_t len, + DES_key_schedule *ks, unsigned char iv[8]); +void des_t4_ede3_cbc_decrypt(const void *inp, void *out, size_t len, + DES_key_schedule *ks, unsigned char iv[8]); +# endif static int des_ede_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc); @@ -75,13 +106,7 @@ static int des_ede3_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, static int des3_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr); -typedef struct { - DES_key_schedule ks1; /* key schedule */ - DES_key_schedule ks2; /* key schedule (for ede) */ - DES_key_schedule ks3; /* key schedule (for ede3) */ -} DES_EDE_KEY; - -# define data(ctx) ((DES_EDE_KEY *)(ctx)->cipher_data) +# define data(ctx) ((DES_EDE_KEY *)(ctx)->cipher_data) /* * Because of various casts and different args can't use @@ -123,7 +148,9 @@ static int des_ede_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, static int des_ede_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) { -# ifdef KSSL_DEBUG + DES_EDE_KEY *dat = data(ctx); + +# ifdef KSSL_DEBUG { int i; fprintf(stderr, "des_ede_cbc_cipher(ctx=%p, buflen=%d)\n", ctx, @@ -133,21 +160,24 @@ static int des_ede_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, fprintf(stderr, "%02X", ctx->iv[i]); fprintf(stderr, "\n"); } -# endif /* KSSL_DEBUG */ +# endif /* KSSL_DEBUG */ + if (dat->stream.cbc) { + (*dat->stream.cbc) (in, out, inl, &dat->ks, ctx->iv); + return 1; + } + while (inl >= EVP_MAXCHUNK) { DES_ede3_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, - &data(ctx)->ks1, &data(ctx)->ks2, - &data(ctx)->ks3, (DES_cblock *)ctx->iv, - ctx->encrypt); + &dat->ks1, &dat->ks2, &dat->ks3, + (DES_cblock *)ctx->iv, ctx->encrypt); inl -= EVP_MAXCHUNK; in += EVP_MAXCHUNK; out += EVP_MAXCHUNK; } if (inl) DES_ede3_cbc_encrypt(in, out, (long)inl, - &data(ctx)->ks1, &data(ctx)->ks2, - &data(ctx)->ks3, (DES_cblock *)ctx->iv, - ctx->encrypt); + &dat->ks1, &dat->ks2, &dat->ks3, + (DES_cblock *)ctx->iv, ctx->encrypt); return 1; } @@ -215,39 +245,57 @@ static int des_ede3_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } BLOCK_CIPHER_defs(des_ede, DES_EDE_KEY, NID_des_ede, 8, 16, 8, 64, - EVP_CIPH_RAND_KEY, des_ede_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, des3_ctrl) -# define des_ede3_cfb64_cipher des_ede_cfb64_cipher -# define des_ede3_ofb_cipher des_ede_ofb_cipher -# define des_ede3_cbc_cipher des_ede_cbc_cipher -# define des_ede3_ecb_cipher des_ede_ecb_cipher + EVP_CIPH_RAND_KEY | EVP_CIPH_FLAG_DEFAULT_ASN1, + des_ede_init_key, NULL, NULL, NULL, des3_ctrl) +# define des_ede3_cfb64_cipher des_ede_cfb64_cipher +# define des_ede3_ofb_cipher des_ede_ofb_cipher +# define des_ede3_cbc_cipher des_ede_cbc_cipher +# define des_ede3_ecb_cipher des_ede_ecb_cipher BLOCK_CIPHER_defs(des_ede3, DES_EDE_KEY, NID_des_ede3, 8, 24, 8, 64, - EVP_CIPH_RAND_KEY, des_ede3_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, des3_ctrl) + EVP_CIPH_RAND_KEY | EVP_CIPH_FLAG_FIPS | + EVP_CIPH_FLAG_DEFAULT_ASN1, des_ede3_init_key, NULL, NULL, NULL, + des3_ctrl) BLOCK_CIPHER_def_cfb(des_ede3, DES_EDE_KEY, NID_des_ede3, 24, 8, 1, - EVP_CIPH_RAND_KEY, des_ede3_init_key, NULL, - EVP_CIPHER_set_asn1_iv, - EVP_CIPHER_get_asn1_iv, des3_ctrl) + EVP_CIPH_RAND_KEY | EVP_CIPH_FLAG_FIPS | + EVP_CIPH_FLAG_DEFAULT_ASN1, des_ede3_init_key, NULL, NULL, + NULL, des3_ctrl) BLOCK_CIPHER_def_cfb(des_ede3, DES_EDE_KEY, NID_des_ede3, 24, 8, 8, - EVP_CIPH_RAND_KEY, des_ede3_init_key, NULL, - EVP_CIPHER_set_asn1_iv, - EVP_CIPHER_get_asn1_iv, des3_ctrl) + EVP_CIPH_RAND_KEY | EVP_CIPH_FLAG_FIPS | + EVP_CIPH_FLAG_DEFAULT_ASN1, des_ede3_init_key, NULL, NULL, + NULL, des3_ctrl) static int des_ede_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { DES_cblock *deskey = (DES_cblock *)key; -# ifdef EVP_CHECK_DES_KEY - if (DES_set_key_checked(&deskey[0], &data(ctx)->ks1) - ! !DES_set_key_checked(&deskey[1], &data(ctx)->ks2)) + DES_EDE_KEY *dat = data(ctx); + + dat->stream.cbc = NULL; +# if defined(SPARC_DES_CAPABLE) + if (SPARC_DES_CAPABLE) { + int mode = ctx->cipher->flags & EVP_CIPH_MODE; + + if (mode == EVP_CIPH_CBC_MODE) { + des_t4_key_expand(&deskey[0], &dat->ks1); + des_t4_key_expand(&deskey[1], &dat->ks2); + memcpy(&dat->ks3, &dat->ks1, sizeof(dat->ks1)); + dat->stream.cbc = enc ? des_t4_ede3_cbc_encrypt : + des_t4_ede3_cbc_decrypt; + return 1; + } + } +# endif +# ifdef EVP_CHECK_DES_KEY + if (DES_set_key_checked(&deskey[0], &dat->ks1) + ! !DES_set_key_checked(&deskey[1], &dat->ks2)) return 0; -# else - DES_set_key_unchecked(&deskey[0], &data(ctx)->ks1); - DES_set_key_unchecked(&deskey[1], &data(ctx)->ks2); -# endif - memcpy(&data(ctx)->ks3, &data(ctx)->ks1, sizeof(data(ctx)->ks1)); +# else + DES_set_key_unchecked(&deskey[0], &dat->ks1); + DES_set_key_unchecked(&deskey[1], &dat->ks2); +# endif + memcpy(&dat->ks3, &dat->ks1, sizeof(dat->ks1)); return 1; } @@ -255,7 +303,9 @@ static int des_ede3_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { DES_cblock *deskey = (DES_cblock *)key; -# ifdef KSSL_DEBUG + DES_EDE_KEY *dat = data(ctx); + +# ifdef KSSL_DEBUG { int i; fprintf(stderr, "des_ede3_init_key(ctx=%p)\n", ctx); @@ -270,18 +320,33 @@ static int des_ede3_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, fprintf(stderr, "\n"); } } -# endif /* KSSL_DEBUG */ +# endif /* KSSL_DEBUG */ + + dat->stream.cbc = NULL; +# if defined(SPARC_DES_CAPABLE) + if (SPARC_DES_CAPABLE) { + int mode = ctx->cipher->flags & EVP_CIPH_MODE; -# ifdef EVP_CHECK_DES_KEY - if (DES_set_key_checked(&deskey[0], &data(ctx)->ks1) - || DES_set_key_checked(&deskey[1], &data(ctx)->ks2) - || DES_set_key_checked(&deskey[2], &data(ctx)->ks3)) + if (mode == EVP_CIPH_CBC_MODE) { + des_t4_key_expand(&deskey[0], &dat->ks1); + des_t4_key_expand(&deskey[1], &dat->ks2); + des_t4_key_expand(&deskey[2], &dat->ks3); + dat->stream.cbc = enc ? des_t4_ede3_cbc_encrypt : + des_t4_ede3_cbc_decrypt; + return 1; + } + } +# endif +# ifdef EVP_CHECK_DES_KEY + if (DES_set_key_checked(&deskey[0], &dat->ks1) + || DES_set_key_checked(&deskey[1], &dat->ks2) + || DES_set_key_checked(&deskey[2], &dat->ks3)) return 0; -# else - DES_set_key_unchecked(&deskey[0], &data(ctx)->ks1); - DES_set_key_unchecked(&deskey[1], &data(ctx)->ks2); - DES_set_key_unchecked(&deskey[2], &data(ctx)->ks3); -# endif +# else + DES_set_key_unchecked(&deskey[0], &dat->ks1); + DES_set_key_unchecked(&deskey[1], &dat->ks2); + DES_set_key_unchecked(&deskey[2], &dat->ks3); +# endif return 1; } @@ -315,5 +380,115 @@ const EVP_CIPHER *EVP_des_ede3(void) { return &des_ede3_ecb; } + +# ifndef OPENSSL_NO_SHA + +# include + +static const unsigned char wrap_iv[8] = + { 0x4a, 0xdd, 0xa2, 0x2c, 0x79, 0xe8, 0x21, 0x05 }; + +static int des_ede3_unwrap(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t inl) +{ + unsigned char icv[8], iv[8], sha1tmp[SHA_DIGEST_LENGTH]; + int rv = -1; + if (inl < 24) + return -1; + if (!out) + return inl - 16; + memcpy(ctx->iv, wrap_iv, 8); + /* Decrypt first block which will end up as icv */ + des_ede_cbc_cipher(ctx, icv, in, 8); + /* Decrypt central blocks */ + /* + * If decrypting in place move whole output along a block so the next + * des_ede_cbc_cipher is in place. + */ + if (out == in) { + memmove(out, out + 8, inl - 8); + in -= 8; + } + des_ede_cbc_cipher(ctx, out, in + 8, inl - 16); + /* Decrypt final block which will be IV */ + des_ede_cbc_cipher(ctx, iv, in + inl - 8, 8); + /* Reverse order of everything */ + BUF_reverse(icv, NULL, 8); + BUF_reverse(out, NULL, inl - 16); + BUF_reverse(ctx->iv, iv, 8); + /* Decrypt again using new IV */ + des_ede_cbc_cipher(ctx, out, out, inl - 16); + des_ede_cbc_cipher(ctx, icv, icv, 8); + /* Work out SHA1 hash of first portion */ + SHA1(out, inl - 16, sha1tmp); + + if (!CRYPTO_memcmp(sha1tmp, icv, 8)) + rv = inl - 16; + OPENSSL_cleanse(icv, 8); + OPENSSL_cleanse(sha1tmp, SHA_DIGEST_LENGTH); + OPENSSL_cleanse(iv, 8); + OPENSSL_cleanse(ctx->iv, 8); + if (rv == -1) + OPENSSL_cleanse(out, inl - 16); + + return rv; +} + +static int des_ede3_wrap(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t inl) +{ + unsigned char sha1tmp[SHA_DIGEST_LENGTH]; + if (!out) + return inl + 16; + /* Copy input to output buffer + 8 so we have space for IV */ + memmove(out + 8, in, inl); + /* Work out ICV */ + SHA1(in, inl, sha1tmp); + memcpy(out + inl + 8, sha1tmp, 8); + OPENSSL_cleanse(sha1tmp, SHA_DIGEST_LENGTH); + /* Generate random IV */ + if (RAND_bytes(ctx->iv, 8) <= 0) + return -1; + memcpy(out, ctx->iv, 8); + /* Encrypt everything after IV in place */ + des_ede_cbc_cipher(ctx, out + 8, out + 8, inl + 8); + BUF_reverse(out, NULL, inl + 16); + memcpy(ctx->iv, wrap_iv, 8); + des_ede_cbc_cipher(ctx, out, out, inl + 16); + return inl + 16; +} + +static int des_ede3_wrap_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t inl) +{ + /* + * Sanity check input length: we typically only wrap keys so EVP_MAXCHUNK + * is more than will ever be needed. Also input length must be a multiple + * of 8 bits. + */ + if (inl >= EVP_MAXCHUNK || inl % 8) + return -1; + if (ctx->encrypt) + return des_ede3_wrap(ctx, out, in, inl); + else + return des_ede3_unwrap(ctx, out, in, inl); +} + +static const EVP_CIPHER des3_wrap = { + NID_id_smime_alg_CMS3DESwrap, + 8, 24, 0, + EVP_CIPH_WRAP_MODE | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER + | EVP_CIPH_FLAG_DEFAULT_ASN1, + des_ede3_init_key, des_ede3_wrap_cipher, + NULL, + sizeof(DES_EDE_KEY), + NULL, NULL, NULL, NULL +}; + +const EVP_CIPHER *EVP_des_ede3_wrap(void) +{ + return &des3_wrap; +} + # endif #endif diff --git a/crypto/evp/e_null.c b/crypto/evp/e_null.c index af90ce326bd9..599fcb808d21 100644 --- a/crypto/evp/e_null.c +++ b/crypto/evp/e_null.c @@ -61,8 +61,6 @@ #include #include -#ifndef OPENSSL_FIPS - static int null_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc); static int null_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, @@ -100,4 +98,3 @@ static int null_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, memcpy((char *)out, (const char *)in, inl); return 1; } -#endif diff --git a/crypto/evp/encode.c b/crypto/evp/encode.c index 5c5988fc45e1..c361d1f01269 100644 --- a/crypto/evp/encode.c +++ b/crypto/evp/encode.c @@ -248,7 +248,7 @@ int EVP_DecodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl, /* We parse the input data */ for (i = 0; i < inl; i++) { - /* If the current line is > 80 characters, scream alot */ + /* If the current line is > 80 characters, scream a lot */ if (ln >= 80) { rv = -1; goto end; diff --git a/crypto/evp/evp.h b/crypto/evp/evp.h index 6cf98acc0b79..39ab7937d256 100644 --- a/crypto/evp/evp.h +++ b/crypto/evp/evp.h @@ -113,6 +113,7 @@ # define EVP_PKEY_DSA3 NID_dsaWithSHA1 # define EVP_PKEY_DSA4 NID_dsaWithSHA1_2 # define EVP_PKEY_DH NID_dhKeyAgreement +# define EVP_PKEY_DHX NID_dhpublicnumber # define EVP_PKEY_EC NID_X9_62_id_ecPublicKey # define EVP_PKEY_HMAC NID_hmac # define EVP_PKEY_CMAC NID_cmac @@ -345,6 +346,7 @@ struct evp_cipher_st { # define EVP_CIPH_GCM_MODE 0x6 # define EVP_CIPH_CCM_MODE 0x7 # define EVP_CIPH_XTS_MODE 0x10001 +# define EVP_CIPH_WRAP_MODE 0x10002 # define EVP_CIPH_MODE 0xF0007 /* Set if variable length cipher */ # define EVP_CIPH_VARIABLE_LENGTH 0x8 @@ -375,6 +377,14 @@ struct evp_cipher_st { */ # define EVP_CIPH_FLAG_CUSTOM_CIPHER 0x100000 # define EVP_CIPH_FLAG_AEAD_CIPHER 0x200000 +# define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0x400000 + +/* + * Cipher context flag to indicate we can handle wrap mode: if allowed in + * older applications it could overflow buffers. + */ + +# define EVP_CIPHER_CTX_FLAG_WRAP_ALLOW 0x1 /* ctrl() values */ @@ -408,9 +418,21 @@ struct evp_cipher_st { /* Set the GCM invocation field, decrypt only */ # define EVP_CTRL_GCM_SET_IV_INV 0x18 +# define EVP_CTRL_TLS1_1_MULTIBLOCK_AAD 0x19 +# define EVP_CTRL_TLS1_1_MULTIBLOCK_ENCRYPT 0x1a +# define EVP_CTRL_TLS1_1_MULTIBLOCK_DECRYPT 0x1b +# define EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE 0x1c + /* RFC 5246 defines additional data to be 13 bytes in length */ # define EVP_AEAD_TLS1_AAD_LEN 13 +typedef struct { + unsigned char *out; + const unsigned char *inp; + size_t len; + unsigned int interleave; +} EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM; + /* GCM TLS constants */ /* Length of fixed part of IV derived from PRF */ # define EVP_GCM_TLS_FIXED_IV_LEN 4 @@ -639,7 +661,8 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, int EVP_DigestVerifyInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey); -int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, unsigned char *sig, size_t siglen); +int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, + const unsigned char *sig, size_t siglen); int EVP_OpenInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, const unsigned char *ek, int ekl, const unsigned char *iv, @@ -744,6 +767,7 @@ const EVP_CIPHER *EVP_des_cbc(void); const EVP_CIPHER *EVP_des_ede_cbc(void); const EVP_CIPHER *EVP_des_ede3_cbc(void); const EVP_CIPHER *EVP_desx_cbc(void); +const EVP_CIPHER *EVP_des_ede3_wrap(void); /* * This should now be supported through the dev_crypto ENGINE. But also, why * are rc4 and md5 declarations made here inside a "NO_DES" precompiler @@ -813,6 +837,7 @@ const EVP_CIPHER *EVP_aes_128_ctr(void); const EVP_CIPHER *EVP_aes_128_ccm(void); const EVP_CIPHER *EVP_aes_128_gcm(void); const EVP_CIPHER *EVP_aes_128_xts(void); +const EVP_CIPHER *EVP_aes_128_wrap(void); const EVP_CIPHER *EVP_aes_192_ecb(void); const EVP_CIPHER *EVP_aes_192_cbc(void); const EVP_CIPHER *EVP_aes_192_cfb1(void); @@ -823,6 +848,7 @@ const EVP_CIPHER *EVP_aes_192_ofb(void); const EVP_CIPHER *EVP_aes_192_ctr(void); const EVP_CIPHER *EVP_aes_192_ccm(void); const EVP_CIPHER *EVP_aes_192_gcm(void); +const EVP_CIPHER *EVP_aes_192_wrap(void); const EVP_CIPHER *EVP_aes_256_ecb(void); const EVP_CIPHER *EVP_aes_256_cbc(void); const EVP_CIPHER *EVP_aes_256_cfb1(void); @@ -834,10 +860,15 @@ const EVP_CIPHER *EVP_aes_256_ctr(void); const EVP_CIPHER *EVP_aes_256_ccm(void); const EVP_CIPHER *EVP_aes_256_gcm(void); const EVP_CIPHER *EVP_aes_256_xts(void); +const EVP_CIPHER *EVP_aes_256_wrap(void); # if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA1) const EVP_CIPHER *EVP_aes_128_cbc_hmac_sha1(void); const EVP_CIPHER *EVP_aes_256_cbc_hmac_sha1(void); # endif +# ifndef OPENSSL_NO_SHA256 +const EVP_CIPHER *EVP_aes_128_cbc_hmac_sha256(void); +const EVP_CIPHER *EVP_aes_256_cbc_hmac_sha256(void); +# endif # endif # ifndef OPENSSL_NO_CAMELLIA const EVP_CIPHER *EVP_camellia_128_ecb(void); @@ -1028,6 +1059,7 @@ void EVP_PBE_cleanup(void); # define ASN1_PKEY_CTRL_DEFAULT_MD_NID 0x3 # define ASN1_PKEY_CTRL_CMS_SIGN 0x5 # define ASN1_PKEY_CTRL_CMS_ENVELOPE 0x7 +# define ASN1_PKEY_CTRL_CMS_RI_TYPE 0x8 int EVP_PKEY_asn1_get_count(void); const EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_get0(int idx); @@ -1091,6 +1123,19 @@ void EVP_PKEY_asn1_set_free(EVP_PKEY_ASN1_METHOD *ameth, void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth, int (*pkey_ctrl) (EVP_PKEY *pkey, int op, long arg1, void *arg2)); +void EVP_PKEY_asn1_set_item(EVP_PKEY_ASN1_METHOD *ameth, + int (*item_verify) (EVP_MD_CTX *ctx, + const ASN1_ITEM *it, + void *asn, + X509_ALGOR *a, + ASN1_BIT_STRING *sig, + EVP_PKEY *pkey), + int (*item_sign) (EVP_MD_CTX *ctx, + const ASN1_ITEM *it, + void *asn, + X509_ALGOR *alg1, + X509_ALGOR *alg2, + ASN1_BIT_STRING *sig)); # define EVP_PKEY_OP_UNDEFINED 0 # define EVP_PKEY_OP_PARAMGEN (1<<1) @@ -1121,6 +1166,10 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth, EVP_PKEY_CTX_ctrl(ctx, -1, EVP_PKEY_OP_TYPE_SIG, \ EVP_PKEY_CTRL_MD, 0, (void *)md) +# define EVP_PKEY_CTX_get_signature_md(ctx, pmd) \ + EVP_PKEY_CTX_ctrl(ctx, -1, EVP_PKEY_OP_TYPE_SIG, \ + EVP_PKEY_CTRL_GET_MD, 0, (void *)pmd) + # define EVP_PKEY_CTRL_MD 1 # define EVP_PKEY_CTRL_PEER_KEY 2 @@ -1142,6 +1191,8 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth, # define EVP_PKEY_CTRL_CIPHER 12 +# define EVP_PKEY_CTRL_GET_MD 13 + # define EVP_PKEY_ALG_CTRL 0x1000 # define EVP_PKEY_FLAG_AUTOARGLEN 2 @@ -1327,11 +1378,13 @@ void ERR_load_EVP_strings(void); # define EVP_F_AESNI_INIT_KEY 165 # define EVP_F_AESNI_XTS_CIPHER 176 # define EVP_F_AES_INIT_KEY 133 +# define EVP_F_AES_T4_INIT_KEY 178 # define EVP_F_AES_XTS 172 # define EVP_F_AES_XTS_CIPHER 175 # define EVP_F_ALG_MODULE_INIT 177 # define EVP_F_CAMELLIA_INIT_KEY 159 # define EVP_F_CMAC_INIT 173 +# define EVP_F_CMLL_T4_INIT_KEY 179 # define EVP_F_D2I_PKEY 100 # define EVP_F_DO_SIGVER_INIT 161 # define EVP_F_DSAPKEY2PKCS8 134 @@ -1471,6 +1524,7 @@ void ERR_load_EVP_strings(void); # define EVP_R_UNSUPPORTED_PRF 125 # define EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM 118 # define EVP_R_UNSUPPORTED_SALT_TYPE 126 +# define EVP_R_WRAP_MODE_NOT_ALLOWED 170 # define EVP_R_WRONG_FINAL_BLOCK_LENGTH 109 # define EVP_R_WRONG_PUBLIC_KEY_TYPE 110 diff --git a/crypto/evp/evp_enc.c b/crypto/evp/evp_enc.c index 4e983c4bda23..65f0e0244dce 100644 --- a/crypto/evp/evp_enc.c +++ b/crypto/evp/evp_enc.c @@ -169,8 +169,14 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, #endif #ifdef OPENSSL_FIPS - if (FIPS_mode()) + if (FIPS_mode()) { + const EVP_CIPHER *fcipher; + if (cipher) + fcipher = evp_get_fips_cipher(cipher); + if (fcipher) + cipher = fcipher; return FIPS_cipherinit(ctx, cipher, key, iv, enc); + } #endif ctx->cipher = cipher; if (ctx->cipher->ctx_size) { @@ -183,7 +189,8 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ctx->cipher_data = NULL; } ctx->key_len = cipher->key_len; - ctx->flags = 0; + /* Preserve wrap enable flag, zero everything else */ + ctx->flags &= EVP_CIPHER_CTX_FLAG_WRAP_ALLOW; if (ctx->cipher->flags & EVP_CIPH_CTRL_INIT) { if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_INIT, 0, NULL)) { EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_INITIALIZATION_ERROR); @@ -206,6 +213,12 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, || ctx->cipher->block_size == 8 || ctx->cipher->block_size == 16); + if (!(ctx->flags & EVP_CIPHER_CTX_FLAG_WRAP_ALLOW) + && EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_WRAP_MODE) { + EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_WRAP_MODE_NOT_ALLOWED); + return 0; + } + if (!(EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_CUSTOM_IV)) { switch (EVP_CIPHER_CTX_mode(ctx)) { diff --git a/crypto/evp/evp_err.c b/crypto/evp/evp_err.c index 686a69958dd7..15cf5532b382 100644 --- a/crypto/evp/evp_err.c +++ b/crypto/evp/evp_err.c @@ -1,6 +1,6 @@ /* crypto/evp/evp_err.c */ /* ==================================================================== - * Copyright (c) 1999-2011 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2013 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -73,11 +73,13 @@ static ERR_STRING_DATA EVP_str_functs[] = { {ERR_FUNC(EVP_F_AESNI_INIT_KEY), "AESNI_INIT_KEY"}, {ERR_FUNC(EVP_F_AESNI_XTS_CIPHER), "AESNI_XTS_CIPHER"}, {ERR_FUNC(EVP_F_AES_INIT_KEY), "AES_INIT_KEY"}, + {ERR_FUNC(EVP_F_AES_T4_INIT_KEY), "AES_T4_INIT_KEY"}, {ERR_FUNC(EVP_F_AES_XTS), "AES_XTS"}, {ERR_FUNC(EVP_F_AES_XTS_CIPHER), "AES_XTS_CIPHER"}, {ERR_FUNC(EVP_F_ALG_MODULE_INIT), "ALG_MODULE_INIT"}, {ERR_FUNC(EVP_F_CAMELLIA_INIT_KEY), "CAMELLIA_INIT_KEY"}, {ERR_FUNC(EVP_F_CMAC_INIT), "CMAC_INIT"}, + {ERR_FUNC(EVP_F_CMLL_T4_INIT_KEY), "CMLL_T4_INIT_KEY"}, {ERR_FUNC(EVP_F_D2I_PKEY), "D2I_PKEY"}, {ERR_FUNC(EVP_F_DO_SIGVER_INIT), "DO_SIGVER_INIT"}, {ERR_FUNC(EVP_F_DSAPKEY2PKCS8), "DSAPKEY2PKCS8"}, @@ -232,6 +234,7 @@ static ERR_STRING_DATA EVP_str_reasons[] = { {ERR_REASON(EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM), "unsupported private key algorithm"}, {ERR_REASON(EVP_R_UNSUPPORTED_SALT_TYPE), "unsupported salt type"}, + {ERR_REASON(EVP_R_WRAP_MODE_NOT_ALLOWED), "wrap mode not allowed"}, {ERR_REASON(EVP_R_WRONG_FINAL_BLOCK_LENGTH), "wrong final block length"}, {ERR_REASON(EVP_R_WRONG_PUBLIC_KEY_TYPE), "wrong public key type"}, {0, NULL} diff --git a/crypto/evp/evp_extra_test.c b/crypto/evp/evp_extra_test.c index 21688b0cd383..0f7b011ce85e 100644 --- a/crypto/evp/evp_extra_test.c +++ b/crypto/evp/evp_extra_test.c @@ -345,7 +345,7 @@ static int test_EVP_DigestVerifyInit(void) if (pkey == NULL || !EVP_DigestVerifyInit(&md_ctx, NULL, EVP_sha256(), NULL, pkey) || !EVP_DigestVerifyUpdate(&md_ctx, kMsg, sizeof(kMsg)) || - !EVP_DigestVerifyFinal(&md_ctx, (unsigned char *)kSignature, sizeof(kSignature))) { + !EVP_DigestVerifyFinal(&md_ctx, kSignature, sizeof(kSignature))) { goto out; } ret = 1; diff --git a/crypto/evp/evp_fips.c b/crypto/evp/evp_fips.c deleted file mode 100644 index 71a32fca8d2a..000000000000 --- a/crypto/evp/evp_fips.c +++ /dev/null @@ -1,310 +0,0 @@ -/* crypto/evp/evp_fips.c */ -/* - * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL - * project. - */ -/* ==================================================================== - * Copyright (c) 2011 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * licensing@OpenSSL.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - */ - -#include - -#ifdef OPENSSL_FIPS -# include - -const EVP_CIPHER *EVP_aes_128_cbc(void) -{ - return FIPS_evp_aes_128_cbc(); -} - -const EVP_CIPHER *EVP_aes_128_ccm(void) -{ - return FIPS_evp_aes_128_ccm(); -} - -const EVP_CIPHER *EVP_aes_128_cfb1(void) -{ - return FIPS_evp_aes_128_cfb1(); -} - -const EVP_CIPHER *EVP_aes_128_cfb128(void) -{ - return FIPS_evp_aes_128_cfb128(); -} - -const EVP_CIPHER *EVP_aes_128_cfb8(void) -{ - return FIPS_evp_aes_128_cfb8(); -} - -const EVP_CIPHER *EVP_aes_128_ctr(void) -{ - return FIPS_evp_aes_128_ctr(); -} - -const EVP_CIPHER *EVP_aes_128_ecb(void) -{ - return FIPS_evp_aes_128_ecb(); -} - -const EVP_CIPHER *EVP_aes_128_gcm(void) -{ - return FIPS_evp_aes_128_gcm(); -} - -const EVP_CIPHER *EVP_aes_128_ofb(void) -{ - return FIPS_evp_aes_128_ofb(); -} - -const EVP_CIPHER *EVP_aes_128_xts(void) -{ - return FIPS_evp_aes_128_xts(); -} - -const EVP_CIPHER *EVP_aes_192_cbc(void) -{ - return FIPS_evp_aes_192_cbc(); -} - -const EVP_CIPHER *EVP_aes_192_ccm(void) -{ - return FIPS_evp_aes_192_ccm(); -} - -const EVP_CIPHER *EVP_aes_192_cfb1(void) -{ - return FIPS_evp_aes_192_cfb1(); -} - -const EVP_CIPHER *EVP_aes_192_cfb128(void) -{ - return FIPS_evp_aes_192_cfb128(); -} - -const EVP_CIPHER *EVP_aes_192_cfb8(void) -{ - return FIPS_evp_aes_192_cfb8(); -} - -const EVP_CIPHER *EVP_aes_192_ctr(void) -{ - return FIPS_evp_aes_192_ctr(); -} - -const EVP_CIPHER *EVP_aes_192_ecb(void) -{ - return FIPS_evp_aes_192_ecb(); -} - -const EVP_CIPHER *EVP_aes_192_gcm(void) -{ - return FIPS_evp_aes_192_gcm(); -} - -const EVP_CIPHER *EVP_aes_192_ofb(void) -{ - return FIPS_evp_aes_192_ofb(); -} - -const EVP_CIPHER *EVP_aes_256_cbc(void) -{ - return FIPS_evp_aes_256_cbc(); -} - -const EVP_CIPHER *EVP_aes_256_ccm(void) -{ - return FIPS_evp_aes_256_ccm(); -} - -const EVP_CIPHER *EVP_aes_256_cfb1(void) -{ - return FIPS_evp_aes_256_cfb1(); -} - -const EVP_CIPHER *EVP_aes_256_cfb128(void) -{ - return FIPS_evp_aes_256_cfb128(); -} - -const EVP_CIPHER *EVP_aes_256_cfb8(void) -{ - return FIPS_evp_aes_256_cfb8(); -} - -const EVP_CIPHER *EVP_aes_256_ctr(void) -{ - return FIPS_evp_aes_256_ctr(); -} - -const EVP_CIPHER *EVP_aes_256_ecb(void) -{ - return FIPS_evp_aes_256_ecb(); -} - -const EVP_CIPHER *EVP_aes_256_gcm(void) -{ - return FIPS_evp_aes_256_gcm(); -} - -const EVP_CIPHER *EVP_aes_256_ofb(void) -{ - return FIPS_evp_aes_256_ofb(); -} - -const EVP_CIPHER *EVP_aes_256_xts(void) -{ - return FIPS_evp_aes_256_xts(); -} - -const EVP_CIPHER *EVP_des_ede(void) -{ - return FIPS_evp_des_ede(); -} - -const EVP_CIPHER *EVP_des_ede3(void) -{ - return FIPS_evp_des_ede3(); -} - -const EVP_CIPHER *EVP_des_ede3_cbc(void) -{ - return FIPS_evp_des_ede3_cbc(); -} - -const EVP_CIPHER *EVP_des_ede3_cfb1(void) -{ - return FIPS_evp_des_ede3_cfb1(); -} - -const EVP_CIPHER *EVP_des_ede3_cfb64(void) -{ - return FIPS_evp_des_ede3_cfb64(); -} - -const EVP_CIPHER *EVP_des_ede3_cfb8(void) -{ - return FIPS_evp_des_ede3_cfb8(); -} - -const EVP_CIPHER *EVP_des_ede3_ecb(void) -{ - return FIPS_evp_des_ede3_ecb(); -} - -const EVP_CIPHER *EVP_des_ede3_ofb(void) -{ - return FIPS_evp_des_ede3_ofb(); -} - -const EVP_CIPHER *EVP_des_ede_cbc(void) -{ - return FIPS_evp_des_ede_cbc(); -} - -const EVP_CIPHER *EVP_des_ede_cfb64(void) -{ - return FIPS_evp_des_ede_cfb64(); -} - -const EVP_CIPHER *EVP_des_ede_ecb(void) -{ - return FIPS_evp_des_ede_ecb(); -} - -const EVP_CIPHER *EVP_des_ede_ofb(void) -{ - return FIPS_evp_des_ede_ofb(); -} - -const EVP_CIPHER *EVP_enc_null(void) -{ - return FIPS_evp_enc_null(); -} - -const EVP_MD *EVP_sha1(void) -{ - return FIPS_evp_sha1(); -} - -const EVP_MD *EVP_sha224(void) -{ - return FIPS_evp_sha224(); -} - -const EVP_MD *EVP_sha256(void) -{ - return FIPS_evp_sha256(); -} - -const EVP_MD *EVP_sha384(void) -{ - return FIPS_evp_sha384(); -} - -const EVP_MD *EVP_sha512(void) -{ - return FIPS_evp_sha512(); -} - -const EVP_MD *EVP_dss(void) -{ - return FIPS_evp_dss(); -} - -const EVP_MD *EVP_dss1(void) -{ - return FIPS_evp_dss1(); -} - -const EVP_MD *EVP_ecdsa(void) -{ - return FIPS_evp_ecdsa(); -} - -#endif diff --git a/crypto/evp/evp_lib.c b/crypto/evp/evp_lib.c index d4d2b4b28081..a53a27ca0c92 100644 --- a/crypto/evp/evp_lib.c +++ b/crypto/evp/evp_lib.c @@ -60,6 +60,10 @@ #include "cryptlib.h" #include #include +#ifdef OPENSSL_FIPS +# include +# include "evp_locl.h" +#endif int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type) { @@ -67,9 +71,13 @@ int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type) if (c->cipher->set_asn1_parameters != NULL) ret = c->cipher->set_asn1_parameters(c, type); - else if (c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) - ret = EVP_CIPHER_set_asn1_iv(c, type); - else + else if (c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) { + if (EVP_CIPHER_CTX_mode(c) == EVP_CIPH_WRAP_MODE) { + ASN1_TYPE_set(type, V_ASN1_NULL, NULL); + ret = 1; + } else + ret = EVP_CIPHER_set_asn1_iv(c, type); + } else ret = -1; return (ret); } @@ -80,9 +88,11 @@ int EVP_CIPHER_asn1_to_param(EVP_CIPHER_CTX *c, ASN1_TYPE *type) if (c->cipher->get_asn1_parameters != NULL) ret = c->cipher->get_asn1_parameters(c, type); - else if (c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) + else if (c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) { + if (EVP_CIPHER_CTX_mode(c) == EVP_CIPH_WRAP_MODE) + return 1; ret = EVP_CIPHER_get_asn1_iv(c, type); - else + } else ret = -1; return (ret); } @@ -200,12 +210,22 @@ const EVP_CIPHER *EVP_CIPHER_CTX_cipher(const EVP_CIPHER_CTX *ctx) unsigned long EVP_CIPHER_flags(const EVP_CIPHER *cipher) { +#ifdef OPENSSL_FIPS + const EVP_CIPHER *fcipher; + fcipher = evp_get_fips_cipher(cipher); + if (fcipher && fcipher->flags & EVP_CIPH_FLAG_FIPS) + return cipher->flags | EVP_CIPH_FLAG_FIPS; +#endif return cipher->flags; } unsigned long EVP_CIPHER_CTX_flags(const EVP_CIPHER_CTX *ctx) { +#ifdef OPENSSL_FIPS + return EVP_CIPHER_flags(ctx->cipher); +#else return ctx->cipher->flags; +#endif } void *EVP_CIPHER_CTX_get_app_data(const EVP_CIPHER_CTX *ctx) @@ -272,8 +292,40 @@ int EVP_MD_size(const EVP_MD *md) return md->md_size; } +#ifdef OPENSSL_FIPS + +const EVP_MD *evp_get_fips_md(const EVP_MD *md) +{ + int nid = EVP_MD_type(md); + if (nid == NID_dsa) + return FIPS_evp_dss1(); + else if (nid == NID_dsaWithSHA) + return FIPS_evp_dss(); + else if (nid == NID_ecdsa_with_SHA1) + return FIPS_evp_ecdsa(); + else + return FIPS_get_digestbynid(nid); +} + +const EVP_CIPHER *evp_get_fips_cipher(const EVP_CIPHER *cipher) +{ + int nid = cipher->nid; + if (nid == NID_undef) + return FIPS_evp_enc_null(); + else + return FIPS_get_cipherbynid(nid); +} + +#endif + unsigned long EVP_MD_flags(const EVP_MD *md) { +#ifdef OPENSSL_FIPS + const EVP_MD *fmd; + fmd = evp_get_fips_md(md); + if (fmd && fmd->flags & EVP_MD_FLAG_FIPS) + return md->flags | EVP_MD_FLAG_FIPS; +#endif return md->flags; } diff --git a/crypto/evp/evp_locl.h b/crypto/evp/evp_locl.h index 980dadab23d4..2bb709a065d5 100644 --- a/crypto/evp/evp_locl.h +++ b/crypto/evp/evp_locl.h @@ -333,6 +333,9 @@ int PKCS5_v2_PBKDF2_keyivgen(EVP_CIPHER_CTX *ctx, const char *pass, const EVP_CIPHER *c, const EVP_MD *md, int en_de); +const EVP_MD *evp_get_fips_md(const EVP_MD *md); +const EVP_CIPHER *evp_get_fips_cipher(const EVP_CIPHER *cipher); + #ifdef OPENSSL_FIPS # ifdef OPENSSL_DOING_MAKEDEPEND diff --git a/crypto/evp/evp_test.c b/crypto/evp/evp_test.c index d06b4ee50914..d7441ec7b702 100644 --- a/crypto/evp/evp_test.c +++ b/crypto/evp/evp_test.c @@ -132,11 +132,13 @@ static int test1_exit(int ec) static void test1(const EVP_CIPHER *c, const unsigned char *key, int kn, const unsigned char *iv, int in, const unsigned char *plaintext, int pn, - const unsigned char *ciphertext, int cn, int encdec) + const unsigned char *ciphertext, int cn, + const unsigned char *aad, int an, + const unsigned char *tag, int tn, int encdec) { EVP_CIPHER_CTX ctx; unsigned char out[4096]; - int outl, outl2; + int outl, outl2, mode; printf("Testing cipher %s%s\n", EVP_CIPHER_name(c), (encdec == @@ -147,15 +149,78 @@ static void test1(const EVP_CIPHER *c, const unsigned char *key, int kn, hexdump(stdout, "IV", iv, in); hexdump(stdout, "Plaintext", plaintext, pn); hexdump(stdout, "Ciphertext", ciphertext, cn); - - if (kn != c->key_len) { + if (an) + hexdump(stdout, "AAD", aad, an); + if (tn) + hexdump(stdout, "Tag", tag, tn); + mode = EVP_CIPHER_mode(c); + if (kn != EVP_CIPHER_key_length(c)) { fprintf(stderr, "Key length doesn't match, got %d expected %lu\n", kn, - (unsigned long)c->key_len); + (unsigned long)EVP_CIPHER_key_length(c)); test1_exit(5); } EVP_CIPHER_CTX_init(&ctx); + EVP_CIPHER_CTX_set_flags(&ctx, EVP_CIPHER_CTX_FLAG_WRAP_ALLOW); if (encdec != 0) { - if (!EVP_EncryptInit_ex(&ctx, c, NULL, key, iv)) { + if (mode == EVP_CIPH_GCM_MODE) { + if (!EVP_EncryptInit_ex(&ctx, c, NULL, NULL, NULL)) { + fprintf(stderr, "EncryptInit failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(10); + } + if (!EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_GCM_SET_IVLEN, in, NULL)) { + fprintf(stderr, "IV length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(11); + } + if (!EVP_EncryptInit_ex(&ctx, NULL, NULL, key, iv)) { + fprintf(stderr, "Key/IV set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(12); + } + if (an && !EVP_EncryptUpdate(&ctx, NULL, &outl, aad, an)) { + fprintf(stderr, "AAD set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(13); + } + } else if (mode == EVP_CIPH_CCM_MODE) { + if (!EVP_EncryptInit_ex(&ctx, c, NULL, NULL, NULL)) { + fprintf(stderr, "EncryptInit failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(10); + } + if (!EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_CCM_SET_IVLEN, in, NULL)) { + fprintf(stderr, "IV length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(11); + } + if (!EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_CCM_SET_TAG, tn, NULL)) { + fprintf(stderr, "Tag length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(11); + } + if (!EVP_EncryptInit_ex(&ctx, NULL, NULL, key, iv)) { + fprintf(stderr, "Key/IV set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(12); + } + if (!EVP_EncryptUpdate(&ctx, NULL, &outl, NULL, pn)) { + fprintf(stderr, "Plaintext length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(12); + } + if (an && !EVP_EncryptUpdate(&ctx, NULL, &outl, aad, an)) { + fprintf(stderr, "AAD set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(13); + } + } else if (mode == EVP_CIPH_WRAP_MODE) { + if (!EVP_EncryptInit_ex(&ctx, c, NULL, key, in ? iv : NULL)) { + fprintf(stderr, "EncryptInit failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(10); + } + } else if (!EVP_EncryptInit_ex(&ctx, c, NULL, key, iv)) { fprintf(stderr, "EncryptInit failed\n"); ERR_print_errors_fp(stderr); test1_exit(10); @@ -185,10 +250,93 @@ static void test1(const EVP_CIPHER *c, const unsigned char *key, int kn, hexdump(stderr, "Expected", ciphertext, cn); test1_exit(9); } + if (mode == EVP_CIPH_GCM_MODE || mode == EVP_CIPH_CCM_MODE) { + unsigned char rtag[16]; + /* + * Note: EVP_CTRL_CCM_GET_TAG has same value as + * EVP_CTRL_GCM_GET_TAG + */ + if (!EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_GCM_GET_TAG, tn, rtag)) { + fprintf(stderr, "Get tag failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(14); + } + if (memcmp(rtag, tag, tn)) { + fprintf(stderr, "Tag mismatch\n"); + hexdump(stderr, "Got", rtag, tn); + hexdump(stderr, "Expected", tag, tn); + test1_exit(9); + } + } } if (encdec <= 0) { - if (!EVP_DecryptInit_ex(&ctx, c, NULL, key, iv)) { + if (mode == EVP_CIPH_GCM_MODE) { + if (!EVP_DecryptInit_ex(&ctx, c, NULL, NULL, NULL)) { + fprintf(stderr, "EncryptInit failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(10); + } + if (!EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_GCM_SET_IVLEN, in, NULL)) { + fprintf(stderr, "IV length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(11); + } + if (!EVP_DecryptInit_ex(&ctx, NULL, NULL, key, iv)) { + fprintf(stderr, "Key/IV set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(12); + } + if (!EVP_CIPHER_CTX_ctrl + (&ctx, EVP_CTRL_GCM_SET_TAG, tn, (void *)tag)) { + fprintf(stderr, "Set tag failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(14); + } + if (an && !EVP_DecryptUpdate(&ctx, NULL, &outl, aad, an)) { + fprintf(stderr, "AAD set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(13); + } + } else if (mode == EVP_CIPH_CCM_MODE) { + if (!EVP_DecryptInit_ex(&ctx, c, NULL, NULL, NULL)) { + fprintf(stderr, "DecryptInit failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(10); + } + if (!EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_CCM_SET_IVLEN, in, NULL)) { + fprintf(stderr, "IV length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(11); + } + if (!EVP_CIPHER_CTX_ctrl + (&ctx, EVP_CTRL_CCM_SET_TAG, tn, (void *)tag)) { + fprintf(stderr, "Tag length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(11); + } + if (!EVP_DecryptInit_ex(&ctx, NULL, NULL, key, iv)) { + fprintf(stderr, "Key/Nonce set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(12); + } + if (!EVP_DecryptUpdate(&ctx, NULL, &outl, NULL, pn)) { + fprintf(stderr, "Plaintext length set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(12); + } + if (an && !EVP_EncryptUpdate(&ctx, NULL, &outl, aad, an)) { + fprintf(stderr, "AAD set failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(13); + } + } else if (mode == EVP_CIPH_WRAP_MODE) { + if (!EVP_DecryptInit_ex(&ctx, c, NULL, key, in ? iv : NULL)) { + fprintf(stderr, "EncryptInit failed\n"); + ERR_print_errors_fp(stderr); + test1_exit(10); + } + } else if (!EVP_DecryptInit_ex(&ctx, c, NULL, key, iv)) { fprintf(stderr, "DecryptInit failed\n"); ERR_print_errors_fp(stderr); test1_exit(11); @@ -200,7 +348,8 @@ static void test1(const EVP_CIPHER *c, const unsigned char *key, int kn, ERR_print_errors_fp(stderr); test1_exit(6); } - if (!EVP_DecryptFinal_ex(&ctx, out + outl, &outl2)) { + if (mode != EVP_CIPH_CCM_MODE + && !EVP_DecryptFinal_ex(&ctx, out + outl, &outl2)) { fprintf(stderr, "DecryptFinal failed\n"); ERR_print_errors_fp(stderr); test1_exit(7); @@ -228,7 +377,9 @@ static void test1(const EVP_CIPHER *c, const unsigned char *key, int kn, static int test_cipher(const char *cipher, const unsigned char *key, int kn, const unsigned char *iv, int in, const unsigned char *plaintext, int pn, - const unsigned char *ciphertext, int cn, int encdec) + const unsigned char *ciphertext, int cn, + const unsigned char *aad, int an, + const unsigned char *tag, int tn, int encdec) { const EVP_CIPHER *c; @@ -236,7 +387,8 @@ static int test_cipher(const char *cipher, const unsigned char *key, int kn, if (!c) return 0; - test1(c, key, kn, iv, in, plaintext, pn, ciphertext, cn, encdec); + test1(c, key, kn, iv, in, plaintext, pn, ciphertext, cn, aad, an, tag, tn, + encdec); return 1; } @@ -316,7 +468,7 @@ int main(int argc, char **argv) perror(szTestFile); EXIT(2); } - + ERR_load_crypto_strings(); /* Load up the software EVP_CIPHER and EVP_MD definitions */ OpenSSL_add_all_ciphers(); OpenSSL_add_all_digests(); @@ -346,9 +498,11 @@ int main(int argc, char **argv) char line[4096]; char *p; char *cipher; - unsigned char *iv, *key, *plaintext, *ciphertext; + unsigned char *iv, *key, *plaintext, *ciphertext, *aad, *tag; int encdec; int kn, in, pn, cn; + int an = 0; + int tn = 0; if (!fgets((char *)line, sizeof line, f)) break; @@ -361,19 +515,37 @@ int main(int argc, char **argv) plaintext = ustrsep(&p, ":"); ciphertext = ustrsep(&p, ":"); if (p[-1] == '\n') { - p[-1] = '\0'; encdec = -1; + p[-1] = '\0'; + tag = aad = NULL; + an = tn = 0; } else { - encdec = atoi(sstrsep(&p, "\n")); + aad = ustrsep(&p, ":"); + tag = ustrsep(&p, ":"); + if (tag == NULL) { + p = (char *)aad; + tag = aad = NULL; + an = tn = 0; + } + if (p[-1] == '\n') { + encdec = -1; + p[-1] = '\0'; + } else + encdec = atoi(sstrsep(&p, "\n")); } kn = convert(key); in = convert(iv); pn = convert(plaintext); cn = convert(ciphertext); + if (aad) { + an = convert(aad); + tn = convert(tag); + } if (!test_cipher - (cipher, key, kn, iv, in, plaintext, pn, ciphertext, cn, encdec) + (cipher, key, kn, iv, in, plaintext, pn, ciphertext, cn, aad, an, + tag, tn, encdec) && !test_digest(cipher, plaintext, pn, ciphertext, cn)) { #ifdef OPENSSL_NO_AES if (strstr(cipher, "AES") == cipher) { diff --git a/crypto/evp/evptests.txt b/crypto/evp/evptests.txt index c273707c1444..4e9958b3b5bc 100644 --- a/crypto/evp/evptests.txt +++ b/crypto/evp/evptests.txt @@ -1,4 +1,5 @@ #cipher:key:iv:plaintext:ciphertext:0/1(decrypt/encrypt) +#aadcipher:key:iv:plaintext:ciphertext:aad:tag:0/1(decrypt/encrypt) #digest:::input:output # SHA(1) tests (from shatest.c) @@ -332,3 +333,69 @@ SEED-ECB:00000000000000000000000000000000::000102030405060708090A0B0C0D0E0F:5EBA SEED-ECB:000102030405060708090A0B0C0D0E0F::00000000000000000000000000000000:C11F22F20140505084483597E4370F43:1 SEED-ECB:4706480851E61BE85D74BFB3FD956185::83A2F8A288641FB9A4E9A5CC2F131C7D:EE54D13EBCAE706D226BC3142CD40D4A:1 SEED-ECB:28DBC3BC49FFD87DCFA509B11D422BE7::B41E6BE2EBA84A148E2EED84593C5EC7:9B9B7BFCD1813CB95D0B3618F40F5122:1 + +# AES CCM 256 bit key +aes-256-ccm:1bde3251d41a8b5ea013c195ae128b218b3e0306376357077ef1c1c78548b92e:5b8e40746f6b98e00f1d13ff41:53bd72a97089e312422bf72e242377b3c6ee3e2075389b999c4ef7f28bd2b80a:9a5fcccdb4cf04e7293d2775cc76a488f042382d949b43b7d6bb2b9864786726:c17a32514eb6103f3249e076d4c871dc97e04b286699e54491dc18f6d734d4c0:2024931d73bca480c24a24ece6b6c2bf + +# AES GCM test vectors from http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-spec.pdf +aes-128-gcm:00000000000000000000000000000000:000000000000000000000000::::58e2fccefa7e3061367f1d57a4e7455a +aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:00000000000000000000000000000000:0388dace60b6a392f328c2b971b2fe78::ab6e47d42cec13bdf53a67b21257bddf +aes-128-gcm:feffe9928665731c6d6a8f9467308308:cafebabefacedbaddecaf888:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255:42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091473f5985::4d5c2af327cd64a62cf35abd2ba6fab4 +aes-128-gcm:feffe9928665731c6d6a8f9467308308:cafebabefacedbaddecaf888:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091:feedfacedeadbeeffeedfacedeadbeefabaddad2:5bc94fbc3221a5db94fae95ae7121a47 +aes-128-gcm:feffe9928665731c6d6a8f9467308308:cafebabefacedbad:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:61353b4c2806934a777ff51fa22a4755699b2a714fcdc6f83766e5f97b6c742373806900e49f24b22b097544d4896b424989b5e1ebac0f07c23f4598:feedfacedeadbeeffeedfacedeadbeefabaddad2:3612d2e79e3b0785561be14aaca2fccb +aes-128-gcm:feffe9928665731c6d6a8f9467308308:9313225df88406e555909c5aff5269aa6a7a9538534f7da1e4c303d2a318a728c3c0c95156809539fcf0e2429a6b525416aedbf5a0de6a57a637b39b:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:8ce24998625615b603a033aca13fb894be9112a5c3a211a8ba262a3cca7e2ca701e4a9a4fba43c90ccdcb281d48c7c6fd62875d2aca417034c34aee5:feedfacedeadbeeffeedfacedeadbeefabaddad2:619cc5aefffe0bfa462af43c1699d050 +aes-192-gcm:000000000000000000000000000000000000000000000000:000000000000000000000000::::cd33b28ac773f74ba00ed1f312572435 +aes-192-gcm:000000000000000000000000000000000000000000000000:000000000000000000000000:00000000000000000000000000000000:98e7247c07f0fe411c267e4384b0f600::2ff58d80033927ab8ef4d4587514f0fb +aes-192-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c:cafebabefacedbaddecaf888:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255:3980ca0b3c00e841eb06fac4872a2757859e1ceaa6efd984628593b40ca1e19c7d773d00c144c525ac619d18c84a3f4718e2448b2fe324d9ccda2710acade256::9924a7c8587336bfb118024db8674a14 +aes-192-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c:cafebabefacedbaddecaf888:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:3980ca0b3c00e841eb06fac4872a2757859e1ceaa6efd984628593b40ca1e19c7d773d00c144c525ac619d18c84a3f4718e2448b2fe324d9ccda2710:feedfacedeadbeeffeedfacedeadbeefabaddad2:2519498e80f1478f37ba55bd6d27618c +aes-192-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c:cafebabefacedbad:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:0f10f599ae14a154ed24b36e25324db8c566632ef2bbb34f8347280fc4507057fddc29df9a471f75c66541d4d4dad1c9e93a19a58e8b473fa0f062f7:feedfacedeadbeeffeedfacedeadbeefabaddad2:65dcc57fcf623a24094fcca40d3533f8 +aes-192-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c:9313225df88406e555909c5aff5269aa6a7a9538534f7da1e4c303d2a318a728c3c0c95156809539fcf0e2429a6b525416aedbf5a0de6a57a637b39b:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:d27e88681ce3243c4830165a8fdcf9ff1de9a1d8e6b447ef6ef7b79828666e4581e79012af34ddd9e2f037589b292db3e67c036745fa22e7e9b7373b:feedfacedeadbeeffeedfacedeadbeefabaddad2:dcf566ff291c25bbb8568fc3d376a6d9 +aes-256-gcm:0000000000000000000000000000000000000000000000000000000000000000:000000000000000000000000::::530f8afbc74536b9a963b4f1c4cb738b +aes-256-gcm:0000000000000000000000000000000000000000000000000000000000000000:000000000000000000000000:00000000000000000000000000000000:cea7403d4d606b6e074ec5d3baf39d18::d0d1c8a799996bf0265b98b5d48ab919 +aes-256-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308:cafebabefacedbaddecaf888:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255:522dc1f099567d07f47f37a32a84427d643a8cdcbfe5c0c97598a2bd2555d1aa8cb08e48590dbb3da7b08b1056828838c5f61e6393ba7a0abcc9f662898015ad::b094dac5d93471bdec1a502270e3cc6c +aes-256-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308:cafebabefacedbaddecaf888:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:522dc1f099567d07f47f37a32a84427d643a8cdcbfe5c0c97598a2bd2555d1aa8cb08e48590dbb3da7b08b1056828838c5f61e6393ba7a0abcc9f662:feedfacedeadbeeffeedfacedeadbeefabaddad2:76fc6ece0f4e1768cddf8853bb2d551b +aes-256-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308:cafebabefacedbad:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:c3762df1ca787d32ae47c13bf19844cbaf1ae14d0b976afac52ff7d79bba9de0feb582d33934a4f0954cc2363bc73f7862ac430e64abe499f47c9b1f:feedfacedeadbeeffeedfacedeadbeefabaddad2:3a337dbf46a792c45e454913fe2ea8f2 +aes-256-gcm:feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308:9313225df88406e555909c5aff5269aa6a7a9538534f7da1e4c303d2a318a728c3c0c95156809539fcf0e2429a6b525416aedbf5a0de6a57a637b39b:d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39:5a8def2f0c9e53f1f75d7853659e2a20eeb2b22aafde6419a058ab4f6f746bf40fc0c3b780f244452da3ebf1c5d82cdea2418997200ef82e44ae7e3f:feedfacedeadbeeffeedfacedeadbeefabaddad2:a44a8266ee1c8eb0c8b5d4cf5ae9f19a +# local add-ons, primarily streaming ghash tests +# 128 bytes aad +aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:::d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255522dc1f099567d07f47f37a32a84427d643a8cdcbfe5c0c97598a2bd2555d1aa8cb08e48590dbb3da7b08b1056828838c5f61e6393ba7a0abcc9f662898015ad:5fea793a2d6f974d37e68e0cb8ff9492 +# 48 bytes plaintext +aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:0388dace60b6a392f328c2b971b2fe78f795aaab494b5923f7fd89ff948bc1e0200211214e7394da2089b6acd093abe0::9dd0a376b08e40eb00c35f29f9ea61a4 +# 80 bytes plaintext +aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:0388dace60b6a392f328c2b971b2fe78f795aaab494b5923f7fd89ff948bc1e0200211214e7394da2089b6acd093abe0c94da219118e297d7b7ebcbcc9c388f28ade7d85a8ee35616f7124a9d5270291::98885a3a22bd4742fe7b72172193b163 +# 128 bytes plaintext +aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:0388dace60b6a392f328c2b971b2fe78f795aaab494b5923f7fd89ff948bc1e0200211214e7394da2089b6acd093abe0c94da219118e297d7b7ebcbcc9c388f28ade7d85a8ee35616f7124a9d527029195b84d1b96c690ff2f2de30bf2ec89e00253786e126504f0dab90c48a30321de3345e6b0461e7c9e6c6b7afedde83f40::cac45f60e31efd3b5a43b98a22ce1aa1 +# 192 bytes plaintext, iv is chosen so that initial counter LSB is 0xFF +aes-128-gcm:00000000000000000000000000000000:ffffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:56b3373ca9ef6e4a2b64fe1e9a17b61425f10d47a75a5fce13efc6bc784af24f4141bdd48cf7c770887afd573cca5418a9aeffcd7c5ceddfc6a78397b9a85b499da558257267caab2ad0b23ca476a53cb17fb41c4b8b475cb4f3f7165094c229c9e8c4dc0a2a5ff1903e501511221376a1cdb8364c5061a20cae74bc4acd76ceb0abc9fd3217ef9f8c90be402ddf6d8697f4f880dff15bfb7a6b28241ec8fe183c2d59e3f9dfff653c7126f0acb9e64211f42bae12af462b1070bef1ab5e3606::566f8ef683078bfdeeffa869d751a017 +# 80 bytes plaintext, submitted by Intel +aes-128-gcm:843ffcf5d2b72694d19ed01d01249412:dbcca32ebf9b804617c3aa9e:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f:6268c6fa2a80b2d137467f092f657ac04d89be2beaa623d61b5a868c8f03ff95d3dcee23ad2f1ab3a6c80eaf4b140eb05de3457f0fbc111a6b43d0763aa422a3013cf1dc37fe417d1fbfc449b75d4cc5:00000000000000000000000000000000101112131415161718191a1b1c1d1e1f:3b629ccfbc1119b7319e1dce2cd6fd6d + +# AES XTS test vectors from IEEE Std 1619-2007 +aes-128-xts:0000000000000000000000000000000000000000000000000000000000000000:00000000000000000000000000000000:0000000000000000000000000000000000000000000000000000000000000000:917cf69ebd68b2ec9b9fe9a3eadda692cd43d2f59598ed858c02c2652fbf922e +aes-128-xts:1111111111111111111111111111111122222222222222222222222222222222:33333333330000000000000000000000:4444444444444444444444444444444444444444444444444444444444444444:c454185e6a16936e39334038acef838bfb186fff7480adc4289382ecd6d394f0 +aes-128-xts:fffefdfcfbfaf9f8f7f6f5f4f3f2f1f022222222222222222222222222222222:33333333330000000000000000000000:4444444444444444444444444444444444444444444444444444444444444444:af85336b597afc1a900b2eb21ec949d292df4c047e0b21532186a5971a227a89 +aes-128-xts:2718281828459045235360287471352631415926535897932384626433832795:00000000000000000000000000000000:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:27a7479befa1d476489f308cd4cfa6e2a96e4bbe3208ff25287dd3819616e89cc78cf7f5e543445f8333d8fa7f56000005279fa5d8b5e4ad40e736ddb4d35412328063fd2aab53e5ea1e0a9f332500a5df9487d07a5c92cc512c8866c7e860ce93fdf166a24912b422976146ae20ce846bb7dc9ba94a767aaef20c0d61ad02655ea92dc4c4e41a8952c651d33174be51a10c421110e6d81588ede82103a252d8a750e8768defffed9122810aaeb99f9172af82b604dc4b8e51bcb08235a6f4341332e4ca60482a4ba1a03b3e65008fc5da76b70bf1690db4eae29c5f1badd03c5ccf2a55d705ddcd86d449511ceb7ec30bf12b1fa35b913f9f747a8afd1b130e94bff94effd01a91735ca1726acd0b197c4e5b03393697e126826fb6bbde8ecc1e08298516e2c9ed03ff3c1b7860f6de76d4cecd94c8119855ef5297ca67e9f3e7ff72b1e99785ca0a7e7720c5b36dc6d72cac9574c8cbbc2f801e23e56fd344b07f22154beba0f08ce8891e643ed995c94d9a69c9f1b5f499027a78572aeebd74d20cc39881c213ee770b1010e4bea718846977ae119f7a023ab58cca0ad752afe656bb3c17256a9f6e9bf19fdd5a38fc82bbe872c5539edb609ef4f79c203ebb140f2e583cb2ad15b4aa5b655016a8449277dbd477ef2c8d6c017db738b18deb4a427d1923ce3ff262735779a418f20a282df920147beabe421ee5319d0568 +aes-128-xts:2718281828459045235360287471352631415926535897932384626433832795:01000000000000000000000000000000:27a7479befa1d476489f308cd4cfa6e2a96e4bbe3208ff25287dd3819616e89cc78cf7f5e543445f8333d8fa7f56000005279fa5d8b5e4ad40e736ddb4d35412328063fd2aab53e5ea1e0a9f332500a5df9487d07a5c92cc512c8866c7e860ce93fdf166a24912b422976146ae20ce846bb7dc9ba94a767aaef20c0d61ad02655ea92dc4c4e41a8952c651d33174be51a10c421110e6d81588ede82103a252d8a750e8768defffed9122810aaeb99f9172af82b604dc4b8e51bcb08235a6f4341332e4ca60482a4ba1a03b3e65008fc5da76b70bf1690db4eae29c5f1badd03c5ccf2a55d705ddcd86d449511ceb7ec30bf12b1fa35b913f9f747a8afd1b130e94bff94effd01a91735ca1726acd0b197c4e5b03393697e126826fb6bbde8ecc1e08298516e2c9ed03ff3c1b7860f6de76d4cecd94c8119855ef5297ca67e9f3e7ff72b1e99785ca0a7e7720c5b36dc6d72cac9574c8cbbc2f801e23e56fd344b07f22154beba0f08ce8891e643ed995c94d9a69c9f1b5f499027a78572aeebd74d20cc39881c213ee770b1010e4bea718846977ae119f7a023ab58cca0ad752afe656bb3c17256a9f6e9bf19fdd5a38fc82bbe872c5539edb609ef4f79c203ebb140f2e583cb2ad15b4aa5b655016a8449277dbd477ef2c8d6c017db738b18deb4a427d1923ce3ff262735779a418f20a282df920147beabe421ee5319d0568:264d3ca8512194fec312c8c9891f279fefdd608d0c027b60483a3fa811d65ee59d52d9e40ec5672d81532b38b6b089ce951f0f9c35590b8b978d175213f329bb1c2fd30f2f7f30492a61a532a79f51d36f5e31a7c9a12c286082ff7d2394d18f783e1a8e72c722caaaa52d8f065657d2631fd25bfd8e5baad6e527d763517501c68c5edc3cdd55435c532d7125c8614deed9adaa3acade5888b87bef641c4c994c8091b5bcd387f3963fb5bc37aa922fbfe3df4e5b915e6eb514717bdd2a74079a5073f5c4bfd46adf7d282e7a393a52579d11a028da4d9cd9c77124f9648ee383b1ac763930e7162a8d37f350b2f74b8472cf09902063c6b32e8c2d9290cefbd7346d1c779a0df50edcde4531da07b099c638e83a755944df2aef1aa31752fd323dcb710fb4bfbb9d22b925bc3577e1b8949e729a90bbafeacf7f7879e7b1147e28ba0bae940db795a61b15ecf4df8db07b824bb062802cc98a9545bb2aaeed77cb3fc6db15dcd7d80d7d5bc406c4970a3478ada8899b329198eb61c193fb6275aa8ca340344a75a862aebe92eee1ce032fd950b47d7704a3876923b4ad62844bf4a09c4dbe8b4397184b7471360c9564880aedddb9baa4af2e75394b08cd32ff479c57a07d3eab5d54de5f9738b8d27f27a9f0ab11799d7b7ffefb2704c95c6ad12c39f1e867a4b7b1d7818a4b753dfd2a89ccb45e001a03a867b187f225dd +aes-128-xts:2718281828459045235360287471352631415926535897932384626433832795:02000000000000000000000000000000:264d3ca8512194fec312c8c9891f279fefdd608d0c027b60483a3fa811d65ee59d52d9e40ec5672d81532b38b6b089ce951f0f9c35590b8b978d175213f329bb1c2fd30f2f7f30492a61a532a79f51d36f5e31a7c9a12c286082ff7d2394d18f783e1a8e72c722caaaa52d8f065657d2631fd25bfd8e5baad6e527d763517501c68c5edc3cdd55435c532d7125c8614deed9adaa3acade5888b87bef641c4c994c8091b5bcd387f3963fb5bc37aa922fbfe3df4e5b915e6eb514717bdd2a74079a5073f5c4bfd46adf7d282e7a393a52579d11a028da4d9cd9c77124f9648ee383b1ac763930e7162a8d37f350b2f74b8472cf09902063c6b32e8c2d9290cefbd7346d1c779a0df50edcde4531da07b099c638e83a755944df2aef1aa31752fd323dcb710fb4bfbb9d22b925bc3577e1b8949e729a90bbafeacf7f7879e7b1147e28ba0bae940db795a61b15ecf4df8db07b824bb062802cc98a9545bb2aaeed77cb3fc6db15dcd7d80d7d5bc406c4970a3478ada8899b329198eb61c193fb6275aa8ca340344a75a862aebe92eee1ce032fd950b47d7704a3876923b4ad62844bf4a09c4dbe8b4397184b7471360c9564880aedddb9baa4af2e75394b08cd32ff479c57a07d3eab5d54de5f9738b8d27f27a9f0ab11799d7b7ffefb2704c95c6ad12c39f1e867a4b7b1d7818a4b753dfd2a89ccb45e001a03a867b187f225dd:fa762a3680b76007928ed4a4f49a9456031b704782e65e16cecb54ed7d017b5e18abd67b338e81078f21edb7868d901ebe9c731a7c18b5e6dec1d6a72e078ac9a4262f860beefa14f4e821018272e411a951502b6e79066e84252c3346f3aa62344351a291d4bedc7a07618bdea2af63145cc7a4b8d4070691ae890cd65733e7946e9021a1dffc4c59f159425ee6d50ca9b135fa6162cea18a939838dc000fb386fad086acce5ac07cb2ece7fd580b00cfa5e98589631dc25e8e2a3daf2ffdec26531659912c9d8f7a15e5865ea8fb5816d6207052bd7128cd743c12c8118791a4736811935eb982a532349e31dd401e0b660a568cb1a4711f552f55ded59f1f15bf7196b3ca12a91e488ef59d64f3a02bf45239499ac6176ae321c4a211ec545365971c5d3f4f09d4eb139bfdf2073d33180b21002b65cc9865e76cb24cd92c874c24c18350399a936ab3637079295d76c417776b94efce3a0ef7206b15110519655c956cbd8b2489405ee2b09a6b6eebe0c53790a12a8998378b33a5b71159625f4ba49d2a2fdba59fbf0897bc7aabd8d707dc140a80f0f309f835d3da54ab584e501dfa0ee977fec543f74186a802b9a37adb3e8291eca04d66520d229e60401e7282bef486ae059aa70696e0e305d777140a7a883ecdcb69b9ff938e8a4231864c69ca2c2043bed007ff3e605e014bcf518138dc3a25c5e236171a2d01d6 +aes-128-xts:2718281828459045235360287471352631415926535897932384626433832795:fd000000000000000000000000000000:8e41b78c390b5af9d758bb214a67e9f6bf7727b09ac6124084c37611398fa45daad94868600ed391fb1acd4857a95b466e62ef9f4b377244d1c152e7b30d731aad30c716d214b707aed99eb5b5e580b3e887cf7497465651d4b60e6042051da3693c3b78c14489543be8b6ad0ba629565bba202313ba7b0d0c94a3252b676f46cc02ce0f8a7d34c0ed229129673c1f61aed579d08a9203a25aac3a77e9db60267996db38df637356d9dcd1632e369939f2a29d89345c66e05066f1a3677aef18dea4113faeb629e46721a66d0a7e785d3e29af2594eb67dfa982affe0aac058f6e15864269b135418261fc3afb089472cf68c45dd7f231c6249ba0255e1e033833fc4d00a3fe02132d7bc3873614b8aee34273581ea0325c81f0270affa13641d052d36f0757d484014354d02d6883ca15c24d8c3956b1bd027bcf41f151fd8023c5340e5606f37e90fdb87c86fb4fa634b3718a30bace06a66eaf8f63c4aa3b637826a87fe8cfa44282e92cb1615af3a28e53bc74c7cba1a0977be9065d0c1a5dec6c54ae38d37f37aa35283e048e5530a85c4e7a29d7b92ec0c3169cdf2a805c7604bce60049b9fb7b8eaac10f51ae23794ceba68bb58112e293b9b692ca721b37c662f8574ed4dba6f88e170881c82cddc1034a0ca7e284bf0962b6b26292d836fa9f73c1ac770eef0f2d3a1eaf61d3e03555fd424eedd67e18a18094f888:d55f684f81f4426e9fde92a5ff02df2ac896af63962888a97910c1379e20b0a3b1db613fb7fe2e07004329ea5c22bfd33e3dbe4cf58cc608c2c26c19a2e2fe22f98732c2b5cb844cc6c0702d91e1d50fc4382a7eba5635cd602432a2306ac4ce82f8d70c8d9bc15f918fe71e74c622d5cf71178bf6e0b9cc9f2b41dd8dbe441c41cd0c73a6dc47a348f6702f9d0e9b1b1431e948e299b9ec2272ab2c5f0c7be86affa5dec87a0bee81d3d50007edaa2bcfccb35605155ff36ed8edd4a40dcd4b243acd11b2b987bdbfaf91a7cac27e9c5aea525ee53de7b2d3332c8644402b823e94a7db26276d2d23aa07180f76b4fd29b9c0823099c9d62c519880aee7e9697617c1497d47bf3e571950311421b6b734d38b0db91eb85331b91ea9f61530f54512a5a52a4bad589eb69781d537f23297bb459bdad2948a29e1550bf4787e0be95bb173cf5fab17dab7a13a052a63453d97ccec1a321954886b7a1299faaeecae35c6eaaca753b041b5e5f093bf83397fd21dd6b3012066fcc058cc32c3b09d7562dee29509b5839392c9ff05f51f3166aaac4ac5f238038a3045e6f72e48ef0fe8bc675e82c318a268e43970271bf119b81bf6a982746554f84e72b9f00280a320a08142923c23c883423ff949827f29bbacdc1ccdb04938ce6098c95ba6b32528f4ef78eed778b2e122ddfd1cbdd11d1c0a6783e011fc536d63d053260637 +aes-128-xts:2718281828459045235360287471352631415926535897932384626433832795:fe000000000000000000000000000000:d55f684f81f4426e9fde92a5ff02df2ac896af63962888a97910c1379e20b0a3b1db613fb7fe2e07004329ea5c22bfd33e3dbe4cf58cc608c2c26c19a2e2fe22f98732c2b5cb844cc6c0702d91e1d50fc4382a7eba5635cd602432a2306ac4ce82f8d70c8d9bc15f918fe71e74c622d5cf71178bf6e0b9cc9f2b41dd8dbe441c41cd0c73a6dc47a348f6702f9d0e9b1b1431e948e299b9ec2272ab2c5f0c7be86affa5dec87a0bee81d3d50007edaa2bcfccb35605155ff36ed8edd4a40dcd4b243acd11b2b987bdbfaf91a7cac27e9c5aea525ee53de7b2d3332c8644402b823e94a7db26276d2d23aa07180f76b4fd29b9c0823099c9d62c519880aee7e9697617c1497d47bf3e571950311421b6b734d38b0db91eb85331b91ea9f61530f54512a5a52a4bad589eb69781d537f23297bb459bdad2948a29e1550bf4787e0be95bb173cf5fab17dab7a13a052a63453d97ccec1a321954886b7a1299faaeecae35c6eaaca753b041b5e5f093bf83397fd21dd6b3012066fcc058cc32c3b09d7562dee29509b5839392c9ff05f51f3166aaac4ac5f238038a3045e6f72e48ef0fe8bc675e82c318a268e43970271bf119b81bf6a982746554f84e72b9f00280a320a08142923c23c883423ff949827f29bbacdc1ccdb04938ce6098c95ba6b32528f4ef78eed778b2e122ddfd1cbdd11d1c0a6783e011fc536d63d053260637:72efc1ebfe1ee25975a6eb3aa8589dda2b261f1c85bdab442a9e5b2dd1d7c3957a16fc08e526d4b1223f1b1232a11af274c3d70dac57f83e0983c498f1a6f1aecb021c3e70085a1e527f1ce41ee5911a82020161529cd82773762daf5459de94a0a82adae7e1703c808543c29ed6fb32d9e004327c1355180c995a07741493a09c21ba01a387882da4f62534b87bb15d60d197201c0fd3bf30c1500a3ecfecdd66d8721f90bcc4c17ee925c61b0a03727a9c0d5f5ca462fbfa0af1c2513a9d9d4b5345bd27a5f6e653f751693e6b6a2b8ead57d511e00e58c45b7b8d005af79288f5c7c22fd4f1bf7a898b03a5634c6a1ae3f9fae5de4f296a2896b23e7ed43ed14fa5a2803f4d28f0d3ffcf24757677aebdb47bb388378708948a8d4126ed1839e0da29a537a8c198b3c66ab00712dd261674bf45a73d67f76914f830ca014b65596f27e4cf62de66125a5566df9975155628b400fbfb3a29040ed50faffdbb18aece7c5c44693260aab386c0a37b11b114f1c415aebb653be468179428d43a4d8bc3ec38813eca30a13cf1bb18d524f1992d44d8b1a42ea30b22e6c95b199d8d182f8840b09d059585c31ad691fa0619ff038aca2c39a943421157361717c49d322028a74648113bd8c9d7ec77cf3c89c1ec8718ceff8516d96b34c3c614f10699c9abc4ed0411506223bea16af35c883accdbe1104eef0cfdb54e12fb230a +aes-128-xts:2718281828459045235360287471352631415926535897932384626433832795:ff000000000000000000000000000000:72efc1ebfe1ee25975a6eb3aa8589dda2b261f1c85bdab442a9e5b2dd1d7c3957a16fc08e526d4b1223f1b1232a11af274c3d70dac57f83e0983c498f1a6f1aecb021c3e70085a1e527f1ce41ee5911a82020161529cd82773762daf5459de94a0a82adae7e1703c808543c29ed6fb32d9e004327c1355180c995a07741493a09c21ba01a387882da4f62534b87bb15d60d197201c0fd3bf30c1500a3ecfecdd66d8721f90bcc4c17ee925c61b0a03727a9c0d5f5ca462fbfa0af1c2513a9d9d4b5345bd27a5f6e653f751693e6b6a2b8ead57d511e00e58c45b7b8d005af79288f5c7c22fd4f1bf7a898b03a5634c6a1ae3f9fae5de4f296a2896b23e7ed43ed14fa5a2803f4d28f0d3ffcf24757677aebdb47bb388378708948a8d4126ed1839e0da29a537a8c198b3c66ab00712dd261674bf45a73d67f76914f830ca014b65596f27e4cf62de66125a5566df9975155628b400fbfb3a29040ed50faffdbb18aece7c5c44693260aab386c0a37b11b114f1c415aebb653be468179428d43a4d8bc3ec38813eca30a13cf1bb18d524f1992d44d8b1a42ea30b22e6c95b199d8d182f8840b09d059585c31ad691fa0619ff038aca2c39a943421157361717c49d322028a74648113bd8c9d7ec77cf3c89c1ec8718ceff8516d96b34c3c614f10699c9abc4ed0411506223bea16af35c883accdbe1104eef0cfdb54e12fb230a:3260ae8dad1f4a32c5cafe3ab0eb95549d461a67ceb9e5aa2d3afb62dece0553193ba50c75be251e08d1d08f1088576c7efdfaaf3f459559571e12511753b07af073f35da06af0ce0bbf6b8f5ccc5cea500ec1b211bd51f63b606bf6528796ca12173ba39b8935ee44ccce646f90a45bf9ccc567f0ace13dc2d53ebeedc81f58b2e41179dddf0d5a5c42f5d8506c1a5d2f8f59f3ea873cbcd0eec19acbf325423bd3dcb8c2b1bf1d1eaed0eba7f0698e4314fbeb2f1566d1b9253008cbccf45a2b0d9c5c9c21474f4076e02be26050b99dee4fd68a4cf890e496e4fcae7b70f94ea5a9062da0daeba1993d2ccd1dd3c244b8428801495a58b216547e7e847c46d1d756377b6242d2e5fb83bf752b54e0df71e889f3a2bb0f4c10805bf3c590376e3c24e22ff57f7fa965577375325cea5d920db94b9c336b455f6e894c01866fe9fbb8c8d3f70a2957285f6dfb5dcd8cbf54782f8fe7766d4723819913ac773421e3a31095866bad22c86a6036b2518b2059b4229d18c8c2ccbdf906c6cc6e82464ee57bddb0bebcb1dc645325bfb3e665ef7251082c88ebb1cf203bd779fdd38675713c8daadd17e1cabee432b09787b6ddf3304e38b731b45df5df51b78fcfb3d32466028d0ba36555e7e11ab0ee0666061d1645d962444bc47a38188930a84b4d561395c73c087021927ca638b7afc8a8679ccb84c26555440ec7f10445cd + +aes-256-xts:27182818284590452353602874713526624977572470936999595749669676273141592653589793238462643383279502884197169399375105820974944592:ff000000000000000000000000000000:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:1c3b3a102f770386e4836c99e370cf9bea00803f5e482357a4ae12d414a3e63b5d31e276f8fe4a8d66b317f9ac683f44680a86ac35adfc3345befecb4bb188fd5776926c49a3095eb108fd1098baec70aaa66999a72a82f27d848b21d4a741b0c5cd4d5fff9dac89aeba122961d03a757123e9870f8acf1000020887891429ca2a3e7a7d7df7b10355165c8b9a6d0a7de8b062c4500dc4cd120c0f7418dae3d0b5781c34803fa75421c790dfe1de1834f280d7667b327f6c8cd7557e12ac3a0f93ec05c52e0493ef31a12d3d9260f79a289d6a379bc70c50841473d1a8cc81ec583e9645e07b8d9670655ba5bbcfecc6dc3966380ad8fecb17b6ba02469a020a84e18e8f84252070c13e9f1f289be54fbc481457778f616015e1327a02b140f1505eb309326d68378f8374595c849d84f4c333ec4423885143cb47bd71c5edae9be69a2ffeceb1bec9de244fbe15992b11b77c040f12bd8f6a975a44a0f90c29a9abc3d4d893927284c58754cce294529f8614dcd2aba991925fedc4ae74ffac6e333b93eb4aff0479da9a410e4450e0dd7ae4c6e2910900575da401fc07059f645e8b7e9bfdef33943054ff84011493c27b3429eaedb4ed5376441a77ed43851ad77f16f541dfd269d50d6a5f14fb0aab1cbb4c1550be97f7ab4066193c4caa773dad38014bd2092fa755c824bb5e54c4f36ffda9fcea70b9c6e693e148c151 +aes-256-xts:27182818284590452353602874713526624977572470936999595749669676273141592653589793238462643383279502884197169399375105820974944592:ffff0000000000000000000000000000:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:77a31251618a15e6b92d1d66dffe7b50b50bad552305ba0217a610688eff7e11e1d0225438e093242d6db274fde801d4cae06f2092c728b2478559df58e837c2469ee4a4fa794e4bbc7f39bc026e3cb72c33b0888f25b4acf56a2a9804f1ce6d3d6e1dc6ca181d4b546179d55544aa7760c40d06741539c7e3cd9d2f6650b2013fd0eeb8c2b8e3d8d240ccae2d4c98320a7442e1c8d75a42d6e6cfa4c2eca1798d158c7aecdf82490f24bb9b38e108bcda12c3faf9a21141c3613b58367f922aaa26cd22f23d708dae699ad7cb40a8ad0b6e2784973dcb605684c08b8d6998c69aac049921871ebb65301a4619ca80ecb485a31d744223ce8ddc2394828d6a80470c092f5ba413c3378fa6054255c6f9df4495862bbb3287681f931b687c888abf844dfc8fc28331e579928cd12bd2390ae123cf03818d14dedde5c0c24c8ab018bfca75ca096f2d531f3d1619e785f1ada437cab92e980558b3dce1474afb75bfedbf8ff54cb2618e0244c9ac0d3c66fb51598cd2db11f9be39791abe447c63094f7c453b7ff87cb5bb36b7c79efb0872d17058b83b15ab0866ad8a58656c5a7e20dbdf308b2461d97c0ec0024a2715055249cf3b478ddd4740de654f75ca686e0d7345c69ed50cdc2a8b332b1f8824108ac937eb050585608ee734097fc09054fbff89eeaeea791f4a7ab1f9868294a4f9e27b42af8100cb9d59cef9645803 +aes-256-xts:27182818284590452353602874713526624977572470936999595749669676273141592653589793238462643383279502884197169399375105820974944592:ffffff00000000000000000000000000:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:e387aaa58ba483afa7e8eb469778317ecf4cf573aa9d4eac23f2cdf914e4e200a8b490e42ee646802dc6ee2b471b278195d60918ececb44bf79966f83faba0499298ebc699c0c8634715a320bb4f075d622e74c8c932004f25b41e361025b5a87815391f6108fc4afa6a05d9303c6ba68a128a55705d415985832fdeaae6c8e19110e84d1b1f199a2692119edc96132658f09da7c623efcec712537a3d94c0bf5d7e352ec94ae5797fdb377dc1551150721adf15bd26a8efc2fcaad56881fa9e62462c28f30ae1ceaca93c345cf243b73f542e2074a705bd2643bb9f7cc79bb6e7091ea6e232df0f9ad0d6cf502327876d82207abf2115cdacf6d5a48f6c1879a65b115f0f8b3cb3c59d15dd8c769bc014795a1837f3901b5845eb491adfefe097b1fa30a12fc1f65ba22905031539971a10f2f36c321bb51331cdefb39e3964c7ef079994f5b69b2edd83a71ef549971ee93f44eac3938fcdd61d01fa71799da3a8091c4c48aa9ed263ff0749df95d44fef6a0bb578ec69456aa5408ae32c7af08ad7ba8921287e3bbee31b767be06a0e705c864a769137df28292283ea81a2480241b44d9921cdbec1bc28dc1fda114bd8e5217ac9d8ebafa720e9da4f9ace231cc949e5b96fe76ffc21063fddc83a6b8679c00d35e09576a875305bed5f36ed242c8900dd1fa965bc950dfce09b132263a1eef52dd6888c309f5a7d712826 +aes-256-xts:27182818284590452353602874713526624977572470936999595749669676273141592653589793238462643383279502884197169399375105820974944592:ffffffff000000000000000000000000:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:bf53d2dade78e822a4d949a9bc6766b01b06a8ef70d26748c6a7fc36d80ae4c5520f7c4ab0ac8544424fa405162fef5a6b7f229498063618d39f0003cb5fb8d1c86b643497da1ff945c8d3bedeca4f479702a7a735f043ddb1d6aaade3c4a0ac7ca7f3fa5279bef56f82cd7a2f38672e824814e10700300a055e1630b8f1cb0e919f5e942010a416e2bf48cb46993d3cb6a51c19bacf864785a00bc2ecff15d350875b246ed53e68be6f55bd7e05cfc2b2ed6432198a6444b6d8c247fab941f569768b5c429366f1d3f00f0345b96123d56204c01c63b22ce78baf116e525ed90fdea39fa469494d3866c31e05f295ff21fea8d4e6e13d67e47ce722e9698a1c1048d68ebcde76b86fcf976eab8aa9790268b7068e017a8b9b749409514f1053027fd16c3786ea1bac5f15cb79711ee2abe82f5cf8b13ae73030ef5b9e4457e75d1304f988d62dd6fc4b94ed38ba831da4b7634971b6cd8ec325d9c61c00f1df73627ed3745a5e8489f3a95c69639c32cd6e1d537a85f75cc844726e8a72fc0077ad22000f1d5078f6b866318c668f1ad03d5a5fced5219f2eabbd0aa5c0f460d183f04404a0d6f469558e81fab24a167905ab4c7878502ad3e38fdbe62a41556cec37325759533ce8f25f367c87bb5578d667ae93f9e2fd99bcbc5f2fbba88cf6516139420fcff3b7361d86322c4bd84c82f335abb152c4a93411373aaa8220 +aes-256-xts:27182818284590452353602874713526624977572470936999595749669676273141592653589793238462643383279502884197169399375105820974944592:ffffffffff0000000000000000000000:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:64497e5a831e4a932c09be3e5393376daa599548b816031d224bbf50a818ed2350eae7e96087c8a0db51ad290bd00c1ac1620857635bf246c176ab463be30b808da548081ac847b158e1264be25bb0910bbc92647108089415d45fab1b3d2604e8a8eff1ae4020cfa39936b66827b23f371b92200be90251e6d73c5f86de5fd4a950781933d79a28272b782a2ec313efdfcc0628f43d744c2dc2ff3dcb66999b50c7ca895b0c64791eeaa5f29499fb1c026f84ce5b5c72ba1083cddb5ce45434631665c333b60b11593fb253c5179a2c8db813782a004856a1653011e93fb6d876c18366dd8683f53412c0c180f9c848592d593f8609ca736317d356e13e2bff3a9f59cd9aeb19cd482593d8c46128bb32423b37a9adfb482b99453fbe25a41bf6feb4aa0bef5ed24bf73c762978025482c13115e4015aac992e5613a3b5c2f685b84795cb6e9b2656d8c88157e52c42f978d8634c43d06fea928f2822e465aa6576e9bf419384506cc3ce3c54ac1a6f67dc66f3b30191e698380bc999b05abce19dc0c6dcc2dd001ec535ba18deb2df1a101023108318c75dc98611a09dc48a0acdec676fabdf222f07e026f059b672b56e5cbc8e1d21bbd867dd927212054681d70ea737134cdfce93b6f82ae22423274e58a0821cc5502e2d0ab4585e94de6975be5e0b4efce51cd3e70c25a1fbbbd609d273ad5b0d59631c531f6a0a57b9 + +aes-128-xts:fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0bfbebdbcbbbab9b8b7b6b5b4b3b2b1b0:9a785634120000000000000000000000:000102030405060708090a0b0c0d0e0f10:6c1625db4671522d3d7599601de7ca09ed +aes-128-xts:fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0bfbebdbcbbbab9b8b7b6b5b4b3b2b1b0:9a785634120000000000000000000000:000102030405060708090a0b0c0d0e0f1011:d069444b7a7e0cab09e24447d24deb1fedbf +aes-128-xts:fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0bfbebdbcbbbab9b8b7b6b5b4b3b2b1b0:9a785634120000000000000000000000:000102030405060708090a0b0c0d0e0f101112:e5df1351c0544ba1350b3363cd8ef4beedbf9d +aes-128-xts:fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0bfbebdbcbbbab9b8b7b6b5b4b3b2b1b0:9a785634120000000000000000000000:000102030405060708090a0b0c0d0e0f10111213:9d84c813f719aa2c7be3f66171c7c5c2edbf9dac +aes-128-xts:e0e1e2e3e4e5e6e7e8e9eaebecedeeefc0c1c2c3c4c5c6c7c8c9cacbcccdcecf:21436587a90000000000000000000000:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:38b45812ef43a05bd957e545907e223b954ab4aaf088303ad910eadf14b42be68b2461149d8c8ba85f992be970bc621f1b06573f63e867bf5875acafa04e42ccbd7bd3c2a0fb1fff791ec5ec36c66ae4ac1e806d81fbf709dbe29e471fad38549c8e66f5345d7c1eb94f405d1ec785cc6f6a68f6254dd8339f9d84057e01a17741990482999516b5611a38f41bb6478e6f173f320805dd71b1932fc333cb9ee39936beea9ad96fa10fb4112b901734ddad40bc1878995f8e11aee7d141a2f5d48b7a4e1e7f0b2c04830e69a4fd1378411c2f287edf48c6c4e5c247a19680f7fe41cefbd49b582106e3616cbbe4dfb2344b2ae9519391f3e0fb4922254b1d6d2d19c6d4d537b3a26f3bcc51588b32f3eca0829b6a5ac72578fb814fb43cf80d64a233e3f997a3f02683342f2b33d25b492536b93becb2f5e1a8b82f5b883342729e8ae09d16938841a21a97fb543eea3bbff59f13c1a18449e398701c1ad51648346cbc04c27bb2da3b93a1372ccae548fb53bee476f9e9c91773b1bb19828394d55d3e1a20ed69113a860b6829ffa847224604435070221b257e8dff783615d2cae4803a93aa4334ab482a0afac9c0aeda70b45a481df5dec5df8cc0f423c77a5fd46cd312021d4b438862419a791be03bb4d97c0e59578542531ba466a83baf92cefc151b5cc1611a167893819b63fb8a6b18e86de60290fa72b797b0ce59f3 +# AES wrap tests from RFC3394 +id-aes128-wrap:000102030405060708090A0B0C0D0E0F::00112233445566778899AABBCCDDEEFF:1FA68B0A8112B447AEF34BD8FB5A7B829D3E862371D2CFE5 +id-aes192-wrap:000102030405060708090A0B0C0D0E0F1011121314151617::00112233445566778899AABBCCDDEEFF:96778B25AE6CA435F92B5B97C050AED2468AB8A17AD84E5D +id-aes256-wrap:000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F::00112233445566778899AABBCCDDEEFF:64E8C3F9CE0F5BA263E9777905818A2A93C8191E7D6E8AE7 +id-aes192-wrap:000102030405060708090A0B0C0D0E0F1011121314151617::00112233445566778899AABBCCDDEEFF0001020304050607:031D33264E15D33268F24EC260743EDCE1C6C7DDEE725A936BA814915C6762D2 +id-aes256-wrap:000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F::00112233445566778899AABBCCDDEEFF0001020304050607:A8F9BC1612C68B3FF6E6F4FBE30E71E4769C8B80A32CB8958CD5D17D6B254DA1 +id-aes256-wrap:000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F::00112233445566778899AABBCCDDEEFF000102030405060708090A0B0C0D0E0F:28C9F404C4B810F4CBCCB35CFB87F8263F5786E2D80ED326CBC7F0E71A99F43BFB988B9B7A02DD21 diff --git a/crypto/evp/m_dss.c b/crypto/evp/m_dss.c index f22ba522c2f2..147844862d17 100644 --- a/crypto/evp/m_dss.c +++ b/crypto/evp/m_dss.c @@ -66,7 +66,6 @@ #endif #ifndef OPENSSL_NO_SHA -# ifndef OPENSSL_FIPS static int init(EVP_MD_CTX *ctx) { @@ -102,5 +101,4 @@ const EVP_MD *EVP_dss(void) { return (&dsa_md); } -# endif #endif diff --git a/crypto/evp/m_dss1.c b/crypto/evp/m_dss1.c index 976148e436a0..e36fabff700e 100644 --- a/crypto/evp/m_dss1.c +++ b/crypto/evp/m_dss1.c @@ -68,8 +68,6 @@ # include # endif -# ifndef OPENSSL_FIPS - static int init(EVP_MD_CTX *ctx) { return SHA1_Init(ctx->md_data); @@ -104,5 +102,4 @@ const EVP_MD *EVP_dss1(void) { return (&dss1_md); } -# endif #endif diff --git a/crypto/evp/m_ecdsa.c b/crypto/evp/m_ecdsa.c index d11a13a6d7a7..803d31495577 100644 --- a/crypto/evp/m_ecdsa.c +++ b/crypto/evp/m_ecdsa.c @@ -116,7 +116,6 @@ #include #ifndef OPENSSL_NO_SHA -# ifndef OPENSSL_FIPS static int init(EVP_MD_CTX *ctx) { @@ -152,5 +151,4 @@ const EVP_MD *EVP_ecdsa(void) { return (&ecdsa_md); } -# endif #endif diff --git a/crypto/evp/m_sha1.c b/crypto/evp/m_sha1.c index 0cc63555ad48..a74e6b77948e 100644 --- a/crypto/evp/m_sha1.c +++ b/crypto/evp/m_sha1.c @@ -59,16 +59,14 @@ #include #include "cryptlib.h" -#ifndef OPENSSL_FIPS +#ifndef OPENSSL_NO_SHA -# ifndef OPENSSL_NO_SHA - -# include -# include -# include -# ifndef OPENSSL_NO_RSA -# include -# endif +# include +# include +# include +# ifndef OPENSSL_NO_RSA +# include +# endif static int init(EVP_MD_CTX *ctx) { @@ -104,9 +102,9 @@ const EVP_MD *EVP_sha1(void) { return (&sha1_md); } -# endif +#endif -# ifndef OPENSSL_NO_SHA256 +#ifndef OPENSSL_NO_SHA256 static int init224(EVP_MD_CTX *ctx) { return SHA224_Init(ctx->md_data); @@ -171,9 +169,9 @@ const EVP_MD *EVP_sha256(void) { return (&sha256_md); } -# endif /* ifndef OPENSSL_NO_SHA256 */ +#endif /* ifndef OPENSSL_NO_SHA256 */ -# ifndef OPENSSL_NO_SHA512 +#ifndef OPENSSL_NO_SHA512 static int init384(EVP_MD_CTX *ctx) { return SHA384_Init(ctx->md_data); @@ -234,6 +232,4 @@ const EVP_MD *EVP_sha512(void) { return (&sha512_md); } -# endif /* ifndef OPENSSL_NO_SHA512 */ - -#endif +#endif /* ifndef OPENSSL_NO_SHA512 */ diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c index e153a1833e07..4492d207f28e 100644 --- a/crypto/evp/m_sigver.c +++ b/crypto/evp/m_sigver.c @@ -73,15 +73,18 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, if (ctx->pctx == NULL) return 0; - if (type == NULL) { - int def_nid; - if (EVP_PKEY_get_default_digest_nid(pkey, &def_nid) > 0) - type = EVP_get_digestbynid(def_nid); - } + if (!(ctx->pctx->pmeth->flags & EVP_PKEY_FLAG_SIGCTX_CUSTOM)) { - if (type == NULL) { - EVPerr(EVP_F_DO_SIGVER_INIT, EVP_R_NO_DEFAULT_DIGEST); - return 0; + if (type == NULL) { + int def_nid; + if (EVP_PKEY_get_default_digest_nid(pkey, &def_nid) > 0) + type = EVP_get_digestbynid(def_nid); + } + + if (type == NULL) { + EVPerr(EVP_F_DO_SIGVER_INIT, EVP_R_NO_DEFAULT_DIGEST); + return 0; + } } if (ver) { @@ -103,6 +106,8 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, return 0; if (pctx) *pctx = ctx->pctx; + if (ctx->pctx->pmeth->flags & EVP_PKEY_FLAG_SIGCTX_CUSTOM) + return 1; if (!EVP_DigestInit_ex(ctx, type, e)) return 0; return 1; @@ -124,7 +129,19 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, size_t *siglen) { int sctx, r = 0; - if (ctx->pctx->pmeth->signctx) + EVP_PKEY_CTX *pctx = ctx->pctx; + if (pctx->pmeth->flags & EVP_PKEY_FLAG_SIGCTX_CUSTOM) { + EVP_PKEY_CTX *dctx; + if (!sigret) + return pctx->pmeth->signctx(pctx, sigret, siglen, ctx); + dctx = EVP_PKEY_CTX_dup(ctx->pctx); + if (!dctx) + return 0; + r = dctx->pmeth->signctx(dctx, sigret, siglen, ctx); + EVP_PKEY_CTX_free(dctx); + return r; + } + if (pctx->pmeth->signctx) sctx = 1; else sctx = 0; @@ -147,20 +164,19 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, return 0; } else { if (sctx) { - if (ctx->pctx->pmeth->signctx(ctx->pctx, sigret, siglen, ctx) <= - 0) + if (pctx->pmeth->signctx(pctx, sigret, siglen, ctx) <= 0) return 0; } else { int s = EVP_MD_size(ctx->digest); - if (s < 0 - || EVP_PKEY_sign(ctx->pctx, sigret, siglen, NULL, s) <= 0) + if (s < 0 || EVP_PKEY_sign(pctx, sigret, siglen, NULL, s) <= 0) return 0; } } return 1; } -int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, unsigned char *sig, size_t siglen) +int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sig, + size_t siglen) { EVP_MD_CTX tmp_ctx; unsigned char md[EVP_MAX_MD_SIZE]; diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c index 2b84dc75ec3a..1171d3086d0b 100644 --- a/crypto/evp/p_lib.c +++ b/crypto/evp/p_lib.c @@ -337,7 +337,7 @@ int EVP_PKEY_set1_DH(EVP_PKEY *pkey, DH *key) DH *EVP_PKEY_get1_DH(EVP_PKEY *pkey) { - if (pkey->type != EVP_PKEY_DH) { + if (pkey->type != EVP_PKEY_DH && pkey->type != EVP_PKEY_DHX) { EVPerr(EVP_F_EVP_PKEY_GET1_DH, EVP_R_EXPECTING_A_DH_KEY); return NULL; } diff --git a/crypto/evp/pmeth_lib.c b/crypto/evp/pmeth_lib.c index ae8bccb0c6eb..9f81d10021a0 100644 --- a/crypto/evp/pmeth_lib.c +++ b/crypto/evp/pmeth_lib.c @@ -75,6 +75,7 @@ STACK_OF(EVP_PKEY_METHOD) *app_pkey_methods = NULL; extern const EVP_PKEY_METHOD rsa_pkey_meth, dh_pkey_meth, dsa_pkey_meth; extern const EVP_PKEY_METHOD ec_pkey_meth, hmac_pkey_meth, cmac_pkey_meth; +extern const EVP_PKEY_METHOD dhx_pkey_meth; static const EVP_PKEY_METHOD *standard_methods[] = { #ifndef OPENSSL_NO_RSA @@ -90,7 +91,10 @@ static const EVP_PKEY_METHOD *standard_methods[] = { &ec_pkey_meth, #endif &hmac_pkey_meth, - &cmac_pkey_meth + &cmac_pkey_meth, +#ifndef OPENSSL_NO_DH + &dhx_pkey_meth +#endif }; DECLARE_OBJ_BSEARCH_CMP_FN(const EVP_PKEY_METHOD *, const EVP_PKEY_METHOD *, diff --git a/crypto/hmac/hm_ameth.c b/crypto/hmac/hm_ameth.c index 641c797ef1d5..29b2b5dffcf7 100644 --- a/crypto/hmac/hm_ameth.c +++ b/crypto/hmac/hm_ameth.c @@ -87,7 +87,7 @@ static int hmac_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2) { switch (op) { case ASN1_PKEY_CTRL_DEFAULT_MD_NID: - *(int *)arg2 = NID_sha1; + *(int *)arg2 = NID_sha256; return 1; default: diff --git a/crypto/hmac/hmac.c b/crypto/hmac/hmac.c index 33d88be1179b..51a0a3efcd67 100644 --- a/crypto/hmac/hmac.c +++ b/crypto/hmac/hmac.c @@ -72,6 +72,16 @@ int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len, unsigned char pad[HMAC_MAX_MD_CBLOCK]; #ifdef OPENSSL_FIPS + /* If FIPS mode switch to approved implementation if possible */ + if (FIPS_mode()) { + const EVP_MD *fipsmd; + if (md) { + fipsmd = FIPS_get_digestbynid(EVP_MD_type(md)); + if (fipsmd) + md = fipsmd; + } + } + if (FIPS_mode()) { /* If we have an ENGINE need to allow non FIPS */ if ((impl || ctx->i_ctx.engine) diff --git a/crypto/hmac/hmactest.c b/crypto/hmac/hmactest.c index 271d0ebf264c..5a573950a453 100644 --- a/crypto/hmac/hmactest.c +++ b/crypto/hmac/hmactest.c @@ -195,6 +195,7 @@ int main(int argc, char *argv[]) } printf("test 4 ok\n"); test5: + HMAC_CTX_cleanup(&ctx); HMAC_CTX_init(&ctx); if (HMAC_Init_ex(&ctx, test[4].key, test[4].key_len, NULL, NULL)) { printf("Should fail to initialise HMAC with empty MD (test 5)\n"); @@ -284,6 +285,7 @@ test5: printf("test 5 ok\n"); } test6: + HMAC_CTX_cleanup(&ctx); HMAC_CTX_init(&ctx); if (!HMAC_Init_ex(&ctx, test[7].key, test[7].key_len, EVP_sha1(), NULL)) { printf("Failed to initialise HMAC (test 6)\n"); @@ -314,6 +316,7 @@ test6: printf("test 6 ok\n"); } end: + HMAC_CTX_cleanup(&ctx); EXIT(err); return (0); } diff --git a/crypto/jpake/jpake.c b/crypto/jpake/jpake.c index ed2e888eb4c0..8c38727e20fd 100644 --- a/crypto/jpake/jpake.c +++ b/crypto/jpake/jpake.c @@ -4,6 +4,7 @@ #include #include #include +#include /* * In the definition, (xa, xb, xc, xd) are Alice's (x1, x2, x3, x4) or diff --git a/crypto/md32_common.h b/crypto/md32_common.h index 1823833419c4..96828d2693a1 100644 --- a/crypto/md32_common.h +++ b/crypto/md32_common.h @@ -215,12 +215,30 @@ asm ("bswapl %0":"=r"(r):"0"(r)); \ *((unsigned int *)(c))=r; (c)+=4; r; }) # endif +# elif defined(__aarch64__) +# if defined(__BYTE_ORDER__) +# if defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define HOST_c2l(c,l) ({ unsigned int r; \ + asm ("rev %w0,%w1" \ + :"=r"(r) \ + :"r"(*((const unsigned int *)(c))));\ + (c)+=4; (l)=r; }) +# define HOST_l2c(l,c) ({ unsigned int r; \ + asm ("rev %w0,%w1" \ + :"=r"(r) \ + :"r"((unsigned int)(l)));\ + *((unsigned int *)(c))=r; (c)+=4; r; }) +# elif defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, (l)) +# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, (l)) +# endif +# endif # endif # endif -# endif -# if defined(__s390__) || defined(__s390x__) -# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, (l)) -# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, (l)) +# if defined(__s390__) || defined(__s390x__) +# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, (l)) +# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, (l)) +# endif # endif # ifndef HOST_c2l @@ -250,12 +268,12 @@ (c)+=4; (l); }) # endif # endif -# endif -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) -# ifndef B_ENDIAN - /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ -# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) -# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) +# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) +# ifndef B_ENDIAN + /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ +# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) +# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) +# endif # endif # endif diff --git a/crypto/md5/Makefile b/crypto/md5/Makefile index 0f87dbd54230..f5240da74cd7 100644 --- a/crypto/md5/Makefile +++ b/crypto/md5/Makefile @@ -52,6 +52,9 @@ md5-ia64.s: asm/md5-ia64.S $(CC) $(CFLAGS) -E asm/md5-ia64.S | \ $(PERL) -ne 's/;\s+/;\n/g; print;' > $@ +md5-sparcv9.S: asm/md5-sparcv9.pl + $(PERL) asm/md5-sparcv9.pl $@ $(CFLAGS) + files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/md5/asm/md5-sparcv9.pl b/crypto/md5/asm/md5-sparcv9.pl new file mode 100755 index 000000000000..407da3c1b0df --- /dev/null +++ b/crypto/md5/asm/md5-sparcv9.pl @@ -0,0 +1,430 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# +# Hardware SPARC T4 support by David S. Miller . +# ==================================================================== + +# MD5 for SPARCv9, 6.9 cycles per byte on UltraSPARC, >40% faster than +# code generated by Sun C 5.2. + +# SPARC T4 MD5 hardware achieves 3.20 cycles per byte, which is 2.1x +# faster than software. Multi-process benchmark saturates at 12x +# single-process result on 8-core processor, or ~11GBps per 2.85GHz +# socket. + +$output=shift; +open STDOUT,">$output"; + +use integer; + +($ctx,$inp,$len)=("%i0","%i1","%i2"); # input arguments + +# 64-bit values +@X=("%o0","%o1","%o2","%o3","%o4","%o5","%o7","%g1","%g2"); +$tx="%g3"; +($AB,$CD)=("%g4","%g5"); + +# 32-bit values +@V=($A,$B,$C,$D)=map("%l$_",(0..3)); +($t1,$t2,$t3,$saved_asi)=map("%l$_",(4..7)); +($shr,$shl1,$shl2)=("%i3","%i4","%i5"); + +my @K=( 0xd76aa478,0xe8c7b756,0x242070db,0xc1bdceee, + 0xf57c0faf,0x4787c62a,0xa8304613,0xfd469501, + 0x698098d8,0x8b44f7af,0xffff5bb1,0x895cd7be, + 0x6b901122,0xfd987193,0xa679438e,0x49b40821, + + 0xf61e2562,0xc040b340,0x265e5a51,0xe9b6c7aa, + 0xd62f105d,0x02441453,0xd8a1e681,0xe7d3fbc8, + 0x21e1cde6,0xc33707d6,0xf4d50d87,0x455a14ed, + 0xa9e3e905,0xfcefa3f8,0x676f02d9,0x8d2a4c8a, + + 0xfffa3942,0x8771f681,0x6d9d6122,0xfde5380c, + 0xa4beea44,0x4bdecfa9,0xf6bb4b60,0xbebfbc70, + 0x289b7ec6,0xeaa127fa,0xd4ef3085,0x04881d05, + 0xd9d4d039,0xe6db99e5,0x1fa27cf8,0xc4ac5665, + + 0xf4292244,0x432aff97,0xab9423a7,0xfc93a039, + 0x655b59c3,0x8f0ccc92,0xffeff47d,0x85845dd1, + 0x6fa87e4f,0xfe2ce6e0,0xa3014314,0x4e0811a1, + 0xf7537e82,0xbd3af235,0x2ad7d2bb,0xeb86d391, 0 ); + +sub R0 { + my ($i,$a,$b,$c,$d) = @_; + my $rot = (7,12,17,22)[$i%4]; + my $j = ($i+1)/2; + + if ($i&1) { + $code.=<<___; + srlx @X[$j],$shr,@X[$j] ! align X[`$i+1`] + and $b,$t1,$t1 ! round $i + sllx @X[$j+1],$shl1,$tx + add $t2,$a,$a + sllx $tx,$shl2,$tx + xor $d,$t1,$t1 + or $tx,@X[$j],@X[$j] + sethi %hi(@K[$i+1]),$t2 + add $t1,$a,$a + or $t2,%lo(@K[$i+1]),$t2 + sll $a,$rot,$t3 + add @X[$j],$t2,$t2 ! X[`$i+1`]+K[`$i+1`] + srl $a,32-$rot,$a + add $b,$t3,$t3 + xor $b,$c,$t1 + add $t3,$a,$a +___ + } else { + $code.=<<___; + srlx @X[$j],32,$tx ! extract X[`2*$j+1`] + and $b,$t1,$t1 ! round $i + add $t2,$a,$a + xor $d,$t1,$t1 + sethi %hi(@K[$i+1]),$t2 + add $t1,$a,$a + or $t2,%lo(@K[$i+1]),$t2 + sll $a,$rot,$t3 + add $tx,$t2,$t2 ! X[`2*$j+1`]+K[`$i+1`] + srl $a,32-$rot,$a + add $b,$t3,$t3 + xor $b,$c,$t1 + add $t3,$a,$a +___ + } +} + +sub R0_1 { + my ($i,$a,$b,$c,$d) = @_; + my $rot = (7,12,17,22)[$i%4]; + +$code.=<<___; + srlx @X[0],32,$tx ! extract X[1] + and $b,$t1,$t1 ! round $i + add $t2,$a,$a + xor $d,$t1,$t1 + sethi %hi(@K[$i+1]),$t2 + add $t1,$a,$a + or $t2,%lo(@K[$i+1]),$t2 + sll $a,$rot,$t3 + add $tx,$t2,$t2 ! X[1]+K[`$i+1`] + srl $a,32-$rot,$a + add $b,$t3,$t3 + andn $b,$c,$t1 + add $t3,$a,$a +___ +} + +sub R1 { + my ($i,$a,$b,$c,$d) = @_; + my $rot = (5,9,14,20)[$i%4]; + my $j = $i<31 ? (1+5*($i+1))%16 : (5+3*($i+1))%16; + my $xi = @X[$j/2]; + +$code.=<<___ if ($j&1 && ($xi=$tx)); + srlx @X[$j/2],32,$xi ! extract X[$j] +___ +$code.=<<___; + and $b,$d,$t3 ! round $i + add $t2,$a,$a + or $t3,$t1,$t1 + sethi %hi(@K[$i+1]),$t2 + add $t1,$a,$a + or $t2,%lo(@K[$i+1]),$t2 + sll $a,$rot,$t3 + add $xi,$t2,$t2 ! X[$j]+K[`$i+1`] + srl $a,32-$rot,$a + add $b,$t3,$t3 + `$i<31?"andn":"xor"` $b,$c,$t1 + add $t3,$a,$a +___ +} + +sub R2 { + my ($i,$a,$b,$c,$d) = @_; + my $rot = (4,11,16,23)[$i%4]; + my $j = $i<47 ? (5+3*($i+1))%16 : (0+7*($i+1))%16; + my $xi = @X[$j/2]; + +$code.=<<___ if ($j&1 && ($xi=$tx)); + srlx @X[$j/2],32,$xi ! extract X[$j] +___ +$code.=<<___; + add $t2,$a,$a ! round $i + xor $b,$t1,$t1 + sethi %hi(@K[$i+1]),$t2 + add $t1,$a,$a + or $t2,%lo(@K[$i+1]),$t2 + sll $a,$rot,$t3 + add $xi,$t2,$t2 ! X[$j]+K[`$i+1`] + srl $a,32-$rot,$a + add $b,$t3,$t3 + xor $b,$c,$t1 + add $t3,$a,$a +___ +} + +sub R3 { + my ($i,$a,$b,$c,$d) = @_; + my $rot = (6,10,15,21)[$i%4]; + my $j = (0+7*($i+1))%16; + my $xi = @X[$j/2]; + +$code.=<<___; + add $t2,$a,$a ! round $i +___ +$code.=<<___ if ($j&1 && ($xi=$tx)); + srlx @X[$j/2],32,$xi ! extract X[$j] +___ +$code.=<<___; + orn $b,$d,$t1 + sethi %hi(@K[$i+1]),$t2 + xor $c,$t1,$t1 + or $t2,%lo(@K[$i+1]),$t2 + add $t1,$a,$a + sll $a,$rot,$t3 + add $xi,$t2,$t2 ! X[$j]+K[`$i+1`] + srl $a,32-$rot,$a + add $b,$t3,$t3 + add $t3,$a,$a +___ +} + +$code.=<<___; +#include "sparc_arch.h" + +#ifdef __arch64__ +.register %g2,#scratch +.register %g3,#scratch +#endif + +.section ".text",#alloc,#execinstr + +#ifdef __PIC__ +SPARC_PIC_THUNK(%g1) +#endif + +.globl md5_block_asm_data_order +.align 32 +md5_block_asm_data_order: + SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5) + ld [%g1+4],%g1 ! OPENSSL_sparcv9cap_P[1] + + andcc %g1, CFR_MD5, %g0 + be .Lsoftware + nop + + mov 4, %g1 + andcc %o1, 0x7, %g0 + lda [%o0 + %g0]0x88, %f0 ! load context + lda [%o0 + %g1]0x88, %f1 + add %o0, 8, %o0 + lda [%o0 + %g0]0x88, %f2 + lda [%o0 + %g1]0x88, %f3 + bne,pn %icc, .Lhwunaligned + sub %o0, 8, %o0 + +.Lhw_loop: + ldd [%o1 + 0x00], %f8 + ldd [%o1 + 0x08], %f10 + ldd [%o1 + 0x10], %f12 + ldd [%o1 + 0x18], %f14 + ldd [%o1 + 0x20], %f16 + ldd [%o1 + 0x28], %f18 + ldd [%o1 + 0x30], %f20 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x38], %f22 + add %o1, 0x40, %o1 + prefetch [%o1 + 63], 20 + + .word 0x81b02800 ! MD5 + + bne,pt SIZE_T_CC, .Lhw_loop + nop + +.Lhwfinish: + sta %f0, [%o0 + %g0]0x88 ! store context + sta %f1, [%o0 + %g1]0x88 + add %o0, 8, %o0 + sta %f2, [%o0 + %g0]0x88 + sta %f3, [%o0 + %g1]0x88 + retl + nop + +.align 8 +.Lhwunaligned: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f10 +.Lhwunaligned_loop: + ldd [%o1 + 0x08], %f12 + ldd [%o1 + 0x10], %f14 + ldd [%o1 + 0x18], %f16 + ldd [%o1 + 0x20], %f18 + ldd [%o1 + 0x28], %f20 + ldd [%o1 + 0x30], %f22 + ldd [%o1 + 0x38], %f24 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x40], %f26 + add %o1, 0x40, %o1 + prefetch [%o1 + 63], 20 + + faligndata %f10, %f12, %f8 + faligndata %f12, %f14, %f10 + faligndata %f14, %f16, %f12 + faligndata %f16, %f18, %f14 + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + + .word 0x81b02800 ! MD5 + + bne,pt SIZE_T_CC, .Lhwunaligned_loop + for %f26, %f26, %f10 ! %f10=%f26 + + ba .Lhwfinish + nop + +.align 16 +.Lsoftware: + save %sp,-STACK_FRAME,%sp + + rd %asi,$saved_asi + wr %g0,0x88,%asi ! ASI_PRIMARY_LITTLE + and $inp,7,$shr + andn $inp,7,$inp + + sll $shr,3,$shr ! *=8 + mov 56,$shl2 + ld [$ctx+0],$A + sub $shl2,$shr,$shl2 + ld [$ctx+4],$B + and $shl2,32,$shl1 + add $shl2,8,$shl2 + ld [$ctx+8],$C + sub $shl2,$shl1,$shl2 ! shr+shl1+shl2==64 + ld [$ctx+12],$D + nop + +.Loop: + cmp $shr,0 ! was inp aligned? + ldxa [$inp+0]%asi,@X[0] ! load little-endian input + ldxa [$inp+8]%asi,@X[1] + ldxa [$inp+16]%asi,@X[2] + ldxa [$inp+24]%asi,@X[3] + ldxa [$inp+32]%asi,@X[4] + sllx $A,32,$AB ! pack A,B + ldxa [$inp+40]%asi,@X[5] + sllx $C,32,$CD ! pack C,D + ldxa [$inp+48]%asi,@X[6] + or $B,$AB,$AB + ldxa [$inp+56]%asi,@X[7] + or $D,$CD,$CD + bnz,a,pn %icc,.+8 + ldxa [$inp+64]%asi,@X[8] + + srlx @X[0],$shr,@X[0] ! align X[0] + sllx @X[1],$shl1,$tx + sethi %hi(@K[0]),$t2 + sllx $tx,$shl2,$tx + or $t2,%lo(@K[0]),$t2 + or $tx,@X[0],@X[0] + xor $C,$D,$t1 + add @X[0],$t2,$t2 ! X[0]+K[0] +___ + for ($i=0;$i<15;$i++) { &R0($i,@V); unshift(@V,pop(@V)); } + for (;$i<16;$i++) { &R0_1($i,@V); unshift(@V,pop(@V)); } + for (;$i<32;$i++) { &R1($i,@V); unshift(@V,pop(@V)); } + for (;$i<48;$i++) { &R2($i,@V); unshift(@V,pop(@V)); } + for (;$i<64;$i++) { &R3($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + srlx $AB,32,$t1 ! unpack A,B,C,D and accumulate + add $inp,64,$inp ! advance inp + srlx $CD,32,$t2 + add $t1,$A,$A + subcc $len,1,$len ! done yet? + add $AB,$B,$B + add $t2,$C,$C + add $CD,$D,$D + srl $B,0,$B ! clruw $B + bne SIZE_T_CC,.Loop + srl $D,0,$D ! clruw $D + + st $A,[$ctx+0] ! write out ctx + st $B,[$ctx+4] + st $C,[$ctx+8] + st $D,[$ctx+12] + + wr %g0,$saved_asi,%asi + ret + restore +.type md5_block_asm_data_order,#function +.size md5_block_asm_data_order,(.-md5_block_asm_data_order) + +.asciz "MD5 block transform for SPARCv9, CRYPTOGAMS by " +.align 4 +___ + +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my $ref,$opf; +my %visopf = ( "faligndata" => 0x048, + "for" => 0x07c ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} +sub unalignaddr { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my $ref="$mnemonic\t$rs1,$rs2,$rd"; + + foreach ($rs1,$rs2,$rd) { + if (/%([goli])([0-7])/) { $_=$bias{$1}+$2; } + else { return $ref; } + } + return sprintf ".word\t0x%08x !%s", + 0x81b00300|$rd<<25|$rs1<<14|$rs2, + $ref; +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/\b(f[^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ + &unvis($1,$2,$3,$4) + /ge; + s/\b(alignaddr)\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ + &unalignaddr($1,$2,$3,$4) + /ge; + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/md5/md5_locl.h b/crypto/md5/md5_locl.h index 5f6f2fdedd78..82e69218dae1 100644 --- a/crypto/md5/md5_locl.h +++ b/crypto/md5/md5_locl.h @@ -71,6 +71,8 @@ # define md5_block_data_order md5_block_asm_data_order # elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64) # define md5_block_data_order md5_block_asm_data_order +# elif defined(__sparc) || defined(__sparc__) +# define md5_block_data_order md5_block_asm_data_order # endif #endif diff --git a/crypto/modes/Makefile b/crypto/modes/Makefile index e278fa6a2528..a7863d98be2f 100644 --- a/crypto/modes/Makefile +++ b/crypto/modes/Makefile @@ -22,9 +22,9 @@ APPS= LIB=$(TOP)/libcrypto.a LIBSRC= cbc128.c ctr128.c cts128.c cfb128.c ofb128.c gcm128.c \ - ccm128.c xts128.c + ccm128.c xts128.c wrap128.c LIBOBJ= cbc128.o ctr128.o cts128.o cfb128.o ofb128.o gcm128.o \ - ccm128.o xts128.o $(MODES_ASM_OBJ) + ccm128.o xts128.o wrap128.o $(MODES_ASM_OBJ) SRC= $(LIBSRC) @@ -50,20 +50,26 @@ ghash-x86.s: asm/ghash-x86.pl $(PERL) asm/ghash-x86.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@ ghash-x86_64.s: asm/ghash-x86_64.pl $(PERL) asm/ghash-x86_64.pl $(PERLASM_SCHEME) > $@ +aesni-gcm-x86_64.s: asm/aesni-gcm-x86_64.pl + $(PERL) asm/aesni-gcm-x86_64.pl $(PERLASM_SCHEME) > $@ ghash-sparcv9.s: asm/ghash-sparcv9.pl $(PERL) asm/ghash-sparcv9.pl $@ $(CFLAGS) ghash-alpha.s: asm/ghash-alpha.pl - (preproc=/tmp/$$$$.$@; trap "rm $$preproc" INT; \ + (preproc=$$$$.$@.S; trap "rm $$preproc" INT; \ $(PERL) asm/ghash-alpha.pl > $$preproc && \ - $(CC) -E $$preproc > $@ && rm $$preproc) - + $(CC) -E -P $$preproc > $@ && rm $$preproc) ghash-parisc.s: asm/ghash-parisc.pl $(PERL) asm/ghash-parisc.pl $(PERLASM_SCHEME) $@ +ghashv8-armx.S: asm/ghashv8-armx.pl + $(PERL) asm/ghashv8-armx.pl $(PERLASM_SCHEME) $@ +ghashp8-ppc.s: asm/ghashp8-ppc.pl + $(PERL) asm/ghashp8-ppc.pl $(PERLASM_SCHEME) $@ # GNU make "catch all" ghash-%.S: asm/ghash-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ ghash-armv4.o: ghash-armv4.S +ghashv8-armx.o: ghashv8-armx.S files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO @@ -139,6 +145,14 @@ ofb128.o: ../../include/openssl/modes.h ../../include/openssl/opensslconf.h ofb128.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h ofb128.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h ofb128.o: ../../include/openssl/symhacks.h modes_lcl.h ofb128.c +wrap128.o: ../../e_os.h ../../include/openssl/bio.h +wrap128.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h +wrap128.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h +wrap128.o: ../../include/openssl/lhash.h ../../include/openssl/modes.h +wrap128.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h +wrap128.o: ../../include/openssl/ossl_typ.h ../../include/openssl/safestack.h +wrap128.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h +wrap128.o: ../cryptlib.h wrap128.c xts128.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h xts128.o: ../../include/openssl/modes.h ../../include/openssl/opensslconf.h xts128.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h diff --git a/crypto/modes/asm/aesni-gcm-x86_64.pl b/crypto/modes/asm/aesni-gcm-x86_64.pl new file mode 100755 index 000000000000..7e4e04ea2530 --- /dev/null +++ b/crypto/modes/asm/aesni-gcm-x86_64.pl @@ -0,0 +1,1057 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# +# AES-NI-CTR+GHASH stitch. +# +# February 2013 +# +# OpenSSL GCM implementation is organized in such way that its +# performance is rather close to the sum of its streamed components, +# in the context parallelized AES-NI CTR and modulo-scheduled +# PCLMULQDQ-enabled GHASH. Unfortunately, as no stitch implementation +# was observed to perform significantly better than the sum of the +# components on contemporary CPUs, the effort was deemed impossible to +# justify. This module is based on combination of Intel submissions, +# [1] and [2], with MOVBE twist suggested by Ilya Albrekht and Max +# Locktyukhin of Intel Corp. who verified that it reduces shuffles +# pressure with notable relative improvement, achieving 1.0 cycle per +# byte processed with 128-bit key on Haswell processor, and 0.74 - +# on Broadwell. [Mentioned results are raw profiled measurements for +# favourable packet size, one divisible by 96. Applications using the +# EVP interface will observe a few percent worse performance.] +# +# [1] http://rt.openssl.org/Ticket/Display.html?id=2900&user=guest&pass=guest +# [2] http://www.intel.com/content/dam/www/public/us/en/documents/software-support/enabling-high-performance-gcm.pdf + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +if ($avx>1) {{{ + +($inp,$out,$len,$key,$ivp,$Xip)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); + +($Ii,$T1,$T2,$Hkey, + $Z0,$Z1,$Z2,$Z3,$Xi) = map("%xmm$_",(0..8)); + +($inout0,$inout1,$inout2,$inout3,$inout4,$inout5,$rndkey) = map("%xmm$_",(9..15)); + +($counter,$rounds,$ret,$const,$in0,$end0)=("%ebx","%ebp","%r10","%r11","%r14","%r15"); + +$code=<<___; +.text + +.type _aesni_ctr32_ghash_6x,\@abi-omnipotent +.align 32 +_aesni_ctr32_ghash_6x: + vmovdqu 0x20($const),$T2 # borrow $T2, .Lone_msb + sub \$6,$len + vpxor $Z0,$Z0,$Z0 # $Z0 = 0 + vmovdqu 0x00-0x80($key),$rndkey + vpaddb $T2,$T1,$inout1 + vpaddb $T2,$inout1,$inout2 + vpaddb $T2,$inout2,$inout3 + vpaddb $T2,$inout3,$inout4 + vpaddb $T2,$inout4,$inout5 + vpxor $rndkey,$T1,$inout0 + vmovdqu $Z0,16+8(%rsp) # "$Z3" = 0 + jmp .Loop6x + +.align 32 +.Loop6x: + add \$`6<<24`,$counter + jc .Lhandle_ctr32 # discard $inout[1-5]? + vmovdqu 0x00-0x20($Xip),$Hkey # $Hkey^1 + vpaddb $T2,$inout5,$T1 # next counter value + vpxor $rndkey,$inout1,$inout1 + vpxor $rndkey,$inout2,$inout2 + +.Lresume_ctr32: + vmovdqu $T1,($ivp) # save next counter value + vpclmulqdq \$0x10,$Hkey,$Z3,$Z1 + vpxor $rndkey,$inout3,$inout3 + vmovups 0x10-0x80($key),$T2 # borrow $T2 for $rndkey + vpclmulqdq \$0x01,$Hkey,$Z3,$Z2 + xor %r12,%r12 + cmp $in0,$end0 + + vaesenc $T2,$inout0,$inout0 + vmovdqu 0x30+8(%rsp),$Ii # I[4] + vpxor $rndkey,$inout4,$inout4 + vpclmulqdq \$0x00,$Hkey,$Z3,$T1 + vaesenc $T2,$inout1,$inout1 + vpxor $rndkey,$inout5,$inout5 + setnc %r12b + vpclmulqdq \$0x11,$Hkey,$Z3,$Z3 + vaesenc $T2,$inout2,$inout2 + vmovdqu 0x10-0x20($Xip),$Hkey # $Hkey^2 + neg %r12 + vaesenc $T2,$inout3,$inout3 + vpxor $Z1,$Z2,$Z2 + vpclmulqdq \$0x00,$Hkey,$Ii,$Z1 + vpxor $Z0,$Xi,$Xi # modulo-scheduled + vaesenc $T2,$inout4,$inout4 + vpxor $Z1,$T1,$Z0 + and \$0x60,%r12 + vmovups 0x20-0x80($key),$rndkey + vpclmulqdq \$0x10,$Hkey,$Ii,$T1 + vaesenc $T2,$inout5,$inout5 + + vpclmulqdq \$0x01,$Hkey,$Ii,$T2 + lea ($in0,%r12),$in0 + vaesenc $rndkey,$inout0,$inout0 + vpxor 16+8(%rsp),$Xi,$Xi # modulo-scheduled [vpxor $Z3,$Xi,$Xi] + vpclmulqdq \$0x11,$Hkey,$Ii,$Hkey + vmovdqu 0x40+8(%rsp),$Ii # I[3] + vaesenc $rndkey,$inout1,$inout1 + movbe 0x58($in0),%r13 + vaesenc $rndkey,$inout2,$inout2 + movbe 0x50($in0),%r12 + vaesenc $rndkey,$inout3,$inout3 + mov %r13,0x20+8(%rsp) + vaesenc $rndkey,$inout4,$inout4 + mov %r12,0x28+8(%rsp) + vmovdqu 0x30-0x20($Xip),$Z1 # borrow $Z1 for $Hkey^3 + vaesenc $rndkey,$inout5,$inout5 + + vmovups 0x30-0x80($key),$rndkey + vpxor $T1,$Z2,$Z2 + vpclmulqdq \$0x00,$Z1,$Ii,$T1 + vaesenc $rndkey,$inout0,$inout0 + vpxor $T2,$Z2,$Z2 + vpclmulqdq \$0x10,$Z1,$Ii,$T2 + vaesenc $rndkey,$inout1,$inout1 + vpxor $Hkey,$Z3,$Z3 + vpclmulqdq \$0x01,$Z1,$Ii,$Hkey + vaesenc $rndkey,$inout2,$inout2 + vpclmulqdq \$0x11,$Z1,$Ii,$Z1 + vmovdqu 0x50+8(%rsp),$Ii # I[2] + vaesenc $rndkey,$inout3,$inout3 + vaesenc $rndkey,$inout4,$inout4 + vpxor $T1,$Z0,$Z0 + vmovdqu 0x40-0x20($Xip),$T1 # borrow $T1 for $Hkey^4 + vaesenc $rndkey,$inout5,$inout5 + + vmovups 0x40-0x80($key),$rndkey + vpxor $T2,$Z2,$Z2 + vpclmulqdq \$0x00,$T1,$Ii,$T2 + vaesenc $rndkey,$inout0,$inout0 + vpxor $Hkey,$Z2,$Z2 + vpclmulqdq \$0x10,$T1,$Ii,$Hkey + vaesenc $rndkey,$inout1,$inout1 + movbe 0x48($in0),%r13 + vpxor $Z1,$Z3,$Z3 + vpclmulqdq \$0x01,$T1,$Ii,$Z1 + vaesenc $rndkey,$inout2,$inout2 + movbe 0x40($in0),%r12 + vpclmulqdq \$0x11,$T1,$Ii,$T1 + vmovdqu 0x60+8(%rsp),$Ii # I[1] + vaesenc $rndkey,$inout3,$inout3 + mov %r13,0x30+8(%rsp) + vaesenc $rndkey,$inout4,$inout4 + mov %r12,0x38+8(%rsp) + vpxor $T2,$Z0,$Z0 + vmovdqu 0x60-0x20($Xip),$T2 # borrow $T2 for $Hkey^5 + vaesenc $rndkey,$inout5,$inout5 + + vmovups 0x50-0x80($key),$rndkey + vpxor $Hkey,$Z2,$Z2 + vpclmulqdq \$0x00,$T2,$Ii,$Hkey + vaesenc $rndkey,$inout0,$inout0 + vpxor $Z1,$Z2,$Z2 + vpclmulqdq \$0x10,$T2,$Ii,$Z1 + vaesenc $rndkey,$inout1,$inout1 + movbe 0x38($in0),%r13 + vpxor $T1,$Z3,$Z3 + vpclmulqdq \$0x01,$T2,$Ii,$T1 + vpxor 0x70+8(%rsp),$Xi,$Xi # accumulate I[0] + vaesenc $rndkey,$inout2,$inout2 + movbe 0x30($in0),%r12 + vpclmulqdq \$0x11,$T2,$Ii,$T2 + vaesenc $rndkey,$inout3,$inout3 + mov %r13,0x40+8(%rsp) + vaesenc $rndkey,$inout4,$inout4 + mov %r12,0x48+8(%rsp) + vpxor $Hkey,$Z0,$Z0 + vmovdqu 0x70-0x20($Xip),$Hkey # $Hkey^6 + vaesenc $rndkey,$inout5,$inout5 + + vmovups 0x60-0x80($key),$rndkey + vpxor $Z1,$Z2,$Z2 + vpclmulqdq \$0x10,$Hkey,$Xi,$Z1 + vaesenc $rndkey,$inout0,$inout0 + vpxor $T1,$Z2,$Z2 + vpclmulqdq \$0x01,$Hkey,$Xi,$T1 + vaesenc $rndkey,$inout1,$inout1 + movbe 0x28($in0),%r13 + vpxor $T2,$Z3,$Z3 + vpclmulqdq \$0x00,$Hkey,$Xi,$T2 + vaesenc $rndkey,$inout2,$inout2 + movbe 0x20($in0),%r12 + vpclmulqdq \$0x11,$Hkey,$Xi,$Xi + vaesenc $rndkey,$inout3,$inout3 + mov %r13,0x50+8(%rsp) + vaesenc $rndkey,$inout4,$inout4 + mov %r12,0x58+8(%rsp) + vpxor $Z1,$Z2,$Z2 + vaesenc $rndkey,$inout5,$inout5 + vpxor $T1,$Z2,$Z2 + + vmovups 0x70-0x80($key),$rndkey + vpslldq \$8,$Z2,$Z1 + vpxor $T2,$Z0,$Z0 + vmovdqu 0x10($const),$Hkey # .Lpoly + + vaesenc $rndkey,$inout0,$inout0 + vpxor $Xi,$Z3,$Z3 + vaesenc $rndkey,$inout1,$inout1 + vpxor $Z1,$Z0,$Z0 + movbe 0x18($in0),%r13 + vaesenc $rndkey,$inout2,$inout2 + movbe 0x10($in0),%r12 + vpalignr \$8,$Z0,$Z0,$Ii # 1st phase + vpclmulqdq \$0x10,$Hkey,$Z0,$Z0 + mov %r13,0x60+8(%rsp) + vaesenc $rndkey,$inout3,$inout3 + mov %r12,0x68+8(%rsp) + vaesenc $rndkey,$inout4,$inout4 + vmovups 0x80-0x80($key),$T1 # borrow $T1 for $rndkey + vaesenc $rndkey,$inout5,$inout5 + + vaesenc $T1,$inout0,$inout0 + vmovups 0x90-0x80($key),$rndkey + vaesenc $T1,$inout1,$inout1 + vpsrldq \$8,$Z2,$Z2 + vaesenc $T1,$inout2,$inout2 + vpxor $Z2,$Z3,$Z3 + vaesenc $T1,$inout3,$inout3 + vpxor $Ii,$Z0,$Z0 + movbe 0x08($in0),%r13 + vaesenc $T1,$inout4,$inout4 + movbe 0x00($in0),%r12 + vaesenc $T1,$inout5,$inout5 + vmovups 0xa0-0x80($key),$T1 + cmp \$11,$rounds + jb .Lenc_tail # 128-bit key + + vaesenc $rndkey,$inout0,$inout0 + vaesenc $rndkey,$inout1,$inout1 + vaesenc $rndkey,$inout2,$inout2 + vaesenc $rndkey,$inout3,$inout3 + vaesenc $rndkey,$inout4,$inout4 + vaesenc $rndkey,$inout5,$inout5 + + vaesenc $T1,$inout0,$inout0 + vaesenc $T1,$inout1,$inout1 + vaesenc $T1,$inout2,$inout2 + vaesenc $T1,$inout3,$inout3 + vaesenc $T1,$inout4,$inout4 + vmovups 0xb0-0x80($key),$rndkey + vaesenc $T1,$inout5,$inout5 + vmovups 0xc0-0x80($key),$T1 + je .Lenc_tail # 192-bit key + + vaesenc $rndkey,$inout0,$inout0 + vaesenc $rndkey,$inout1,$inout1 + vaesenc $rndkey,$inout2,$inout2 + vaesenc $rndkey,$inout3,$inout3 + vaesenc $rndkey,$inout4,$inout4 + vaesenc $rndkey,$inout5,$inout5 + + vaesenc $T1,$inout0,$inout0 + vaesenc $T1,$inout1,$inout1 + vaesenc $T1,$inout2,$inout2 + vaesenc $T1,$inout3,$inout3 + vaesenc $T1,$inout4,$inout4 + vmovups 0xd0-0x80($key),$rndkey + vaesenc $T1,$inout5,$inout5 + vmovups 0xe0-0x80($key),$T1 + jmp .Lenc_tail # 256-bit key + +.align 32 +.Lhandle_ctr32: + vmovdqu ($const),$Ii # borrow $Ii for .Lbswap_mask + vpshufb $Ii,$T1,$Z2 # byte-swap counter + vmovdqu 0x30($const),$Z1 # borrow $Z1, .Ltwo_lsb + vpaddd 0x40($const),$Z2,$inout1 # .Lone_lsb + vpaddd $Z1,$Z2,$inout2 + vmovdqu 0x00-0x20($Xip),$Hkey # $Hkey^1 + vpaddd $Z1,$inout1,$inout3 + vpshufb $Ii,$inout1,$inout1 + vpaddd $Z1,$inout2,$inout4 + vpshufb $Ii,$inout2,$inout2 + vpxor $rndkey,$inout1,$inout1 + vpaddd $Z1,$inout3,$inout5 + vpshufb $Ii,$inout3,$inout3 + vpxor $rndkey,$inout2,$inout2 + vpaddd $Z1,$inout4,$T1 # byte-swapped next counter value + vpshufb $Ii,$inout4,$inout4 + vpshufb $Ii,$inout5,$inout5 + vpshufb $Ii,$T1,$T1 # next counter value + jmp .Lresume_ctr32 + +.align 32 +.Lenc_tail: + vaesenc $rndkey,$inout0,$inout0 + vmovdqu $Z3,16+8(%rsp) # postpone vpxor $Z3,$Xi,$Xi + vpalignr \$8,$Z0,$Z0,$Xi # 2nd phase + vaesenc $rndkey,$inout1,$inout1 + vpclmulqdq \$0x10,$Hkey,$Z0,$Z0 + vpxor 0x00($inp),$T1,$T2 + vaesenc $rndkey,$inout2,$inout2 + vpxor 0x10($inp),$T1,$Ii + vaesenc $rndkey,$inout3,$inout3 + vpxor 0x20($inp),$T1,$Z1 + vaesenc $rndkey,$inout4,$inout4 + vpxor 0x30($inp),$T1,$Z2 + vaesenc $rndkey,$inout5,$inout5 + vpxor 0x40($inp),$T1,$Z3 + vpxor 0x50($inp),$T1,$Hkey + vmovdqu ($ivp),$T1 # load next counter value + + vaesenclast $T2,$inout0,$inout0 + vmovdqu 0x20($const),$T2 # borrow $T2, .Lone_msb + vaesenclast $Ii,$inout1,$inout1 + vpaddb $T2,$T1,$Ii + mov %r13,0x70+8(%rsp) + lea 0x60($inp),$inp + vaesenclast $Z1,$inout2,$inout2 + vpaddb $T2,$Ii,$Z1 + mov %r12,0x78+8(%rsp) + lea 0x60($out),$out + vmovdqu 0x00-0x80($key),$rndkey + vaesenclast $Z2,$inout3,$inout3 + vpaddb $T2,$Z1,$Z2 + vaesenclast $Z3, $inout4,$inout4 + vpaddb $T2,$Z2,$Z3 + vaesenclast $Hkey,$inout5,$inout5 + vpaddb $T2,$Z3,$Hkey + + add \$0x60,$ret + sub \$0x6,$len + jc .L6x_done + + vmovups $inout0,-0x60($out) # save output + vpxor $rndkey,$T1,$inout0 + vmovups $inout1,-0x50($out) + vmovdqa $Ii,$inout1 # 0 latency + vmovups $inout2,-0x40($out) + vmovdqa $Z1,$inout2 # 0 latency + vmovups $inout3,-0x30($out) + vmovdqa $Z2,$inout3 # 0 latency + vmovups $inout4,-0x20($out) + vmovdqa $Z3,$inout4 # 0 latency + vmovups $inout5,-0x10($out) + vmovdqa $Hkey,$inout5 # 0 latency + vmovdqu 0x20+8(%rsp),$Z3 # I[5] + jmp .Loop6x + +.L6x_done: + vpxor 16+8(%rsp),$Xi,$Xi # modulo-scheduled + vpxor $Z0,$Xi,$Xi # modulo-scheduled + + ret +.size _aesni_ctr32_ghash_6x,.-_aesni_ctr32_ghash_6x +___ +###################################################################### +# +# size_t aesni_gcm_[en|de]crypt(const void *inp, void *out, size_t len, +# const AES_KEY *key, unsigned char iv[16], +# struct { u128 Xi,H,Htbl[9]; } *Xip); +$code.=<<___; +.globl aesni_gcm_decrypt +.type aesni_gcm_decrypt,\@function,6 +.align 32 +aesni_gcm_decrypt: + xor $ret,$ret + cmp \$0x60,$len # minimal accepted length + jb .Lgcm_dec_abort + + lea (%rsp),%rax # save stack pointer + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,-0xd8(%rax) + movaps %xmm7,-0xc8(%rax) + movaps %xmm8,-0xb8(%rax) + movaps %xmm9,-0xa8(%rax) + movaps %xmm10,-0x98(%rax) + movaps %xmm11,-0x88(%rax) + movaps %xmm12,-0x78(%rax) + movaps %xmm13,-0x68(%rax) + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +.Lgcm_dec_body: +___ +$code.=<<___; + vzeroupper + + vmovdqu ($ivp),$T1 # input counter value + add \$-128,%rsp + mov 12($ivp),$counter + lea .Lbswap_mask(%rip),$const + lea -0x80($key),$in0 # borrow $in0 + mov \$0xf80,$end0 # borrow $end0 + vmovdqu ($Xip),$Xi # load Xi + and \$-128,%rsp # ensure stack alignment + vmovdqu ($const),$Ii # borrow $Ii for .Lbswap_mask + lea 0x80($key),$key # size optimization + lea 0x20+0x20($Xip),$Xip # size optimization + mov 0xf0-0x80($key),$rounds + vpshufb $Ii,$Xi,$Xi + + and $end0,$in0 + and %rsp,$end0 + sub $in0,$end0 + jc .Ldec_no_key_aliasing + cmp \$768,$end0 + jnc .Ldec_no_key_aliasing + sub $end0,%rsp # avoid aliasing with key +.Ldec_no_key_aliasing: + + vmovdqu 0x50($inp),$Z3 # I[5] + lea ($inp),$in0 + vmovdqu 0x40($inp),$Z0 + lea -0xc0($inp,$len),$end0 + vmovdqu 0x30($inp),$Z1 + shr \$4,$len + xor $ret,$ret + vmovdqu 0x20($inp),$Z2 + vpshufb $Ii,$Z3,$Z3 # passed to _aesni_ctr32_ghash_6x + vmovdqu 0x10($inp),$T2 + vpshufb $Ii,$Z0,$Z0 + vmovdqu ($inp),$Hkey + vpshufb $Ii,$Z1,$Z1 + vmovdqu $Z0,0x30(%rsp) + vpshufb $Ii,$Z2,$Z2 + vmovdqu $Z1,0x40(%rsp) + vpshufb $Ii,$T2,$T2 + vmovdqu $Z2,0x50(%rsp) + vpshufb $Ii,$Hkey,$Hkey + vmovdqu $T2,0x60(%rsp) + vmovdqu $Hkey,0x70(%rsp) + + call _aesni_ctr32_ghash_6x + + vmovups $inout0,-0x60($out) # save output + vmovups $inout1,-0x50($out) + vmovups $inout2,-0x40($out) + vmovups $inout3,-0x30($out) + vmovups $inout4,-0x20($out) + vmovups $inout5,-0x10($out) + + vpshufb ($const),$Xi,$Xi # .Lbswap_mask + vmovdqu $Xi,-0x40($Xip) # output Xi + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xd8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp # restore %rsp +.Lgcm_dec_abort: + mov $ret,%rax # return value + ret +.size aesni_gcm_decrypt,.-aesni_gcm_decrypt +___ + +$code.=<<___; +.type _aesni_ctr32_6x,\@abi-omnipotent +.align 32 +_aesni_ctr32_6x: + vmovdqu 0x00-0x80($key),$Z0 # borrow $Z0 for $rndkey + vmovdqu 0x20($const),$T2 # borrow $T2, .Lone_msb + lea -1($rounds),%r13 + vmovups 0x10-0x80($key),$rndkey + lea 0x20-0x80($key),%r12 + vpxor $Z0,$T1,$inout0 + add \$`6<<24`,$counter + jc .Lhandle_ctr32_2 + vpaddb $T2,$T1,$inout1 + vpaddb $T2,$inout1,$inout2 + vpxor $Z0,$inout1,$inout1 + vpaddb $T2,$inout2,$inout3 + vpxor $Z0,$inout2,$inout2 + vpaddb $T2,$inout3,$inout4 + vpxor $Z0,$inout3,$inout3 + vpaddb $T2,$inout4,$inout5 + vpxor $Z0,$inout4,$inout4 + vpaddb $T2,$inout5,$T1 + vpxor $Z0,$inout5,$inout5 + jmp .Loop_ctr32 + +.align 16 +.Loop_ctr32: + vaesenc $rndkey,$inout0,$inout0 + vaesenc $rndkey,$inout1,$inout1 + vaesenc $rndkey,$inout2,$inout2 + vaesenc $rndkey,$inout3,$inout3 + vaesenc $rndkey,$inout4,$inout4 + vaesenc $rndkey,$inout5,$inout5 + vmovups (%r12),$rndkey + lea 0x10(%r12),%r12 + dec %r13d + jnz .Loop_ctr32 + + vmovdqu (%r12),$Hkey # last round key + vaesenc $rndkey,$inout0,$inout0 + vpxor 0x00($inp),$Hkey,$Z0 + vaesenc $rndkey,$inout1,$inout1 + vpxor 0x10($inp),$Hkey,$Z1 + vaesenc $rndkey,$inout2,$inout2 + vpxor 0x20($inp),$Hkey,$Z2 + vaesenc $rndkey,$inout3,$inout3 + vpxor 0x30($inp),$Hkey,$Xi + vaesenc $rndkey,$inout4,$inout4 + vpxor 0x40($inp),$Hkey,$T2 + vaesenc $rndkey,$inout5,$inout5 + vpxor 0x50($inp),$Hkey,$Hkey + lea 0x60($inp),$inp + + vaesenclast $Z0,$inout0,$inout0 + vaesenclast $Z1,$inout1,$inout1 + vaesenclast $Z2,$inout2,$inout2 + vaesenclast $Xi,$inout3,$inout3 + vaesenclast $T2,$inout4,$inout4 + vaesenclast $Hkey,$inout5,$inout5 + vmovups $inout0,0x00($out) + vmovups $inout1,0x10($out) + vmovups $inout2,0x20($out) + vmovups $inout3,0x30($out) + vmovups $inout4,0x40($out) + vmovups $inout5,0x50($out) + lea 0x60($out),$out + + ret +.align 32 +.Lhandle_ctr32_2: + vpshufb $Ii,$T1,$Z2 # byte-swap counter + vmovdqu 0x30($const),$Z1 # borrow $Z1, .Ltwo_lsb + vpaddd 0x40($const),$Z2,$inout1 # .Lone_lsb + vpaddd $Z1,$Z2,$inout2 + vpaddd $Z1,$inout1,$inout3 + vpshufb $Ii,$inout1,$inout1 + vpaddd $Z1,$inout2,$inout4 + vpshufb $Ii,$inout2,$inout2 + vpxor $Z0,$inout1,$inout1 + vpaddd $Z1,$inout3,$inout5 + vpshufb $Ii,$inout3,$inout3 + vpxor $Z0,$inout2,$inout2 + vpaddd $Z1,$inout4,$T1 # byte-swapped next counter value + vpshufb $Ii,$inout4,$inout4 + vpxor $Z0,$inout3,$inout3 + vpshufb $Ii,$inout5,$inout5 + vpxor $Z0,$inout4,$inout4 + vpshufb $Ii,$T1,$T1 # next counter value + vpxor $Z0,$inout5,$inout5 + jmp .Loop_ctr32 +.size _aesni_ctr32_6x,.-_aesni_ctr32_6x + +.globl aesni_gcm_encrypt +.type aesni_gcm_encrypt,\@function,6 +.align 32 +aesni_gcm_encrypt: + xor $ret,$ret + cmp \$0x60*3,$len # minimal accepted length + jb .Lgcm_enc_abort + + lea (%rsp),%rax # save stack pointer + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,-0xd8(%rax) + movaps %xmm7,-0xc8(%rax) + movaps %xmm8,-0xb8(%rax) + movaps %xmm9,-0xa8(%rax) + movaps %xmm10,-0x98(%rax) + movaps %xmm11,-0x88(%rax) + movaps %xmm12,-0x78(%rax) + movaps %xmm13,-0x68(%rax) + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +.Lgcm_enc_body: +___ +$code.=<<___; + vzeroupper + + vmovdqu ($ivp),$T1 # input counter value + add \$-128,%rsp + mov 12($ivp),$counter + lea .Lbswap_mask(%rip),$const + lea -0x80($key),$in0 # borrow $in0 + mov \$0xf80,$end0 # borrow $end0 + lea 0x80($key),$key # size optimization + vmovdqu ($const),$Ii # borrow $Ii for .Lbswap_mask + and \$-128,%rsp # ensure stack alignment + mov 0xf0-0x80($key),$rounds + + and $end0,$in0 + and %rsp,$end0 + sub $in0,$end0 + jc .Lenc_no_key_aliasing + cmp \$768,$end0 + jnc .Lenc_no_key_aliasing + sub $end0,%rsp # avoid aliasing with key +.Lenc_no_key_aliasing: + + lea ($out),$in0 + lea -0xc0($out,$len),$end0 + shr \$4,$len + + call _aesni_ctr32_6x + vpshufb $Ii,$inout0,$Xi # save bswapped output on stack + vpshufb $Ii,$inout1,$T2 + vmovdqu $Xi,0x70(%rsp) + vpshufb $Ii,$inout2,$Z0 + vmovdqu $T2,0x60(%rsp) + vpshufb $Ii,$inout3,$Z1 + vmovdqu $Z0,0x50(%rsp) + vpshufb $Ii,$inout4,$Z2 + vmovdqu $Z1,0x40(%rsp) + vpshufb $Ii,$inout5,$Z3 # passed to _aesni_ctr32_ghash_6x + vmovdqu $Z2,0x30(%rsp) + + call _aesni_ctr32_6x + + vmovdqu ($Xip),$Xi # load Xi + lea 0x20+0x20($Xip),$Xip # size optimization + sub \$12,$len + mov \$0x60*2,$ret + vpshufb $Ii,$Xi,$Xi + + call _aesni_ctr32_ghash_6x + vmovdqu 0x20(%rsp),$Z3 # I[5] + vmovdqu ($const),$Ii # borrow $Ii for .Lbswap_mask + vmovdqu 0x00-0x20($Xip),$Hkey # $Hkey^1 + vpunpckhqdq $Z3,$Z3,$T1 + vmovdqu 0x20-0x20($Xip),$rndkey # borrow $rndkey for $HK + vmovups $inout0,-0x60($out) # save output + vpshufb $Ii,$inout0,$inout0 # but keep bswapped copy + vpxor $Z3,$T1,$T1 + vmovups $inout1,-0x50($out) + vpshufb $Ii,$inout1,$inout1 + vmovups $inout2,-0x40($out) + vpshufb $Ii,$inout2,$inout2 + vmovups $inout3,-0x30($out) + vpshufb $Ii,$inout3,$inout3 + vmovups $inout4,-0x20($out) + vpshufb $Ii,$inout4,$inout4 + vmovups $inout5,-0x10($out) + vpshufb $Ii,$inout5,$inout5 + vmovdqu $inout0,0x10(%rsp) # free $inout0 +___ +{ my ($HK,$T3)=($rndkey,$inout0); + +$code.=<<___; + vmovdqu 0x30(%rsp),$Z2 # I[4] + vmovdqu 0x10-0x20($Xip),$Ii # borrow $Ii for $Hkey^2 + vpunpckhqdq $Z2,$Z2,$T2 + vpclmulqdq \$0x00,$Hkey,$Z3,$Z1 + vpxor $Z2,$T2,$T2 + vpclmulqdq \$0x11,$Hkey,$Z3,$Z3 + vpclmulqdq \$0x00,$HK,$T1,$T1 + + vmovdqu 0x40(%rsp),$T3 # I[3] + vpclmulqdq \$0x00,$Ii,$Z2,$Z0 + vmovdqu 0x30-0x20($Xip),$Hkey # $Hkey^3 + vpxor $Z1,$Z0,$Z0 + vpunpckhqdq $T3,$T3,$Z1 + vpclmulqdq \$0x11,$Ii,$Z2,$Z2 + vpxor $T3,$Z1,$Z1 + vpxor $Z3,$Z2,$Z2 + vpclmulqdq \$0x10,$HK,$T2,$T2 + vmovdqu 0x50-0x20($Xip),$HK + vpxor $T1,$T2,$T2 + + vmovdqu 0x50(%rsp),$T1 # I[2] + vpclmulqdq \$0x00,$Hkey,$T3,$Z3 + vmovdqu 0x40-0x20($Xip),$Ii # borrow $Ii for $Hkey^4 + vpxor $Z0,$Z3,$Z3 + vpunpckhqdq $T1,$T1,$Z0 + vpclmulqdq \$0x11,$Hkey,$T3,$T3 + vpxor $T1,$Z0,$Z0 + vpxor $Z2,$T3,$T3 + vpclmulqdq \$0x00,$HK,$Z1,$Z1 + vpxor $T2,$Z1,$Z1 + + vmovdqu 0x60(%rsp),$T2 # I[1] + vpclmulqdq \$0x00,$Ii,$T1,$Z2 + vmovdqu 0x60-0x20($Xip),$Hkey # $Hkey^5 + vpxor $Z3,$Z2,$Z2 + vpunpckhqdq $T2,$T2,$Z3 + vpclmulqdq \$0x11,$Ii,$T1,$T1 + vpxor $T2,$Z3,$Z3 + vpxor $T3,$T1,$T1 + vpclmulqdq \$0x10,$HK,$Z0,$Z0 + vmovdqu 0x80-0x20($Xip),$HK + vpxor $Z1,$Z0,$Z0 + + vpxor 0x70(%rsp),$Xi,$Xi # accumulate I[0] + vpclmulqdq \$0x00,$Hkey,$T2,$Z1 + vmovdqu 0x70-0x20($Xip),$Ii # borrow $Ii for $Hkey^6 + vpunpckhqdq $Xi,$Xi,$T3 + vpxor $Z2,$Z1,$Z1 + vpclmulqdq \$0x11,$Hkey,$T2,$T2 + vpxor $Xi,$T3,$T3 + vpxor $T1,$T2,$T2 + vpclmulqdq \$0x00,$HK,$Z3,$Z3 + vpxor $Z0,$Z3,$Z0 + + vpclmulqdq \$0x00,$Ii,$Xi,$Z2 + vmovdqu 0x00-0x20($Xip),$Hkey # $Hkey^1 + vpunpckhqdq $inout5,$inout5,$T1 + vpclmulqdq \$0x11,$Ii,$Xi,$Xi + vpxor $inout5,$T1,$T1 + vpxor $Z1,$Z2,$Z1 + vpclmulqdq \$0x10,$HK,$T3,$T3 + vmovdqu 0x20-0x20($Xip),$HK + vpxor $T2,$Xi,$Z3 + vpxor $Z0,$T3,$Z2 + + vmovdqu 0x10-0x20($Xip),$Ii # borrow $Ii for $Hkey^2 + vpxor $Z1,$Z3,$T3 # aggregated Karatsuba post-processing + vpclmulqdq \$0x00,$Hkey,$inout5,$Z0 + vpxor $T3,$Z2,$Z2 + vpunpckhqdq $inout4,$inout4,$T2 + vpclmulqdq \$0x11,$Hkey,$inout5,$inout5 + vpxor $inout4,$T2,$T2 + vpslldq \$8,$Z2,$T3 + vpclmulqdq \$0x00,$HK,$T1,$T1 + vpxor $T3,$Z1,$Xi + vpsrldq \$8,$Z2,$Z2 + vpxor $Z2,$Z3,$Z3 + + vpclmulqdq \$0x00,$Ii,$inout4,$Z1 + vmovdqu 0x30-0x20($Xip),$Hkey # $Hkey^3 + vpxor $Z0,$Z1,$Z1 + vpunpckhqdq $inout3,$inout3,$T3 + vpclmulqdq \$0x11,$Ii,$inout4,$inout4 + vpxor $inout3,$T3,$T3 + vpxor $inout5,$inout4,$inout4 + vpalignr \$8,$Xi,$Xi,$inout5 # 1st phase + vpclmulqdq \$0x10,$HK,$T2,$T2 + vmovdqu 0x50-0x20($Xip),$HK + vpxor $T1,$T2,$T2 + + vpclmulqdq \$0x00,$Hkey,$inout3,$Z0 + vmovdqu 0x40-0x20($Xip),$Ii # borrow $Ii for $Hkey^4 + vpxor $Z1,$Z0,$Z0 + vpunpckhqdq $inout2,$inout2,$T1 + vpclmulqdq \$0x11,$Hkey,$inout3,$inout3 + vpxor $inout2,$T1,$T1 + vpxor $inout4,$inout3,$inout3 + vxorps 0x10(%rsp),$Z3,$Z3 # accumulate $inout0 + vpclmulqdq \$0x00,$HK,$T3,$T3 + vpxor $T2,$T3,$T3 + + vpclmulqdq \$0x10,0x10($const),$Xi,$Xi + vxorps $inout5,$Xi,$Xi + + vpclmulqdq \$0x00,$Ii,$inout2,$Z1 + vmovdqu 0x60-0x20($Xip),$Hkey # $Hkey^5 + vpxor $Z0,$Z1,$Z1 + vpunpckhqdq $inout1,$inout1,$T2 + vpclmulqdq \$0x11,$Ii,$inout2,$inout2 + vpxor $inout1,$T2,$T2 + vpalignr \$8,$Xi,$Xi,$inout5 # 2nd phase + vpxor $inout3,$inout2,$inout2 + vpclmulqdq \$0x10,$HK,$T1,$T1 + vmovdqu 0x80-0x20($Xip),$HK + vpxor $T3,$T1,$T1 + + vxorps $Z3,$inout5,$inout5 + vpclmulqdq \$0x10,0x10($const),$Xi,$Xi + vxorps $inout5,$Xi,$Xi + + vpclmulqdq \$0x00,$Hkey,$inout1,$Z0 + vmovdqu 0x70-0x20($Xip),$Ii # borrow $Ii for $Hkey^6 + vpxor $Z1,$Z0,$Z0 + vpunpckhqdq $Xi,$Xi,$T3 + vpclmulqdq \$0x11,$Hkey,$inout1,$inout1 + vpxor $Xi,$T3,$T3 + vpxor $inout2,$inout1,$inout1 + vpclmulqdq \$0x00,$HK,$T2,$T2 + vpxor $T1,$T2,$T2 + + vpclmulqdq \$0x00,$Ii,$Xi,$Z1 + vpclmulqdq \$0x11,$Ii,$Xi,$Z3 + vpxor $Z0,$Z1,$Z1 + vpclmulqdq \$0x10,$HK,$T3,$Z2 + vpxor $inout1,$Z3,$Z3 + vpxor $T2,$Z2,$Z2 + + vpxor $Z1,$Z3,$Z0 # aggregated Karatsuba post-processing + vpxor $Z0,$Z2,$Z2 + vpslldq \$8,$Z2,$T1 + vmovdqu 0x10($const),$Hkey # .Lpoly + vpsrldq \$8,$Z2,$Z2 + vpxor $T1,$Z1,$Xi + vpxor $Z2,$Z3,$Z3 + + vpalignr \$8,$Xi,$Xi,$T2 # 1st phase + vpclmulqdq \$0x10,$Hkey,$Xi,$Xi + vpxor $T2,$Xi,$Xi + + vpalignr \$8,$Xi,$Xi,$T2 # 2nd phase + vpclmulqdq \$0x10,$Hkey,$Xi,$Xi + vpxor $Z3,$T2,$T2 + vpxor $T2,$Xi,$Xi +___ +} +$code.=<<___; + vpshufb ($const),$Xi,$Xi # .Lbswap_mask + vmovdqu $Xi,-0x40($Xip) # output Xi + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp # restore %rsp +.Lgcm_enc_abort: + mov $ret,%rax # return value + ret +.size aesni_gcm_encrypt,.-aesni_gcm_encrypt +___ + +$code.=<<___; +.align 64 +.Lbswap_mask: + .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 +.Lpoly: + .byte 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 +.Lone_msb: + .byte 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 +.Ltwo_lsb: + .byte 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +.Lone_lsb: + .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +.asciz "AES-NI GCM module for x86_64, CRYPTOGAMS by " +.align 64 +___ +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___ +.extern __imp_RtlVirtualUnwind +.type gcm_se_handler,\@abi-omnipotent +.align 16 +gcm_se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + mov 120($context),%rax # pull context->Rax + + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + mov %r15,240($context) + mov %r14,232($context) + mov %r13,224($context) + mov %r12,216($context) + mov %rbp,160($context) + mov %rbx,144($context) + + lea -0xd8(%rax),%rsi # %xmm save area + lea 512($context),%rdi # & context.Xmm6 + mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) + .long 0xa548f3fc # cld; rep movsq + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size gcm_se_handler,.-gcm_se_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_aesni_gcm_decrypt + .rva .LSEH_end_aesni_gcm_decrypt + .rva .LSEH_gcm_dec_info + + .rva .LSEH_begin_aesni_gcm_encrypt + .rva .LSEH_end_aesni_gcm_encrypt + .rva .LSEH_gcm_enc_info +.section .xdata +.align 8 +.LSEH_gcm_dec_info: + .byte 9,0,0,0 + .rva gcm_se_handler + .rva .Lgcm_dec_body,.Lgcm_dec_abort +.LSEH_gcm_enc_info: + .byte 9,0,0,0 + .rva gcm_se_handler + .rva .Lgcm_enc_body,.Lgcm_enc_abort +___ +} +}}} else {{{ +$code=<<___; # assembler is too old +.text + +.globl aesni_gcm_encrypt +.type aesni_gcm_encrypt,\@abi-omnipotent +aesni_gcm_encrypt: + xor %eax,%eax + ret +.size aesni_gcm_encrypt,.-aesni_gcm_encrypt + +.globl aesni_gcm_decrypt +.type aesni_gcm_decrypt,\@abi-omnipotent +aesni_gcm_decrypt: + xor %eax,%eax + ret +.size aesni_gcm_decrypt,.-aesni_gcm_decrypt +___ +}}} + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; + +print $code; + +close STDOUT; diff --git a/crypto/modes/asm/ghash-armv4.pl b/crypto/modes/asm/ghash-armv4.pl index d91586ee2925..77fbf34465db 100755 --- a/crypto/modes/asm/ghash-armv4.pl +++ b/crypto/modes/asm/ghash-armv4.pl @@ -35,6 +35,20 @@ # Add NEON implementation featuring polynomial multiplication, i.e. no # lookup tables involved. On Cortex A8 it was measured to process one # byte in 15 cycles or 55% faster than integer-only code. +# +# April 2014 +# +# Switch to multiplication algorithm suggested in paper referred +# below and combine it with reduction algorithm from x86 module. +# Performance improvement over previous version varies from 65% on +# Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8 +# processes one byte in 8.45 cycles, A9 - in 10.2, Snapdragon S4 - +# in 9.33. +# +# Câmara, D.; Gouvêa, C. P. L.; López, J. & Dahab, R.: Fast Software +# Polynomial Multiplication on ARM Processors using the NEON Engine. +# +# http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf # ==================================================================== # Note about "528B" variant. In ARM case it makes lesser sense to @@ -303,117 +317,161 @@ $code.=<<___; .size gcm_gmult_4bit,.-gcm_gmult_4bit ___ { -my $cnt=$Htbl; # $Htbl is used once in the very beginning - -my ($Hhi, $Hlo, $Zo, $T, $xi, $mod) = map("d$_",(0..7)); -my ($Qhi, $Qlo, $Z, $R, $zero, $Qpost, $IN) = map("q$_",(8..15)); - -# Z:Zo keeps 128-bit result shifted by 1 to the right, with bottom bit -# in Zo. Or should I say "top bit", because GHASH is specified in -# reverse bit order? Otherwise straightforward 128-bt H by one input -# byte multiplication and modulo-reduction, times 16. +my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3)); +my ($t0,$t1,$t2,$t3)=map("q$_",(8..12)); +my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31)); -sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } -sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } -sub Q() { shift=~m|d([1-3]?[02468])|?"q".($1/2):""; } +sub clmul64x64 { +my ($r,$a,$b)=@_; +$code.=<<___; + vext.8 $t0#lo, $a, $a, #1 @ A1 + vmull.p8 $t0, $t0#lo, $b @ F = A1*B + vext.8 $r#lo, $b, $b, #1 @ B1 + vmull.p8 $r, $a, $r#lo @ E = A*B1 + vext.8 $t1#lo, $a, $a, #2 @ A2 + vmull.p8 $t1, $t1#lo, $b @ H = A2*B + vext.8 $t3#lo, $b, $b, #2 @ B2 + vmull.p8 $t3, $a, $t3#lo @ G = A*B2 + vext.8 $t2#lo, $a, $a, #3 @ A3 + veor $t0, $t0, $r @ L = E + F + vmull.p8 $t2, $t2#lo, $b @ J = A3*B + vext.8 $r#lo, $b, $b, #3 @ B3 + veor $t1, $t1, $t3 @ M = G + H + vmull.p8 $r, $a, $r#lo @ I = A*B3 + veor $t0#lo, $t0#lo, $t0#hi @ t0 = (L) (P0 + P1) << 8 + vand $t0#hi, $t0#hi, $k48 + vext.8 $t3#lo, $b, $b, #4 @ B4 + veor $t1#lo, $t1#lo, $t1#hi @ t1 = (M) (P2 + P3) << 16 + vand $t1#hi, $t1#hi, $k32 + vmull.p8 $t3, $a, $t3#lo @ K = A*B4 + veor $t2, $t2, $r @ N = I + J + veor $t0#lo, $t0#lo, $t0#hi + veor $t1#lo, $t1#lo, $t1#hi + veor $t2#lo, $t2#lo, $t2#hi @ t2 = (N) (P4 + P5) << 24 + vand $t2#hi, $t2#hi, $k16 + vext.8 $t0, $t0, $t0, #15 + veor $t3#lo, $t3#lo, $t3#hi @ t3 = (K) (P6 + P7) << 32 + vmov.i64 $t3#hi, #0 + vext.8 $t1, $t1, $t1, #14 + veor $t2#lo, $t2#lo, $t2#hi + vmull.p8 $r, $a, $b @ D = A*B + vext.8 $t3, $t3, $t3, #12 + vext.8 $t2, $t2, $t2, #13 + veor $t0, $t0, $t1 + veor $t2, $t2, $t3 + veor $r, $r, $t0 + veor $r, $r, $t2 +___ +} $code.=<<___; -#if __ARM_ARCH__>=7 +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a .fpu neon +.global gcm_init_neon +.type gcm_init_neon,%function +.align 4 +gcm_init_neon: + vld1.64 $IN#hi,[r1,:64]! @ load H + vmov.i8 $t0,#0xe1 + vld1.64 $IN#lo,[r1,:64] + vshl.i64 $t0#hi,#57 + vshr.u64 $t0#lo,#63 @ t0=0xc2....01 + vdup.8 $t1,$IN#hi[7] + vshr.u64 $Hlo,$IN#lo,#63 + vshr.s8 $t1,#7 @ broadcast carry bit + vshl.i64 $IN,$IN,#1 + vand $t0,$t0,$t1 + vorr $IN#hi,$Hlo @ H<<<=1 + veor $IN,$IN,$t0 @ twisted H + vstmia r0,{$IN} + + ret @ bx lr +.size gcm_init_neon,.-gcm_init_neon + .global gcm_gmult_neon .type gcm_gmult_neon,%function .align 4 gcm_gmult_neon: - sub $Htbl,#16 @ point at H in GCM128_CTX - vld1.64 `&Dhi("$IN")`,[$Xi,:64]!@ load Xi - vmov.i32 $mod,#0xe1 @ our irreducible polynomial - vld1.64 `&Dlo("$IN")`,[$Xi,:64]! - vshr.u64 $mod,#32 - vldmia $Htbl,{$Hhi-$Hlo} @ load H - veor $zero,$zero + vld1.64 $IN#hi,[$Xi,:64]! @ load Xi + vld1.64 $IN#lo,[$Xi,:64]! + vmov.i64 $k48,#0x0000ffffffffffff + vldmia $Htbl,{$Hlo-$Hhi} @ load twisted H + vmov.i64 $k32,#0x00000000ffffffff #ifdef __ARMEL__ vrev64.8 $IN,$IN #endif - veor $Qpost,$Qpost - veor $R,$R - mov $cnt,#16 - veor $Z,$Z + vmov.i64 $k16,#0x000000000000ffff + veor $Hhl,$Hlo,$Hhi @ Karatsuba pre-processing mov $len,#16 - veor $Zo,$Zo - vdup.8 $xi,`&Dlo("$IN")`[0] @ broadcast lowest byte - b .Linner_neon + b .Lgmult_neon .size gcm_gmult_neon,.-gcm_gmult_neon .global gcm_ghash_neon .type gcm_ghash_neon,%function .align 4 gcm_ghash_neon: - vld1.64 `&Dhi("$Z")`,[$Xi,:64]! @ load Xi - vmov.i32 $mod,#0xe1 @ our irreducible polynomial - vld1.64 `&Dlo("$Z")`,[$Xi,:64]! - vshr.u64 $mod,#32 - vldmia $Xi,{$Hhi-$Hlo} @ load H - veor $zero,$zero - nop + vld1.64 $Xl#hi,[$Xi,:64]! @ load Xi + vld1.64 $Xl#lo,[$Xi,:64]! + vmov.i64 $k48,#0x0000ffffffffffff + vldmia $Htbl,{$Hlo-$Hhi} @ load twisted H + vmov.i64 $k32,#0x00000000ffffffff #ifdef __ARMEL__ - vrev64.8 $Z,$Z + vrev64.8 $Xl,$Xl #endif -.Louter_neon: - vld1.64 `&Dhi($IN)`,[$inp]! @ load inp - veor $Qpost,$Qpost - vld1.64 `&Dlo($IN)`,[$inp]! - veor $R,$R - mov $cnt,#16 + vmov.i64 $k16,#0x000000000000ffff + veor $Hhl,$Hlo,$Hhi @ Karatsuba pre-processing + +.Loop_neon: + vld1.64 $IN#hi,[$inp]! @ load inp + vld1.64 $IN#lo,[$inp]! #ifdef __ARMEL__ vrev64.8 $IN,$IN #endif - veor $Zo,$Zo - veor $IN,$Z @ inp^=Xi - veor $Z,$Z - vdup.8 $xi,`&Dlo("$IN")`[0] @ broadcast lowest byte -.Linner_neon: - subs $cnt,$cnt,#1 - vmull.p8 $Qlo,$Hlo,$xi @ H.lo·Xi[i] - vmull.p8 $Qhi,$Hhi,$xi @ H.hi·Xi[i] - vext.8 $IN,$zero,#1 @ IN>>=8 - - veor $Z,$Qpost @ modulo-scheduled part - vshl.i64 `&Dlo("$R")`,#48 - vdup.8 $xi,`&Dlo("$IN")`[0] @ broadcast lowest byte - veor $T,`&Dlo("$Qlo")`,`&Dlo("$Z")` - - veor `&Dhi("$Z")`,`&Dlo("$R")` - vuzp.8 $Qlo,$Qhi - vsli.8 $Zo,$T,#1 @ compose the "carry" byte - vext.8 $Z,$zero,#1 @ Z>>=8 - - vmull.p8 $R,$Zo,$mod @ "carry"·0xe1 - vshr.u8 $Zo,$T,#7 @ save Z's bottom bit - vext.8 $Qpost,$Qlo,$zero,#1 @ Qlo>>=8 - veor $Z,$Qhi - bne .Linner_neon - - veor $Z,$Qpost @ modulo-scheduled artefact - vshl.i64 `&Dlo("$R")`,#48 - veor `&Dhi("$Z")`,`&Dlo("$R")` - - @ finalization, normalize Z:Zo - vand $Zo,$mod @ suffices to mask the bit - vshr.u64 `&Dhi(&Q("$Zo"))`,`&Dlo("$Z")`,#63 - vshl.i64 $Z,#1 + veor $IN,$Xl @ inp^=Xi +.Lgmult_neon: +___ + &clmul64x64 ($Xl,$Hlo,"$IN#lo"); # H.lo·Xi.lo +$code.=<<___; + veor $IN#lo,$IN#lo,$IN#hi @ Karatsuba pre-processing +___ + &clmul64x64 ($Xm,$Hhl,"$IN#lo"); # (H.lo+H.hi)·(Xi.lo+Xi.hi) + &clmul64x64 ($Xh,$Hhi,"$IN#hi"); # H.hi·Xi.hi +$code.=<<___; + veor $Xm,$Xm,$Xl @ Karatsuba post-processing + veor $Xm,$Xm,$Xh + veor $Xl#hi,$Xl#hi,$Xm#lo + veor $Xh#lo,$Xh#lo,$Xm#hi @ Xh|Xl - 256-bit result + + @ equivalent of reduction_avx from ghash-x86_64.pl + vshl.i64 $t1,$Xl,#57 @ 1st phase + vshl.i64 $t2,$Xl,#62 + veor $t2,$t2,$t1 @ + vshl.i64 $t1,$Xl,#63 + veor $t2, $t2, $t1 @ + veor $Xl#hi,$Xl#hi,$t2#lo @ + veor $Xh#lo,$Xh#lo,$t2#hi + + vshr.u64 $t2,$Xl,#1 @ 2nd phase + veor $Xh,$Xh,$Xl + veor $Xl,$Xl,$t2 @ + vshr.u64 $t2,$t2,#6 + vshr.u64 $Xl,$Xl,#1 @ + veor $Xl,$Xl,$Xh @ + veor $Xl,$Xl,$t2 @ + subs $len,#16 - vorr $Z,`&Q("$Zo")` @ Z=Z:Zo<<1 - bne .Louter_neon + bne .Loop_neon #ifdef __ARMEL__ - vrev64.8 $Z,$Z + vrev64.8 $Xl,$Xl #endif sub $Xi,#16 - vst1.64 `&Dhi("$Z")`,[$Xi,:64]! @ write out Xi - vst1.64 `&Dlo("$Z")`,[$Xi,:64] + vst1.64 $Xl#hi,[$Xi,:64]! @ write out Xi + vst1.64 $Xl#lo,[$Xi,:64] - bx lr + ret @ bx lr .size gcm_ghash_neon,.-gcm_ghash_neon #endif ___ @@ -423,7 +481,13 @@ $code.=<<___; .align 2 ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 -print $code; +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or + s/\bret\b/bx lr/go or + s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4 + + print $_,"\n"; +} close STDOUT; # enforce flush diff --git a/crypto/modes/asm/ghash-s390x.pl b/crypto/modes/asm/ghash-s390x.pl index 6a40d5d89c0c..39096b423ad8 100755 --- a/crypto/modes/asm/ghash-s390x.pl +++ b/crypto/modes/asm/ghash-s390x.pl @@ -186,13 +186,13 @@ $code.=<<___; sllg $rem1,$Zlo,3 xgr $Zlo,$tmp ngr $rem1,$x78 + sllg $tmp,$Zhi,60 j .Lghash_inner .align 16 .Lghash_inner: srlg $Zlo,$Zlo,4 - sllg $tmp,$Zhi,60 - xg $Zlo,8($nlo,$Htbl) srlg $Zhi,$Zhi,4 + xg $Zlo,8($nlo,$Htbl) llgc $xi,0($cnt,$Xi) xg $Zhi,0($nlo,$Htbl) sllg $nlo,$xi,4 @@ -213,9 +213,9 @@ $code.=<<___; sllg $rem1,$Zlo,3 xgr $Zlo,$tmp ngr $rem1,$x78 + sllg $tmp,$Zhi,60 brct $cnt,.Lghash_inner - sllg $tmp,$Zhi,60 srlg $Zlo,$Zlo,4 srlg $Zhi,$Zhi,4 xg $Zlo,8($nlo,$Htbl) diff --git a/crypto/modes/asm/ghash-sparcv9.pl b/crypto/modes/asm/ghash-sparcv9.pl index 70e7b044a3ec..0365e0f1ff42 100755 --- a/crypto/modes/asm/ghash-sparcv9.pl +++ b/crypto/modes/asm/ghash-sparcv9.pl @@ -36,6 +36,15 @@ # references to input data and Z.hi updates to achieve 12 cycles # timing. To anchor to something else, sha1-sparcv9.pl spends 11.6 # cycles to process one byte on UltraSPARC pre-Tx CPU and ~24 on T1. +# +# October 2012 +# +# Add VIS3 lookup-table-free implementation using polynomial +# multiplication xmulx[hi] and extended addition addxc[cc] +# instructions. 4.52/7.63x improvement on T3/T4 or in absolute +# terms 7.90/2.14 cycles per byte. On T4 multi-process benchmark +# saturates at ~15.5x single-process result on 8-core processor, +# or ~20.5GBps per 2.85GHz socket. $bits=32; for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } @@ -66,6 +75,10 @@ $Htbl="%i1"; $inp="%i2"; $len="%i3"; +$code.=<<___ if ($bits==64); +.register %g2,#scratch +.register %g3,#scratch +___ $code.=<<___; .section ".text",#alloc,#execinstr @@ -321,10 +334,238 @@ gcm_gmult_4bit: restore .type gcm_gmult_4bit,#function .size gcm_gmult_4bit,(.-gcm_gmult_4bit) -.asciz "GHASH for SPARCv9, CRYPTOGAMS by " +___ + +{{{ +# Straightforward 128x128-bit multiplication using Karatsuba algorithm +# followed by pair of 64-bit reductions [with a shortcut in first one, +# which allowed to break dependency between reductions and remove one +# multiplication from critical path]. While it might be suboptimal +# with regard to sheer number of multiplications, other methods [such +# as aggregate reduction] would require more 64-bit registers, which +# we don't have in 32-bit application context. + +($Xip,$Htable,$inp,$len)=map("%i$_",(0..3)); + +($Hhl,$Hlo,$Hhi,$Xlo,$Xhi,$xE1,$sqr, $C0,$C1,$C2,$C3,$V)= + (map("%o$_",(0..5,7)),map("%g$_",(1..5))); + +($shl,$shr)=map("%l$_",(0..7)); + +# For details regarding "twisted H" see ghash-x86.pl. +$code.=<<___; +.globl gcm_init_vis3 +.align 32 +gcm_init_vis3: + save %sp,-$frame,%sp + + ldx [%i1+0],$Hhi + ldx [%i1+8],$Hlo + mov 0xE1,$Xhi + mov 1,$Xlo + sllx $Xhi,57,$Xhi + srax $Hhi,63,$C0 ! broadcast carry + addcc $Hlo,$Hlo,$Hlo ! H<<=1 + addxc $Hhi,$Hhi,$Hhi + and $C0,$Xlo,$Xlo + and $C0,$Xhi,$Xhi + xor $Xlo,$Hlo,$Hlo + xor $Xhi,$Hhi,$Hhi + stx $Hlo,[%i0+8] ! save twisted H + stx $Hhi,[%i0+0] + + sethi %hi(0xA0406080),$V + sethi %hi(0x20C0E000),%l0 + or $V,%lo(0xA0406080),$V + or %l0,%lo(0x20C0E000),%l0 + sllx $V,32,$V + or %l0,$V,$V ! (0xE0·i)&0xff=0xA040608020C0E000 + stx $V,[%i0+16] + + ret + restore +.type gcm_init_vis3,#function +.size gcm_init_vis3,.-gcm_init_vis3 + +.globl gcm_gmult_vis3 +.align 32 +gcm_gmult_vis3: + save %sp,-$frame,%sp + + ldx [$Xip+8],$Xlo ! load Xi + ldx [$Xip+0],$Xhi + ldx [$Htable+8],$Hlo ! load twisted H + ldx [$Htable+0],$Hhi + + mov 0xE1,%l7 + sllx %l7,57,$xE1 ! 57 is not a typo + ldx [$Htable+16],$V ! (0xE0·i)&0xff=0xA040608020C0E000 + + xor $Hhi,$Hlo,$Hhl ! Karatsuba pre-processing + xmulx $Xlo,$Hlo,$C0 + xor $Xlo,$Xhi,$C2 ! Karatsuba pre-processing + xmulx $C2,$Hhl,$C1 + xmulxhi $Xlo,$Hlo,$Xlo + xmulxhi $C2,$Hhl,$C2 + xmulxhi $Xhi,$Hhi,$C3 + xmulx $Xhi,$Hhi,$Xhi + + sll $C0,3,$sqr + srlx $V,$sqr,$sqr ! ·0xE0 [implicit &(7<<3)] + xor $C0,$sqr,$sqr + sllx $sqr,57,$sqr ! ($C0·0xE1)<<1<<56 [implicit &0x7f] + + xor $C0,$C1,$C1 ! Karatsuba post-processing + xor $Xlo,$C2,$C2 + xor $sqr,$Xlo,$Xlo ! real destination is $C1 + xor $C3,$C2,$C2 + xor $Xlo,$C1,$C1 + xor $Xhi,$C2,$C2 + xor $Xhi,$C1,$C1 + + xmulxhi $C0,$xE1,$Xlo ! ·0xE1<<1<<56 + xor $C0,$C2,$C2 + xmulx $C1,$xE1,$C0 + xor $C1,$C3,$C3 + xmulxhi $C1,$xE1,$C1 + + xor $Xlo,$C2,$C2 + xor $C0,$C2,$C2 + xor $C1,$C3,$C3 + + stx $C2,[$Xip+8] ! save Xi + stx $C3,[$Xip+0] + + ret + restore +.type gcm_gmult_vis3,#function +.size gcm_gmult_vis3,.-gcm_gmult_vis3 + +.globl gcm_ghash_vis3 +.align 32 +gcm_ghash_vis3: + save %sp,-$frame,%sp + + ldx [$Xip+8],$C2 ! load Xi + ldx [$Xip+0],$C3 + ldx [$Htable+8],$Hlo ! load twisted H + ldx [$Htable+0],$Hhi + + mov 0xE1,%l7 + sllx %l7,57,$xE1 ! 57 is not a typo + ldx [$Htable+16],$V ! (0xE0·i)&0xff=0xA040608020C0E000 + + and $inp,7,$shl + andn $inp,7,$inp + sll $shl,3,$shl + prefetch [$inp+63], 20 + sub %g0,$shl,$shr + + xor $Hhi,$Hlo,$Hhl ! Karatsuba pre-processing +.Loop: + ldx [$inp+8],$Xlo + brz,pt $shl,1f + ldx [$inp+0],$Xhi + + ldx [$inp+16],$C1 ! align data + srlx $Xlo,$shr,$C0 + sllx $Xlo,$shl,$Xlo + sllx $Xhi,$shl,$Xhi + srlx $C1,$shr,$C1 + or $C0,$Xhi,$Xhi + or $C1,$Xlo,$Xlo +1: + add $inp,16,$inp + sub $len,16,$len + xor $C2,$Xlo,$Xlo + xor $C3,$Xhi,$Xhi + prefetch [$inp+63], 20 + + xmulx $Xlo,$Hlo,$C0 + xor $Xlo,$Xhi,$C2 ! Karatsuba pre-processing + xmulx $C2,$Hhl,$C1 + xmulxhi $Xlo,$Hlo,$Xlo + xmulxhi $C2,$Hhl,$C2 + xmulxhi $Xhi,$Hhi,$C3 + xmulx $Xhi,$Hhi,$Xhi + + sll $C0,3,$sqr + srlx $V,$sqr,$sqr ! ·0xE0 [implicit &(7<<3)] + xor $C0,$sqr,$sqr + sllx $sqr,57,$sqr ! ($C0·0xE1)<<1<<56 [implicit &0x7f] + + xor $C0,$C1,$C1 ! Karatsuba post-processing + xor $Xlo,$C2,$C2 + xor $sqr,$Xlo,$Xlo ! real destination is $C1 + xor $C3,$C2,$C2 + xor $Xlo,$C1,$C1 + xor $Xhi,$C2,$C2 + xor $Xhi,$C1,$C1 + + xmulxhi $C0,$xE1,$Xlo ! ·0xE1<<1<<56 + xor $C0,$C2,$C2 + xmulx $C1,$xE1,$C0 + xor $C1,$C3,$C3 + xmulxhi $C1,$xE1,$C1 + + xor $Xlo,$C2,$C2 + xor $C0,$C2,$C2 + brnz,pt $len,.Loop + xor $C1,$C3,$C3 + + stx $C2,[$Xip+8] ! save Xi + stx $C3,[$Xip+0] + + ret + restore +.type gcm_ghash_vis3,#function +.size gcm_ghash_vis3,.-gcm_ghash_vis3 +___ +}}} +$code.=<<___; +.asciz "GHASH for SPARCv9/VIS3, CRYPTOGAMS by " .align 4 ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -print $code; + +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis3 { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my ($ref,$opf); +my %visopf = ( "addxc" => 0x011, + "addxccc" => 0x013, + "xmulx" => 0x115, + "xmulxhi" => 0x116 ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%([goli])([0-9])/); + $_=$bias{$1}+$2; + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/\b(xmulx[hi]*|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ + &unvis3($1,$2,$3,$4) + /ge; + + print $_,"\n"; +} + close STDOUT; diff --git a/crypto/modes/asm/ghash-x86.pl b/crypto/modes/asm/ghash-x86.pl index 83c727e07f95..23a5527b30af 100755 --- a/crypto/modes/asm/ghash-x86.pl +++ b/crypto/modes/asm/ghash-x86.pl @@ -12,25 +12,27 @@ # The module implements "4-bit" GCM GHASH function and underlying # single multiplication operation in GF(2^128). "4-bit" means that it # uses 256 bytes per-key table [+64/128 bytes fixed table]. It has two -# code paths: vanilla x86 and vanilla MMX. Former will be executed on -# 486 and Pentium, latter on all others. MMX GHASH features so called +# code paths: vanilla x86 and vanilla SSE. Former will be executed on +# 486 and Pentium, latter on all others. SSE GHASH features so called # "528B" variant of "4-bit" method utilizing additional 256+16 bytes # of per-key storage [+512 bytes shared table]. Performance results # are for streamed GHASH subroutine and are expressed in cycles per # processed byte, less is better: # -# gcc 2.95.3(*) MMX assembler x86 assembler +# gcc 2.95.3(*) SSE assembler x86 assembler # # Pentium 105/111(**) - 50 # PIII 68 /75 12.2 24 # P4 125/125 17.8 84(***) # Opteron 66 /70 10.1 30 # Core2 54 /67 8.4 18 +# Atom 105/105 16.8 53 +# VIA Nano 69 /71 13.0 27 # # (*) gcc 3.4.x was observed to generate few percent slower code, # which is one of reasons why 2.95.3 results were chosen, # another reason is lack of 3.4.x results for older CPUs; -# comparison with MMX results is not completely fair, because C +# comparison with SSE results is not completely fair, because C # results are for vanilla "256B" implementation, while # assembler results are for "528B";-) # (**) second number is result for code compiled with -fPIC flag, @@ -40,8 +42,8 @@ # # To summarize, it's >2-5 times faster than gcc-generated code. To # anchor it to something else SHA1 assembler processes one byte in -# 11-13 cycles on contemporary x86 cores. As for choice of MMX in -# particular, see comment at the end of the file... +# ~7 cycles on contemporary x86 cores. As for choice of MMX/SSE +# in particular, see comment at the end of the file... # May 2010 # @@ -113,6 +115,16 @@ # similar manner resulted in almost 20% degradation on Sandy Bridge, # where original 64-bit code processes one byte in 1.95 cycles. +##################################################################### +# For reference, AMD Bulldozer processes one byte in 1.98 cycles in +# 32-bit mode and 1.89 in 64-bit. + +# February 2013 +# +# Overhaul: aggregate Karatsuba post-processing, improve ILP in +# reduction_alg9. Resulting performance is 1.96 cycles per byte on +# Westmere, 1.95 - on Sandy/Ivy Bridge, 1.76 - on Bulldozer. + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; push(@INC,"${dir}","${dir}../../perlasm"); require "x86asm.pl"; @@ -822,17 +834,18 @@ $len="ebx"; &static_label("bswap"); sub clmul64x64_T2 { # minimal "register" pressure -my ($Xhi,$Xi,$Hkey)=@_; +my ($Xhi,$Xi,$Hkey,$HK)=@_; &movdqa ($Xhi,$Xi); # &pshufd ($T1,$Xi,0b01001110); - &pshufd ($T2,$Hkey,0b01001110); + &pshufd ($T2,$Hkey,0b01001110) if (!defined($HK)); &pxor ($T1,$Xi); # - &pxor ($T2,$Hkey); + &pxor ($T2,$Hkey) if (!defined($HK)); + $HK=$T2 if (!defined($HK)); &pclmulqdq ($Xi,$Hkey,0x00); ####### &pclmulqdq ($Xhi,$Hkey,0x11); ####### - &pclmulqdq ($T1,$T2,0x00); ####### + &pclmulqdq ($T1,$HK,0x00); ####### &xorps ($T1,$Xi); # &xorps ($T1,$Xhi); # @@ -879,31 +892,32 @@ if (1) { # Algorithm 9 with <<1 twist. # below. Algorithm 9 was therefore chosen for # further optimization... -sub reduction_alg9 { # 17/13 times faster than Intel version +sub reduction_alg9 { # 17/11 times faster than Intel version my ($Xhi,$Xi) = @_; # 1st phase - &movdqa ($T1,$Xi); # + &movdqa ($T2,$Xi); # + &movdqa ($T1,$Xi); + &psllq ($Xi,5); + &pxor ($T1,$Xi); # &psllq ($Xi,1); &pxor ($Xi,$T1); # - &psllq ($Xi,5); # - &pxor ($Xi,$T1); # &psllq ($Xi,57); # - &movdqa ($T2,$Xi); # + &movdqa ($T1,$Xi); # &pslldq ($Xi,8); - &psrldq ($T2,8); # - &pxor ($Xi,$T1); - &pxor ($Xhi,$T2); # + &psrldq ($T1,8); # + &pxor ($Xi,$T2); + &pxor ($Xhi,$T1); # # 2nd phase &movdqa ($T2,$Xi); + &psrlq ($Xi,1); + &pxor ($Xhi,$T2); # + &pxor ($T2,$Xi); &psrlq ($Xi,5); &pxor ($Xi,$T2); # &psrlq ($Xi,1); # - &pxor ($Xi,$T2); # - &pxor ($T2,$Xhi); - &psrlq ($Xi,1); # - &pxor ($Xi,$T2); # + &pxor ($Xi,$Xhi) # } &function_begin_B("gcm_init_clmul"); @@ -937,8 +951,14 @@ my ($Xhi,$Xi) = @_; &clmul64x64_T2 ($Xhi,$Xi,$Hkey); &reduction_alg9 ($Xhi,$Xi); + &pshufd ($T1,$Hkey,0b01001110); + &pshufd ($T2,$Xi,0b01001110); + &pxor ($T1,$Hkey); # Karatsuba pre-processing &movdqu (&QWP(0,$Htbl),$Hkey); # save H + &pxor ($T2,$Xi); # Karatsuba pre-processing &movdqu (&QWP(16,$Htbl),$Xi); # save H^2 + &palignr ($T2,$T1,8); # low part is H.lo^H.hi + &movdqu (&QWP(32,$Htbl),$T2); # save Karatsuba "salt" &ret (); &function_end_B("gcm_init_clmul"); @@ -956,8 +976,9 @@ my ($Xhi,$Xi) = @_; &movdqa ($T3,&QWP(0,$const)); &movups ($Hkey,&QWP(0,$Htbl)); &pshufb ($Xi,$T3); + &movups ($T2,&QWP(32,$Htbl)); - &clmul64x64_T2 ($Xhi,$Xi,$Hkey); + &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$T2); &reduction_alg9 ($Xhi,$Xi); &pshufb ($Xi,$T3); @@ -994,79 +1015,109 @@ my ($Xhi,$Xi) = @_; &movdqu ($Xn,&QWP(16,$inp)); # Ii+1 &pshufb ($T1,$T3); &pshufb ($Xn,$T3); + &movdqu ($T3,&QWP(32,$Htbl)); &pxor ($Xi,$T1); # Ii+Xi - &clmul64x64_T2 ($Xhn,$Xn,$Hkey); # H*Ii+1 + &pshufd ($T1,$Xn,0b01001110); # H*Ii+1 + &movdqa ($Xhn,$Xn); + &pxor ($T1,$Xn); # + &lea ($inp,&DWP(32,$inp)); # i+=2 + + &pclmulqdq ($Xn,$Hkey,0x00); ####### + &pclmulqdq ($Xhn,$Hkey,0x11); ####### + &pclmulqdq ($T1,$T3,0x00); ####### &movups ($Hkey,&QWP(16,$Htbl)); # load H^2 + &nop (); - &lea ($inp,&DWP(32,$inp)); # i+=2 &sub ($len,0x20); &jbe (&label("even_tail")); + &jmp (&label("mod_loop")); -&set_label("mod_loop"); - &clmul64x64_T2 ($Xhi,$Xi,$Hkey); # H^2*(Ii+Xi) - &movdqu ($T1,&QWP(0,$inp)); # Ii - &movups ($Hkey,&QWP(0,$Htbl)); # load H +&set_label("mod_loop",32); + &pshufd ($T2,$Xi,0b01001110); # H^2*(Ii+Xi) + &movdqa ($Xhi,$Xi); + &pxor ($T2,$Xi); # + &nop (); - &pxor ($Xi,$Xn); # (H*Ii+1) + H^2*(Ii+Xi) - &pxor ($Xhi,$Xhn); + &pclmulqdq ($Xi,$Hkey,0x00); ####### + &pclmulqdq ($Xhi,$Hkey,0x11); ####### + &pclmulqdq ($T2,$T3,0x10); ####### + &movups ($Hkey,&QWP(0,$Htbl)); # load H - &movdqu ($Xn,&QWP(16,$inp)); # Ii+1 - &pshufb ($T1,$T3); - &pshufb ($Xn,$T3); + &xorps ($Xi,$Xn); # (H*Ii+1) + H^2*(Ii+Xi) + &movdqa ($T3,&QWP(0,$const)); + &xorps ($Xhi,$Xhn); + &movdqu ($Xhn,&QWP(0,$inp)); # Ii + &pxor ($T1,$Xi); # aggregated Karatsuba post-processing + &movdqu ($Xn,&QWP(16,$inp)); # Ii+1 + &pxor ($T1,$Xhi); # - &movdqa ($T3,$Xn); #&clmul64x64_TX ($Xhn,$Xn,$Hkey); H*Ii+1 - &movdqa ($Xhn,$Xn); - &pxor ($Xhi,$T1); # "Ii+Xi", consume early + &pshufb ($Xhn,$T3); + &pxor ($T2,$T1); # - &movdqa ($T1,$Xi); #&reduction_alg9($Xhi,$Xi); 1st phase + &movdqa ($T1,$T2); # + &psrldq ($T2,8); + &pslldq ($T1,8); # + &pxor ($Xhi,$T2); + &pxor ($Xi,$T1); # + &pshufb ($Xn,$T3); + &pxor ($Xhi,$Xhn); # "Ii+Xi", consume early + + &movdqa ($Xhn,$Xn); #&clmul64x64_TX ($Xhn,$Xn,$Hkey); H*Ii+1 + &movdqa ($T2,$Xi); #&reduction_alg9($Xhi,$Xi); 1st phase + &movdqa ($T1,$Xi); + &psllq ($Xi,5); + &pxor ($T1,$Xi); # &psllq ($Xi,1); &pxor ($Xi,$T1); # - &psllq ($Xi,5); # - &pxor ($Xi,$T1); # &pclmulqdq ($Xn,$Hkey,0x00); ####### + &movups ($T3,&QWP(32,$Htbl)); &psllq ($Xi,57); # - &movdqa ($T2,$Xi); # + &movdqa ($T1,$Xi); # &pslldq ($Xi,8); - &psrldq ($T2,8); # - &pxor ($Xi,$T1); - &pshufd ($T1,$T3,0b01001110); + &psrldq ($T1,8); # + &pxor ($Xi,$T2); + &pxor ($Xhi,$T1); # + &pshufd ($T1,$Xhn,0b01001110); + &movdqa ($T2,$Xi); # 2nd phase + &psrlq ($Xi,1); + &pxor ($T1,$Xhn); &pxor ($Xhi,$T2); # - &pxor ($T1,$T3); - &pshufd ($T3,$Hkey,0b01001110); - &pxor ($T3,$Hkey); # - &pclmulqdq ($Xhn,$Hkey,0x11); ####### - &movdqa ($T2,$Xi); # 2nd phase + &movups ($Hkey,&QWP(16,$Htbl)); # load H^2 + &pxor ($T2,$Xi); &psrlq ($Xi,5); &pxor ($Xi,$T2); # &psrlq ($Xi,1); # - &pxor ($Xi,$T2); # - &pxor ($T2,$Xhi); - &psrlq ($Xi,1); # - &pxor ($Xi,$T2); # - + &pxor ($Xi,$Xhi) # &pclmulqdq ($T1,$T3,0x00); ####### - &movups ($Hkey,&QWP(16,$Htbl)); # load H^2 - &xorps ($T1,$Xn); # - &xorps ($T1,$Xhn); # - - &movdqa ($T3,$T1); # - &psrldq ($T1,8); - &pslldq ($T3,8); # - &pxor ($Xhn,$T1); - &pxor ($Xn,$T3); # - &movdqa ($T3,&QWP(0,$const)); &lea ($inp,&DWP(32,$inp)); &sub ($len,0x20); &ja (&label("mod_loop")); &set_label("even_tail"); - &clmul64x64_T2 ($Xhi,$Xi,$Hkey); # H^2*(Ii+Xi) + &pshufd ($T2,$Xi,0b01001110); # H^2*(Ii+Xi) + &movdqa ($Xhi,$Xi); + &pxor ($T2,$Xi); # - &pxor ($Xi,$Xn); # (H*Ii+1) + H^2*(Ii+Xi) - &pxor ($Xhi,$Xhn); + &pclmulqdq ($Xi,$Hkey,0x00); ####### + &pclmulqdq ($Xhi,$Hkey,0x11); ####### + &pclmulqdq ($T2,$T3,0x10); ####### + &movdqa ($T3,&QWP(0,$const)); + + &xorps ($Xi,$Xn); # (H*Ii+1) + H^2*(Ii+Xi) + &xorps ($Xhi,$Xhn); + &pxor ($T1,$Xi); # aggregated Karatsuba post-processing + &pxor ($T1,$Xhi); # + + &pxor ($T2,$T1); # + + &movdqa ($T1,$T2); # + &psrldq ($T2,8); + &pslldq ($T1,8); # + &pxor ($Xhi,$T2); + &pxor ($Xi,$T1); # &reduction_alg9 ($Xhi,$Xi); @@ -1273,13 +1324,6 @@ my ($Xhi,$Xi)=@_; &set_label("bswap",64); &data_byte(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0); &data_byte(1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2); # 0x1c2_polynomial -}} # $sse2 - -&set_label("rem_4bit",64); - &data_word(0,0x0000<<$S,0,0x1C20<<$S,0,0x3840<<$S,0,0x2460<<$S); - &data_word(0,0x7080<<$S,0,0x6CA0<<$S,0,0x48C0<<$S,0,0x54E0<<$S); - &data_word(0,0xE100<<$S,0,0xFD20<<$S,0,0xD940<<$S,0,0xC560<<$S); - &data_word(0,0x9180<<$S,0,0x8DA0<<$S,0,0xA9C0<<$S,0,0xB5E0<<$S); &set_label("rem_8bit",64); &data_short(0x0000,0x01C2,0x0384,0x0246,0x0708,0x06CA,0x048C,0x054E); &data_short(0x0E10,0x0FD2,0x0D94,0x0C56,0x0918,0x08DA,0x0A9C,0x0B5E); @@ -1313,6 +1357,13 @@ my ($Xhi,$Xi)=@_; &data_short(0xA7D0,0xA612,0xA454,0xA596,0xA0D8,0xA11A,0xA35C,0xA29E); &data_short(0xB5E0,0xB422,0xB664,0xB7A6,0xB2E8,0xB32A,0xB16C,0xB0AE); &data_short(0xBBF0,0xBA32,0xB874,0xB9B6,0xBCF8,0xBD3A,0xBF7C,0xBEBE); +}} # $sse2 + +&set_label("rem_4bit",64); + &data_word(0,0x0000<<$S,0,0x1C20<<$S,0,0x3840<<$S,0,0x2460<<$S); + &data_word(0,0x7080<<$S,0,0x6CA0<<$S,0,0x48C0<<$S,0,0x54E0<<$S); + &data_word(0,0xE100<<$S,0,0xFD20<<$S,0,0xD940<<$S,0,0xC560<<$S); + &data_word(0,0x9180<<$S,0,0x8DA0<<$S,0,0xA9C0<<$S,0,0xB5E0<<$S); }}} # !$x86only &asciz("GHASH for x86, CRYPTOGAMS by "); diff --git a/crypto/modes/asm/ghash-x86_64.pl b/crypto/modes/asm/ghash-x86_64.pl index 38d779edbcfd..6e656ca13b80 100755 --- a/crypto/modes/asm/ghash-x86_64.pl +++ b/crypto/modes/asm/ghash-x86_64.pl @@ -22,6 +22,8 @@ # P4 28.6 14.0 +100% # Opteron 19.3 7.7 +150% # Core2 17.8 8.1(**) +120% +# Atom 31.6 16.8 +88% +# VIA Nano 21.8 10.1 +115% # # (*) comparison is not completely fair, because C results are # for vanilla "256B" implementation, while assembler results @@ -39,6 +41,44 @@ # providing access to a Westmere-based system on behalf of Intel # Open Source Technology Centre. +# December 2012 +# +# Overhaul: aggregate Karatsuba post-processing, improve ILP in +# reduction_alg9, increase reduction aggregate factor to 4x. As for +# the latter. ghash-x86.pl discusses that it makes lesser sense to +# increase aggregate factor. Then why increase here? Critical path +# consists of 3 independent pclmulqdq instructions, Karatsuba post- +# processing and reduction. "On top" of this we lay down aggregated +# multiplication operations, triplets of independent pclmulqdq's. As +# issue rate for pclmulqdq is limited, it makes lesser sense to +# aggregate more multiplications than it takes to perform remaining +# non-multiplication operations. 2x is near-optimal coefficient for +# contemporary Intel CPUs (therefore modest improvement coefficient), +# but not for Bulldozer. Latter is because logical SIMD operations +# are twice as slow in comparison to Intel, so that critical path is +# longer. A CPU with higher pclmulqdq issue rate would also benefit +# from higher aggregate factor... +# +# Westmere 1.78(+13%) +# Sandy Bridge 1.80(+8%) +# Ivy Bridge 1.80(+7%) +# Haswell 0.55(+93%) (if system doesn't support AVX) +# Broadwell 0.45(+110%)(if system doesn't support AVX) +# Bulldozer 1.49(+27%) +# Silvermont 2.88(+13%) + +# March 2013 +# +# ... 8x aggregate factor AVX code path is using reduction algorithm +# suggested by Shay Gueron[1]. Even though contemporary AVX-capable +# CPUs such as Sandy and Ivy Bridge can execute it, the code performs +# sub-optimally in comparison to above mentioned version. But thanks +# to Ilya Albrekht and Max Locktyukhin of Intel Corp. we knew that +# it performs in 0.41 cycles per byte on Haswell processor, and in +# 0.29 on Broadwell. +# +# [1] http://rt.openssl.org/Ticket/Display.html?id=2900&user=guest&pass=guest + $flavour = shift; $output = shift; if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } @@ -50,9 +90,30 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or die "can't locate x86_64-xlate.pl"; +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + open OUT,"| \"$^X\" $xlate $flavour $output"; *STDOUT=*OUT; +$do4xaggr=1; + # common register layout $nlo="%rax"; $nhi="%rbx"; @@ -160,6 +221,7 @@ ___ $code=<<___; .text +.extern OPENSSL_ia32cap_P .globl gcm_gmult_4bit .type gcm_gmult_4bit,\@function,2 @@ -352,19 +414,27 @@ ___ ($T1,$T2,$T3)=("%xmm3","%xmm4","%xmm5"); sub clmul64x64_T2 { # minimal register pressure -my ($Xhi,$Xi,$Hkey,$modulo)=@_; +my ($Xhi,$Xi,$Hkey,$HK)=@_; -$code.=<<___ if (!defined($modulo)); +if (!defined($HK)) { $HK = $T2; +$code.=<<___; movdqa $Xi,$Xhi # pshufd \$0b01001110,$Xi,$T1 pshufd \$0b01001110,$Hkey,$T2 pxor $Xi,$T1 # pxor $Hkey,$T2 ___ +} else { +$code.=<<___; + movdqa $Xi,$Xhi # + pshufd \$0b01001110,$Xi,$T1 + pxor $Xi,$T1 # +___ +} $code.=<<___; pclmulqdq \$0x00,$Hkey,$Xi ####### pclmulqdq \$0x11,$Hkey,$Xhi ####### - pclmulqdq \$0x00,$T2,$T1 ####### + pclmulqdq \$0x00,$HK,$T1 ####### pxor $Xi,$T1 # pxor $Xhi,$T1 # @@ -376,42 +446,53 @@ $code.=<<___; ___ } -sub reduction_alg9 { # 17/13 times faster than Intel version +sub reduction_alg9 { # 17/11 times faster than Intel version my ($Xhi,$Xi) = @_; $code.=<<___; # 1st phase - movdqa $Xi,$T1 # + movdqa $Xi,$T2 # + movdqa $Xi,$T1 + psllq \$5,$Xi + pxor $Xi,$T1 # psllq \$1,$Xi pxor $T1,$Xi # - psllq \$5,$Xi # - pxor $T1,$Xi # psllq \$57,$Xi # - movdqa $Xi,$T2 # + movdqa $Xi,$T1 # pslldq \$8,$Xi - psrldq \$8,$T2 # - pxor $T1,$Xi - pxor $T2,$Xhi # + psrldq \$8,$T1 # + pxor $T2,$Xi + pxor $T1,$Xhi # # 2nd phase movdqa $Xi,$T2 + psrlq \$1,$Xi + pxor $T2,$Xhi # + pxor $Xi,$T2 psrlq \$5,$Xi pxor $T2,$Xi # psrlq \$1,$Xi # - pxor $T2,$Xi # - pxor $Xhi,$T2 - psrlq \$1,$Xi # - pxor $T2,$Xi # + pxor $Xhi,$Xi # ___ } { my ($Htbl,$Xip)=@_4args; + my $HK="%xmm6"; $code.=<<___; .globl gcm_init_clmul .type gcm_init_clmul,\@abi-omnipotent .align 16 gcm_init_clmul: +.L_init_clmul: +___ +$code.=<<___ if ($win64); +.LSEH_begin_gcm_init_clmul: + # I can't trust assembler to use specific encoding:-( + .byte 0x48,0x83,0xec,0x18 #sub $0x18,%rsp + .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp) +___ +$code.=<<___; movdqu ($Xip),$Hkey pshufd \$0b01001110,$Hkey,$Hkey # dword swap @@ -430,13 +511,47 @@ gcm_init_clmul: pxor $T3,$Hkey # if(carry) H^=0x1c2_polynomial # calculate H^2 + pshufd \$0b01001110,$Hkey,$HK movdqa $Hkey,$Xi + pxor $Hkey,$HK ___ - &clmul64x64_T2 ($Xhi,$Xi,$Hkey); + &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); &reduction_alg9 ($Xhi,$Xi); $code.=<<___; - movdqu $Hkey,($Htbl) # save H - movdqu $Xi,16($Htbl) # save H^2 + pshufd \$0b01001110,$Hkey,$T1 + pshufd \$0b01001110,$Xi,$T2 + pxor $Hkey,$T1 # Karatsuba pre-processing + movdqu $Hkey,0x00($Htbl) # save H + pxor $Xi,$T2 # Karatsuba pre-processing + movdqu $Xi,0x10($Htbl) # save H^2 + palignr \$8,$T1,$T2 # low part is H.lo^H.hi... + movdqu $T2,0x20($Htbl) # save Karatsuba "salt" +___ +if ($do4xaggr) { + &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); # H^3 + &reduction_alg9 ($Xhi,$Xi); +$code.=<<___; + movdqa $Xi,$T3 +___ + &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); # H^4 + &reduction_alg9 ($Xhi,$Xi); +$code.=<<___; + pshufd \$0b01001110,$T3,$T1 + pshufd \$0b01001110,$Xi,$T2 + pxor $T3,$T1 # Karatsuba pre-processing + movdqu $T3,0x30($Htbl) # save H^3 + pxor $Xi,$T2 # Karatsuba pre-processing + movdqu $Xi,0x40($Htbl) # save H^4 + palignr \$8,$T1,$T2 # low part is H^3.lo^H^3.hi... + movdqu $T2,0x50($Htbl) # save Karatsuba "salt" +___ +} +$code.=<<___ if ($win64); + movaps (%rsp),%xmm6 + lea 0x18(%rsp),%rsp +.LSEH_end_gcm_init_clmul: +___ +$code.=<<___; ret .size gcm_init_clmul,.-gcm_init_clmul ___ @@ -449,13 +564,38 @@ $code.=<<___; .type gcm_gmult_clmul,\@abi-omnipotent .align 16 gcm_gmult_clmul: +.L_gmult_clmul: movdqu ($Xip),$Xi movdqa .Lbswap_mask(%rip),$T3 movdqu ($Htbl),$Hkey + movdqu 0x20($Htbl),$T2 pshufb $T3,$Xi ___ - &clmul64x64_T2 ($Xhi,$Xi,$Hkey); - &reduction_alg9 ($Xhi,$Xi); + &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$T2); +$code.=<<___ if (0 || (&reduction_alg9($Xhi,$Xi)&&0)); + # experimental alternative. special thing about is that there + # no dependency between the two multiplications... + mov \$`0xE1<<1`,%eax + mov \$0xA040608020C0E000,%r10 # ((7..0)·0xE0)&0xff + mov \$0x07,%r11d + movq %rax,$T1 + movq %r10,$T2 + movq %r11,$T3 # borrow $T3 + pand $Xi,$T3 + pshufb $T3,$T2 # ($Xi&7)·0xE0 + movq %rax,$T3 + pclmulqdq \$0x00,$Xi,$T1 # ·(0xE1<<1) + pxor $Xi,$T2 + pslldq \$15,$T2 + paddd $T2,$T2 # <<(64+56+1) + pxor $T2,$Xi + pclmulqdq \$0x01,$T3,$Xi + movdqa .Lbswap_mask(%rip),$T3 # reload $T3 + psrldq \$1,$T1 + pxor $T1,$Xhi + pslldq \$7,$Xi + pxor $Xhi,$Xi +___ $code.=<<___; pshufb $T3,$Xi movdqu $Xi,($Xip) @@ -465,129 +605,327 @@ ___ } { my ($Xip,$Htbl,$inp,$len)=@_4args; - my $Xn="%xmm6"; - my $Xhn="%xmm7"; - my $Hkey2="%xmm8"; - my $T1n="%xmm9"; - my $T2n="%xmm10"; + my ($Xln,$Xmn,$Xhn,$Hkey2,$HK) = map("%xmm$_",(3..7)); + my ($T1,$T2,$T3)=map("%xmm$_",(8..10)); $code.=<<___; .globl gcm_ghash_clmul .type gcm_ghash_clmul,\@abi-omnipotent -.align 16 +.align 32 gcm_ghash_clmul: +.L_ghash_clmul: ___ $code.=<<___ if ($win64); + lea -0x88(%rsp),%rax .LSEH_begin_gcm_ghash_clmul: # I can't trust assembler to use specific encoding:-( - .byte 0x48,0x83,0xec,0x58 #sub \$0x58,%rsp - .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp) - .byte 0x0f,0x29,0x7c,0x24,0x10 #movdqa %xmm7,0x10(%rsp) - .byte 0x44,0x0f,0x29,0x44,0x24,0x20 #movaps %xmm8,0x20(%rsp) - .byte 0x44,0x0f,0x29,0x4c,0x24,0x30 #movaps %xmm9,0x30(%rsp) - .byte 0x44,0x0f,0x29,0x54,0x24,0x40 #movaps %xmm10,0x40(%rsp) + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax),%rsp + .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6,-0x20(%rax) + .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7,-0x10(%rax) + .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8,0(%rax) + .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9,0x10(%rax) + .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10,0x20(%rax) + .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11,0x30(%rax) + .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12,0x40(%rax) + .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13,0x50(%rax) + .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14,0x60(%rax) + .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15,0x70(%rax) ___ $code.=<<___; movdqa .Lbswap_mask(%rip),$T3 movdqu ($Xip),$Xi movdqu ($Htbl),$Hkey + movdqu 0x20($Htbl),$HK pshufb $T3,$Xi sub \$0x10,$len jz .Lodd_tail - movdqu 16($Htbl),$Hkey2 + movdqu 0x10($Htbl),$Hkey2 +___ +if ($do4xaggr) { +my ($Xl,$Xm,$Xh,$Hkey3,$Hkey4)=map("%xmm$_",(11..15)); + +$code.=<<___; + mov OPENSSL_ia32cap_P+4(%rip),%eax + cmp \$0x30,$len + jb .Lskip4x + + and \$`1<<26|1<<22`,%eax # isolate MOVBE+XSAVE + cmp \$`1<<22`,%eax # check for MOVBE without XSAVE + je .Lskip4x + + sub \$0x30,$len + mov \$0xA040608020C0E000,%rax # ((7..0)·0xE0)&0xff + movdqu 0x30($Htbl),$Hkey3 + movdqu 0x40($Htbl),$Hkey4 + + ####### + # Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P + # + movdqu 0x30($inp),$Xln + movdqu 0x20($inp),$Xl + pshufb $T3,$Xln + pshufb $T3,$Xl + movdqa $Xln,$Xhn + pshufd \$0b01001110,$Xln,$Xmn + pxor $Xln,$Xmn + pclmulqdq \$0x00,$Hkey,$Xln + pclmulqdq \$0x11,$Hkey,$Xhn + pclmulqdq \$0x00,$HK,$Xmn + + movdqa $Xl,$Xh + pshufd \$0b01001110,$Xl,$Xm + pxor $Xl,$Xm + pclmulqdq \$0x00,$Hkey2,$Xl + pclmulqdq \$0x11,$Hkey2,$Xh + pclmulqdq \$0x10,$HK,$Xm + xorps $Xl,$Xln + xorps $Xh,$Xhn + movups 0x50($Htbl),$HK + xorps $Xm,$Xmn + + movdqu 0x10($inp),$Xl + movdqu 0($inp),$T1 + pshufb $T3,$Xl + pshufb $T3,$T1 + movdqa $Xl,$Xh + pshufd \$0b01001110,$Xl,$Xm + pxor $T1,$Xi + pxor $Xl,$Xm + pclmulqdq \$0x00,$Hkey3,$Xl + movdqa $Xi,$Xhi + pshufd \$0b01001110,$Xi,$T1 + pxor $Xi,$T1 + pclmulqdq \$0x11,$Hkey3,$Xh + pclmulqdq \$0x00,$HK,$Xm + xorps $Xl,$Xln + xorps $Xh,$Xhn + + lea 0x40($inp),$inp + sub \$0x40,$len + jc .Ltail4x + + jmp .Lmod4_loop +.align 32 +.Lmod4_loop: + pclmulqdq \$0x00,$Hkey4,$Xi + xorps $Xm,$Xmn + movdqu 0x30($inp),$Xl + pshufb $T3,$Xl + pclmulqdq \$0x11,$Hkey4,$Xhi + xorps $Xln,$Xi + movdqu 0x20($inp),$Xln + movdqa $Xl,$Xh + pclmulqdq \$0x10,$HK,$T1 + pshufd \$0b01001110,$Xl,$Xm + xorps $Xhn,$Xhi + pxor $Xl,$Xm + pshufb $T3,$Xln + movups 0x20($Htbl),$HK + xorps $Xmn,$T1 + pclmulqdq \$0x00,$Hkey,$Xl + pshufd \$0b01001110,$Xln,$Xmn + + pxor $Xi,$T1 # aggregated Karatsuba post-processing + movdqa $Xln,$Xhn + pxor $Xhi,$T1 # + pxor $Xln,$Xmn + movdqa $T1,$T2 # + pclmulqdq \$0x11,$Hkey,$Xh + pslldq \$8,$T1 + psrldq \$8,$T2 # + pxor $T1,$Xi + movdqa .L7_mask(%rip),$T1 + pxor $T2,$Xhi # + movq %rax,$T2 + + pand $Xi,$T1 # 1st phase + pshufb $T1,$T2 # + pxor $Xi,$T2 # + pclmulqdq \$0x00,$HK,$Xm + psllq \$57,$T2 # + movdqa $T2,$T1 # + pslldq \$8,$T2 + pclmulqdq \$0x00,$Hkey2,$Xln + psrldq \$8,$T1 # + pxor $T2,$Xi + pxor $T1,$Xhi # + movdqu 0($inp),$T1 + + movdqa $Xi,$T2 # 2nd phase + psrlq \$1,$Xi + pclmulqdq \$0x11,$Hkey2,$Xhn + xorps $Xl,$Xln + movdqu 0x10($inp),$Xl + pshufb $T3,$Xl + pclmulqdq \$0x10,$HK,$Xmn + xorps $Xh,$Xhn + movups 0x50($Htbl),$HK + pshufb $T3,$T1 + pxor $T2,$Xhi # + pxor $Xi,$T2 + psrlq \$5,$Xi + + movdqa $Xl,$Xh + pxor $Xm,$Xmn + pshufd \$0b01001110,$Xl,$Xm + pxor $T2,$Xi # + pxor $T1,$Xhi + pxor $Xl,$Xm + pclmulqdq \$0x00,$Hkey3,$Xl + psrlq \$1,$Xi # + pxor $Xhi,$Xi # + movdqa $Xi,$Xhi + pclmulqdq \$0x11,$Hkey3,$Xh + xorps $Xl,$Xln + pshufd \$0b01001110,$Xi,$T1 + pxor $Xi,$T1 + + pclmulqdq \$0x00,$HK,$Xm + xorps $Xh,$Xhn + + lea 0x40($inp),$inp + sub \$0x40,$len + jnc .Lmod4_loop + +.Ltail4x: + pclmulqdq \$0x00,$Hkey4,$Xi + pclmulqdq \$0x11,$Hkey4,$Xhi + pclmulqdq \$0x10,$HK,$T1 + xorps $Xm,$Xmn + xorps $Xln,$Xi + xorps $Xhn,$Xhi + pxor $Xi,$Xhi # aggregated Karatsuba post-processing + pxor $Xmn,$T1 + + pxor $Xhi,$T1 # + pxor $Xi,$Xhi + + movdqa $T1,$T2 # + psrldq \$8,$T1 + pslldq \$8,$T2 # + pxor $T1,$Xhi + pxor $T2,$Xi # +___ + &reduction_alg9($Xhi,$Xi); +$code.=<<___; + add \$0x40,$len + jz .Ldone + movdqu 0x20($Htbl),$HK + sub \$0x10,$len + jz .Lodd_tail +.Lskip4x: +___ +} +$code.=<<___; ####### # Xi+2 =[H*(Ii+1 + Xi+1)] mod P = # [(H*Ii+1) + (H*Xi+1)] mod P = # [(H*Ii+1) + H^2*(Ii+Xi)] mod P # movdqu ($inp),$T1 # Ii - movdqu 16($inp),$Xn # Ii+1 + movdqu 16($inp),$Xln # Ii+1 pshufb $T3,$T1 - pshufb $T3,$Xn + pshufb $T3,$Xln pxor $T1,$Xi # Ii+Xi -___ - &clmul64x64_T2 ($Xhn,$Xn,$Hkey); # H*Ii+1 -$code.=<<___; - movdqa $Xi,$Xhi # - pshufd \$0b01001110,$Xi,$T1 - pshufd \$0b01001110,$Hkey2,$T2 - pxor $Xi,$T1 # - pxor $Hkey2,$T2 + + movdqa $Xln,$Xhn + pshufd \$0b01001110,$Xln,$Xmn + pxor $Xln,$Xmn + pclmulqdq \$0x00,$Hkey,$Xln + pclmulqdq \$0x11,$Hkey,$Xhn + pclmulqdq \$0x00,$HK,$Xmn lea 32($inp),$inp # i+=2 + nop sub \$0x20,$len jbe .Leven_tail + nop + jmp .Lmod_loop +.align 32 .Lmod_loop: -___ - &clmul64x64_T2 ($Xhi,$Xi,$Hkey2,1); # H^2*(Ii+Xi) -$code.=<<___; - movdqu ($inp),$T1 # Ii - pxor $Xn,$Xi # (H*Ii+1) + H^2*(Ii+Xi) - pxor $Xhn,$Xhi + movdqa $Xi,$Xhi + movdqa $Xmn,$T1 + pshufd \$0b01001110,$Xi,$Xmn # + pxor $Xi,$Xmn # - movdqu 16($inp),$Xn # Ii+1 - pshufb $T3,$T1 - pshufb $T3,$Xn + pclmulqdq \$0x00,$Hkey2,$Xi + pclmulqdq \$0x11,$Hkey2,$Xhi + pclmulqdq \$0x10,$HK,$Xmn - movdqa $Xn,$Xhn # - pshufd \$0b01001110,$Xn,$T1n - pshufd \$0b01001110,$Hkey,$T2n - pxor $Xn,$T1n # - pxor $Hkey,$T2n - pxor $T1,$Xhi # "Ii+Xi", consume early + pxor $Xln,$Xi # (H*Ii+1) + H^2*(Ii+Xi) + pxor $Xhn,$Xhi + movdqu ($inp),$T2 # Ii + pxor $Xi,$T1 # aggregated Karatsuba post-processing + pshufb $T3,$T2 + movdqu 16($inp),$Xln # Ii+1 + + pxor $Xhi,$T1 + pxor $T2,$Xhi # "Ii+Xi", consume early + pxor $T1,$Xmn + pshufb $T3,$Xln + movdqa $Xmn,$T1 # + psrldq \$8,$T1 + pslldq \$8,$Xmn # + pxor $T1,$Xhi + pxor $Xmn,$Xi # + + movdqa $Xln,$Xhn # - movdqa $Xi,$T1 # 1st phase + movdqa $Xi,$T2 # 1st phase + movdqa $Xi,$T1 + psllq \$5,$Xi + pxor $Xi,$T1 # + pclmulqdq \$0x00,$Hkey,$Xln ####### psllq \$1,$Xi pxor $T1,$Xi # - psllq \$5,$Xi # - pxor $T1,$Xi # - pclmulqdq \$0x00,$Hkey,$Xn ####### psllq \$57,$Xi # - movdqa $Xi,$T2 # + movdqa $Xi,$T1 # pslldq \$8,$Xi - psrldq \$8,$T2 # - pxor $T1,$Xi - pxor $T2,$Xhi # + psrldq \$8,$T1 # + pxor $T2,$Xi + pshufd \$0b01001110,$Xhn,$Xmn + pxor $T1,$Xhi # + pxor $Xhn,$Xmn # - pclmulqdq \$0x11,$Hkey,$Xhn ####### movdqa $Xi,$T2 # 2nd phase + psrlq \$1,$Xi + pclmulqdq \$0x11,$Hkey,$Xhn ####### + pxor $T2,$Xhi # + pxor $Xi,$T2 psrlq \$5,$Xi pxor $T2,$Xi # + lea 32($inp),$inp psrlq \$1,$Xi # - pxor $T2,$Xi # - pxor $Xhi,$T2 - psrlq \$1,$Xi # - pxor $T2,$Xi # + pclmulqdq \$0x00,$HK,$Xmn ####### + pxor $Xhi,$Xi # - pclmulqdq \$0x00,$T2n,$T1n ####### - movdqa $Xi,$Xhi # - pshufd \$0b01001110,$Xi,$T1 - pshufd \$0b01001110,$Hkey2,$T2 - pxor $Xi,$T1 # - pxor $Hkey2,$T2 - - pxor $Xn,$T1n # - pxor $Xhn,$T1n # - movdqa $T1n,$T2n # - psrldq \$8,$T1n - pslldq \$8,$T2n # - pxor $T1n,$Xhn - pxor $T2n,$Xn # - - lea 32($inp),$inp sub \$0x20,$len ja .Lmod_loop .Leven_tail: -___ - &clmul64x64_T2 ($Xhi,$Xi,$Hkey2,1); # H^2*(Ii+Xi) -$code.=<<___; - pxor $Xn,$Xi # (H*Ii+1) + H^2*(Ii+Xi) + movdqa $Xi,$Xhi + movdqa $Xmn,$T1 + pshufd \$0b01001110,$Xi,$Xmn # + pxor $Xi,$Xmn # + + pclmulqdq \$0x00,$Hkey2,$Xi + pclmulqdq \$0x11,$Hkey2,$Xhi + pclmulqdq \$0x10,$HK,$Xmn + + pxor $Xln,$Xi # (H*Ii+1) + H^2*(Ii+Xi) pxor $Xhn,$Xhi + pxor $Xi,$T1 + pxor $Xhi,$T1 + pxor $T1,$Xmn + movdqa $Xmn,$T1 # + psrldq \$8,$T1 + pslldq \$8,$Xmn # + pxor $T1,$Xhi + pxor $Xmn,$Xi # ___ &reduction_alg9 ($Xhi,$Xi); $code.=<<___; @@ -599,7 +937,7 @@ $code.=<<___; pshufb $T3,$T1 pxor $T1,$Xi # Ii+Xi ___ - &clmul64x64_T2 ($Xhi,$Xi,$Hkey); # H*(Ii+Xi) + &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); # H*(Ii+Xi) &reduction_alg9 ($Xhi,$Xi); $code.=<<___; .Ldone: @@ -612,21 +950,607 @@ $code.=<<___ if ($win64); movaps 0x20(%rsp),%xmm8 movaps 0x30(%rsp),%xmm9 movaps 0x40(%rsp),%xmm10 - add \$0x58,%rsp + movaps 0x50(%rsp),%xmm11 + movaps 0x60(%rsp),%xmm12 + movaps 0x70(%rsp),%xmm13 + movaps 0x80(%rsp),%xmm14 + movaps 0x90(%rsp),%xmm15 + lea 0xa8(%rsp),%rsp +.LSEH_end_gcm_ghash_clmul: ___ $code.=<<___; ret -.LSEH_end_gcm_ghash_clmul: .size gcm_ghash_clmul,.-gcm_ghash_clmul ___ } + +$code.=<<___; +.globl gcm_init_avx +.type gcm_init_avx,\@abi-omnipotent +.align 32 +gcm_init_avx: +___ +if ($avx) { +my ($Htbl,$Xip)=@_4args; +my $HK="%xmm6"; + +$code.=<<___ if ($win64); +.LSEH_begin_gcm_init_avx: + # I can't trust assembler to use specific encoding:-( + .byte 0x48,0x83,0xec,0x18 #sub $0x18,%rsp + .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp) +___ +$code.=<<___; + vzeroupper + + vmovdqu ($Xip),$Hkey + vpshufd \$0b01001110,$Hkey,$Hkey # dword swap + + # <<1 twist + vpshufd \$0b11111111,$Hkey,$T2 # broadcast uppermost dword + vpsrlq \$63,$Hkey,$T1 + vpsllq \$1,$Hkey,$Hkey + vpxor $T3,$T3,$T3 # + vpcmpgtd $T2,$T3,$T3 # broadcast carry bit + vpslldq \$8,$T1,$T1 + vpor $T1,$Hkey,$Hkey # H<<=1 + + # magic reduction + vpand .L0x1c2_polynomial(%rip),$T3,$T3 + vpxor $T3,$Hkey,$Hkey # if(carry) H^=0x1c2_polynomial + + vpunpckhqdq $Hkey,$Hkey,$HK + vmovdqa $Hkey,$Xi + vpxor $Hkey,$HK,$HK + mov \$4,%r10 # up to H^8 + jmp .Linit_start_avx +___ + +sub clmul64x64_avx { +my ($Xhi,$Xi,$Hkey,$HK)=@_; + +if (!defined($HK)) { $HK = $T2; +$code.=<<___; + vpunpckhqdq $Xi,$Xi,$T1 + vpunpckhqdq $Hkey,$Hkey,$T2 + vpxor $Xi,$T1,$T1 # + vpxor $Hkey,$T2,$T2 +___ +} else { +$code.=<<___; + vpunpckhqdq $Xi,$Xi,$T1 + vpxor $Xi,$T1,$T1 # +___ +} +$code.=<<___; + vpclmulqdq \$0x11,$Hkey,$Xi,$Xhi ####### + vpclmulqdq \$0x00,$Hkey,$Xi,$Xi ####### + vpclmulqdq \$0x00,$HK,$T1,$T1 ####### + vpxor $Xi,$Xhi,$T2 # + vpxor $T2,$T1,$T1 # + + vpslldq \$8,$T1,$T2 # + vpsrldq \$8,$T1,$T1 + vpxor $T2,$Xi,$Xi # + vpxor $T1,$Xhi,$Xhi +___ +} + +sub reduction_avx { +my ($Xhi,$Xi) = @_; + +$code.=<<___; + vpsllq \$57,$Xi,$T1 # 1st phase + vpsllq \$62,$Xi,$T2 + vpxor $T1,$T2,$T2 # + vpsllq \$63,$Xi,$T1 + vpxor $T1,$T2,$T2 # + vpslldq \$8,$T2,$T1 # + vpsrldq \$8,$T2,$T2 + vpxor $T1,$Xi,$Xi # + vpxor $T2,$Xhi,$Xhi + + vpsrlq \$1,$Xi,$T2 # 2nd phase + vpxor $Xi,$Xhi,$Xhi + vpxor $T2,$Xi,$Xi # + vpsrlq \$5,$T2,$T2 + vpxor $T2,$Xi,$Xi # + vpsrlq \$1,$Xi,$Xi # + vpxor $Xhi,$Xi,$Xi # +___ +} +$code.=<<___; +.align 32 +.Linit_loop_avx: + vpalignr \$8,$T1,$T2,$T3 # low part is H.lo^H.hi... + vmovdqu $T3,-0x10($Htbl) # save Karatsuba "salt" +___ + &clmul64x64_avx ($Xhi,$Xi,$Hkey,$HK); # calculate H^3,5,7 + &reduction_avx ($Xhi,$Xi); +$code.=<<___; +.Linit_start_avx: + vmovdqa $Xi,$T3 +___ + &clmul64x64_avx ($Xhi,$Xi,$Hkey,$HK); # calculate H^2,4,6,8 + &reduction_avx ($Xhi,$Xi); +$code.=<<___; + vpshufd \$0b01001110,$T3,$T1 + vpshufd \$0b01001110,$Xi,$T2 + vpxor $T3,$T1,$T1 # Karatsuba pre-processing + vmovdqu $T3,0x00($Htbl) # save H^1,3,5,7 + vpxor $Xi,$T2,$T2 # Karatsuba pre-processing + vmovdqu $Xi,0x10($Htbl) # save H^2,4,6,8 + lea 0x30($Htbl),$Htbl + sub \$1,%r10 + jnz .Linit_loop_avx + + vpalignr \$8,$T2,$T1,$T3 # last "salt" is flipped + vmovdqu $T3,-0x10($Htbl) + + vzeroupper +___ +$code.=<<___ if ($win64); + movaps (%rsp),%xmm6 + lea 0x18(%rsp),%rsp +.LSEH_end_gcm_init_avx: +___ +$code.=<<___; + ret +.size gcm_init_avx,.-gcm_init_avx +___ +} else { +$code.=<<___; + jmp .L_init_clmul +.size gcm_init_avx,.-gcm_init_avx +___ +} + +$code.=<<___; +.globl gcm_gmult_avx +.type gcm_gmult_avx,\@abi-omnipotent +.align 32 +gcm_gmult_avx: + jmp .L_gmult_clmul +.size gcm_gmult_avx,.-gcm_gmult_avx +___ + +$code.=<<___; +.globl gcm_ghash_avx +.type gcm_ghash_avx,\@abi-omnipotent +.align 32 +gcm_ghash_avx: +___ +if ($avx) { +my ($Xip,$Htbl,$inp,$len)=@_4args; +my ($Xlo,$Xhi,$Xmi, + $Zlo,$Zhi,$Zmi, + $Hkey,$HK,$T1,$T2, + $Xi,$Xo,$Tred,$bswap,$Ii,$Ij) = map("%xmm$_",(0..15)); + +$code.=<<___ if ($win64); + lea -0x88(%rsp),%rax +.LSEH_begin_gcm_ghash_avx: + # I can't trust assembler to use specific encoding:-( + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax),%rsp + .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6,-0x20(%rax) + .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7,-0x10(%rax) + .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8,0(%rax) + .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9,0x10(%rax) + .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10,0x20(%rax) + .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11,0x30(%rax) + .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12,0x40(%rax) + .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13,0x50(%rax) + .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14,0x60(%rax) + .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15,0x70(%rax) +___ +$code.=<<___; + vzeroupper + + vmovdqu ($Xip),$Xi # load $Xi + lea .L0x1c2_polynomial(%rip),%r10 + lea 0x40($Htbl),$Htbl # size optimization + vmovdqu .Lbswap_mask(%rip),$bswap + vpshufb $bswap,$Xi,$Xi + cmp \$0x80,$len + jb .Lshort_avx + sub \$0x80,$len + + vmovdqu 0x70($inp),$Ii # I[7] + vmovdqu 0x00-0x40($Htbl),$Hkey # $Hkey^1 + vpshufb $bswap,$Ii,$Ii + vmovdqu 0x20-0x40($Htbl),$HK + + vpunpckhqdq $Ii,$Ii,$T2 + vmovdqu 0x60($inp),$Ij # I[6] + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpxor $Ii,$T2,$T2 + vpshufb $bswap,$Ij,$Ij + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vmovdqu 0x10-0x40($Htbl),$Hkey # $Hkey^2 + vpunpckhqdq $Ij,$Ij,$T1 + vmovdqu 0x50($inp),$Ii # I[5] + vpclmulqdq \$0x00,$HK,$T2,$Xmi + vpxor $Ij,$T1,$T1 + + vpshufb $bswap,$Ii,$Ii + vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo + vpunpckhqdq $Ii,$Ii,$T2 + vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi + vmovdqu 0x30-0x40($Htbl),$Hkey # $Hkey^3 + vpxor $Ii,$T2,$T2 + vmovdqu 0x40($inp),$Ij # I[4] + vpclmulqdq \$0x10,$HK,$T1,$Zmi + vmovdqu 0x50-0x40($Htbl),$HK + + vpshufb $bswap,$Ij,$Ij + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpxor $Xhi,$Zhi,$Zhi + vpunpckhqdq $Ij,$Ij,$T1 + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vmovdqu 0x40-0x40($Htbl),$Hkey # $Hkey^4 + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T2,$Xmi + vpxor $Ij,$T1,$T1 + + vmovdqu 0x30($inp),$Ii # I[3] + vpxor $Zlo,$Xlo,$Xlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo + vpxor $Zhi,$Xhi,$Xhi + vpshufb $bswap,$Ii,$Ii + vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi + vmovdqu 0x60-0x40($Htbl),$Hkey # $Hkey^5 + vpxor $Zmi,$Xmi,$Xmi + vpunpckhqdq $Ii,$Ii,$T2 + vpclmulqdq \$0x10,$HK,$T1,$Zmi + vmovdqu 0x80-0x40($Htbl),$HK + vpxor $Ii,$T2,$T2 + + vmovdqu 0x20($inp),$Ij # I[2] + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpxor $Xhi,$Zhi,$Zhi + vpshufb $bswap,$Ij,$Ij + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vmovdqu 0x70-0x40($Htbl),$Hkey # $Hkey^6 + vpxor $Xmi,$Zmi,$Zmi + vpunpckhqdq $Ij,$Ij,$T1 + vpclmulqdq \$0x00,$HK,$T2,$Xmi + vpxor $Ij,$T1,$T1 + + vmovdqu 0x10($inp),$Ii # I[1] + vpxor $Zlo,$Xlo,$Xlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo + vpxor $Zhi,$Xhi,$Xhi + vpshufb $bswap,$Ii,$Ii + vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi + vmovdqu 0x90-0x40($Htbl),$Hkey # $Hkey^7 + vpxor $Zmi,$Xmi,$Xmi + vpunpckhqdq $Ii,$Ii,$T2 + vpclmulqdq \$0x10,$HK,$T1,$Zmi + vmovdqu 0xb0-0x40($Htbl),$HK + vpxor $Ii,$T2,$T2 + + vmovdqu ($inp),$Ij # I[0] + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpxor $Xhi,$Zhi,$Zhi + vpshufb $bswap,$Ij,$Ij + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vmovdqu 0xa0-0x40($Htbl),$Hkey # $Hkey^8 + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x10,$HK,$T2,$Xmi + + lea 0x80($inp),$inp + cmp \$0x80,$len + jb .Ltail_avx + + vpxor $Xi,$Ij,$Ij # accumulate $Xi + sub \$0x80,$len + jmp .Loop8x_avx + +.align 32 +.Loop8x_avx: + vpunpckhqdq $Ij,$Ij,$T1 + vmovdqu 0x70($inp),$Ii # I[7] + vpxor $Xlo,$Zlo,$Zlo + vpxor $Ij,$T1,$T1 + vpclmulqdq \$0x00,$Hkey,$Ij,$Xi + vpshufb $bswap,$Ii,$Ii + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xo + vmovdqu 0x00-0x40($Htbl),$Hkey # $Hkey^1 + vpunpckhqdq $Ii,$Ii,$T2 + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Tred + vmovdqu 0x20-0x40($Htbl),$HK + vpxor $Ii,$T2,$T2 + + vmovdqu 0x60($inp),$Ij # I[6] + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpxor $Zlo,$Xi,$Xi # collect result + vpshufb $bswap,$Ij,$Ij + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vxorps $Zhi,$Xo,$Xo + vmovdqu 0x10-0x40($Htbl),$Hkey # $Hkey^2 + vpunpckhqdq $Ij,$Ij,$T1 + vpclmulqdq \$0x00,$HK, $T2,$Xmi + vpxor $Zmi,$Tred,$Tred + vxorps $Ij,$T1,$T1 + + vmovdqu 0x50($inp),$Ii # I[5] + vpxor $Xi,$Tred,$Tred # aggregated Karatsuba post-processing + vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo + vpxor $Xo,$Tred,$Tred + vpslldq \$8,$Tred,$T2 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi + vpsrldq \$8,$Tred,$Tred + vpxor $T2, $Xi, $Xi + vmovdqu 0x30-0x40($Htbl),$Hkey # $Hkey^3 + vpshufb $bswap,$Ii,$Ii + vxorps $Tred,$Xo, $Xo + vpxor $Xhi,$Zhi,$Zhi + vpunpckhqdq $Ii,$Ii,$T2 + vpclmulqdq \$0x10,$HK, $T1,$Zmi + vmovdqu 0x50-0x40($Htbl),$HK + vpxor $Ii,$T2,$T2 + vpxor $Xmi,$Zmi,$Zmi + + vmovdqu 0x40($inp),$Ij # I[4] + vpalignr \$8,$Xi,$Xi,$Tred # 1st phase + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpshufb $bswap,$Ij,$Ij + vpxor $Zlo,$Xlo,$Xlo + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vmovdqu 0x40-0x40($Htbl),$Hkey # $Hkey^4 + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Zhi,$Xhi,$Xhi + vpclmulqdq \$0x00,$HK, $T2,$Xmi + vxorps $Ij,$T1,$T1 + vpxor $Zmi,$Xmi,$Xmi + + vmovdqu 0x30($inp),$Ii # I[3] + vpclmulqdq \$0x10,(%r10),$Xi,$Xi + vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo + vpshufb $bswap,$Ii,$Ii + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi + vmovdqu 0x60-0x40($Htbl),$Hkey # $Hkey^5 + vpunpckhqdq $Ii,$Ii,$T2 + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x10,$HK, $T1,$Zmi + vmovdqu 0x80-0x40($Htbl),$HK + vpxor $Ii,$T2,$T2 + vpxor $Xmi,$Zmi,$Zmi + + vmovdqu 0x20($inp),$Ij # I[2] + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpshufb $bswap,$Ij,$Ij + vpxor $Zlo,$Xlo,$Xlo + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vmovdqu 0x70-0x40($Htbl),$Hkey # $Hkey^6 + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Zhi,$Xhi,$Xhi + vpclmulqdq \$0x00,$HK, $T2,$Xmi + vpxor $Ij,$T1,$T1 + vpxor $Zmi,$Xmi,$Xmi + vxorps $Tred,$Xi,$Xi + + vmovdqu 0x10($inp),$Ii # I[1] + vpalignr \$8,$Xi,$Xi,$Tred # 2nd phase + vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo + vpshufb $bswap,$Ii,$Ii + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi + vmovdqu 0x90-0x40($Htbl),$Hkey # $Hkey^7 + vpclmulqdq \$0x10,(%r10),$Xi,$Xi + vxorps $Xo,$Tred,$Tred + vpunpckhqdq $Ii,$Ii,$T2 + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x10,$HK, $T1,$Zmi + vmovdqu 0xb0-0x40($Htbl),$HK + vpxor $Ii,$T2,$T2 + vpxor $Xmi,$Zmi,$Zmi + + vmovdqu ($inp),$Ij # I[0] + vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo + vpshufb $bswap,$Ij,$Ij + vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi + vmovdqu 0xa0-0x40($Htbl),$Hkey # $Hkey^8 + vpxor $Tred,$Ij,$Ij + vpclmulqdq \$0x10,$HK, $T2,$Xmi + vpxor $Xi,$Ij,$Ij # accumulate $Xi + + lea 0x80($inp),$inp + sub \$0x80,$len + jnc .Loop8x_avx + + add \$0x80,$len + jmp .Ltail_no_xor_avx + +.align 32 +.Lshort_avx: + vmovdqu -0x10($inp,$len),$Ii # very last word + lea ($inp,$len),$inp + vmovdqu 0x00-0x40($Htbl),$Hkey # $Hkey^1 + vmovdqu 0x20-0x40($Htbl),$HK + vpshufb $bswap,$Ii,$Ij + + vmovdqa $Xlo,$Zlo # subtle way to zero $Zlo, + vmovdqa $Xhi,$Zhi # $Zhi and + vmovdqa $Xmi,$Zmi # $Zmi + sub \$0x10,$len + jz .Ltail_avx + + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo + vpxor $Ij,$T1,$T1 + vmovdqu -0x20($inp),$Ii + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi + vmovdqu 0x10-0x40($Htbl),$Hkey # $Hkey^2 + vpshufb $bswap,$Ii,$Ij + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Xmi + vpsrldq \$8,$HK,$HK + sub \$0x10,$len + jz .Ltail_avx + + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo + vpxor $Ij,$T1,$T1 + vmovdqu -0x30($inp),$Ii + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi + vmovdqu 0x30-0x40($Htbl),$Hkey # $Hkey^3 + vpshufb $bswap,$Ii,$Ij + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Xmi + vmovdqu 0x50-0x40($Htbl),$HK + sub \$0x10,$len + jz .Ltail_avx + + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo + vpxor $Ij,$T1,$T1 + vmovdqu -0x40($inp),$Ii + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi + vmovdqu 0x40-0x40($Htbl),$Hkey # $Hkey^4 + vpshufb $bswap,$Ii,$Ij + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Xmi + vpsrldq \$8,$HK,$HK + sub \$0x10,$len + jz .Ltail_avx + + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo + vpxor $Ij,$T1,$T1 + vmovdqu -0x50($inp),$Ii + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi + vmovdqu 0x60-0x40($Htbl),$Hkey # $Hkey^5 + vpshufb $bswap,$Ii,$Ij + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Xmi + vmovdqu 0x80-0x40($Htbl),$HK + sub \$0x10,$len + jz .Ltail_avx + + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo + vpxor $Ij,$T1,$T1 + vmovdqu -0x60($inp),$Ii + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi + vmovdqu 0x70-0x40($Htbl),$Hkey # $Hkey^6 + vpshufb $bswap,$Ii,$Ij + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Xmi + vpsrldq \$8,$HK,$HK + sub \$0x10,$len + jz .Ltail_avx + + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo + vpxor $Ij,$T1,$T1 + vmovdqu -0x70($inp),$Ii + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi + vmovdqu 0x90-0x40($Htbl),$Hkey # $Hkey^7 + vpshufb $bswap,$Ii,$Ij + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Xmi + vmovq 0xb8-0x40($Htbl),$HK + sub \$0x10,$len + jmp .Ltail_avx + +.align 32 +.Ltail_avx: + vpxor $Xi,$Ij,$Ij # accumulate $Xi +.Ltail_no_xor_avx: + vpunpckhqdq $Ij,$Ij,$T1 + vpxor $Xlo,$Zlo,$Zlo + vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo + vpxor $Ij,$T1,$T1 + vpxor $Xhi,$Zhi,$Zhi + vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi + vpxor $Xmi,$Zmi,$Zmi + vpclmulqdq \$0x00,$HK,$T1,$Xmi + + vmovdqu (%r10),$Tred + + vpxor $Xlo,$Zlo,$Xi + vpxor $Xhi,$Zhi,$Xo + vpxor $Xmi,$Zmi,$Zmi + + vpxor $Xi, $Zmi,$Zmi # aggregated Karatsuba post-processing + vpxor $Xo, $Zmi,$Zmi + vpslldq \$8, $Zmi,$T2 + vpsrldq \$8, $Zmi,$Zmi + vpxor $T2, $Xi, $Xi + vpxor $Zmi,$Xo, $Xo + + vpclmulqdq \$0x10,$Tred,$Xi,$T2 # 1st phase + vpalignr \$8,$Xi,$Xi,$Xi + vpxor $T2,$Xi,$Xi + + vpclmulqdq \$0x10,$Tred,$Xi,$T2 # 2nd phase + vpalignr \$8,$Xi,$Xi,$Xi + vpxor $Xo,$Xi,$Xi + vpxor $T2,$Xi,$Xi + + cmp \$0,$len + jne .Lshort_avx + + vpshufb $bswap,$Xi,$Xi + vmovdqu $Xi,($Xip) + vzeroupper +___ +$code.=<<___ if ($win64); + movaps (%rsp),%xmm6 + movaps 0x10(%rsp),%xmm7 + movaps 0x20(%rsp),%xmm8 + movaps 0x30(%rsp),%xmm9 + movaps 0x40(%rsp),%xmm10 + movaps 0x50(%rsp),%xmm11 + movaps 0x60(%rsp),%xmm12 + movaps 0x70(%rsp),%xmm13 + movaps 0x80(%rsp),%xmm14 + movaps 0x90(%rsp),%xmm15 + lea 0xa8(%rsp),%rsp +.LSEH_end_gcm_ghash_avx: +___ +$code.=<<___; + ret +.size gcm_ghash_avx,.-gcm_ghash_avx +___ +} else { +$code.=<<___; + jmp .L_ghash_clmul +.size gcm_ghash_avx,.-gcm_ghash_avx +___ +} + $code.=<<___; .align 64 .Lbswap_mask: .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 .L0x1c2_polynomial: .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 +.L7_mask: + .long 7,0,7,0 +.L7_mask_poly: + .long 7,0,`0xE1<<1`,0 .align 64 .type .Lrem_4bit,\@object .Lrem_4bit: @@ -774,10 +1698,24 @@ se_handler: .rva .LSEH_end_gcm_ghash_4bit .rva .LSEH_info_gcm_ghash_4bit + .rva .LSEH_begin_gcm_init_clmul + .rva .LSEH_end_gcm_init_clmul + .rva .LSEH_info_gcm_init_clmul + .rva .LSEH_begin_gcm_ghash_clmul .rva .LSEH_end_gcm_ghash_clmul .rva .LSEH_info_gcm_ghash_clmul +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_gcm_init_avx + .rva .LSEH_end_gcm_init_avx + .rva .LSEH_info_gcm_init_clmul + .rva .LSEH_begin_gcm_ghash_avx + .rva .LSEH_end_gcm_ghash_avx + .rva .LSEH_info_gcm_ghash_clmul +___ +$code.=<<___; .section .xdata .align 8 .LSEH_info_gcm_gmult_4bit: @@ -788,14 +1726,23 @@ se_handler: .byte 9,0,0,0 .rva se_handler .rva .Lghash_prologue,.Lghash_epilogue # HandlerData +.LSEH_info_gcm_init_clmul: + .byte 0x01,0x08,0x03,0x00 + .byte 0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6 + .byte 0x04,0x22,0x00,0x00 #sub rsp,0x18 .LSEH_info_gcm_ghash_clmul: - .byte 0x01,0x1f,0x0b,0x00 - .byte 0x1f,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10 - .byte 0x19,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9 - .byte 0x13,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8 - .byte 0x0d,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7 - .byte 0x08,0x68,0x00,0x00 #movaps (rsp),xmm6 - .byte 0x04,0xa2,0x00,0x00 #sub rsp,0x58 + .byte 0x01,0x33,0x16,0x00 + .byte 0x33,0xf8,0x09,0x00 #movaps 0x90(rsp),xmm15 + .byte 0x2e,0xe8,0x08,0x00 #movaps 0x80(rsp),xmm14 + .byte 0x29,0xd8,0x07,0x00 #movaps 0x70(rsp),xmm13 + .byte 0x24,0xc8,0x06,0x00 #movaps 0x60(rsp),xmm12 + .byte 0x1f,0xb8,0x05,0x00 #movaps 0x50(rsp),xmm11 + .byte 0x1a,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10 + .byte 0x15,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9 + .byte 0x10,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8 + .byte 0x0c,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7 + .byte 0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6 + .byte 0x04,0x01,0x15,0x00 #sub rsp,0xa8 ___ } diff --git a/crypto/modes/asm/ghashp8-ppc.pl b/crypto/modes/asm/ghashp8-ppc.pl new file mode 100755 index 000000000000..e76a58c343c1 --- /dev/null +++ b/crypto/modes/asm/ghashp8-ppc.pl @@ -0,0 +1,234 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# GHASH for for PowerISA v2.07. +# +# July 2014 +# +# Accurate performance measurements are problematic, because it's +# always virtualized setup with possibly throttled processor. +# Relative comparison is therefore more informative. This initial +# version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x +# faster than "4-bit" integer-only compiler-generated 64-bit code. +# "Initial version" means that there is room for futher improvement. + +$flavour=shift; +$output =shift; + +if ($flavour =~ /64/) { + $SIZE_T=8; + $LRSAVE=2*$SIZE_T; + $STU="stdu"; + $POP="ld"; + $PUSH="std"; +} elsif ($flavour =~ /32/) { + $SIZE_T=4; + $LRSAVE=$SIZE_T; + $STU="stwu"; + $POP="lwz"; + $PUSH="stw"; +} else { die "nonsense $flavour"; } + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; + +my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6)); # argument block + +my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3)); +my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12)); +my $vrsave="r12"; + +$code=<<___; +.machine "any" + +.text + +.globl .gcm_init_p8 +.align 5 +.gcm_init_p8: + lis r0,0xfff0 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $H,0,r4 # load H + + vspltisb $xC2,-16 # 0xf0 + vspltisb $t0,1 # one + vaddubm $xC2,$xC2,$xC2 # 0xe0 + vxor $zero,$zero,$zero + vor $xC2,$xC2,$t0 # 0xe1 + vsldoi $xC2,$xC2,$zero,15 # 0xe1... + vsldoi $t1,$zero,$t0,1 # ...1 + vaddubm $xC2,$xC2,$xC2 # 0xc2... + vspltisb $t2,7 + vor $xC2,$xC2,$t1 # 0xc2....01 + vspltb $t1,$H,0 # most significant byte + vsl $H,$H,$t0 # H<<=1 + vsrab $t1,$t1,$t2 # broadcast carry bit + vand $t1,$t1,$xC2 + vxor $H,$H,$t1 # twisted H + + vsldoi $H,$H,$H,8 # twist even more ... + vsldoi $xC2,$zero,$xC2,8 # 0xc2.0 + vsldoi $Hl,$zero,$H,8 # ... and split + vsldoi $Hh,$H,$zero,8 + + stvx_u $xC2,0,r3 # save pre-computed table + stvx_u $Hl,r8,r3 + stvx_u $H, r9,r3 + stvx_u $Hh,r10,r3 + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,2,0 + .long 0 +.size .gcm_init_p8,.-.gcm_init_p8 + +.globl .gcm_gmult_p8 +.align 5 +.gcm_gmult_p8: + lis r0,0xfff8 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $IN,0,$Xip # load Xi + + lvx_u $Hl,r8,$Htbl # load pre-computed table + le?lvsl $lemask,r0,r0 + lvx_u $H, r9,$Htbl + le?vspltisb $t0,0x07 + lvx_u $Hh,r10,$Htbl + le?vxor $lemask,$lemask,$t0 + lvx_u $xC2,0,$Htbl + le?vperm $IN,$IN,$IN,$lemask + vxor $zero,$zero,$zero + + vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo + vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi + vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi + + vpmsumd $t2,$Xl,$xC2 # 1st phase + + vsldoi $t0,$Xm,$zero,8 + vsldoi $t1,$zero,$Xm,8 + vxor $Xl,$Xl,$t0 + vxor $Xh,$Xh,$t1 + + vsldoi $Xl,$Xl,$Xl,8 + vxor $Xl,$Xl,$t2 + + vsldoi $t1,$Xl,$Xl,8 # 2nd phase + vpmsumd $Xl,$Xl,$xC2 + vxor $t1,$t1,$Xh + vxor $Xl,$Xl,$t1 + + le?vperm $Xl,$Xl,$Xl,$lemask + stvx_u $Xl,0,$Xip # write out Xi + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,2,0 + .long 0 +.size .gcm_gmult_p8,.-.gcm_gmult_p8 + +.globl .gcm_ghash_p8 +.align 5 +.gcm_ghash_p8: + lis r0,0xfff8 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $Xl,0,$Xip # load Xi + + lvx_u $Hl,r8,$Htbl # load pre-computed table + le?lvsl $lemask,r0,r0 + lvx_u $H, r9,$Htbl + le?vspltisb $t0,0x07 + lvx_u $Hh,r10,$Htbl + le?vxor $lemask,$lemask,$t0 + lvx_u $xC2,0,$Htbl + le?vperm $Xl,$Xl,$Xl,$lemask + vxor $zero,$zero,$zero + + lvx_u $IN,0,$inp + addi $inp,$inp,16 + subi $len,$len,16 + le?vperm $IN,$IN,$IN,$lemask + vxor $IN,$IN,$Xl + b Loop + +.align 5 +Loop: + subic $len,$len,16 + vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo + subfe. r0,r0,r0 # borrow?-1:0 + vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi + and r0,r0,$len + vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi + add $inp,$inp,r0 + + vpmsumd $t2,$Xl,$xC2 # 1st phase + + vsldoi $t0,$Xm,$zero,8 + vsldoi $t1,$zero,$Xm,8 + vxor $Xl,$Xl,$t0 + vxor $Xh,$Xh,$t1 + + vsldoi $Xl,$Xl,$Xl,8 + vxor $Xl,$Xl,$t2 + lvx_u $IN,0,$inp + addi $inp,$inp,16 + + vsldoi $t1,$Xl,$Xl,8 # 2nd phase + vpmsumd $Xl,$Xl,$xC2 + le?vperm $IN,$IN,$IN,$lemask + vxor $t1,$t1,$Xh + vxor $IN,$IN,$t1 + vxor $IN,$IN,$Xl + beq Loop # did $len-=16 borrow? + + vxor $Xl,$Xl,$t1 + le?vperm $Xl,$Xl,$Xl,$lemask + stvx_u $Xl,0,$Xip # write out Xi + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,4,0 + .long 0 +.size .gcm_ghash_p8,.-.gcm_ghash_p8 + +.asciz "GHASH for PowerISA 2.07, CRYPTOGAMS by " +.align 2 +___ + +foreach (split("\n",$code)) { + if ($flavour =~ /le$/o) { # little-endian + s/le\?//o or + s/be\?/#be#/o; + } else { + s/le\?/#le#/o or + s/be\?//o; + } + print $_,"\n"; +} + +close STDOUT; # enforce flush diff --git a/crypto/modes/asm/ghashv8-armx.pl b/crypto/modes/asm/ghashv8-armx.pl new file mode 100755 index 000000000000..0b9cd7359fba --- /dev/null +++ b/crypto/modes/asm/ghashv8-armx.pl @@ -0,0 +1,409 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# GHASH for ARMv8 Crypto Extension, 64-bit polynomial multiplication. +# +# June 2014 +# +# Initial version was developed in tight cooperation with Ard +# Biesheuvel from bits-n-pieces from +# other assembly modules. Just like aesv8-armx.pl this module +# supports both AArch32 and AArch64 execution modes. +# +# July 2014 +# +# Implement 2x aggregated reduction [see ghash-x86.pl for background +# information]. +# +# Current performance in cycles per processed byte: +# +# PMULL[2] 32-bit NEON(*) +# Apple A7 0.92 5.62 +# Cortex-A53 1.01 8.39 +# Cortex-A57 1.17 7.61 +# +# (*) presented for reference/comparison purposes; + +$flavour = shift; +open STDOUT,">".shift; + +$Xi="x0"; # argument block +$Htbl="x1"; +$inp="x2"; +$len="x3"; + +$inc="x12"; + +{ +my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3)); +my ($t0,$t1,$t2,$xC2,$H,$Hhl,$H2)=map("q$_",(8..14)); + +$code=<<___; +#include "arm_arch.h" + +.text +___ +$code.=".arch armv8-a+crypto\n" if ($flavour =~ /64/); +$code.=".fpu neon\n.code 32\n" if ($flavour !~ /64/); + +################################################################################ +# void gcm_init_v8(u128 Htable[16],const u64 H[2]); +# +# input: 128-bit H - secret parameter E(K,0^128) +# output: precomputed table filled with degrees of twisted H; +# H is twisted to handle reverse bitness of GHASH; +# only few of 16 slots of Htable[16] are used; +# data is opaque to outside world (which allows to +# optimize the code independently); +# +$code.=<<___; +.global gcm_init_v8 +.type gcm_init_v8,%function +.align 4 +gcm_init_v8: + vld1.64 {$t1},[x1] @ load input H + vmov.i8 $xC2,#0xe1 + vshl.i64 $xC2,$xC2,#57 @ 0xc2.0 + vext.8 $IN,$t1,$t1,#8 + vshr.u64 $t2,$xC2,#63 + vdup.32 $t1,${t1}[1] + vext.8 $t0,$t2,$xC2,#8 @ t0=0xc2....01 + vshr.u64 $t2,$IN,#63 + vshr.s32 $t1,$t1,#31 @ broadcast carry bit + vand $t2,$t2,$t0 + vshl.i64 $IN,$IN,#1 + vext.8 $t2,$t2,$t2,#8 + vand $t0,$t0,$t1 + vorr $IN,$IN,$t2 @ H<<<=1 + veor $H,$IN,$t0 @ twisted H + vst1.64 {$H},[x0],#16 @ store Htable[0] + + @ calculate H^2 + vext.8 $t0,$H,$H,#8 @ Karatsuba pre-processing + vpmull.p64 $Xl,$H,$H + veor $t0,$t0,$H + vpmull2.p64 $Xh,$H,$H + vpmull.p64 $Xm,$t0,$t0 + + vext.8 $t1,$Xl,$Xh,#8 @ Karatsuba post-processing + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + veor $Xl,$Xm,$t2 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase + vpmull.p64 $Xl,$Xl,$xC2 + veor $t2,$t2,$Xh + veor $H2,$Xl,$t2 + + vext.8 $t1,$H2,$H2,#8 @ Karatsuba pre-processing + veor $t1,$t1,$H2 + vext.8 $Hhl,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$Hhl-$H2},[x0] @ store Htable[1..2] + + ret +.size gcm_init_v8,.-gcm_init_v8 +___ +################################################################################ +# void gcm_gmult_v8(u64 Xi[2],const u128 Htable[16]); +# +# input: Xi - current hash value; +# Htable - table precomputed in gcm_init_v8; +# output: Xi - next hash value Xi; +# +$code.=<<___; +.global gcm_gmult_v8 +.type gcm_gmult_v8,%function +.align 4 +gcm_gmult_v8: + vld1.64 {$t1},[$Xi] @ load Xi + vmov.i8 $xC2,#0xe1 + vld1.64 {$H-$Hhl},[$Htbl] @ load twisted H, ... + vshl.u64 $xC2,$xC2,#57 +#ifndef __ARMEB__ + vrev64.8 $t1,$t1 +#endif + vext.8 $IN,$t1,$t1,#8 + + vpmull.p64 $Xl,$H,$IN @ H.lo·Xi.lo + veor $t1,$t1,$IN @ Karatsuba pre-processing + vpmull2.p64 $Xh,$H,$IN @ H.hi·Xi.hi + vpmull.p64 $Xm,$Hhl,$t1 @ (H.lo+H.hi)·(Xi.lo+Xi.hi) + + vext.8 $t1,$Xl,$Xh,#8 @ Karatsuba post-processing + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase of reduction + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + veor $Xl,$Xm,$t2 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase of reduction + vpmull.p64 $Xl,$Xl,$xC2 + veor $t2,$t2,$Xh + veor $Xl,$Xl,$t2 + +#ifndef __ARMEB__ + vrev64.8 $Xl,$Xl +#endif + vext.8 $Xl,$Xl,$Xl,#8 + vst1.64 {$Xl},[$Xi] @ write out Xi + + ret +.size gcm_gmult_v8,.-gcm_gmult_v8 +___ +################################################################################ +# void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); +# +# input: table precomputed in gcm_init_v8; +# current hash value Xi; +# pointer to input data; +# length of input data in bytes, but divisible by block size; +# output: next hash value Xi; +# +$code.=<<___; +.global gcm_ghash_v8 +.type gcm_ghash_v8,%function +.align 4 +gcm_ghash_v8: +___ +$code.=<<___ if ($flavour !~ /64/); + vstmdb sp!,{d8-d15} @ 32-bit ABI says so +___ +$code.=<<___; + vld1.64 {$Xl},[$Xi] @ load [rotated] Xi + @ "[rotated]" means that + @ loaded value would have + @ to be rotated in order to + @ make it appear as in + @ alorithm specification + subs $len,$len,#32 @ see if $len is 32 or larger + mov $inc,#16 @ $inc is used as post- + @ increment for input pointer; + @ as loop is modulo-scheduled + @ $inc is zeroed just in time + @ to preclude oversteping + @ inp[len], which means that + @ last block[s] are actually + @ loaded twice, but last + @ copy is not processed + vld1.64 {$H-$Hhl},[$Htbl],#32 @ load twisted H, ..., H^2 + vmov.i8 $xC2,#0xe1 + vld1.64 {$H2},[$Htbl] + cclr $inc,eq @ is it time to zero $inc? + vext.8 $Xl,$Xl,$Xl,#8 @ rotate Xi + vld1.64 {$t0},[$inp],#16 @ load [rotated] I[0] + vshl.u64 $xC2,$xC2,#57 @ compose 0xc2.0 constant +#ifndef __ARMEB__ + vrev64.8 $t0,$t0 + vrev64.8 $Xl,$Xl +#endif + vext.8 $IN,$t0,$t0,#8 @ rotate I[0] + b.lo .Lodd_tail_v8 @ $len was less than 32 +___ +{ my ($Xln,$Xmn,$Xhn,$In) = map("q$_",(4..7)); + ####### + # Xi+2 =[H*(Ii+1 + Xi+1)] mod P = + # [(H*Ii+1) + (H*Xi+1)] mod P = + # [(H*Ii+1) + H^2*(Ii+Xi)] mod P + # +$code.=<<___; + vld1.64 {$t1},[$inp],$inc @ load [rotated] I[1] +#ifndef __ARMEB__ + vrev64.8 $t1,$t1 +#endif + vext.8 $In,$t1,$t1,#8 + veor $IN,$IN,$Xl @ I[i]^=Xi + vpmull.p64 $Xln,$H,$In @ H·Ii+1 + veor $t1,$t1,$In @ Karatsuba pre-processing + vpmull2.p64 $Xhn,$H,$In + b .Loop_mod2x_v8 + +.align 4 +.Loop_mod2x_v8: + vext.8 $t2,$IN,$IN,#8 + subs $len,$len,#32 @ is there more data? + vpmull.p64 $Xl,$H2,$IN @ H^2.lo·Xi.lo + cclr $inc,lo @ is it time to zero $inc? + + vpmull.p64 $Xmn,$Hhl,$t1 + veor $t2,$t2,$IN @ Karatsuba pre-processing + vpmull2.p64 $Xh,$H2,$IN @ H^2.hi·Xi.hi + veor $Xl,$Xl,$Xln @ accumulate + vpmull2.p64 $Xm,$Hhl,$t2 @ (H^2.lo+H^2.hi)·(Xi.lo+Xi.hi) + vld1.64 {$t0},[$inp],$inc @ load [rotated] I[i+2] + + veor $Xh,$Xh,$Xhn + cclr $inc,eq @ is it time to zero $inc? + veor $Xm,$Xm,$Xmn + + vext.8 $t1,$Xl,$Xh,#8 @ Karatsuba post-processing + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t1 + vld1.64 {$t1},[$inp],$inc @ load [rotated] I[i+3] +#ifndef __ARMEB__ + vrev64.8 $t0,$t0 +#endif + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase of reduction + +#ifndef __ARMEB__ + vrev64.8 $t1,$t1 +#endif + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + vext.8 $In,$t1,$t1,#8 + vext.8 $IN,$t0,$t0,#8 + veor $Xl,$Xm,$t2 + vpmull.p64 $Xln,$H,$In @ H·Ii+1 + veor $IN,$IN,$Xh @ accumulate $IN early + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase of reduction + vpmull.p64 $Xl,$Xl,$xC2 + veor $IN,$IN,$t2 + veor $t1,$t1,$In @ Karatsuba pre-processing + veor $IN,$IN,$Xl + vpmull2.p64 $Xhn,$H,$In + b.hs .Loop_mod2x_v8 @ there was at least 32 more bytes + + veor $Xh,$Xh,$t2 + vext.8 $IN,$t0,$t0,#8 @ re-construct $IN + adds $len,$len,#32 @ re-construct $len + veor $Xl,$Xl,$Xh @ re-construct $Xl + b.eq .Ldone_v8 @ is $len zero? +___ +} +$code.=<<___; +.Lodd_tail_v8: + vext.8 $t2,$Xl,$Xl,#8 + veor $IN,$IN,$Xl @ inp^=Xi + veor $t1,$t0,$t2 @ $t1 is rotated inp^Xi + + vpmull.p64 $Xl,$H,$IN @ H.lo·Xi.lo + veor $t1,$t1,$IN @ Karatsuba pre-processing + vpmull2.p64 $Xh,$H,$IN @ H.hi·Xi.hi + vpmull.p64 $Xm,$Hhl,$t1 @ (H.lo+H.hi)·(Xi.lo+Xi.hi) + + vext.8 $t1,$Xl,$Xh,#8 @ Karatsuba post-processing + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase of reduction + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + veor $Xl,$Xm,$t2 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase of reduction + vpmull.p64 $Xl,$Xl,$xC2 + veor $t2,$t2,$Xh + veor $Xl,$Xl,$t2 + +.Ldone_v8: +#ifndef __ARMEB__ + vrev64.8 $Xl,$Xl +#endif + vext.8 $Xl,$Xl,$Xl,#8 + vst1.64 {$Xl},[$Xi] @ write out Xi + +___ +$code.=<<___ if ($flavour !~ /64/); + vldmia sp!,{d8-d15} @ 32-bit ABI says so +___ +$code.=<<___; + ret +.size gcm_ghash_v8,.-gcm_ghash_v8 +___ +} +$code.=<<___; +.asciz "GHASH for ARMv8, CRYPTOGAMS by " +.align 2 +___ + +if ($flavour =~ /64/) { ######## 64-bit code + sub unvmov { + my $arg=shift; + + $arg =~ m/q([0-9]+)#(lo|hi),\s*q([0-9]+)#(lo|hi)/o && + sprintf "ins v%d.d[%d],v%d.d[%d]",$1,($2 eq "lo")?0:1,$3,($4 eq "lo")?0:1; + } + foreach(split("\n",$code)) { + s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel $1$2,$1zr,$1$2,$3/o or + s/vmov\.i8/movi/o or # fix up legacy mnemonics + s/vmov\s+(.*)/unvmov($1)/geo or + s/vext\.8/ext/o or + s/vshr\.s/sshr\.s/o or + s/vshr/ushr/o or + s/^(\s+)v/$1/o or # strip off v prefix + s/\bbx\s+lr\b/ret/o; + + s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo; # old->new registers + s/@\s/\/\//o; # old->new style commentary + + # fix up remainig legacy suffixes + s/\.[ui]?8(\s)/$1/o; + s/\.[uis]?32//o and s/\.16b/\.4s/go; + m/\.p64/o and s/\.16b/\.1q/o; # 1st pmull argument + m/l\.p64/o and s/\.16b/\.1d/go; # 2nd and 3rd pmull arguments + s/\.[uisp]?64//o and s/\.16b/\.2d/go; + s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o; + + print $_,"\n"; + } +} else { ######## 32-bit code + sub unvdup32 { + my $arg=shift; + + $arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o && + sprintf "vdup.32 q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1; + } + sub unvpmullp64 { + my ($mnemonic,$arg)=@_; + + if ($arg =~ m/q([0-9]+),\s*q([0-9]+),\s*q([0-9]+)/o) { + my $word = 0xf2a00e00|(($1&7)<<13)|(($1&8)<<19) + |(($2&7)<<17)|(($2&8)<<4) + |(($3&7)<<1) |(($3&8)<<2); + $word |= 0x00010001 if ($mnemonic =~ "2"); + # since ARMv7 instructions are always encoded little-endian. + # correct solution is to use .inst directive, but older + # assemblers don't implement it:-( + sprintf ".byte\t0x%02x,0x%02x,0x%02x,0x%02x\t@ %s %s", + $word&0xff,($word>>8)&0xff, + ($word>>16)&0xff,($word>>24)&0xff, + $mnemonic,$arg; + } + } + + foreach(split("\n",$code)) { + s/\b[wx]([0-9]+)\b/r$1/go; # new->old registers + s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go; # new->old registers + s/\/\/\s?/@ /o; # new->old style commentary + + # fix up remainig new-style suffixes + s/\],#[0-9]+/]!/o; + + s/cclr\s+([^,]+),\s*([a-z]+)/mov$2 $1,#0/o or + s/vdup\.32\s+(.*)/unvdup32($1)/geo or + s/v?(pmull2?)\.p64\s+(.*)/unvpmullp64($1,$2)/geo or + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or + s/^(\s+)b\./$1b/o or + s/^(\s+)ret/$1bx\tlr/o; + + print $_,"\n"; + } +} + +close STDOUT; # enforce flush diff --git a/crypto/modes/cbc128.c b/crypto/modes/cbc128.c index 1ed79672749a..c13caea5355b 100644 --- a/crypto/modes/cbc128.c +++ b/crypto/modes/cbc128.c @@ -59,7 +59,7 @@ #endif #include -#ifndef STRICT_ALIGNMENT +#if !defined(STRICT_ALIGNMENT) && !defined(PEDANTIC) # define STRICT_ALIGNMENT 0 #endif diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c index 0ee569fb7af0..e299131c1382 100644 --- a/crypto/modes/gcm128.c +++ b/crypto/modes/gcm128.c @@ -687,20 +687,31 @@ static void gcm_gmult_1bit(u64 Xi[2], const u64 H[2]) #endif -#if TABLE_BITS==4 && defined(GHASH_ASM) +#if TABLE_BITS==4 && (defined(GHASH_ASM) || defined(OPENSSL_CPUID_OBJ)) # if !defined(I386_ONLY) && \ (defined(__i386) || defined(__i386__) || \ defined(__x86_64) || defined(__x86_64__) || \ defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64)) # define GHASH_ASM_X86_OR_64 # define GCM_FUNCREF_4BIT -extern unsigned int OPENSSL_ia32cap_P[2]; +extern unsigned int OPENSSL_ia32cap_P[]; void gcm_init_clmul(u128 Htable[16], const u64 Xi[2]); void gcm_gmult_clmul(u64 Xi[2], const u128 Htable[16]); void gcm_ghash_clmul(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); +# if defined(__i386) || defined(__i386__) || defined(_M_IX86) +# define gcm_init_avx gcm_init_clmul +# define gcm_gmult_avx gcm_gmult_clmul +# define gcm_ghash_avx gcm_ghash_clmul +# else +void gcm_init_avx(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_avx(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +# endif + # if defined(__i386) || defined(__i386__) || defined(_M_IX86) # define GHASH_ASM_X86 void gcm_gmult_4bit_mmx(u64 Xi[2], const u128 Htable[16]); @@ -711,15 +722,41 @@ void gcm_gmult_4bit_x86(u64 Xi[2], const u128 Htable[16]); void gcm_ghash_4bit_x86(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); # endif -# elif defined(__arm__) || defined(__arm) +# elif defined(__arm__) || defined(__arm) || defined(__aarch64__) # include "arm_arch.h" -# if __ARM_ARCH__>=7 +# if __ARM_MAX_ARCH__>=7 # define GHASH_ASM_ARM # define GCM_FUNCREF_4BIT +# define PMULL_CAPABLE (OPENSSL_armcap_P & ARMV8_PMULL) +# if defined(__arm__) || defined(__arm) +# define NEON_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) +# endif +void gcm_init_neon(u128 Htable[16], const u64 Xi[2]); void gcm_gmult_neon(u64 Xi[2], const u128 Htable[16]); void gcm_ghash_neon(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); +void gcm_init_v8(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_v8(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_v8(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); # endif +# elif defined(__sparc__) || defined(__sparc) +# include "sparc_arch.h" +# define GHASH_ASM_SPARC +# define GCM_FUNCREF_4BIT +extern unsigned int OPENSSL_sparcv9cap_P[]; +void gcm_init_vis3(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_vis3(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_vis3(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +# elif defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC)) +# include "ppc_arch.h" +# define GHASH_ASM_PPC +# define GCM_FUNCREF_4BIT +void gcm_init_p8(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_p8(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_p8(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); # endif #endif @@ -768,9 +805,15 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, void *key, block128_f block) # if !defined(GHASH_ASM_X86) || defined(OPENSSL_IA32_SSE2) if (OPENSSL_ia32cap_P[0] & (1 << 24) && /* check FXSR bit */ OPENSSL_ia32cap_P[1] & (1 << 1)) { /* check PCLMULQDQ bit */ - gcm_init_clmul(ctx->Htable, ctx->H.u); - ctx->gmult = gcm_gmult_clmul; - ctx->ghash = gcm_ghash_clmul; + if (((OPENSSL_ia32cap_P[1] >> 22) & 0x41) == 0x41) { /* AVX+MOVBE */ + gcm_init_avx(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_avx; + ctx->ghash = gcm_ghash_avx; + } else { + gcm_init_clmul(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_clmul; + ctx->ghash = gcm_ghash_clmul; + } return; } # endif @@ -792,13 +835,52 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, void *key, block128_f block) ctx->ghash = gcm_ghash_4bit; # endif # elif defined(GHASH_ASM_ARM) - if (OPENSSL_armcap_P & ARMV7_NEON) { +# ifdef PMULL_CAPABLE + if (PMULL_CAPABLE) { + gcm_init_v8(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_v8; + ctx->ghash = gcm_ghash_v8; + } else +# endif +# ifdef NEON_CAPABLE + if (NEON_CAPABLE) { + gcm_init_neon(ctx->Htable, ctx->H.u); ctx->gmult = gcm_gmult_neon; ctx->ghash = gcm_ghash_neon; + } else +# endif + { + gcm_init_4bit(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_4bit; +# if defined(GHASH) + ctx->ghash = gcm_ghash_4bit; +# else + ctx->ghash = NULL; +# endif + } +# elif defined(GHASH_ASM_SPARC) + if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) { + gcm_init_vis3(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_vis3; + ctx->ghash = gcm_ghash_vis3; + } else { + gcm_init_4bit(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_4bit; + ctx->ghash = gcm_ghash_4bit; + } +# elif defined(GHASH_ASM_PPC) + if (OPENSSL_ppccap_P & PPC_CRYPTO207) { + gcm_init_p8(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_p8; + ctx->ghash = gcm_ghash_p8; } else { gcm_init_4bit(ctx->Htable, ctx->H.u); ctx->gmult = gcm_gmult_4bit; +# if defined(GHASH) ctx->ghash = gcm_ghash_4bit; +# else + ctx->ghash = NULL; +# endif } # else gcm_init_4bit(ctx->Htable, ctx->H.u); diff --git a/crypto/modes/modes.h b/crypto/modes/modes.h index 880f020d58e2..fd488499a0b4 100644 --- a/crypto/modes/modes.h +++ b/crypto/modes/modes.h @@ -148,6 +148,16 @@ int CRYPTO_xts128_encrypt(const XTS128_CONTEXT *ctx, const unsigned char *inp, unsigned char *out, size_t len, int enc); +size_t CRYPTO_128_wrap(void *key, const unsigned char *iv, + unsigned char *out, + const unsigned char *in, size_t inlen, + block128_f block); + +size_t CRYPTO_128_unwrap(void *key, const unsigned char *iv, + unsigned char *out, + const unsigned char *in, size_t inlen, + block128_f block); + #ifdef __cplusplus } #endif diff --git a/crypto/modes/modes_lcl.h b/crypto/modes/modes_lcl.h index 296849b8670c..fe14ec7002f0 100644 --- a/crypto/modes/modes_lcl.h +++ b/crypto/modes/modes_lcl.h @@ -25,39 +25,49 @@ typedef unsigned int u32; typedef unsigned char u8; #define STRICT_ALIGNMENT 1 -#if defined(__i386) || defined(__i386__) || \ - defined(__x86_64) || defined(__x86_64__) || \ - defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \ - defined(__s390__) || defined(__s390x__) -# undef STRICT_ALIGNMENT +#ifndef PEDANTIC +# if defined(__i386) || defined(__i386__) || \ + defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \ + defined(__aarch64__) || \ + defined(__s390__) || defined(__s390x__) +# undef STRICT_ALIGNMENT +# endif #endif #if !defined(PEDANTIC) && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) # if defined(__GNUC__) && __GNUC__>=2 # if defined(__x86_64) || defined(__x86_64__) -# define BSWAP8(x) ({ u64 ret=(x); \ +# define BSWAP8(x) ({ u64 ret_=(x); \ asm ("bswapq %0" \ - : "+r"(ret)); ret; }) -# define BSWAP4(x) ({ u32 ret=(x); \ + : "+r"(ret_)); ret_; }) +# define BSWAP4(x) ({ u32 ret_=(x); \ asm ("bswapl %0" \ - : "+r"(ret)); ret; }) + : "+r"(ret_)); ret_; }) # elif (defined(__i386) || defined(__i386__)) && !defined(I386_ONLY) -# define BSWAP8(x) ({ u32 lo=(u64)(x)>>32,hi=(x); \ +# define BSWAP8(x) ({ u32 lo_=(u64)(x)>>32,hi_=(x); \ asm ("bswapl %0; bswapl %1" \ - : "+r"(hi),"+r"(lo)); \ - (u64)hi<<32|lo; }) -# define BSWAP4(x) ({ u32 ret=(x); \ + : "+r"(hi_),"+r"(lo_)); \ + (u64)hi_<<32|lo_; }) +# define BSWAP4(x) ({ u32 ret_=(x); \ asm ("bswapl %0" \ - : "+r"(ret)); ret; }) + : "+r"(ret_)); ret_; }) +# elif defined(__aarch64__) +# define BSWAP8(x) ({ u64 ret_; \ + asm ("rev %0,%1" \ + : "=r"(ret_) : "r"(x)); ret_; }) +# define BSWAP4(x) ({ u32 ret_; \ + asm ("rev %w0,%w1" \ + : "=r"(ret_) : "r"(x)); ret_; }) # elif (defined(__arm__) || defined(__arm)) && !defined(STRICT_ALIGNMENT) -# define BSWAP8(x) ({ u32 lo=(u64)(x)>>32,hi=(x); \ +# define BSWAP8(x) ({ u32 lo_=(u64)(x)>>32,hi_=(x); \ asm ("rev %0,%0; rev %1,%1" \ - : "+r"(hi),"+r"(lo)); \ - (u64)hi<<32|lo; }) -# define BSWAP4(x) ({ u32 ret; \ + : "+r"(hi_),"+r"(lo_)); \ + (u64)hi_<<32|lo_; }) +# define BSWAP4(x) ({ u32 ret_; \ asm ("rev %0,%1" \ - : "=r"(ret) : "r"((u32)(x))); \ - ret; }) + : "=r"(ret_) : "r"((u32)(x))); \ + ret_; }) # endif # elif defined(_MSC_VER) # if _MSC_VER>=1300 diff --git a/crypto/modes/wrap128.c b/crypto/modes/wrap128.c new file mode 100644 index 000000000000..4dcaf0326fa8 --- /dev/null +++ b/crypto/modes/wrap128.c @@ -0,0 +1,138 @@ +/* crypto/modes/wrap128.c */ +/* + * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL + * project. + */ +/* ==================================================================== + * Copyright (c) 2013 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + */ + +#include "cryptlib.h" +#include + +static const unsigned char default_iv[] = { + 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, +}; + +/* + * Input size limit: lower than maximum of standards but far larger than + * anything that will be used in practice. + */ +#define CRYPTO128_WRAP_MAX (1UL << 31) + +size_t CRYPTO_128_wrap(void *key, const unsigned char *iv, + unsigned char *out, + const unsigned char *in, size_t inlen, + block128_f block) +{ + unsigned char *A, B[16], *R; + size_t i, j, t; + if ((inlen & 0x7) || (inlen < 8) || (inlen > CRYPTO128_WRAP_MAX)) + return 0; + A = B; + t = 1; + memcpy(out + 8, in, inlen); + if (!iv) + iv = default_iv; + + memcpy(A, iv, 8); + + for (j = 0; j < 6; j++) { + R = out + 8; + for (i = 0; i < inlen; i += 8, t++, R += 8) { + memcpy(B + 8, R, 8); + block(B, B, key); + A[7] ^= (unsigned char)(t & 0xff); + if (t > 0xff) { + A[6] ^= (unsigned char)((t >> 8) & 0xff); + A[5] ^= (unsigned char)((t >> 16) & 0xff); + A[4] ^= (unsigned char)((t >> 24) & 0xff); + } + memcpy(R, B + 8, 8); + } + } + memcpy(out, A, 8); + return inlen + 8; +} + +size_t CRYPTO_128_unwrap(void *key, const unsigned char *iv, + unsigned char *out, + const unsigned char *in, size_t inlen, + block128_f block) +{ + unsigned char *A, B[16], *R; + size_t i, j, t; + inlen -= 8; + if ((inlen & 0x7) || (inlen < 16) || (inlen > CRYPTO128_WRAP_MAX)) + return 0; + A = B; + t = 6 * (inlen >> 3); + memcpy(A, in, 8); + memcpy(out, in + 8, inlen); + for (j = 0; j < 6; j++) { + R = out + inlen - 8; + for (i = 0; i < inlen; i += 8, t--, R -= 8) { + A[7] ^= (unsigned char)(t & 0xff); + if (t > 0xff) { + A[6] ^= (unsigned char)((t >> 8) & 0xff); + A[5] ^= (unsigned char)((t >> 16) & 0xff); + A[4] ^= (unsigned char)((t >> 24) & 0xff); + } + memcpy(B + 8, R, 8); + block(B, B, key); + memcpy(R, B + 8, 8); + } + } + if (!iv) + iv = default_iv; + if (memcmp(A, iv, 8)) { + OPENSSL_cleanse(out, inlen); + return 0; + } + return inlen; +} diff --git a/crypto/o_str.c b/crypto/o_str.c index b23ef323b124..4e2d096704f0 100644 --- a/crypto/o_str.c +++ b/crypto/o_str.c @@ -62,7 +62,7 @@ #include "o_str.h" #if !defined(OPENSSL_IMPLEMENTS_strncasecmp) && \ - !defined(OPENSSL_SYSNAME_WIN32) && \ + !defined(OPENSSL_SYSNAME_WIN32) && !defined(OPENSSL_SYSNAME_WINCE) && \ !defined(NETWARE_CLIB) # include #endif diff --git a/crypto/o_time.c b/crypto/o_time.c index e18b71d484ff..58413fe97d09 100644 --- a/crypto/o_time.c +++ b/crypto/o_time.c @@ -246,8 +246,72 @@ struct tm *OPENSSL_gmtime(const time_t *timer, struct tm *result) static long date_to_julian(int y, int m, int d); static void julian_to_date(long jd, int *y, int *m, int *d); +static int julian_adj(const struct tm *tm, int off_day, long offset_sec, + long *pday, int *psec); int OPENSSL_gmtime_adj(struct tm *tm, int off_day, long offset_sec) +{ + int time_sec, time_year, time_month, time_day; + long time_jd; + + /* Convert time and offset into julian day and seconds */ + if (!julian_adj(tm, off_day, offset_sec, &time_jd, &time_sec)) + return 0; + + /* Convert Julian day back to date */ + + julian_to_date(time_jd, &time_year, &time_month, &time_day); + + if (time_year < 1900 || time_year > 9999) + return 0; + + /* Update tm structure */ + + tm->tm_year = time_year - 1900; + tm->tm_mon = time_month - 1; + tm->tm_mday = time_day; + + tm->tm_hour = time_sec / 3600; + tm->tm_min = (time_sec / 60) % 60; + tm->tm_sec = time_sec % 60; + + return 1; + +} + +int OPENSSL_gmtime_diff(int *pday, int *psec, + const struct tm *from, const struct tm *to) +{ + int from_sec, to_sec, diff_sec; + long from_jd, to_jd, diff_day; + if (!julian_adj(from, 0, 0, &from_jd, &from_sec)) + return 0; + if (!julian_adj(to, 0, 0, &to_jd, &to_sec)) + return 0; + diff_day = to_jd - from_jd; + diff_sec = to_sec - from_sec; + /* Adjust differences so both positive or both negative */ + if (diff_day > 0 && diff_sec < 0) { + diff_day--; + diff_sec += SECS_PER_DAY; + } + if (diff_day < 0 && diff_sec > 0) { + diff_day++; + diff_sec -= SECS_PER_DAY; + } + + if (pday) + *pday = (int)diff_day; + if (psec) + *psec = diff_sec; + + return 1; + +} + +/* Convert tm structure and offset into julian day and seconds */ +static int julian_adj(const struct tm *tm, int off_day, long offset_sec, + long *pday, int *psec) { int offset_hms, offset_day; long time_jd; @@ -284,25 +348,9 @@ int OPENSSL_gmtime_adj(struct tm *tm, int off_day, long offset_sec) if (time_jd < 0) return 0; - /* Convert Julian day back to date */ - - julian_to_date(time_jd, &time_year, &time_month, &time_day); - - if (time_year < 1900 || time_year > 9999) - return 0; - - /* Update tm structure */ - - tm->tm_year = time_year - 1900; - tm->tm_mon = time_month - 1; - tm->tm_mday = time_day; - - tm->tm_hour = offset_hms / 3600; - tm->tm_min = (offset_hms / 60) % 60; - tm->tm_sec = offset_hms % 60; - + *pday = time_jd; + *psec = offset_hms; return 1; - } /* @@ -354,27 +402,39 @@ int main(int argc, char **argv) int check_time(long offset) { - struct tm tm1, tm2; + struct tm tm1, tm2, o1; + int off_day, off_sec; + long toffset; time_t t1, t2; time(&t1); t2 = t1 + offset; OPENSSL_gmtime(&t2, &tm2); OPENSSL_gmtime(&t1, &tm1); + o1 = tm1; OPENSSL_gmtime_adj(&tm1, 0, offset); - if ((tm1.tm_year == tm2.tm_year) && - (tm1.tm_mon == tm2.tm_mon) && - (tm1.tm_mday == tm2.tm_mday) && - (tm1.tm_hour == tm2.tm_hour) && - (tm1.tm_min == tm2.tm_min) && (tm1.tm_sec == tm2.tm_sec)) - return 1; - fprintf(stderr, "TIME ERROR!!\n"); - fprintf(stderr, "Time1: %d/%d/%d, %d:%02d:%02d\n", - tm2.tm_mday, tm2.tm_mon + 1, tm2.tm_year + 1900, - tm2.tm_hour, tm2.tm_min, tm2.tm_sec); - fprintf(stderr, "Time2: %d/%d/%d, %d:%02d:%02d\n", - tm1.tm_mday, tm1.tm_mon + 1, tm1.tm_year + 1900, - tm1.tm_hour, tm1.tm_min, tm1.tm_sec); - return 0; + if ((tm1.tm_year != tm2.tm_year) || + (tm1.tm_mon != tm2.tm_mon) || + (tm1.tm_mday != tm2.tm_mday) || + (tm1.tm_hour != tm2.tm_hour) || + (tm1.tm_min != tm2.tm_min) || (tm1.tm_sec != tm2.tm_sec)) { + fprintf(stderr, "TIME ERROR!!\n"); + fprintf(stderr, "Time1: %d/%d/%d, %d:%02d:%02d\n", + tm2.tm_mday, tm2.tm_mon + 1, tm2.tm_year + 1900, + tm2.tm_hour, tm2.tm_min, tm2.tm_sec); + fprintf(stderr, "Time2: %d/%d/%d, %d:%02d:%02d\n", + tm1.tm_mday, tm1.tm_mon + 1, tm1.tm_year + 1900, + tm1.tm_hour, tm1.tm_min, tm1.tm_sec); + return 0; + } + OPENSSL_gmtime_diff(&o1, &tm1, &off_day, &off_sec); + toffset = (long)off_day *SECS_PER_DAY + off_sec; + if (offset != toffset) { + fprintf(stderr, "TIME OFFSET ERROR!!\n"); + fprintf(stderr, "Expected %ld, Got %ld (%d:%d)\n", + offset, toffset, off_day, off_sec); + return 0; + } + return 1; } #endif diff --git a/crypto/o_time.h b/crypto/o_time.h index 901b2005677e..a83a3d247d41 100644 --- a/crypto/o_time.h +++ b/crypto/o_time.h @@ -64,5 +64,7 @@ struct tm *OPENSSL_gmtime(const time_t *timer, struct tm *result); int OPENSSL_gmtime_adj(struct tm *tm, int offset_day, long offset_sec); +int OPENSSL_gmtime_diff(int *pday, int *psec, + const struct tm *from, const struct tm *to); #endif diff --git a/crypto/objects/obj_dat.h b/crypto/objects/obj_dat.h index bc69665bc582..b7e3cf280ee0 100644 --- a/crypto/objects/obj_dat.h +++ b/crypto/objects/obj_dat.h @@ -62,12 +62,12 @@ * [including the GNU Public Licence.] */ -#define NUM_NID 920 -#define NUM_SN 913 -#define NUM_LN 913 -#define NUM_OBJ 857 +#define NUM_NID 958 +#define NUM_SN 951 +#define NUM_LN 951 +#define NUM_OBJ 890 -static const unsigned char lvalues[5974]={ +static const unsigned char lvalues[6255]={ 0x2A,0x86,0x48,0x86,0xF7,0x0D, /* [ 0] OBJ_rsadsi */ 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01, /* [ 6] OBJ_pkcs */ 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x02, /* [ 13] OBJ_md2 */ @@ -919,6 +919,39 @@ static const unsigned char lvalues[5974]={ 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x08,/* [5946] OBJ_mgf1 */ 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0A,/* [5955] OBJ_rsassaPss */ 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x07,/* [5964] OBJ_rsaesOaep */ +0x2A,0x86,0x48,0xCE,0x3E,0x02,0x01, /* [5973] OBJ_dhpublicnumber */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x01,/* [5980] OBJ_brainpoolP160r1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x02,/* [5989] OBJ_brainpoolP160t1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x03,/* [5998] OBJ_brainpoolP192r1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x04,/* [6007] OBJ_brainpoolP192t1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x05,/* [6016] OBJ_brainpoolP224r1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x06,/* [6025] OBJ_brainpoolP224t1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x07,/* [6034] OBJ_brainpoolP256r1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x08,/* [6043] OBJ_brainpoolP256t1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x09,/* [6052] OBJ_brainpoolP320r1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0A,/* [6061] OBJ_brainpoolP320t1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0B,/* [6070] OBJ_brainpoolP384r1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0C,/* [6079] OBJ_brainpoolP384t1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0D,/* [6088] OBJ_brainpoolP512r1 */ +0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0E,/* [6097] OBJ_brainpoolP512t1 */ +0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x09,/* [6106] OBJ_pSpecified */ +0x2B,0x81,0x05,0x10,0x86,0x48,0x3F,0x00,0x02,/* [6115] OBJ_dhSinglePass_stdDH_sha1kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0B,0x00, /* [6124] OBJ_dhSinglePass_stdDH_sha224kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0B,0x01, /* [6130] OBJ_dhSinglePass_stdDH_sha256kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0B,0x02, /* [6136] OBJ_dhSinglePass_stdDH_sha384kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0B,0x03, /* [6142] OBJ_dhSinglePass_stdDH_sha512kdf_scheme */ +0x2B,0x81,0x05,0x10,0x86,0x48,0x3F,0x00,0x03,/* [6148] OBJ_dhSinglePass_cofactorDH_sha1kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0E,0x00, /* [6157] OBJ_dhSinglePass_cofactorDH_sha224kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0E,0x01, /* [6163] OBJ_dhSinglePass_cofactorDH_sha256kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0E,0x02, /* [6169] OBJ_dhSinglePass_cofactorDH_sha384kdf_scheme */ +0x2B,0x81,0x04,0x01,0x0E,0x03, /* [6175] OBJ_dhSinglePass_cofactorDH_sha512kdf_scheme */ +0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x02,/* [6181] OBJ_ct_precert_scts */ +0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x03,/* [6191] OBJ_ct_precert_poison */ +0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x04,/* [6201] OBJ_ct_precert_signer */ +0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x05,/* [6211] OBJ_ct_cert_scts */ +0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x01,/* [6221] OBJ_jurisdictionLocalityName */ +0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x02,/* [6232] OBJ_jurisdictionStateOrProvinceName */ +0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x03,/* [6243] OBJ_jurisdictionCountryName */ }; static const ASN1_OBJECT nid_objs[NUM_NID]={ @@ -2399,12 +2432,95 @@ static const ASN1_OBJECT nid_objs[NUM_NID]={ {"AES-256-CBC-HMAC-SHA1","aes-256-cbc-hmac-sha1", NID_aes_256_cbc_hmac_sha1,0,NULL,0}, {"RSAES-OAEP","rsaesOaep",NID_rsaesOaep,9,&(lvalues[5964]),0}, +{"dhpublicnumber","X9.42 DH",NID_dhpublicnumber,7,&(lvalues[5973]),0}, +{"brainpoolP160r1","brainpoolP160r1",NID_brainpoolP160r1,9, + &(lvalues[5980]),0}, +{"brainpoolP160t1","brainpoolP160t1",NID_brainpoolP160t1,9, + &(lvalues[5989]),0}, +{"brainpoolP192r1","brainpoolP192r1",NID_brainpoolP192r1,9, + &(lvalues[5998]),0}, +{"brainpoolP192t1","brainpoolP192t1",NID_brainpoolP192t1,9, + &(lvalues[6007]),0}, +{"brainpoolP224r1","brainpoolP224r1",NID_brainpoolP224r1,9, + &(lvalues[6016]),0}, +{"brainpoolP224t1","brainpoolP224t1",NID_brainpoolP224t1,9, + &(lvalues[6025]),0}, +{"brainpoolP256r1","brainpoolP256r1",NID_brainpoolP256r1,9, + &(lvalues[6034]),0}, +{"brainpoolP256t1","brainpoolP256t1",NID_brainpoolP256t1,9, + &(lvalues[6043]),0}, +{"brainpoolP320r1","brainpoolP320r1",NID_brainpoolP320r1,9, + &(lvalues[6052]),0}, +{"brainpoolP320t1","brainpoolP320t1",NID_brainpoolP320t1,9, + &(lvalues[6061]),0}, +{"brainpoolP384r1","brainpoolP384r1",NID_brainpoolP384r1,9, + &(lvalues[6070]),0}, +{"brainpoolP384t1","brainpoolP384t1",NID_brainpoolP384t1,9, + &(lvalues[6079]),0}, +{"brainpoolP512r1","brainpoolP512r1",NID_brainpoolP512r1,9, + &(lvalues[6088]),0}, +{"brainpoolP512t1","brainpoolP512t1",NID_brainpoolP512t1,9, + &(lvalues[6097]),0}, +{"PSPECIFIED","pSpecified",NID_pSpecified,9,&(lvalues[6106]),0}, +{"dhSinglePass-stdDH-sha1kdf-scheme", + "dhSinglePass-stdDH-sha1kdf-scheme", + NID_dhSinglePass_stdDH_sha1kdf_scheme,9,&(lvalues[6115]),0}, +{"dhSinglePass-stdDH-sha224kdf-scheme", + "dhSinglePass-stdDH-sha224kdf-scheme", + NID_dhSinglePass_stdDH_sha224kdf_scheme,6,&(lvalues[6124]),0}, +{"dhSinglePass-stdDH-sha256kdf-scheme", + "dhSinglePass-stdDH-sha256kdf-scheme", + NID_dhSinglePass_stdDH_sha256kdf_scheme,6,&(lvalues[6130]),0}, +{"dhSinglePass-stdDH-sha384kdf-scheme", + "dhSinglePass-stdDH-sha384kdf-scheme", + NID_dhSinglePass_stdDH_sha384kdf_scheme,6,&(lvalues[6136]),0}, +{"dhSinglePass-stdDH-sha512kdf-scheme", + "dhSinglePass-stdDH-sha512kdf-scheme", + NID_dhSinglePass_stdDH_sha512kdf_scheme,6,&(lvalues[6142]),0}, +{"dhSinglePass-cofactorDH-sha1kdf-scheme", + "dhSinglePass-cofactorDH-sha1kdf-scheme", + NID_dhSinglePass_cofactorDH_sha1kdf_scheme,9,&(lvalues[6148]),0}, +{"dhSinglePass-cofactorDH-sha224kdf-scheme", + "dhSinglePass-cofactorDH-sha224kdf-scheme", + NID_dhSinglePass_cofactorDH_sha224kdf_scheme,6,&(lvalues[6157]),0}, +{"dhSinglePass-cofactorDH-sha256kdf-scheme", + "dhSinglePass-cofactorDH-sha256kdf-scheme", + NID_dhSinglePass_cofactorDH_sha256kdf_scheme,6,&(lvalues[6163]),0}, +{"dhSinglePass-cofactorDH-sha384kdf-scheme", + "dhSinglePass-cofactorDH-sha384kdf-scheme", + NID_dhSinglePass_cofactorDH_sha384kdf_scheme,6,&(lvalues[6169]),0}, +{"dhSinglePass-cofactorDH-sha512kdf-scheme", + "dhSinglePass-cofactorDH-sha512kdf-scheme", + NID_dhSinglePass_cofactorDH_sha512kdf_scheme,6,&(lvalues[6175]),0}, +{"dh-std-kdf","dh-std-kdf",NID_dh_std_kdf,0,NULL,0}, +{"dh-cofactor-kdf","dh-cofactor-kdf",NID_dh_cofactor_kdf,0,NULL,0}, +{"AES-128-CBC-HMAC-SHA256","aes-128-cbc-hmac-sha256", + NID_aes_128_cbc_hmac_sha256,0,NULL,0}, +{"AES-192-CBC-HMAC-SHA256","aes-192-cbc-hmac-sha256", + NID_aes_192_cbc_hmac_sha256,0,NULL,0}, +{"AES-256-CBC-HMAC-SHA256","aes-256-cbc-hmac-sha256", + NID_aes_256_cbc_hmac_sha256,0,NULL,0}, +{"ct_precert_scts","CT Precertificate SCTs",NID_ct_precert_scts,10, + &(lvalues[6181]),0}, +{"ct_precert_poison","CT Precertificate Poison",NID_ct_precert_poison, + 10,&(lvalues[6191]),0}, +{"ct_precert_signer","CT Precertificate Signer",NID_ct_precert_signer, + 10,&(lvalues[6201]),0}, +{"ct_cert_scts","CT Certificate SCTs",NID_ct_cert_scts,10, + &(lvalues[6211]),0}, +{"jurisdictionL","jurisdictionLocalityName", + NID_jurisdictionLocalityName,11,&(lvalues[6221]),0}, +{"jurisdictionST","jurisdictionStateOrProvinceName", + NID_jurisdictionStateOrProvinceName,11,&(lvalues[6232]),0}, +{"jurisdictionC","jurisdictionCountryName", + NID_jurisdictionCountryName,11,&(lvalues[6243]),0}, }; static const unsigned int sn_objs[NUM_SN]={ 364, /* "AD_DVCS" */ 419, /* "AES-128-CBC" */ 916, /* "AES-128-CBC-HMAC-SHA1" */ +948, /* "AES-128-CBC-HMAC-SHA256" */ 421, /* "AES-128-CFB" */ 650, /* "AES-128-CFB1" */ 653, /* "AES-128-CFB8" */ @@ -2414,6 +2530,7 @@ static const unsigned int sn_objs[NUM_SN]={ 913, /* "AES-128-XTS" */ 423, /* "AES-192-CBC" */ 917, /* "AES-192-CBC-HMAC-SHA1" */ +949, /* "AES-192-CBC-HMAC-SHA256" */ 425, /* "AES-192-CFB" */ 651, /* "AES-192-CFB1" */ 654, /* "AES-192-CFB8" */ @@ -2422,6 +2539,7 @@ static const unsigned int sn_objs[NUM_SN]={ 424, /* "AES-192-OFB" */ 427, /* "AES-256-CBC" */ 918, /* "AES-256-CBC-HMAC-SHA1" */ +950, /* "AES-256-CBC-HMAC-SHA256" */ 429, /* "AES-256-CFB" */ 652, /* "AES-256-CFB1" */ 655, /* "AES-256-CFB8" */ @@ -2537,6 +2655,7 @@ static const unsigned int sn_objs[NUM_SN]={ 69, /* "PBKDF2" */ 162, /* "PBMAC1" */ 127, /* "PKIX" */ +935, /* "PSPECIFIED" */ 98, /* "RC2-40-CBC" */ 166, /* "RC2-64-CBC" */ 37, /* "RC2-CBC" */ @@ -2613,6 +2732,20 @@ static const unsigned int sn_objs[NUM_SN]={ 87, /* "basicConstraints" */ 365, /* "basicOCSPResponse" */ 285, /* "biometricInfo" */ +921, /* "brainpoolP160r1" */ +922, /* "brainpoolP160t1" */ +923, /* "brainpoolP192r1" */ +924, /* "brainpoolP192t1" */ +925, /* "brainpoolP224r1" */ +926, /* "brainpoolP224t1" */ +927, /* "brainpoolP256r1" */ +928, /* "brainpoolP256t1" */ +929, /* "brainpoolP320r1" */ +930, /* "brainpoolP320t1" */ +931, /* "brainpoolP384r1" */ +932, /* "brainpoolP384t1" */ +933, /* "brainpoolP512r1" */ +934, /* "brainpoolP512t1" */ 494, /* "buildingName" */ 860, /* "businessCategory" */ 691, /* "c2onb191v4" */ @@ -2658,6 +2791,10 @@ static const unsigned int sn_objs[NUM_SN]={ 884, /* "crossCertificatePair" */ 806, /* "cryptocom" */ 805, /* "cryptopro" */ +954, /* "ct_cert_scts" */ +952, /* "ct_precert_poison" */ +951, /* "ct_precert_scts" */ +953, /* "ct_precert_signer" */ 500, /* "dITRedirect" */ 451, /* "dNSDomain" */ 495, /* "dSAQuality" */ @@ -2667,7 +2804,20 @@ static const unsigned int sn_objs[NUM_SN]={ 891, /* "deltaRevocationList" */ 107, /* "description" */ 871, /* "destinationIndicator" */ +947, /* "dh-cofactor-kdf" */ +946, /* "dh-std-kdf" */ 28, /* "dhKeyAgreement" */ +941, /* "dhSinglePass-cofactorDH-sha1kdf-scheme" */ +942, /* "dhSinglePass-cofactorDH-sha224kdf-scheme" */ +943, /* "dhSinglePass-cofactorDH-sha256kdf-scheme" */ +944, /* "dhSinglePass-cofactorDH-sha384kdf-scheme" */ +945, /* "dhSinglePass-cofactorDH-sha512kdf-scheme" */ +936, /* "dhSinglePass-stdDH-sha1kdf-scheme" */ +937, /* "dhSinglePass-stdDH-sha224kdf-scheme" */ +938, /* "dhSinglePass-stdDH-sha256kdf-scheme" */ +939, /* "dhSinglePass-stdDH-sha384kdf-scheme" */ +940, /* "dhSinglePass-stdDH-sha512kdf-scheme" */ +920, /* "dhpublicnumber" */ 382, /* "directory" */ 887, /* "distinguishedName" */ 892, /* "dmdName" */ @@ -2978,6 +3128,9 @@ static const unsigned int sn_objs[NUM_SN]={ 86, /* "issuerAltName" */ 770, /* "issuingDistributionPoint" */ 492, /* "janetMailbox" */ +957, /* "jurisdictionC" */ +955, /* "jurisdictionL" */ +956, /* "jurisdictionST" */ 150, /* "keyBag" */ 83, /* "keyUsage" */ 477, /* "lastModifiedBy" */ @@ -3328,6 +3481,10 @@ static const unsigned int ln_objs[NUM_LN]={ 285, /* "Biometric Info" */ 179, /* "CA Issuers" */ 785, /* "CA Repository" */ +954, /* "CT Certificate SCTs" */ +952, /* "CT Precertificate Poison" */ +951, /* "CT Precertificate SCTs" */ +953, /* "CT Precertificate Signer" */ 131, /* "Code Signing" */ 783, /* "Diffie-Hellman based MAC" */ 382, /* "Directory" */ @@ -3451,6 +3608,7 @@ static const unsigned int ln_objs[NUM_LN]={ 85, /* "X509v3 Subject Alternative Name" */ 769, /* "X509v3 Subject Directory Attributes" */ 82, /* "X509v3 Subject Key Identifier" */ +920, /* "X9.42 DH" */ 184, /* "X9.57" */ 185, /* "X9.57 CM ?" */ 478, /* "aRecord" */ @@ -3463,6 +3621,7 @@ static const unsigned int ln_objs[NUM_LN]={ 606, /* "additional verification" */ 419, /* "aes-128-cbc" */ 916, /* "aes-128-cbc-hmac-sha1" */ +948, /* "aes-128-cbc-hmac-sha256" */ 896, /* "aes-128-ccm" */ 421, /* "aes-128-cfb" */ 650, /* "aes-128-cfb1" */ @@ -3474,6 +3633,7 @@ static const unsigned int ln_objs[NUM_LN]={ 913, /* "aes-128-xts" */ 423, /* "aes-192-cbc" */ 917, /* "aes-192-cbc-hmac-sha1" */ +949, /* "aes-192-cbc-hmac-sha256" */ 899, /* "aes-192-ccm" */ 425, /* "aes-192-cfb" */ 651, /* "aes-192-cfb1" */ @@ -3484,6 +3644,7 @@ static const unsigned int ln_objs[NUM_LN]={ 424, /* "aes-192-ofb" */ 427, /* "aes-256-cbc" */ 918, /* "aes-256-cbc-hmac-sha1" */ +950, /* "aes-256-cbc-hmac-sha256" */ 902, /* "aes-256-ccm" */ 429, /* "aes-256-cfb" */ 652, /* "aes-256-cfb1" */ @@ -3502,6 +3663,20 @@ static const unsigned int ln_objs[NUM_LN]={ 93, /* "bf-cfb" */ 92, /* "bf-ecb" */ 94, /* "bf-ofb" */ +921, /* "brainpoolP160r1" */ +922, /* "brainpoolP160t1" */ +923, /* "brainpoolP192r1" */ +924, /* "brainpoolP192t1" */ +925, /* "brainpoolP224r1" */ +926, /* "brainpoolP224t1" */ +927, /* "brainpoolP256r1" */ +928, /* "brainpoolP256t1" */ +929, /* "brainpoolP320r1" */ +930, /* "brainpoolP320t1" */ +931, /* "brainpoolP384r1" */ +932, /* "brainpoolP384t1" */ +933, /* "brainpoolP512r1" */ +934, /* "brainpoolP512t1" */ 494, /* "buildingName" */ 860, /* "businessCategory" */ 691, /* "c2onb191v4" */ @@ -3593,7 +3768,19 @@ static const unsigned int ln_objs[NUM_LN]={ 107, /* "description" */ 871, /* "destinationIndicator" */ 80, /* "desx-cbc" */ +947, /* "dh-cofactor-kdf" */ +946, /* "dh-std-kdf" */ 28, /* "dhKeyAgreement" */ +941, /* "dhSinglePass-cofactorDH-sha1kdf-scheme" */ +942, /* "dhSinglePass-cofactorDH-sha224kdf-scheme" */ +943, /* "dhSinglePass-cofactorDH-sha256kdf-scheme" */ +944, /* "dhSinglePass-cofactorDH-sha384kdf-scheme" */ +945, /* "dhSinglePass-cofactorDH-sha512kdf-scheme" */ +936, /* "dhSinglePass-stdDH-sha1kdf-scheme" */ +937, /* "dhSinglePass-stdDH-sha224kdf-scheme" */ +938, /* "dhSinglePass-stdDH-sha256kdf-scheme" */ +939, /* "dhSinglePass-stdDH-sha384kdf-scheme" */ +940, /* "dhSinglePass-stdDH-sha512kdf-scheme" */ 11, /* "directory services (X.500)" */ 378, /* "directory services - algorithms" */ 887, /* "distinguishedName" */ @@ -3881,6 +4068,9 @@ static const unsigned int ln_objs[NUM_LN]={ 645, /* "itu-t" */ 492, /* "janetMailbox" */ 646, /* "joint-iso-itu-t" */ +957, /* "jurisdictionCountryName" */ +955, /* "jurisdictionLocalityName" */ +956, /* "jurisdictionStateOrProvinceName" */ 150, /* "keyBag" */ 773, /* "kisa" */ 477, /* "lastModifiedBy" */ @@ -3917,6 +4107,7 @@ static const unsigned int ln_objs[NUM_LN]={ 18, /* "organizationalUnitName" */ 475, /* "otherMailbox" */ 876, /* "owner" */ +935, /* "pSpecified" */ 489, /* "pagerTelephoneNumber" */ 782, /* "password based MAC" */ 374, /* "path" */ @@ -4560,6 +4751,14 @@ static const unsigned int obj_objs[NUM_OBJ]={ 505, /* OBJ_mime_mhs_headings 1 3 6 1 7 1 1 */ 506, /* OBJ_mime_mhs_bodies 1 3 6 1 7 1 2 */ 119, /* OBJ_ripemd160WithRSA 1 3 36 3 3 1 2 */ +937, /* OBJ_dhSinglePass_stdDH_sha224kdf_scheme 1 3 132 1 11 0 */ +938, /* OBJ_dhSinglePass_stdDH_sha256kdf_scheme 1 3 132 1 11 1 */ +939, /* OBJ_dhSinglePass_stdDH_sha384kdf_scheme 1 3 132 1 11 2 */ +940, /* OBJ_dhSinglePass_stdDH_sha512kdf_scheme 1 3 132 1 11 3 */ +942, /* OBJ_dhSinglePass_cofactorDH_sha224kdf_scheme 1 3 132 1 14 0 */ +943, /* OBJ_dhSinglePass_cofactorDH_sha256kdf_scheme 1 3 132 1 14 1 */ +944, /* OBJ_dhSinglePass_cofactorDH_sha384kdf_scheme 1 3 132 1 14 2 */ +945, /* OBJ_dhSinglePass_cofactorDH_sha512kdf_scheme 1 3 132 1 14 3 */ 631, /* OBJ_setAttr_GenCryptgrm 2 23 42 3 3 3 1 */ 632, /* OBJ_setAttr_T2Enc 2 23 42 3 3 4 1 */ 633, /* OBJ_setAttr_T2cleartxt 2 23 42 3 3 4 2 */ @@ -4608,6 +4807,7 @@ static const unsigned int obj_objs[NUM_OBJ]={ 416, /* OBJ_ecdsa_with_SHA1 1 2 840 10045 4 1 */ 791, /* OBJ_ecdsa_with_Recommended 1 2 840 10045 4 2 */ 792, /* OBJ_ecdsa_with_Specified 1 2 840 10045 4 3 */ +920, /* OBJ_dhpublicnumber 1 2 840 10046 2 1 */ 258, /* OBJ_id_pkix_mod 1 3 6 1 5 5 7 0 */ 175, /* OBJ_id_pe 1 3 6 1 5 5 7 1 */ 259, /* OBJ_id_qt 1 3 6 1 5 5 7 2 */ @@ -4825,6 +5025,7 @@ static const unsigned int obj_objs[NUM_OBJ]={ 644, /* OBJ_rsaOAEPEncryptionSET 1 2 840 113549 1 1 6 */ 919, /* OBJ_rsaesOaep 1 2 840 113549 1 1 7 */ 911, /* OBJ_mgf1 1 2 840 113549 1 1 8 */ +935, /* OBJ_pSpecified 1 2 840 113549 1 1 9 */ 912, /* OBJ_rsassaPss 1 2 840 113549 1 1 10 */ 668, /* OBJ_sha256WithRSAEncryption 1 2 840 113549 1 1 11 */ 669, /* OBJ_sha384WithRSAEncryption 1 2 840 113549 1 1 12 */ @@ -4886,6 +5087,22 @@ static const unsigned int obj_objs[NUM_OBJ]={ 373, /* OBJ_id_pkix_OCSP_valid 1 3 6 1 5 5 7 48 1 9 */ 374, /* OBJ_id_pkix_OCSP_path 1 3 6 1 5 5 7 48 1 10 */ 375, /* OBJ_id_pkix_OCSP_trustRoot 1 3 6 1 5 5 7 48 1 11 */ +921, /* OBJ_brainpoolP160r1 1 3 36 3 3 2 8 1 1 1 */ +922, /* OBJ_brainpoolP160t1 1 3 36 3 3 2 8 1 1 2 */ +923, /* OBJ_brainpoolP192r1 1 3 36 3 3 2 8 1 1 3 */ +924, /* OBJ_brainpoolP192t1 1 3 36 3 3 2 8 1 1 4 */ +925, /* OBJ_brainpoolP224r1 1 3 36 3 3 2 8 1 1 5 */ +926, /* OBJ_brainpoolP224t1 1 3 36 3 3 2 8 1 1 6 */ +927, /* OBJ_brainpoolP256r1 1 3 36 3 3 2 8 1 1 7 */ +928, /* OBJ_brainpoolP256t1 1 3 36 3 3 2 8 1 1 8 */ +929, /* OBJ_brainpoolP320r1 1 3 36 3 3 2 8 1 1 9 */ +930, /* OBJ_brainpoolP320t1 1 3 36 3 3 2 8 1 1 10 */ +931, /* OBJ_brainpoolP384r1 1 3 36 3 3 2 8 1 1 11 */ +932, /* OBJ_brainpoolP384t1 1 3 36 3 3 2 8 1 1 12 */ +933, /* OBJ_brainpoolP512r1 1 3 36 3 3 2 8 1 1 13 */ +934, /* OBJ_brainpoolP512t1 1 3 36 3 3 2 8 1 1 14 */ +936, /* OBJ_dhSinglePass_stdDH_sha1kdf_scheme 1 3 133 16 840 63 0 2 */ +941, /* OBJ_dhSinglePass_cofactorDH_sha1kdf_scheme 1 3 133 16 840 63 0 3 */ 418, /* OBJ_aes_128_ecb 2 16 840 1 101 3 4 1 1 */ 419, /* OBJ_aes_128_cbc 2 16 840 1 101 3 4 1 2 */ 420, /* OBJ_aes_128_ofb128 2 16 840 1 101 3 4 1 3 */ @@ -5013,6 +5230,10 @@ static const unsigned int obj_objs[NUM_OBJ]={ 138, /* OBJ_ms_efs 1 3 6 1 4 1 311 10 3 4 */ 648, /* OBJ_ms_smartcard_login 1 3 6 1 4 1 311 20 2 2 */ 649, /* OBJ_ms_upn 1 3 6 1 4 1 311 20 2 3 */ +951, /* OBJ_ct_precert_scts 1 3 6 1 4 1 11129 2 4 2 */ +952, /* OBJ_ct_precert_poison 1 3 6 1 4 1 11129 2 4 3 */ +953, /* OBJ_ct_precert_signer 1 3 6 1 4 1 11129 2 4 4 */ +954, /* OBJ_ct_cert_scts 1 3 6 1 4 1 11129 2 4 5 */ 751, /* OBJ_camellia_128_cbc 1 2 392 200011 61 1 1 1 2 */ 752, /* OBJ_camellia_192_cbc 1 2 392 200011 61 1 1 1 3 */ 753, /* OBJ_camellia_256_cbc 1 2 392 200011 61 1 1 1 4 */ @@ -5091,5 +5312,8 @@ static const unsigned int obj_objs[NUM_OBJ]={ 154, /* OBJ_secretBag 1 2 840 113549 1 12 10 1 5 */ 155, /* OBJ_safeContentsBag 1 2 840 113549 1 12 10 1 6 */ 34, /* OBJ_idea_cbc 1 3 6 1 4 1 188 7 1 1 2 */ +955, /* OBJ_jurisdictionLocalityName 1 3 6 1 4 1 311 60 2 1 1 */ +956, /* OBJ_jurisdictionStateOrProvinceName 1 3 6 1 4 1 311 60 2 1 2 */ +957, /* OBJ_jurisdictionCountryName 1 3 6 1 4 1 311 60 2 1 3 */ }; diff --git a/crypto/objects/obj_mac.h b/crypto/objects/obj_mac.h index f752aeff75b1..779c309b869b 100644 --- a/crypto/objects/obj_mac.h +++ b/crypto/objects/obj_mac.h @@ -590,6 +590,11 @@ #define NID_mgf1 911 #define OBJ_mgf1 OBJ_pkcs1,8L +#define SN_pSpecified "PSPECIFIED" +#define LN_pSpecified "pSpecified" +#define NID_pSpecified 935 +#define OBJ_pSpecified OBJ_pkcs1,9L + #define SN_rsassaPss "RSASSA-PSS" #define LN_rsassaPss "rsassaPss" #define NID_rsassaPss 912 @@ -4029,3 +4034,161 @@ #define SN_aes_256_cbc_hmac_sha1 "AES-256-CBC-HMAC-SHA1" #define LN_aes_256_cbc_hmac_sha1 "aes-256-cbc-hmac-sha1" #define NID_aes_256_cbc_hmac_sha1 918 + +#define SN_aes_128_cbc_hmac_sha256 "AES-128-CBC-HMAC-SHA256" +#define LN_aes_128_cbc_hmac_sha256 "aes-128-cbc-hmac-sha256" +#define NID_aes_128_cbc_hmac_sha256 948 + +#define SN_aes_192_cbc_hmac_sha256 "AES-192-CBC-HMAC-SHA256" +#define LN_aes_192_cbc_hmac_sha256 "aes-192-cbc-hmac-sha256" +#define NID_aes_192_cbc_hmac_sha256 949 + +#define SN_aes_256_cbc_hmac_sha256 "AES-256-CBC-HMAC-SHA256" +#define LN_aes_256_cbc_hmac_sha256 "aes-256-cbc-hmac-sha256" +#define NID_aes_256_cbc_hmac_sha256 950 + +#define SN_dhpublicnumber "dhpublicnumber" +#define LN_dhpublicnumber "X9.42 DH" +#define NID_dhpublicnumber 920 +#define OBJ_dhpublicnumber OBJ_ISO_US,10046L,2L,1L + +#define SN_brainpoolP160r1 "brainpoolP160r1" +#define NID_brainpoolP160r1 921 +#define OBJ_brainpoolP160r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,1L + +#define SN_brainpoolP160t1 "brainpoolP160t1" +#define NID_brainpoolP160t1 922 +#define OBJ_brainpoolP160t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,2L + +#define SN_brainpoolP192r1 "brainpoolP192r1" +#define NID_brainpoolP192r1 923 +#define OBJ_brainpoolP192r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,3L + +#define SN_brainpoolP192t1 "brainpoolP192t1" +#define NID_brainpoolP192t1 924 +#define OBJ_brainpoolP192t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,4L + +#define SN_brainpoolP224r1 "brainpoolP224r1" +#define NID_brainpoolP224r1 925 +#define OBJ_brainpoolP224r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,5L + +#define SN_brainpoolP224t1 "brainpoolP224t1" +#define NID_brainpoolP224t1 926 +#define OBJ_brainpoolP224t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,6L + +#define SN_brainpoolP256r1 "brainpoolP256r1" +#define NID_brainpoolP256r1 927 +#define OBJ_brainpoolP256r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,7L + +#define SN_brainpoolP256t1 "brainpoolP256t1" +#define NID_brainpoolP256t1 928 +#define OBJ_brainpoolP256t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,8L + +#define SN_brainpoolP320r1 "brainpoolP320r1" +#define NID_brainpoolP320r1 929 +#define OBJ_brainpoolP320r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,9L + +#define SN_brainpoolP320t1 "brainpoolP320t1" +#define NID_brainpoolP320t1 930 +#define OBJ_brainpoolP320t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,10L + +#define SN_brainpoolP384r1 "brainpoolP384r1" +#define NID_brainpoolP384r1 931 +#define OBJ_brainpoolP384r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,11L + +#define SN_brainpoolP384t1 "brainpoolP384t1" +#define NID_brainpoolP384t1 932 +#define OBJ_brainpoolP384t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,12L + +#define SN_brainpoolP512r1 "brainpoolP512r1" +#define NID_brainpoolP512r1 933 +#define OBJ_brainpoolP512r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,13L + +#define SN_brainpoolP512t1 "brainpoolP512t1" +#define NID_brainpoolP512t1 934 +#define OBJ_brainpoolP512t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,14L + +#define OBJ_x9_63_scheme 1L,3L,133L,16L,840L,63L,0L + +#define OBJ_secg_scheme OBJ_certicom_arc,1L + +#define SN_dhSinglePass_stdDH_sha1kdf_scheme "dhSinglePass-stdDH-sha1kdf-scheme" +#define NID_dhSinglePass_stdDH_sha1kdf_scheme 936 +#define OBJ_dhSinglePass_stdDH_sha1kdf_scheme OBJ_x9_63_scheme,2L + +#define SN_dhSinglePass_stdDH_sha224kdf_scheme "dhSinglePass-stdDH-sha224kdf-scheme" +#define NID_dhSinglePass_stdDH_sha224kdf_scheme 937 +#define OBJ_dhSinglePass_stdDH_sha224kdf_scheme OBJ_secg_scheme,11L,0L + +#define SN_dhSinglePass_stdDH_sha256kdf_scheme "dhSinglePass-stdDH-sha256kdf-scheme" +#define NID_dhSinglePass_stdDH_sha256kdf_scheme 938 +#define OBJ_dhSinglePass_stdDH_sha256kdf_scheme OBJ_secg_scheme,11L,1L + +#define SN_dhSinglePass_stdDH_sha384kdf_scheme "dhSinglePass-stdDH-sha384kdf-scheme" +#define NID_dhSinglePass_stdDH_sha384kdf_scheme 939 +#define OBJ_dhSinglePass_stdDH_sha384kdf_scheme OBJ_secg_scheme,11L,2L + +#define SN_dhSinglePass_stdDH_sha512kdf_scheme "dhSinglePass-stdDH-sha512kdf-scheme" +#define NID_dhSinglePass_stdDH_sha512kdf_scheme 940 +#define OBJ_dhSinglePass_stdDH_sha512kdf_scheme OBJ_secg_scheme,11L,3L + +#define SN_dhSinglePass_cofactorDH_sha1kdf_scheme "dhSinglePass-cofactorDH-sha1kdf-scheme" +#define NID_dhSinglePass_cofactorDH_sha1kdf_scheme 941 +#define OBJ_dhSinglePass_cofactorDH_sha1kdf_scheme OBJ_x9_63_scheme,3L + +#define SN_dhSinglePass_cofactorDH_sha224kdf_scheme "dhSinglePass-cofactorDH-sha224kdf-scheme" +#define NID_dhSinglePass_cofactorDH_sha224kdf_scheme 942 +#define OBJ_dhSinglePass_cofactorDH_sha224kdf_scheme OBJ_secg_scheme,14L,0L + +#define SN_dhSinglePass_cofactorDH_sha256kdf_scheme "dhSinglePass-cofactorDH-sha256kdf-scheme" +#define NID_dhSinglePass_cofactorDH_sha256kdf_scheme 943 +#define OBJ_dhSinglePass_cofactorDH_sha256kdf_scheme OBJ_secg_scheme,14L,1L + +#define SN_dhSinglePass_cofactorDH_sha384kdf_scheme "dhSinglePass-cofactorDH-sha384kdf-scheme" +#define NID_dhSinglePass_cofactorDH_sha384kdf_scheme 944 +#define OBJ_dhSinglePass_cofactorDH_sha384kdf_scheme OBJ_secg_scheme,14L,2L + +#define SN_dhSinglePass_cofactorDH_sha512kdf_scheme "dhSinglePass-cofactorDH-sha512kdf-scheme" +#define NID_dhSinglePass_cofactorDH_sha512kdf_scheme 945 +#define OBJ_dhSinglePass_cofactorDH_sha512kdf_scheme OBJ_secg_scheme,14L,3L + +#define SN_dh_std_kdf "dh-std-kdf" +#define NID_dh_std_kdf 946 + +#define SN_dh_cofactor_kdf "dh-cofactor-kdf" +#define NID_dh_cofactor_kdf 947 + +#define SN_ct_precert_scts "ct_precert_scts" +#define LN_ct_precert_scts "CT Precertificate SCTs" +#define NID_ct_precert_scts 951 +#define OBJ_ct_precert_scts 1L,3L,6L,1L,4L,1L,11129L,2L,4L,2L + +#define SN_ct_precert_poison "ct_precert_poison" +#define LN_ct_precert_poison "CT Precertificate Poison" +#define NID_ct_precert_poison 952 +#define OBJ_ct_precert_poison 1L,3L,6L,1L,4L,1L,11129L,2L,4L,3L + +#define SN_ct_precert_signer "ct_precert_signer" +#define LN_ct_precert_signer "CT Precertificate Signer" +#define NID_ct_precert_signer 953 +#define OBJ_ct_precert_signer 1L,3L,6L,1L,4L,1L,11129L,2L,4L,4L + +#define SN_ct_cert_scts "ct_cert_scts" +#define LN_ct_cert_scts "CT Certificate SCTs" +#define NID_ct_cert_scts 954 +#define OBJ_ct_cert_scts 1L,3L,6L,1L,4L,1L,11129L,2L,4L,5L + +#define SN_jurisdictionLocalityName "jurisdictionL" +#define LN_jurisdictionLocalityName "jurisdictionLocalityName" +#define NID_jurisdictionLocalityName 955 +#define OBJ_jurisdictionLocalityName 1L,3L,6L,1L,4L,1L,311L,60L,2L,1L,1L + +#define SN_jurisdictionStateOrProvinceName "jurisdictionST" +#define LN_jurisdictionStateOrProvinceName "jurisdictionStateOrProvinceName" +#define NID_jurisdictionStateOrProvinceName 956 +#define OBJ_jurisdictionStateOrProvinceName 1L,3L,6L,1L,4L,1L,311L,60L,2L,1L,2L + +#define SN_jurisdictionCountryName "jurisdictionC" +#define LN_jurisdictionCountryName "jurisdictionCountryName" +#define NID_jurisdictionCountryName 957 +#define OBJ_jurisdictionCountryName 1L,3L,6L,1L,4L,1L,311L,60L,2L,1L,3L diff --git a/crypto/objects/obj_mac.num b/crypto/objects/obj_mac.num index 1d0a7c802daf..8e5ea8336331 100644 --- a/crypto/objects/obj_mac.num +++ b/crypto/objects/obj_mac.num @@ -917,3 +917,41 @@ aes_128_cbc_hmac_sha1 916 aes_192_cbc_hmac_sha1 917 aes_256_cbc_hmac_sha1 918 rsaesOaep 919 +dhpublicnumber 920 +brainpoolP160r1 921 +brainpoolP160t1 922 +brainpoolP192r1 923 +brainpoolP192t1 924 +brainpoolP224r1 925 +brainpoolP224t1 926 +brainpoolP256r1 927 +brainpoolP256t1 928 +brainpoolP320r1 929 +brainpoolP320t1 930 +brainpoolP384r1 931 +brainpoolP384t1 932 +brainpoolP512r1 933 +brainpoolP512t1 934 +pSpecified 935 +dhSinglePass_stdDH_sha1kdf_scheme 936 +dhSinglePass_stdDH_sha224kdf_scheme 937 +dhSinglePass_stdDH_sha256kdf_scheme 938 +dhSinglePass_stdDH_sha384kdf_scheme 939 +dhSinglePass_stdDH_sha512kdf_scheme 940 +dhSinglePass_cofactorDH_sha1kdf_scheme 941 +dhSinglePass_cofactorDH_sha224kdf_scheme 942 +dhSinglePass_cofactorDH_sha256kdf_scheme 943 +dhSinglePass_cofactorDH_sha384kdf_scheme 944 +dhSinglePass_cofactorDH_sha512kdf_scheme 945 +dh_std_kdf 946 +dh_cofactor_kdf 947 +aes_128_cbc_hmac_sha256 948 +aes_192_cbc_hmac_sha256 949 +aes_256_cbc_hmac_sha256 950 +ct_precert_scts 951 +ct_precert_poison 952 +ct_precert_signer 953 +ct_cert_scts 954 +jurisdictionLocalityName 955 +jurisdictionStateOrProvinceName 956 +jurisdictionCountryName 957 diff --git a/crypto/objects/obj_xref.h b/crypto/objects/obj_xref.h index b8f7d341f787..e453e99f8336 100644 --- a/crypto/objects/obj_xref.h +++ b/crypto/objects/obj_xref.h @@ -41,6 +41,21 @@ static const nid_triple sigoid_srt[] = { {NID_id_GostR3411_94_with_GostR3410_2001_cc, NID_id_GostR3411_94, NID_id_GostR3410_2001_cc}, {NID_rsassaPss, NID_undef, NID_rsaEncryption}, + {NID_dhSinglePass_stdDH_sha1kdf_scheme, NID_sha1, NID_dh_std_kdf}, + {NID_dhSinglePass_stdDH_sha224kdf_scheme, NID_sha224, NID_dh_std_kdf}, + {NID_dhSinglePass_stdDH_sha256kdf_scheme, NID_sha256, NID_dh_std_kdf}, + {NID_dhSinglePass_stdDH_sha384kdf_scheme, NID_sha384, NID_dh_std_kdf}, + {NID_dhSinglePass_stdDH_sha512kdf_scheme, NID_sha512, NID_dh_std_kdf}, + {NID_dhSinglePass_cofactorDH_sha1kdf_scheme, NID_sha1, + NID_dh_cofactor_kdf}, + {NID_dhSinglePass_cofactorDH_sha224kdf_scheme, NID_sha224, + NID_dh_cofactor_kdf}, + {NID_dhSinglePass_cofactorDH_sha256kdf_scheme, NID_sha256, + NID_dh_cofactor_kdf}, + {NID_dhSinglePass_cofactorDH_sha384kdf_scheme, NID_sha384, + NID_dh_cofactor_kdf}, + {NID_dhSinglePass_cofactorDH_sha512kdf_scheme, NID_sha512, + NID_dh_cofactor_kdf}, }; static const nid_triple *const sigoid_srt_xref[] = { @@ -54,19 +69,29 @@ static const nid_triple *const sigoid_srt_xref[] = { &sigoid_srt[5], &sigoid_srt[8], &sigoid_srt[12], + &sigoid_srt[30], + &sigoid_srt[35], &sigoid_srt[6], &sigoid_srt[10], &sigoid_srt[11], &sigoid_srt[13], &sigoid_srt[24], &sigoid_srt[20], + &sigoid_srt[32], + &sigoid_srt[37], &sigoid_srt[14], &sigoid_srt[21], + &sigoid_srt[33], + &sigoid_srt[38], &sigoid_srt[15], &sigoid_srt[22], + &sigoid_srt[34], + &sigoid_srt[39], &sigoid_srt[16], &sigoid_srt[23], &sigoid_srt[19], + &sigoid_srt[31], + &sigoid_srt[36], &sigoid_srt[25], &sigoid_srt[26], &sigoid_srt[27], diff --git a/crypto/objects/obj_xref.txt b/crypto/objects/obj_xref.txt index cb917182ee2f..19c94226b20c 100644 --- a/crypto/objects/obj_xref.txt +++ b/crypto/objects/obj_xref.txt @@ -44,3 +44,15 @@ id_GostR3411_94_with_GostR3410_2001 id_GostR3411_94 id_GostR3410_2001 id_GostR3411_94_with_GostR3410_94 id_GostR3411_94 id_GostR3410_94 id_GostR3411_94_with_GostR3410_94_cc id_GostR3411_94 id_GostR3410_94_cc id_GostR3411_94_with_GostR3410_2001_cc id_GostR3411_94 id_GostR3410_2001_cc +# ECDH KDFs and their corresponding message digests and schemes +dhSinglePass_stdDH_sha1kdf_scheme sha1 dh_std_kdf +dhSinglePass_stdDH_sha224kdf_scheme sha224 dh_std_kdf +dhSinglePass_stdDH_sha256kdf_scheme sha256 dh_std_kdf +dhSinglePass_stdDH_sha384kdf_scheme sha384 dh_std_kdf +dhSinglePass_stdDH_sha512kdf_scheme sha512 dh_std_kdf + +dhSinglePass_cofactorDH_sha1kdf_scheme sha1 dh_cofactor_kdf +dhSinglePass_cofactorDH_sha224kdf_scheme sha224 dh_cofactor_kdf +dhSinglePass_cofactorDH_sha256kdf_scheme sha256 dh_cofactor_kdf +dhSinglePass_cofactorDH_sha384kdf_scheme sha384 dh_cofactor_kdf +dhSinglePass_cofactorDH_sha512kdf_scheme sha512 dh_cofactor_kdf diff --git a/crypto/objects/objects.txt b/crypto/objects/objects.txt index d3bfad72a2ac..b57aabb22650 100644 --- a/crypto/objects/objects.txt +++ b/crypto/objects/objects.txt @@ -168,6 +168,7 @@ pkcs1 5 : RSA-SHA1 : sha1WithRSAEncryption # According to PKCS #1 version 2.1 pkcs1 7 : RSAES-OAEP : rsaesOaep pkcs1 8 : MGF1 : mgf1 +pkcs1 9 : PSPECIFIED : pSpecified pkcs1 10 : RSASSA-PSS : rsassaPss pkcs1 11 : RSA-SHA256 : sha256WithRSAEncryption @@ -1290,3 +1291,60 @@ kisa 1 6 : SEED-OFB : seed-ofb : AES-128-CBC-HMAC-SHA1 : aes-128-cbc-hmac-sha1 : AES-192-CBC-HMAC-SHA1 : aes-192-cbc-hmac-sha1 : AES-256-CBC-HMAC-SHA1 : aes-256-cbc-hmac-sha1 + : AES-128-CBC-HMAC-SHA256 : aes-128-cbc-hmac-sha256 + : AES-192-CBC-HMAC-SHA256 : aes-192-cbc-hmac-sha256 + : AES-256-CBC-HMAC-SHA256 : aes-256-cbc-hmac-sha256 + +ISO-US 10046 2 1 : dhpublicnumber : X9.42 DH + +# RFC 5639 curve OIDs (see http://www.ietf.org/rfc/rfc5639.txt) +# versionOne OBJECT IDENTIFIER ::= { +# iso(1) identifified-organization(3) teletrust(36) algorithm(3) +# signature-algorithm(3) ecSign(2) ecStdCurvesAndGeneration(8) +# ellipticCurve(1) 1 } +1 3 36 3 3 2 8 1 1 1 : brainpoolP160r1 +1 3 36 3 3 2 8 1 1 2 : brainpoolP160t1 +1 3 36 3 3 2 8 1 1 3 : brainpoolP192r1 +1 3 36 3 3 2 8 1 1 4 : brainpoolP192t1 +1 3 36 3 3 2 8 1 1 5 : brainpoolP224r1 +1 3 36 3 3 2 8 1 1 6 : brainpoolP224t1 +1 3 36 3 3 2 8 1 1 7 : brainpoolP256r1 +1 3 36 3 3 2 8 1 1 8 : brainpoolP256t1 +1 3 36 3 3 2 8 1 1 9 : brainpoolP320r1 +1 3 36 3 3 2 8 1 1 10 : brainpoolP320t1 +1 3 36 3 3 2 8 1 1 11 : brainpoolP384r1 +1 3 36 3 3 2 8 1 1 12 : brainpoolP384t1 +1 3 36 3 3 2 8 1 1 13 : brainpoolP512r1 +1 3 36 3 3 2 8 1 1 14 : brainpoolP512t1 + +# ECDH schemes from RFC5753 +!Alias x9-63-scheme 1 3 133 16 840 63 0 +!Alias secg-scheme certicom-arc 1 + +x9-63-scheme 2 : dhSinglePass-stdDH-sha1kdf-scheme +secg-scheme 11 0 : dhSinglePass-stdDH-sha224kdf-scheme +secg-scheme 11 1 : dhSinglePass-stdDH-sha256kdf-scheme +secg-scheme 11 2 : dhSinglePass-stdDH-sha384kdf-scheme +secg-scheme 11 3 : dhSinglePass-stdDH-sha512kdf-scheme + +x9-63-scheme 3 : dhSinglePass-cofactorDH-sha1kdf-scheme +secg-scheme 14 0 : dhSinglePass-cofactorDH-sha224kdf-scheme +secg-scheme 14 1 : dhSinglePass-cofactorDH-sha256kdf-scheme +secg-scheme 14 2 : dhSinglePass-cofactorDH-sha384kdf-scheme +secg-scheme 14 3 : dhSinglePass-cofactorDH-sha512kdf-scheme +# NIDs for use with lookup tables. + : dh-std-kdf + : dh-cofactor-kdf + +# RFC 6962 Extension OIDs (see http://www.ietf.org/rfc/rfc6962.txt) +1 3 6 1 4 1 11129 2 4 2 : ct_precert_scts : CT Precertificate SCTs +1 3 6 1 4 1 11129 2 4 3 : ct_precert_poison : CT Precertificate Poison +1 3 6 1 4 1 11129 2 4 4 : ct_precert_signer : CT Precertificate Signer +1 3 6 1 4 1 11129 2 4 5 : ct_cert_scts : CT Certificate SCTs + +# CABForum EV SSL Certificate Guidelines +# (see https://cabforum.org/extended-validation/) +# OIDs for Subject Jurisdiction of Incorporation or Registration +1 3 6 1 4 1 311 60 2 1 1 : jurisdictionL : jurisdictionLocalityName +1 3 6 1 4 1 311 60 2 1 2 : jurisdictionST : jurisdictionStateOrProvinceName +1 3 6 1 4 1 311 60 2 1 3 : jurisdictionC : jurisdictionCountryName diff --git a/crypto/objects/objxref.pl b/crypto/objects/objxref.pl index 35c06514b932..1913b9d133c2 100755 --- a/crypto/objects/objxref.pl +++ b/crypto/objects/objxref.pl @@ -39,7 +39,8 @@ my @xrkeys = keys %xref_tbl; my @srt1 = sort { $oid_tbl{$a} <=> $oid_tbl{$b}} @xrkeys; -for(my $i = 0; $i <= $#srt1; $i++) +my $i; +for($i = 0; $i <= $#srt1; $i++) { $xref_tbl{$srt1[$i]}[2] = $i; } diff --git a/crypto/ocsp/ocsp.h b/crypto/ocsp/ocsp.h index 25ef01956be2..ca2ee76dce47 100644 --- a/crypto/ocsp/ocsp.h +++ b/crypto/ocsp/ocsp.h @@ -394,11 +394,22 @@ typedef struct ocsp_service_locator_st { OCSP_CERTID *OCSP_CERTID_dup(OCSP_CERTID *id); -OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, char *path, OCSP_REQUEST *req); -OCSP_REQ_CTX *OCSP_sendreq_new(BIO *io, char *path, OCSP_REQUEST *req, +OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, const char *path, OCSP_REQUEST *req); +OCSP_REQ_CTX *OCSP_sendreq_new(BIO *io, const char *path, OCSP_REQUEST *req, int maxline); +int OCSP_REQ_CTX_nbio(OCSP_REQ_CTX *rctx); int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx); +OCSP_REQ_CTX *OCSP_REQ_CTX_new(BIO *io, int maxline); void OCSP_REQ_CTX_free(OCSP_REQ_CTX *rctx); +void OCSP_set_max_response_length(OCSP_REQ_CTX *rctx, unsigned long len); +int OCSP_REQ_CTX_i2d(OCSP_REQ_CTX *rctx, const ASN1_ITEM *it, + ASN1_VALUE *val); +int OCSP_REQ_CTX_nbio_d2i(OCSP_REQ_CTX *rctx, ASN1_VALUE **pval, + const ASN1_ITEM *it); +BIO *OCSP_REQ_CTX_get0_mem_bio(OCSP_REQ_CTX *rctx); +int OCSP_REQ_CTX_i2d(OCSP_REQ_CTX *rctx, const ASN1_ITEM *it, + ASN1_VALUE *val); +int OCSP_REQ_CTX_http(OCSP_REQ_CTX *rctx, const char *op, const char *path); int OCSP_REQ_CTX_set1_req(OCSP_REQ_CTX *rctx, OCSP_REQUEST *req); int OCSP_REQ_CTX_add1_header(OCSP_REQ_CTX *rctx, const char *name, const char *value); @@ -447,7 +458,7 @@ int OCSP_check_validity(ASN1_GENERALIZEDTIME *thisupd, int OCSP_request_verify(OCSP_REQUEST *req, STACK_OF(X509) *certs, X509_STORE *store, unsigned long flags); -int OCSP_parse_url(char *url, char **phost, char **pport, char **ppath, +int OCSP_parse_url(const char *url, char **phost, char **pport, char **ppath, int *pssl); int OCSP_id_issuer_cmp(OCSP_CERTID *a, OCSP_CERTID *b); diff --git a/crypto/ocsp/ocsp_ht.c b/crypto/ocsp/ocsp_ht.c index 970fea4a70a8..88b26b38e8d9 100644 --- a/crypto/ocsp/ocsp_ht.c +++ b/crypto/ocsp/ocsp_ht.c @@ -81,9 +81,10 @@ struct ocsp_req_ctx_st { BIO *io; /* BIO to perform I/O with */ BIO *mem; /* Memory BIO response is built into */ unsigned long asn1_len; /* ASN1 length of response */ + unsigned long max_resp_len; /* Maximum length of response */ }; -#define OCSP_MAX_REQUEST_LENGTH (100 * 1024) +#define OCSP_MAX_RESP_LENGTH (100 * 1024) #define OCSP_MAX_LINE_LEN 4096; /* OCSP states */ @@ -100,15 +101,42 @@ struct ocsp_req_ctx_st { #define OHS_ASN1_HEADER 3 /* OCSP content octets being read */ #define OHS_ASN1_CONTENT 4 +/* First call: ready to start I/O */ +#define OHS_ASN1_WRITE_INIT (5 | OHS_NOREAD) /* Request being sent */ #define OHS_ASN1_WRITE (6 | OHS_NOREAD) /* Request being flushed */ #define OHS_ASN1_FLUSH (7 | OHS_NOREAD) /* Completed */ #define OHS_DONE (8 | OHS_NOREAD) +/* Headers set, no final \r\n included */ +#define OHS_HTTP_HEADER (9 | OHS_NOREAD) static int parse_http_line1(char *line); +OCSP_REQ_CTX *OCSP_REQ_CTX_new(BIO *io, int maxline) +{ + OCSP_REQ_CTX *rctx; + rctx = OPENSSL_malloc(sizeof(OCSP_REQ_CTX)); + if (!rctx) + return NULL; + rctx->state = OHS_ERROR; + rctx->max_resp_len = OCSP_MAX_RESP_LENGTH; + rctx->mem = BIO_new(BIO_s_mem()); + rctx->io = io; + rctx->asn1_len = 0; + if (maxline > 0) + rctx->iobuflen = maxline; + else + rctx->iobuflen = OCSP_MAX_LINE_LEN; + rctx->iobuf = OPENSSL_malloc(rctx->iobuflen); + if (!rctx->iobuf || !rctx->mem) { + OCSP_REQ_CTX_free(rctx); + return NULL; + } + return rctx; +} + void OCSP_REQ_CTX_free(OCSP_REQ_CTX *rctx) { if (rctx->mem) @@ -118,20 +146,71 @@ void OCSP_REQ_CTX_free(OCSP_REQ_CTX *rctx) OPENSSL_free(rctx); } -int OCSP_REQ_CTX_set1_req(OCSP_REQ_CTX *rctx, OCSP_REQUEST *req) +BIO *OCSP_REQ_CTX_get0_mem_bio(OCSP_REQ_CTX *rctx) +{ + return rctx->mem; +} + +void OCSP_set_max_response_length(OCSP_REQ_CTX *rctx, unsigned long len) +{ + if (len == 0) + rctx->max_resp_len = OCSP_MAX_RESP_LENGTH; + else + rctx->max_resp_len = len; +} + +int OCSP_REQ_CTX_i2d(OCSP_REQ_CTX *rctx, const ASN1_ITEM *it, ASN1_VALUE *val) { static const char req_hdr[] = "Content-Type: application/ocsp-request\r\n" "Content-Length: %d\r\n\r\n"; - if (BIO_printf(rctx->mem, req_hdr, i2d_OCSP_REQUEST(req, NULL)) <= 0) + int reqlen = ASN1_item_i2d(val, NULL, it); + if (BIO_printf(rctx->mem, req_hdr, reqlen) <= 0) + return 0; + if (ASN1_item_i2d_bio(it, rctx->mem, val) <= 0) + return 0; + rctx->state = OHS_ASN1_WRITE_INIT; + return 1; +} + +int OCSP_REQ_CTX_nbio_d2i(OCSP_REQ_CTX *rctx, + ASN1_VALUE **pval, const ASN1_ITEM *it) +{ + int rv, len; + const unsigned char *p; + + rv = OCSP_REQ_CTX_nbio(rctx); + if (rv != 1) + return rv; + + len = BIO_get_mem_data(rctx->mem, &p); + *pval = ASN1_item_d2i(NULL, &p, len, it); + if (*pval == NULL) { + rctx->state = OHS_ERROR; return 0; - if (i2d_OCSP_REQUEST_bio(rctx->mem, req) <= 0) + } + return 1; +} + +int OCSP_REQ_CTX_http(OCSP_REQ_CTX *rctx, const char *op, const char *path) +{ + static const char http_hdr[] = "%s %s HTTP/1.0\r\n"; + + if (!path) + path = "/"; + + if (BIO_printf(rctx->mem, http_hdr, op, path) <= 0) return 0; - rctx->state = OHS_ASN1_WRITE; - rctx->asn1_len = BIO_get_mem_data(rctx->mem, NULL); + rctx->state = OHS_HTTP_HEADER; return 1; } +int OCSP_REQ_CTX_set1_req(OCSP_REQ_CTX *rctx, OCSP_REQUEST *req) +{ + return OCSP_REQ_CTX_i2d(rctx, ASN1_ITEM_rptr(OCSP_REQUEST), + (ASN1_VALUE *)req); +} + int OCSP_REQ_CTX_add1_header(OCSP_REQ_CTX *rctx, const char *name, const char *value) { @@ -147,39 +226,27 @@ int OCSP_REQ_CTX_add1_header(OCSP_REQ_CTX *rctx, } if (BIO_write(rctx->mem, "\r\n", 2) != 2) return 0; + rctx->state = OHS_HTTP_HEADER; return 1; } -OCSP_REQ_CTX *OCSP_sendreq_new(BIO *io, char *path, OCSP_REQUEST *req, +OCSP_REQ_CTX *OCSP_sendreq_new(BIO *io, const char *path, OCSP_REQUEST *req, int maxline) { - static const char post_hdr[] = "POST %s HTTP/1.0\r\n"; - OCSP_REQ_CTX *rctx; - rctx = OPENSSL_malloc(sizeof(OCSP_REQ_CTX)); + OCSP_REQ_CTX *rctx = NULL; + rctx = OCSP_REQ_CTX_new(io, maxline); if (!rctx) return NULL; - rctx->state = OHS_ERROR; - rctx->mem = BIO_new(BIO_s_mem()); - rctx->io = io; - rctx->asn1_len = 0; - if (maxline > 0) - rctx->iobuflen = maxline; - else - rctx->iobuflen = OCSP_MAX_LINE_LEN; - rctx->iobuf = OPENSSL_malloc(rctx->iobuflen); - if (!rctx->mem || !rctx->iobuf) - goto err; - if (!path) - path = "/"; - if (BIO_printf(rctx->mem, post_hdr, path) <= 0) + if (!OCSP_REQ_CTX_http(rctx, "POST", path)) goto err; if (req && !OCSP_REQ_CTX_set1_req(rctx, req)) goto err; return rctx; + err: OCSP_REQ_CTX_free(rctx); return NULL; @@ -256,7 +323,7 @@ static int parse_http_line1(char *line) } -int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx) +int OCSP_REQ_CTX_nbio(OCSP_REQ_CTX *rctx) { int i, n; const unsigned char *p; @@ -277,6 +344,17 @@ int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx) } switch (rctx->state) { + case OHS_HTTP_HEADER: + /* Last operation was adding headers: need a final \r\n */ + if (BIO_write(rctx->mem, "\r\n", 2) != 2) { + rctx->state = OHS_ERROR; + return 0; + } + rctx->state = OHS_ASN1_WRITE_INIT; + + case OHS_ASN1_WRITE_INIT: + rctx->asn1_len = BIO_get_mem_data(rctx->mem, NULL); + rctx->state = OHS_ASN1_WRITE; case OHS_ASN1_WRITE: n = BIO_get_mem_data(rctx->mem, &p); @@ -412,7 +490,7 @@ int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx) rctx->asn1_len |= *p++; } - if (rctx->asn1_len > OCSP_MAX_REQUEST_LENGTH) { + if (rctx->asn1_len > rctx->max_resp_len) { rctx->state = OHS_ERROR; return 0; } @@ -426,18 +504,12 @@ int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx) /* Fall thru */ case OHS_ASN1_CONTENT: - n = BIO_get_mem_data(rctx->mem, &p); + n = BIO_get_mem_data(rctx->mem, NULL); if (n < (int)rctx->asn1_len) goto next_io; - *presp = d2i_OCSP_RESPONSE(NULL, &p, rctx->asn1_len); - if (*presp) { - rctx->state = OHS_DONE; - return 1; - } - - rctx->state = OHS_ERROR; - return 0; + rctx->state = OHS_DONE; + return 1; break; @@ -450,9 +522,16 @@ int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx) } +int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx) +{ + return OCSP_REQ_CTX_nbio_d2i(rctx, + (ASN1_VALUE **)presp, + ASN1_ITEM_rptr(OCSP_RESPONSE)); +} + /* Blocking OCSP request handler: now a special case of non-blocking I/O */ -OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, char *path, OCSP_REQUEST *req) +OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, const char *path, OCSP_REQUEST *req) { OCSP_RESPONSE *resp = NULL; OCSP_REQ_CTX *ctx; diff --git a/crypto/ocsp/ocsp_lib.c b/crypto/ocsp/ocsp_lib.c index d28d6b5c042e..442a5b63d4ba 100755 --- a/crypto/ocsp/ocsp_lib.c +++ b/crypto/ocsp/ocsp_lib.c @@ -175,7 +175,7 @@ int OCSP_id_cmp(OCSP_CERTID *a, OCSP_CERTID *b) * whether it is SSL. */ -int OCSP_parse_url(char *url, char **phost, char **pport, char **ppath, +int OCSP_parse_url(const char *url, char **phost, char **pport, char **ppath, int *pssl) { char *p, *buf; diff --git a/crypto/opensslconf.h b/crypto/opensslconf.h index 4a6e2766e0bb..15487c9f93be 100644 --- a/crypto/opensslconf.h +++ b/crypto/opensslconf.h @@ -20,6 +20,9 @@ extern "C" { #ifndef OPENSSL_NO_KRB5 # define OPENSSL_NO_KRB5 #endif +#ifndef OPENSSL_NO_LIBUNBOUND +# define OPENSSL_NO_LIBUNBOUND +#endif #ifndef OPENSSL_NO_MD2 # define OPENSSL_NO_MD2 #endif @@ -32,6 +35,9 @@ extern "C" { #ifndef OPENSSL_NO_SCTP # define OPENSSL_NO_SCTP #endif +#ifndef OPENSSL_NO_SSL_TRACE +# define OPENSSL_NO_SSL_TRACE +#endif #ifndef OPENSSL_NO_STORE # define OPENSSL_NO_STORE #endif @@ -62,6 +68,9 @@ extern "C" { # if defined(OPENSSL_NO_KRB5) && !defined(NO_KRB5) # define NO_KRB5 # endif +# if defined(OPENSSL_NO_LIBUNBOUND) && !defined(NO_LIBUNBOUND) +# define NO_LIBUNBOUND +# endif # if defined(OPENSSL_NO_MD2) && !defined(NO_MD2) # define NO_MD2 # endif @@ -74,6 +83,9 @@ extern "C" { # if defined(OPENSSL_NO_SCTP) && !defined(NO_SCTP) # define NO_SCTP # endif +# if defined(OPENSSL_NO_SSL_TRACE) && !defined(NO_SSL_TRACE) +# define NO_SSL_TRACE +# endif # if defined(OPENSSL_NO_STORE) && !defined(NO_STORE) # define NO_STORE # endif diff --git a/crypto/opensslv.h b/crypto/opensslv.h index bd66999c1c6b..c06b13ac6b0f 100644 --- a/crypto/opensslv.h +++ b/crypto/opensslv.h @@ -30,11 +30,11 @@ extern "C" { * (Prior to 0.9.5a beta1, a different scheme was used: MMNNFFRBB for * major minor fix final patch/beta) */ -# define OPENSSL_VERSION_NUMBER 0x1000110fL +# define OPENSSL_VERSION_NUMBER 0x1000204fL # ifdef OPENSSL_FIPS -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.1p-fips 9 Jul 2015" +# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2d-fips 9 Jul 2015" # else -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.1p 9 Jul 2015" +# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2d 9 Jul 2015" # endif # define OPENSSL_VERSION_PTEXT " part of " OPENSSL_VERSION_TEXT diff --git a/crypto/ossl_typ.h b/crypto/ossl_typ.h index 0fcb0cea6c61..9144ea2cf60b 100644 --- a/crypto/ossl_typ.h +++ b/crypto/ossl_typ.h @@ -100,6 +100,8 @@ typedef int ASN1_BOOLEAN; typedef int ASN1_NULL; # endif +typedef struct asn1_object_st ASN1_OBJECT; + typedef struct ASN1_ITEM_st ASN1_ITEM; typedef struct asn1_pctx_st ASN1_PCTX; diff --git a/crypto/pem/Makefile b/crypto/pem/Makefile index 7a930eeae774..65de60e2a12e 100644 --- a/crypto/pem/Makefile +++ b/crypto/pem/Makefile @@ -171,12 +171,13 @@ pem_pk8.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h pem_pk8.o: ../cryptlib.h pem_pk8.c pem_pkey.o: ../../e_os.h ../../include/openssl/asn1.h pem_pkey.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h -pem_pkey.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h -pem_pkey.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h -pem_pkey.o: ../../include/openssl/ecdsa.h ../../include/openssl/engine.h -pem_pkey.o: ../../include/openssl/err.h ../../include/openssl/evp.h -pem_pkey.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h -pem_pkey.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h +pem_pkey.o: ../../include/openssl/crypto.h ../../include/openssl/dh.h +pem_pkey.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h +pem_pkey.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h +pem_pkey.o: ../../include/openssl/engine.h ../../include/openssl/err.h +pem_pkey.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h +pem_pkey.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h +pem_pkey.o: ../../include/openssl/opensslconf.h pem_pkey.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h pem_pkey.o: ../../include/openssl/pem.h ../../include/openssl/pem2.h pem_pkey.o: ../../include/openssl/pkcs12.h ../../include/openssl/pkcs7.h diff --git a/crypto/pem/pem.h b/crypto/pem/pem.h index 2cdad8ac2669..d3b23fc997d6 100644 --- a/crypto/pem/pem.h +++ b/crypto/pem/pem.h @@ -129,6 +129,7 @@ extern "C" { # define PEM_STRING_PKCS8 "ENCRYPTED PRIVATE KEY" # define PEM_STRING_PKCS8INF "PRIVATE KEY" # define PEM_STRING_DHPARAMS "DH PARAMETERS" +# define PEM_STRING_DHXPARAMS "X9.42 DH PARAMETERS" # define PEM_STRING_SSL_SESSION "SSL SESSION PARAMETERS" # define PEM_STRING_DSAPARAMS "DSA PARAMETERS" # define PEM_STRING_ECDSA_PUBLIC "ECDSA PUBLIC KEY" @@ -181,7 +182,6 @@ typedef struct pem_ctx_st { int num_recipient; PEM_USER **recipient; - /*- XXX(ben): don#t think this is used! STACK *x509_chain; / * certificate chain */ @@ -399,8 +399,8 @@ int PEM_do_header(EVP_CIPHER_INFO *cipher, unsigned char *data, long *len, # ifndef OPENSSL_NO_BIO int PEM_read_bio(BIO *bp, char **name, char **header, unsigned char **data, long *len); -int PEM_write_bio(BIO *bp, const char *name, char *hdr, unsigned char *data, - long len); +int PEM_write_bio(BIO *bp, const char *name, const char *hdr, + const unsigned char *data, long len); int PEM_bytes_read_bio(unsigned char **pdata, long *plen, char **pnm, const char *name, BIO *bp, pem_password_cb *cb, void *u); @@ -419,7 +419,8 @@ int PEM_X509_INFO_write_bio(BIO *bp, X509_INFO *xi, EVP_CIPHER *enc, int PEM_read(FILE *fp, char **name, char **header, unsigned char **data, long *len); -int PEM_write(FILE *fp, char *name, char *hdr, unsigned char *data, long len); +int PEM_write(FILE *fp, const char *name, const char *hdr, + const unsigned char *data, long len); void *PEM_ASN1_read(d2i_of_void *d2i, const char *name, FILE *fp, void **x, pem_password_cb *cb, void *u); int PEM_ASN1_write(i2d_of_void *i2d, const char *name, FILE *fp, @@ -474,6 +475,7 @@ DECLARE_PEM_rw(EC_PUBKEY, EC_KEY) # endif # ifndef OPENSSL_NO_DH DECLARE_PEM_rw_const(DHparams, DH) +DECLARE_PEM_write_const(DHxparams, DH) # endif DECLARE_PEM_rw_cb(PrivateKey, EVP_PKEY) DECLARE_PEM_rw(PUBKEY, EVP_PKEY) @@ -562,8 +564,10 @@ void ERR_load_PEM_strings(void); # define PEM_F_PEM_PK8PKEY 119 # define PEM_F_PEM_READ 108 # define PEM_F_PEM_READ_BIO 109 +# define PEM_F_PEM_READ_BIO_DHPARAMS 141 # define PEM_F_PEM_READ_BIO_PARAMETERS 140 # define PEM_F_PEM_READ_BIO_PRIVATEKEY 123 +# define PEM_F_PEM_READ_DHPARAMS 142 # define PEM_F_PEM_READ_PRIVATEKEY 124 # define PEM_F_PEM_SEALFINAL 110 # define PEM_F_PEM_SEALINIT 111 diff --git a/crypto/pem/pem_all.c b/crypto/pem/pem_all.c index 64b8ba7c256a..0e5be63ef089 100644 --- a/crypto/pem/pem_all.c +++ b/crypto/pem/pem_all.c @@ -421,6 +421,7 @@ EC_KEY *PEM_read_ECPrivateKey(FILE *fp, EC_KEY **eckey, pem_password_cb *cb, #ifndef OPENSSL_NO_DH -IMPLEMENT_PEM_rw_const(DHparams, DH, PEM_STRING_DHPARAMS, DHparams) +IMPLEMENT_PEM_write_const(DHparams, DH, PEM_STRING_DHPARAMS, DHparams) + IMPLEMENT_PEM_write_const(DHxparams, DH, PEM_STRING_DHXPARAMS, DHxparams) #endif - IMPLEMENT_PEM_rw(PUBKEY, EVP_PKEY, PEM_STRING_PUBLIC, PUBKEY) +IMPLEMENT_PEM_rw(PUBKEY, EVP_PKEY, PEM_STRING_PUBLIC, PUBKEY) diff --git a/crypto/pem/pem_err.c b/crypto/pem/pem_err.c index 702c5adecb86..e1f4fdb432d0 100644 --- a/crypto/pem/pem_err.c +++ b/crypto/pem/pem_err.c @@ -1,6 +1,6 @@ /* crypto/pem/pem_err.c */ /* ==================================================================== - * Copyright (c) 1999-2007 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2011 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -99,8 +99,10 @@ static ERR_STRING_DATA PEM_str_functs[] = { {ERR_FUNC(PEM_F_PEM_PK8PKEY), "PEM_PK8PKEY"}, {ERR_FUNC(PEM_F_PEM_READ), "PEM_read"}, {ERR_FUNC(PEM_F_PEM_READ_BIO), "PEM_read_bio"}, + {ERR_FUNC(PEM_F_PEM_READ_BIO_DHPARAMS), "PEM_READ_BIO_DHPARAMS"}, {ERR_FUNC(PEM_F_PEM_READ_BIO_PARAMETERS), "PEM_read_bio_Parameters"}, {ERR_FUNC(PEM_F_PEM_READ_BIO_PRIVATEKEY), "PEM_READ_BIO_PRIVATEKEY"}, + {ERR_FUNC(PEM_F_PEM_READ_DHPARAMS), "PEM_READ_DHPARAMS"}, {ERR_FUNC(PEM_F_PEM_READ_PRIVATEKEY), "PEM_READ_PRIVATEKEY"}, {ERR_FUNC(PEM_F_PEM_SEALFINAL), "PEM_SealFinal"}, {ERR_FUNC(PEM_F_PEM_SEALINIT), "PEM_SealInit"}, diff --git a/crypto/pem/pem_lib.c b/crypto/pem/pem_lib.c index 55071616e203..a29821aab2eb 100644 --- a/crypto/pem/pem_lib.c +++ b/crypto/pem/pem_lib.c @@ -229,6 +229,10 @@ static int check_pem(const char *nm, const char *name) } return 0; } + /* If reading DH parameters handle X9.42 DH format too */ + if (!strcmp(nm, PEM_STRING_DHXPARAMS) && + !strcmp(name, PEM_STRING_DHPARAMS)) + return 1; /* Permit older strings */ @@ -472,8 +476,9 @@ int PEM_do_header(EVP_CIPHER_INFO *cipher, unsigned char *data, long *plen, EVP_CIPHER_CTX_cleanup(&ctx); OPENSSL_cleanse((char *)buf, sizeof(buf)); OPENSSL_cleanse((char *)key, sizeof(key)); - j += i; - if (!o) { + if (o) + j += i; + else { PEMerr(PEM_F_PEM_DO_HEADER, PEM_R_BAD_DECRYPT); return (0); } @@ -574,8 +579,8 @@ static int load_iv(char **fromp, unsigned char *to, int num) } #ifndef OPENSSL_NO_FP_API -int PEM_write(FILE *fp, char *name, char *header, unsigned char *data, - long len) +int PEM_write(FILE *fp, const char *name, const char *header, + const unsigned char *data, long len) { BIO *b; int ret; @@ -591,8 +596,8 @@ int PEM_write(FILE *fp, char *name, char *header, unsigned char *data, } #endif -int PEM_write_bio(BIO *bp, const char *name, char *header, - unsigned char *data, long len) +int PEM_write_bio(BIO *bp, const char *name, const char *header, + const unsigned char *data, long len) { int nlen, n, i, j, outl; unsigned char *buf = NULL; diff --git a/crypto/pem/pem_pkey.c b/crypto/pem/pem_pkey.c index 0b05e63fd7aa..04d6319a225b 100644 --- a/crypto/pem/pem_pkey.c +++ b/crypto/pem/pem_pkey.c @@ -68,6 +68,9 @@ #ifndef OPENSSL_NO_ENGINE # include #endif +#ifndef OPENSSL_NO_DH +# include +#endif #include "asn1_locl.h" int pem_check_suffix(const char *pem_str, const char *suffix); @@ -241,3 +244,50 @@ int PEM_write_PrivateKey(FILE *fp, EVP_PKEY *x, const EVP_CIPHER *enc, } #endif + +#ifndef OPENSSL_NO_DH + +/* Transparently read in PKCS#3 or X9.42 DH parameters */ + +DH *PEM_read_bio_DHparams(BIO *bp, DH **x, pem_password_cb *cb, void *u) +{ + char *nm = NULL; + const unsigned char *p = NULL; + unsigned char *data = NULL; + long len; + DH *ret = NULL; + + if (!PEM_bytes_read_bio(&data, &len, &nm, PEM_STRING_DHPARAMS, bp, cb, u)) + return NULL; + p = data; + + if (!strcmp(nm, PEM_STRING_DHXPARAMS)) + ret = d2i_DHxparams(x, &p, len); + else + ret = d2i_DHparams(x, &p, len); + + if (ret == NULL) + PEMerr(PEM_F_PEM_READ_BIO_DHPARAMS, ERR_R_ASN1_LIB); + OPENSSL_free(nm); + OPENSSL_free(data); + return ret; +} + +# ifndef OPENSSL_NO_FP_API +DH *PEM_read_DHparams(FILE *fp, DH **x, pem_password_cb *cb, void *u) +{ + BIO *b; + DH *ret; + + if ((b = BIO_new(BIO_s_file())) == NULL) { + PEMerr(PEM_F_PEM_READ_DHPARAMS, ERR_R_BUF_LIB); + return (0); + } + BIO_set_fp(b, fp, BIO_NOCLOSE); + ret = PEM_read_bio_DHparams(b, x, cb, u); + BIO_free(b); + return (ret); +} +# endif + +#endif diff --git a/crypto/perlasm/ppc-xlate.pl b/crypto/perlasm/ppc-xlate.pl index a3edd982b664..f89e81429931 100755 --- a/crypto/perlasm/ppc-xlate.pl +++ b/crypto/perlasm/ppc-xlate.pl @@ -27,7 +27,8 @@ my $globl = sub { /osx/ && do { $name = "_$name"; last; }; - /linux.*32/ && do { $ret .= ".globl $name\n"; + /linux.*(32|64le)/ + && do { $ret .= ".globl $name\n"; $ret .= ".type $name,\@function"; last; }; @@ -37,7 +38,6 @@ my $globl = sub { $ret .= ".align 3\n"; $ret .= "$name:\n"; $ret .= ".quad .$name,.TOC.\@tocbase,0\n"; - $ret .= ".size $name,24\n"; $ret .= ".previous\n"; $name = ".$name"; @@ -50,7 +50,9 @@ my $globl = sub { $ret; }; my $text = sub { - ($flavour =~ /aix/) ? ".csect" : ".text"; + my $ret = ($flavour =~ /aix/) ? ".csect\t.text[PR],7" : ".text"; + $ret = ".abiversion 2\n".$ret if ($flavour =~ /linux.*64le/); + $ret; }; my $machine = sub { my $junk = shift; @@ -62,9 +64,12 @@ my $machine = sub { ".machine $arch"; }; my $size = sub { - if ($flavour =~ /linux.*32/) + if ($flavour =~ /linux/) { shift; - ".size " . join(",",@_); + my $name = shift; $name =~ s|^[\.\_]||; + my $ret = ".size $name,.-".($flavour=~/64$/?".":"").$name; + $ret .= "\n.size .$name,.-.$name" if ($flavour=~/64$/); + $ret; } else { ""; } @@ -77,6 +82,25 @@ my $asciz = sub { else { ""; } }; +my $quad = sub { + shift; + my @ret; + my ($hi,$lo); + for (@_) { + if (/^0x([0-9a-f]*?)([0-9a-f]{1,8})$/io) + { $hi=$1?"0x$1":"0"; $lo="0x$2"; } + elsif (/^([0-9]+)$/o) + { $hi=$1>>32; $lo=$1&0xffffffff; } # error-prone with 32-bit perl + else + { $hi=undef; $lo=$_; } + + if (defined($hi)) + { push(@ret,$flavour=~/le$/o?".long\t$lo,$hi":".long\t$hi,$lo"); } + else + { push(@ret,".quad $lo"); } + } + join("\n",@ret); +}; ################################################################ # simplified mnemonics not handled by at least one assembler @@ -122,6 +146,46 @@ my $extrdi = sub { $b = ($b+$n)&63; $n = 64-$n; " rldicl $ra,$rs,$b,$n"; }; +my $vmr = sub { + my ($f,$vx,$vy) = @_; + " vor $vx,$vy,$vy"; +}; + +# PowerISA 2.06 stuff +sub vsxmem_op { + my ($f, $vrt, $ra, $rb, $op) = @_; + " .long ".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|($rb<<11)|($op*2+1); +} +# made-up unaligned memory reference AltiVec/VMX instructions +my $lvx_u = sub { vsxmem_op(@_, 844); }; # lxvd2x +my $stvx_u = sub { vsxmem_op(@_, 972); }; # stxvd2x +my $lvdx_u = sub { vsxmem_op(@_, 588); }; # lxsdx +my $stvdx_u = sub { vsxmem_op(@_, 716); }; # stxsdx +my $lvx_4w = sub { vsxmem_op(@_, 780); }; # lxvw4x +my $stvx_4w = sub { vsxmem_op(@_, 908); }; # stxvw4x + +# PowerISA 2.07 stuff +sub vcrypto_op { + my ($f, $vrt, $vra, $vrb, $op) = @_; + " .long ".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|$op; +} +my $vcipher = sub { vcrypto_op(@_, 1288); }; +my $vcipherlast = sub { vcrypto_op(@_, 1289); }; +my $vncipher = sub { vcrypto_op(@_, 1352); }; +my $vncipherlast= sub { vcrypto_op(@_, 1353); }; +my $vsbox = sub { vcrypto_op(@_, 0, 1480); }; +my $vshasigmad = sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1730); }; +my $vshasigmaw = sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1666); }; +my $vpmsumb = sub { vcrypto_op(@_, 1032); }; +my $vpmsumd = sub { vcrypto_op(@_, 1224); }; +my $vpmsubh = sub { vcrypto_op(@_, 1096); }; +my $vpmsumw = sub { vcrypto_op(@_, 1160); }; +my $vaddudm = sub { vcrypto_op(@_, 192); }; + +my $mtsle = sub { + my ($f, $arg) = @_; + " .long ".sprintf "0x%X",(31<<26)|($arg<<21)|(147*2); +}; while($line=<>) { @@ -138,7 +202,10 @@ while($line=<>) { { $line =~ s|(^[\.\w]+)\:\s*||; my $label = $1; - printf "%s:",($GLOBALS{$label} or $label) if ($label); + if ($label) { + printf "%s:",($GLOBALS{$label} or $label); + printf "\n.localentry\t$GLOBALS{$label},0" if ($GLOBALS{$label} && $flavour =~ /linux.*64le/); + } } { @@ -147,7 +214,7 @@ while($line=<>) { my $mnemonic = $2; my $f = $3; my $opcode = eval("\$$mnemonic"); - $line =~ s|\bc?[rf]([0-9]+)\b|$1|g if ($c ne "." and $flavour !~ /osx/); + $line =~ s/\b(c?[rf]|v|vs)([0-9]+)\b/$2/g if ($c ne "." and $flavour !~ /osx/); if (ref($opcode) eq 'CODE') { $line = &$opcode($f,split(',',$line)); } elsif ($mnemonic) { $line = $c.$mnemonic.$f."\t".$line; } } diff --git a/crypto/perlasm/sparcv9_modes.pl b/crypto/perlasm/sparcv9_modes.pl new file mode 100755 index 000000000000..eb267a57ed81 --- /dev/null +++ b/crypto/perlasm/sparcv9_modes.pl @@ -0,0 +1,1687 @@ +#!/usr/bin/env perl + +# Specific modes implementations for SPARC Architecture 2011. There +# is T4 dependency though, an ASI value that is not specified in the +# Architecture Manual. But as SPARC universe is rather monocultural, +# we imply that processor capable of executing crypto instructions +# can handle the ASI in question as well. This means that we ought to +# keep eyes open when new processors emerge... +# +# As for above mentioned ASI. It's so called "block initializing +# store" which cancels "read" in "read-update-write" on cache lines. +# This is "cooperative" optimization, as it reduces overall pressure +# on memory interface. Benefits can't be observed/quantified with +# usual benchmarks, on the contrary you can notice that single-thread +# performance for parallelizable modes is ~1.5% worse for largest +# block sizes [though few percent better for not so long ones]. All +# this based on suggestions from David Miller. + +sub asm_init { # to be called with @ARGV as argument + for (@_) { $::abibits=64 if (/\-m64/ || /\-xarch\=v9/); } + if ($::abibits==64) { $::bias=2047; $::frame=192; $::size_t_cc="%xcc"; } + else { $::bias=0; $::frame=112; $::size_t_cc="%icc"; } +} + +# unified interface +my ($inp,$out,$len,$key,$ivec)=map("%i$_",(0..5)); +# local variables +my ($ileft,$iright,$ooff,$omask,$ivoff,$blk_init)=map("%l$_",(0..7)); + +sub alg_cbc_encrypt_implement { +my ($alg,$bits) = @_; + +$::code.=<<___; +.globl ${alg}${bits}_t4_cbc_encrypt +.align 32 +${alg}${bits}_t4_cbc_encrypt: + save %sp, -$::frame, %sp + cmp $len, 0 + be,pn $::size_t_cc, .L${bits}_cbc_enc_abort + sub $inp, $out, $blk_init ! $inp!=$out +___ +$::code.=<<___ if (!$::evp); + andcc $ivec, 7, $ivoff + alignaddr $ivec, %g0, $ivec + + ldd [$ivec + 0], %f0 ! load ivec + bz,pt %icc, 1f + ldd [$ivec + 8], %f2 + ldd [$ivec + 16], %f4 + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 +1: +___ +$::code.=<<___ if ($::evp); + ld [$ivec + 0], %f0 + ld [$ivec + 4], %f1 + ld [$ivec + 8], %f2 + ld [$ivec + 12], %f3 +___ +$::code.=<<___; + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + call _${alg}${bits}_load_enckey + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 64, $iright + mov 0xff, $omask + sub $iright, $ileft, $iright + and $out, 7, $ooff + cmp $len, 127 + movrnz $ooff, 0, $blk_init ! if ( $out&7 || + movleu $::size_t_cc, 0, $blk_init ! $len<128 || + brnz,pn $blk_init, .L${bits}cbc_enc_blk ! $inp==$out) + srl $omask, $ooff, $omask + + alignaddrl $out, %g0, $out + srlx $len, 4, $len + prefetch [$out], 22 + +.L${bits}_cbc_enc_loop: + ldx [$inp + 0], %o0 + brz,pt $ileft, 4f + ldx [$inp + 8], %o1 + + ldx [$inp + 16], %o2 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + sllx %o1, $ileft, %o1 + or %g1, %o0, %o0 + srlx %o2, $iright, %o2 + or %o2, %o1, %o1 +4: + xor %g4, %o0, %o0 ! ^= rk[0] + xor %g5, %o1, %o1 + movxtod %o0, %f12 + movxtod %o1, %f14 + + fxor %f12, %f0, %f0 ! ^= ivec + fxor %f14, %f2, %f2 + prefetch [$out + 63], 22 + prefetch [$inp + 16+63], 20 + call _${alg}${bits}_encrypt_1x + add $inp, 16, $inp + + brnz,pn $ooff, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + std %f2, [$out + 8] + brnz,pt $len, .L${bits}_cbc_enc_loop + add $out, 16, $out +___ +$::code.=<<___ if ($::evp); + st %f0, [$ivec + 0] + st %f1, [$ivec + 4] + st %f2, [$ivec + 8] + st %f3, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, 3f + nop + + std %f0, [$ivec + 0] ! write out ivec + std %f2, [$ivec + 8] +___ +$::code.=<<___; +.L${bits}_cbc_enc_abort: + ret + restore + +.align 16 +2: ldxa [$inp]0x82, %o0 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f4 ! handle unaligned output + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + + stda %f4, [$out + $omask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $omask, $omask + stda %f8, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .L${bits}_cbc_enc_loop+4 + orn %g0, $omask, $omask +___ +$::code.=<<___ if ($::evp); + st %f0, [$ivec + 0] + st %f1, [$ivec + 4] + st %f2, [$ivec + 8] + st %f3, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, 3f + nop + + std %f0, [$ivec + 0] ! write out ivec + std %f2, [$ivec + 8] + ret + restore + +.align 16 +3: alignaddrl $ivec, $ivoff, %g0 ! handle unaligned ivec + mov 0xff, $omask + srl $omask, $ivoff, $omask + faligndata %f0, %f0, %f4 + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + stda %f4, [$ivec + $omask]0xc0 + std %f6, [$ivec + 8] + add $ivec, 16, $ivec + orn %g0, $omask, $omask + stda %f8, [$ivec + $omask]0xc0 +___ +$::code.=<<___; + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}cbc_enc_blk: + add $out, $len, $blk_init + and $blk_init, 63, $blk_init ! tail + sub $len, $blk_init, $len + add $blk_init, 15, $blk_init ! round up to 16n + srlx $len, 4, $len + srl $blk_init, 4, $blk_init + +.L${bits}_cbc_enc_blk_loop: + ldx [$inp + 0], %o0 + brz,pt $ileft, 5f + ldx [$inp + 8], %o1 + + ldx [$inp + 16], %o2 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + sllx %o1, $ileft, %o1 + or %g1, %o0, %o0 + srlx %o2, $iright, %o2 + or %o2, %o1, %o1 +5: + xor %g4, %o0, %o0 ! ^= rk[0] + xor %g5, %o1, %o1 + movxtod %o0, %f12 + movxtod %o1, %f14 + + fxor %f12, %f0, %f0 ! ^= ivec + fxor %f14, %f2, %f2 + prefetch [$inp + 16+63], 20 + call _${alg}${bits}_encrypt_1x + add $inp, 16, $inp + sub $len, 1, $len + + stda %f0, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f2, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + brnz,pt $len, .L${bits}_cbc_enc_blk_loop + add $out, 8, $out + + membar #StoreLoad|#StoreStore + brnz,pt $blk_init, .L${bits}_cbc_enc_loop + mov $blk_init, $len +___ +$::code.=<<___ if ($::evp); + st %f0, [$ivec + 0] + st %f1, [$ivec + 4] + st %f2, [$ivec + 8] + st %f3, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, 3b + nop + + std %f0, [$ivec + 0] ! write out ivec + std %f2, [$ivec + 8] +___ +$::code.=<<___; + ret + restore +.type ${alg}${bits}_t4_cbc_encrypt,#function +.size ${alg}${bits}_t4_cbc_encrypt,.-${alg}${bits}_t4_cbc_encrypt +___ +} + +sub alg_cbc_decrypt_implement { +my ($alg,$bits) = @_; + +$::code.=<<___; +.globl ${alg}${bits}_t4_cbc_decrypt +.align 32 +${alg}${bits}_t4_cbc_decrypt: + save %sp, -$::frame, %sp + cmp $len, 0 + be,pn $::size_t_cc, .L${bits}_cbc_dec_abort + sub $inp, $out, $blk_init ! $inp!=$out +___ +$::code.=<<___ if (!$::evp); + andcc $ivec, 7, $ivoff + alignaddr $ivec, %g0, $ivec + + ldd [$ivec + 0], %f12 ! load ivec + bz,pt %icc, 1f + ldd [$ivec + 8], %f14 + ldd [$ivec + 16], %f0 + faligndata %f12, %f14, %f12 + faligndata %f14, %f0, %f14 +1: +___ +$::code.=<<___ if ($::evp); + ld [$ivec + 0], %f12 ! load ivec + ld [$ivec + 4], %f13 + ld [$ivec + 8], %f14 + ld [$ivec + 12], %f15 +___ +$::code.=<<___; + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + call _${alg}${bits}_load_deckey + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 64, $iright + mov 0xff, $omask + sub $iright, $ileft, $iright + and $out, 7, $ooff + cmp $len, 255 + movrnz $ooff, 0, $blk_init ! if ( $out&7 || + movleu $::size_t_cc, 0, $blk_init ! $len<256 || + brnz,pn $blk_init, .L${bits}cbc_dec_blk ! $inp==$out) + srl $omask, $ooff, $omask + + andcc $len, 16, %g0 ! is number of blocks even? + srlx $len, 4, $len + alignaddrl $out, %g0, $out + bz %icc, .L${bits}_cbc_dec_loop2x + prefetch [$out], 22 +.L${bits}_cbc_dec_loop: + ldx [$inp + 0], %o0 + brz,pt $ileft, 4f + ldx [$inp + 8], %o1 + + ldx [$inp + 16], %o2 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + sllx %o1, $ileft, %o1 + or %g1, %o0, %o0 + srlx %o2, $iright, %o2 + or %o2, %o1, %o1 +4: + xor %g4, %o0, %o2 ! ^= rk[0] + xor %g5, %o1, %o3 + movxtod %o2, %f0 + movxtod %o3, %f2 + + prefetch [$out + 63], 22 + prefetch [$inp + 16+63], 20 + call _${alg}${bits}_decrypt_1x + add $inp, 16, $inp + + fxor %f12, %f0, %f0 ! ^= ivec + fxor %f14, %f2, %f2 + movxtod %o0, %f12 + movxtod %o1, %f14 + + brnz,pn $ooff, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + std %f2, [$out + 8] + brnz,pt $len, .L${bits}_cbc_dec_loop2x + add $out, 16, $out +___ +$::code.=<<___ if ($::evp); + st %f12, [$ivec + 0] + st %f13, [$ivec + 4] + st %f14, [$ivec + 8] + st %f15, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, .L${bits}_cbc_dec_unaligned_ivec + nop + + std %f12, [$ivec + 0] ! write out ivec + std %f14, [$ivec + 8] +___ +$::code.=<<___; +.L${bits}_cbc_dec_abort: + ret + restore + +.align 16 +2: ldxa [$inp]0x82, %o0 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f4 ! handle unaligned output + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + + stda %f4, [$out + $omask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $omask, $omask + stda %f8, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .L${bits}_cbc_dec_loop2x+4 + orn %g0, $omask, $omask +___ +$::code.=<<___ if ($::evp); + st %f12, [$ivec + 0] + st %f13, [$ivec + 4] + st %f14, [$ivec + 8] + st %f15, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, .L${bits}_cbc_dec_unaligned_ivec + nop + + std %f12, [$ivec + 0] ! write out ivec + std %f14, [$ivec + 8] +___ +$::code.=<<___; + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}_cbc_dec_loop2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 4f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +4: + xor %g4, %o0, %o4 ! ^= rk[0] + xor %g5, %o1, %o5 + movxtod %o4, %f0 + movxtod %o5, %f2 + xor %g4, %o2, %o4 + xor %g5, %o3, %o5 + movxtod %o4, %f4 + movxtod %o5, %f6 + + prefetch [$out + 63], 22 + prefetch [$inp + 32+63], 20 + call _${alg}${bits}_decrypt_2x + add $inp, 32, $inp + + movxtod %o0, %f8 + movxtod %o1, %f10 + fxor %f12, %f0, %f0 ! ^= ivec + fxor %f14, %f2, %f2 + movxtod %o2, %f12 + movxtod %o3, %f14 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + + brnz,pn $ooff, 2f + sub $len, 2, $len + + std %f0, [$out + 0] + std %f2, [$out + 8] + std %f4, [$out + 16] + std %f6, [$out + 24] + brnz,pt $len, .L${bits}_cbc_dec_loop2x + add $out, 32, $out +___ +$::code.=<<___ if ($::evp); + st %f12, [$ivec + 0] + st %f13, [$ivec + 4] + st %f14, [$ivec + 8] + st %f15, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, .L${bits}_cbc_dec_unaligned_ivec + nop + + std %f12, [$ivec + 0] ! write out ivec + std %f14, [$ivec + 8] +___ +$::code.=<<___; + ret + restore + +.align 16 +2: ldxa [$inp]0x82, %o0 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f8 ! handle unaligned output + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 + faligndata %f4, %f6, %f4 + faligndata %f6, %f6, %f6 + stda %f8, [$out + $omask]0xc0 ! partial store + std %f0, [$out + 8] + std %f2, [$out + 16] + std %f4, [$out + 24] + add $out, 32, $out + orn %g0, $omask, $omask + stda %f6, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .L${bits}_cbc_dec_loop2x+4 + orn %g0, $omask, $omask +___ +$::code.=<<___ if ($::evp); + st %f12, [$ivec + 0] + st %f13, [$ivec + 4] + st %f14, [$ivec + 8] + st %f15, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, .L${bits}_cbc_dec_unaligned_ivec + nop + + std %f12, [$ivec + 0] ! write out ivec + std %f14, [$ivec + 8] + ret + restore + +.align 16 +.L${bits}_cbc_dec_unaligned_ivec: + alignaddrl $ivec, $ivoff, %g0 ! handle unaligned ivec + mov 0xff, $omask + srl $omask, $ivoff, $omask + faligndata %f12, %f12, %f0 + faligndata %f12, %f14, %f2 + faligndata %f14, %f14, %f4 + stda %f0, [$ivec + $omask]0xc0 + std %f2, [$ivec + 8] + add $ivec, 16, $ivec + orn %g0, $omask, $omask + stda %f4, [$ivec + $omask]0xc0 +___ +$::code.=<<___; + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}cbc_dec_blk: + add $out, $len, $blk_init + and $blk_init, 63, $blk_init ! tail + sub $len, $blk_init, $len + add $blk_init, 15, $blk_init ! round up to 16n + srlx $len, 4, $len + srl $blk_init, 4, $blk_init + sub $len, 1, $len + add $blk_init, 1, $blk_init + +.L${bits}_cbc_dec_blk_loop2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 5f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +5: + xor %g4, %o0, %o4 ! ^= rk[0] + xor %g5, %o1, %o5 + movxtod %o4, %f0 + movxtod %o5, %f2 + xor %g4, %o2, %o4 + xor %g5, %o3, %o5 + movxtod %o4, %f4 + movxtod %o5, %f6 + + prefetch [$inp + 32+63], 20 + call _${alg}${bits}_decrypt_2x + add $inp, 32, $inp + subcc $len, 2, $len + + movxtod %o0, %f8 + movxtod %o1, %f10 + fxor %f12, %f0, %f0 ! ^= ivec + fxor %f14, %f2, %f2 + movxtod %o2, %f12 + movxtod %o3, %f14 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + + stda %f0, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f2, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f4, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f6, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + bgu,pt $::size_t_cc, .L${bits}_cbc_dec_blk_loop2x + add $out, 8, $out + + add $blk_init, $len, $len + andcc $len, 1, %g0 ! is number of blocks even? + membar #StoreLoad|#StoreStore + bnz,pt %icc, .L${bits}_cbc_dec_loop + srl $len, 0, $len + brnz,pn $len, .L${bits}_cbc_dec_loop2x + nop +___ +$::code.=<<___ if ($::evp); + st %f12, [$ivec + 0] ! write out ivec + st %f13, [$ivec + 4] + st %f14, [$ivec + 8] + st %f15, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, 3b + nop + + std %f12, [$ivec + 0] ! write out ivec + std %f14, [$ivec + 8] +___ +$::code.=<<___; + ret + restore +.type ${alg}${bits}_t4_cbc_decrypt,#function +.size ${alg}${bits}_t4_cbc_decrypt,.-${alg}${bits}_t4_cbc_decrypt +___ +} + +sub alg_ctr32_implement { +my ($alg,$bits) = @_; + +$::code.=<<___; +.globl ${alg}${bits}_t4_ctr32_encrypt +.align 32 +${alg}${bits}_t4_ctr32_encrypt: + save %sp, -$::frame, %sp + + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + call _${alg}${bits}_load_enckey + sllx $len, 4, $len + + ld [$ivec + 0], %l4 ! counter + ld [$ivec + 4], %l5 + ld [$ivec + 8], %l6 + ld [$ivec + 12], %l7 + + sllx %l4, 32, %o5 + or %l5, %o5, %o5 + sllx %l6, 32, %g1 + xor %o5, %g4, %g4 ! ^= rk[0] + xor %g1, %g5, %g5 + movxtod %g4, %f14 ! most significant 64 bits + + sub $inp, $out, $blk_init ! $inp!=$out + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 64, $iright + mov 0xff, $omask + sub $iright, $ileft, $iright + and $out, 7, $ooff + cmp $len, 255 + movrnz $ooff, 0, $blk_init ! if ( $out&7 || + movleu $::size_t_cc, 0, $blk_init ! $len<256 || + brnz,pn $blk_init, .L${bits}_ctr32_blk ! $inp==$out) + srl $omask, $ooff, $omask + + andcc $len, 16, %g0 ! is number of blocks even? + alignaddrl $out, %g0, $out + bz %icc, .L${bits}_ctr32_loop2x + srlx $len, 4, $len +.L${bits}_ctr32_loop: + ldx [$inp + 0], %o0 + brz,pt $ileft, 4f + ldx [$inp + 8], %o1 + + ldx [$inp + 16], %o2 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + sllx %o1, $ileft, %o1 + or %g1, %o0, %o0 + srlx %o2, $iright, %o2 + or %o2, %o1, %o1 +4: + xor %g5, %l7, %g1 ! ^= rk[0] + add %l7, 1, %l7 + movxtod %g1, %f2 + srl %l7, 0, %l7 ! clruw + prefetch [$out + 63], 22 + prefetch [$inp + 16+63], 20 +___ +$::code.=<<___ if ($alg eq "aes"); + aes_eround01 %f16, %f14, %f2, %f4 + aes_eround23 %f18, %f14, %f2, %f2 +___ +$::code.=<<___ if ($alg eq "cmll"); + camellia_f %f16, %f2, %f14, %f2 + camellia_f %f18, %f14, %f2, %f0 +___ +$::code.=<<___; + call _${alg}${bits}_encrypt_1x+8 + add $inp, 16, $inp + + movxtod %o0, %f10 + movxtod %o1, %f12 + fxor %f10, %f0, %f0 ! ^= inp + fxor %f12, %f2, %f2 + + brnz,pn $ooff, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + std %f2, [$out + 8] + brnz,pt $len, .L${bits}_ctr32_loop2x + add $out, 16, $out + + ret + restore + +.align 16 +2: ldxa [$inp]0x82, %o0 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f4 ! handle unaligned output + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + stda %f4, [$out + $omask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $omask, $omask + stda %f8, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .L${bits}_ctr32_loop2x+4 + orn %g0, $omask, $omask + + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}_ctr32_loop2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 4f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +4: + xor %g5, %l7, %g1 ! ^= rk[0] + add %l7, 1, %l7 + movxtod %g1, %f2 + srl %l7, 0, %l7 ! clruw + xor %g5, %l7, %g1 + add %l7, 1, %l7 + movxtod %g1, %f6 + srl %l7, 0, %l7 ! clruw + prefetch [$out + 63], 22 + prefetch [$inp + 32+63], 20 +___ +$::code.=<<___ if ($alg eq "aes"); + aes_eround01 %f16, %f14, %f2, %f8 + aes_eround23 %f18, %f14, %f2, %f2 + aes_eround01 %f16, %f14, %f6, %f10 + aes_eround23 %f18, %f14, %f6, %f6 +___ +$::code.=<<___ if ($alg eq "cmll"); + camellia_f %f16, %f2, %f14, %f2 + camellia_f %f16, %f6, %f14, %f6 + camellia_f %f18, %f14, %f2, %f0 + camellia_f %f18, %f14, %f6, %f4 +___ +$::code.=<<___; + call _${alg}${bits}_encrypt_2x+16 + add $inp, 32, $inp + + movxtod %o0, %f8 + movxtod %o1, %f10 + movxtod %o2, %f12 + fxor %f8, %f0, %f0 ! ^= inp + movxtod %o3, %f8 + fxor %f10, %f2, %f2 + fxor %f12, %f4, %f4 + fxor %f8, %f6, %f6 + + brnz,pn $ooff, 2f + sub $len, 2, $len + + std %f0, [$out + 0] + std %f2, [$out + 8] + std %f4, [$out + 16] + std %f6, [$out + 24] + brnz,pt $len, .L${bits}_ctr32_loop2x + add $out, 32, $out + + ret + restore + +.align 16 +2: ldxa [$inp]0x82, %o0 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f8 ! handle unaligned output + faligndata %f0, %f2, %f0 + faligndata %f2, %f4, %f2 + faligndata %f4, %f6, %f4 + faligndata %f6, %f6, %f6 + + stda %f8, [$out + $omask]0xc0 ! partial store + std %f0, [$out + 8] + std %f2, [$out + 16] + std %f4, [$out + 24] + add $out, 32, $out + orn %g0, $omask, $omask + stda %f6, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .L${bits}_ctr32_loop2x+4 + orn %g0, $omask, $omask + + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}_ctr32_blk: + add $out, $len, $blk_init + and $blk_init, 63, $blk_init ! tail + sub $len, $blk_init, $len + add $blk_init, 15, $blk_init ! round up to 16n + srlx $len, 4, $len + srl $blk_init, 4, $blk_init + sub $len, 1, $len + add $blk_init, 1, $blk_init + +.L${bits}_ctr32_blk_loop2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 5f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +5: + xor %g5, %l7, %g1 ! ^= rk[0] + add %l7, 1, %l7 + movxtod %g1, %f2 + srl %l7, 0, %l7 ! clruw + xor %g5, %l7, %g1 + add %l7, 1, %l7 + movxtod %g1, %f6 + srl %l7, 0, %l7 ! clruw + prefetch [$inp + 32+63], 20 +___ +$::code.=<<___ if ($alg eq "aes"); + aes_eround01 %f16, %f14, %f2, %f8 + aes_eround23 %f18, %f14, %f2, %f2 + aes_eround01 %f16, %f14, %f6, %f10 + aes_eround23 %f18, %f14, %f6, %f6 +___ +$::code.=<<___ if ($alg eq "cmll"); + camellia_f %f16, %f2, %f14, %f2 + camellia_f %f16, %f6, %f14, %f6 + camellia_f %f18, %f14, %f2, %f0 + camellia_f %f18, %f14, %f6, %f4 +___ +$::code.=<<___; + call _${alg}${bits}_encrypt_2x+16 + add $inp, 32, $inp + subcc $len, 2, $len + + movxtod %o0, %f8 + movxtod %o1, %f10 + movxtod %o2, %f12 + fxor %f8, %f0, %f0 ! ^= inp + movxtod %o3, %f8 + fxor %f10, %f2, %f2 + fxor %f12, %f4, %f4 + fxor %f8, %f6, %f6 + + stda %f0, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f2, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f4, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f6, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + bgu,pt $::size_t_cc, .L${bits}_ctr32_blk_loop2x + add $out, 8, $out + + add $blk_init, $len, $len + andcc $len, 1, %g0 ! is number of blocks even? + membar #StoreLoad|#StoreStore + bnz,pt %icc, .L${bits}_ctr32_loop + srl $len, 0, $len + brnz,pn $len, .L${bits}_ctr32_loop2x + nop + + ret + restore +.type ${alg}${bits}_t4_ctr32_encrypt,#function +.size ${alg}${bits}_t4_ctr32_encrypt,.-${alg}${bits}_t4_ctr32_encrypt +___ +} + +sub alg_xts_implement { +my ($alg,$bits,$dir) = @_; +my ($inp,$out,$len,$key1,$key2,$ivec)=map("%i$_",(0..5)); +my $rem=$ivec; + +$::code.=<<___; +.globl ${alg}${bits}_t4_xts_${dir}crypt +.align 32 +${alg}${bits}_t4_xts_${dir}crypt: + save %sp, -$::frame-16, %sp + + mov $ivec, %o0 + add %fp, $::bias-16, %o1 + call ${alg}_t4_encrypt + mov $key2, %o2 + + add %fp, $::bias-16, %l7 + ldxa [%l7]0x88, %g2 + add %fp, $::bias-8, %l7 + ldxa [%l7]0x88, %g3 ! %g3:%g2 is tweak + + sethi %hi(0x76543210), %l7 + or %l7, %lo(0x76543210), %l7 + bmask %l7, %g0, %g0 ! byte swap mask + + prefetch [$inp], 20 + prefetch [$inp + 63], 20 + call _${alg}${bits}_load_${dir}ckey + and $len, 15, $rem + and $len, -16, $len +___ +$code.=<<___ if ($dir eq "de"); + mov 0, %l7 + movrnz $rem, 16, %l7 + sub $len, %l7, $len +___ +$code.=<<___; + + sub $inp, $out, $blk_init ! $inp!=$out + and $inp, 7, $ileft + andn $inp, 7, $inp + sll $ileft, 3, $ileft + mov 64, $iright + mov 0xff, $omask + sub $iright, $ileft, $iright + and $out, 7, $ooff + cmp $len, 255 + movrnz $ooff, 0, $blk_init ! if ( $out&7 || + movleu $::size_t_cc, 0, $blk_init ! $len<256 || + brnz,pn $blk_init, .L${bits}_xts_${dir}blk ! $inp==$out) + srl $omask, $ooff, $omask + + andcc $len, 16, %g0 ! is number of blocks even? +___ +$code.=<<___ if ($dir eq "de"); + brz,pn $len, .L${bits}_xts_${dir}steal +___ +$code.=<<___; + alignaddrl $out, %g0, $out + bz %icc, .L${bits}_xts_${dir}loop2x + srlx $len, 4, $len +.L${bits}_xts_${dir}loop: + ldx [$inp + 0], %o0 + brz,pt $ileft, 4f + ldx [$inp + 8], %o1 + + ldx [$inp + 16], %o2 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + sllx %o1, $ileft, %o1 + or %g1, %o0, %o0 + srlx %o2, $iright, %o2 + or %o2, %o1, %o1 +4: + movxtod %g2, %f12 + movxtod %g3, %f14 + bshuffle %f12, %f12, %f12 + bshuffle %f14, %f14, %f14 + + xor %g4, %o0, %o0 ! ^= rk[0] + xor %g5, %o1, %o1 + movxtod %o0, %f0 + movxtod %o1, %f2 + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + + prefetch [$out + 63], 22 + prefetch [$inp + 16+63], 20 + call _${alg}${bits}_${dir}crypt_1x + add $inp, 16, $inp + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + + srax %g3, 63, %l7 ! next tweak value + addcc %g2, %g2, %g2 + and %l7, 0x87, %l7 + addxc %g3, %g3, %g3 + xor %l7, %g2, %g2 + + brnz,pn $ooff, 2f + sub $len, 1, $len + + std %f0, [$out + 0] + std %f2, [$out + 8] + brnz,pt $len, .L${bits}_xts_${dir}loop2x + add $out, 16, $out + + brnz,pn $rem, .L${bits}_xts_${dir}steal + nop + + ret + restore + +.align 16 +2: ldxa [$inp]0x82, %o0 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f4 ! handle unaligned output + faligndata %f0, %f2, %f6 + faligndata %f2, %f2, %f8 + stda %f4, [$out + $omask]0xc0 ! partial store + std %f6, [$out + 8] + add $out, 16, $out + orn %g0, $omask, $omask + stda %f8, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .L${bits}_xts_${dir}loop2x+4 + orn %g0, $omask, $omask + + brnz,pn $rem, .L${bits}_xts_${dir}steal + nop + + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}_xts_${dir}loop2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 4f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +4: + movxtod %g2, %f12 + movxtod %g3, %f14 + bshuffle %f12, %f12, %f12 + bshuffle %f14, %f14, %f14 + + srax %g3, 63, %l7 ! next tweak value + addcc %g2, %g2, %g2 + and %l7, 0x87, %l7 + addxc %g3, %g3, %g3 + xor %l7, %g2, %g2 + + movxtod %g2, %f8 + movxtod %g3, %f10 + bshuffle %f8, %f8, %f8 + bshuffle %f10, %f10, %f10 + + xor %g4, %o0, %o0 ! ^= rk[0] + xor %g5, %o1, %o1 + xor %g4, %o2, %o2 ! ^= rk[0] + xor %g5, %o3, %o3 + movxtod %o0, %f0 + movxtod %o1, %f2 + movxtod %o2, %f4 + movxtod %o3, %f6 + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + fxor %f8, %f4, %f4 ! ^= tweak[0] + fxor %f10, %f6, %f6 + + prefetch [$out + 63], 22 + prefetch [$inp + 32+63], 20 + call _${alg}${bits}_${dir}crypt_2x + add $inp, 32, $inp + + movxtod %g2, %f8 + movxtod %g3, %f10 + + srax %g3, 63, %l7 ! next tweak value + addcc %g2, %g2, %g2 + and %l7, 0x87, %l7 + addxc %g3, %g3, %g3 + xor %l7, %g2, %g2 + + bshuffle %f8, %f8, %f8 + bshuffle %f10, %f10, %f10 + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + + brnz,pn $ooff, 2f + sub $len, 2, $len + + std %f0, [$out + 0] + std %f2, [$out + 8] + std %f4, [$out + 16] + std %f6, [$out + 24] + brnz,pt $len, .L${bits}_xts_${dir}loop2x + add $out, 32, $out + + fsrc2 %f4, %f0 + fsrc2 %f6, %f2 + brnz,pn $rem, .L${bits}_xts_${dir}steal + nop + + ret + restore + +.align 16 +2: ldxa [$inp]0x82, %o0 ! avoid read-after-write hazard + ! and ~3x deterioration + ! in inp==out case + faligndata %f0, %f0, %f8 ! handle unaligned output + faligndata %f0, %f2, %f10 + faligndata %f2, %f4, %f12 + faligndata %f4, %f6, %f14 + faligndata %f6, %f6, %f0 + + stda %f8, [$out + $omask]0xc0 ! partial store + std %f10, [$out + 8] + std %f12, [$out + 16] + std %f14, [$out + 24] + add $out, 32, $out + orn %g0, $omask, $omask + stda %f0, [$out + $omask]0xc0 ! partial store + + brnz,pt $len, .L${bits}_xts_${dir}loop2x+4 + orn %g0, $omask, $omask + + fsrc2 %f4, %f0 + fsrc2 %f6, %f2 + brnz,pn $rem, .L${bits}_xts_${dir}steal + nop + + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}_xts_${dir}blk: + add $out, $len, $blk_init + and $blk_init, 63, $blk_init ! tail + sub $len, $blk_init, $len + add $blk_init, 15, $blk_init ! round up to 16n + srlx $len, 4, $len + srl $blk_init, 4, $blk_init + sub $len, 1, $len + add $blk_init, 1, $blk_init + +.L${bits}_xts_${dir}blk2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 5f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +5: + movxtod %g2, %f12 + movxtod %g3, %f14 + bshuffle %f12, %f12, %f12 + bshuffle %f14, %f14, %f14 + + srax %g3, 63, %l7 ! next tweak value + addcc %g2, %g2, %g2 + and %l7, 0x87, %l7 + addxc %g3, %g3, %g3 + xor %l7, %g2, %g2 + + movxtod %g2, %f8 + movxtod %g3, %f10 + bshuffle %f8, %f8, %f8 + bshuffle %f10, %f10, %f10 + + xor %g4, %o0, %o0 ! ^= rk[0] + xor %g5, %o1, %o1 + xor %g4, %o2, %o2 ! ^= rk[0] + xor %g5, %o3, %o3 + movxtod %o0, %f0 + movxtod %o1, %f2 + movxtod %o2, %f4 + movxtod %o3, %f6 + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + fxor %f8, %f4, %f4 ! ^= tweak[0] + fxor %f10, %f6, %f6 + + prefetch [$inp + 32+63], 20 + call _${alg}${bits}_${dir}crypt_2x + add $inp, 32, $inp + + movxtod %g2, %f8 + movxtod %g3, %f10 + + srax %g3, 63, %l7 ! next tweak value + addcc %g2, %g2, %g2 + and %l7, 0x87, %l7 + addxc %g3, %g3, %g3 + xor %l7, %g2, %g2 + + bshuffle %f8, %f8, %f8 + bshuffle %f10, %f10, %f10 + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + + subcc $len, 2, $len + stda %f0, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f2, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f4, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f6, [$out]0xe2 ! ASI_BLK_INIT, T4-specific + bgu,pt $::size_t_cc, .L${bits}_xts_${dir}blk2x + add $out, 8, $out + + add $blk_init, $len, $len + andcc $len, 1, %g0 ! is number of blocks even? + membar #StoreLoad|#StoreStore + bnz,pt %icc, .L${bits}_xts_${dir}loop + srl $len, 0, $len + brnz,pn $len, .L${bits}_xts_${dir}loop2x + nop + + fsrc2 %f4, %f0 + fsrc2 %f6, %f2 + brnz,pn $rem, .L${bits}_xts_${dir}steal + nop + + ret + restore +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +___ +$code.=<<___ if ($dir eq "en"); +.align 32 +.L${bits}_xts_${dir}steal: + std %f0, [%fp + $::bias-16] ! copy of output + std %f2, [%fp + $::bias-8] + + srl $ileft, 3, $ileft + add %fp, $::bias-16, %l7 + add $inp, $ileft, $inp ! original $inp+$len&-15 + add $out, $ooff, $out ! original $out+$len&-15 + mov 0, $ileft + nop ! align + +.L${bits}_xts_${dir}stealing: + ldub [$inp + $ileft], %o0 + ldub [%l7 + $ileft], %o1 + dec $rem + stb %o0, [%l7 + $ileft] + stb %o1, [$out + $ileft] + brnz $rem, .L${bits}_xts_${dir}stealing + inc $ileft + + mov %l7, $inp + sub $out, 16, $out + mov 0, $ileft + sub $out, $ooff, $out + ba .L${bits}_xts_${dir}loop ! one more time + mov 1, $len ! $rem is 0 +___ +$code.=<<___ if ($dir eq "de"); +.align 32 +.L${bits}_xts_${dir}steal: + ldx [$inp + 0], %o0 + brz,pt $ileft, 8f + ldx [$inp + 8], %o1 + + ldx [$inp + 16], %o2 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + sllx %o1, $ileft, %o1 + or %g1, %o0, %o0 + srlx %o2, $iright, %o2 + or %o2, %o1, %o1 +8: + srax %g3, 63, %l7 ! next tweak value + addcc %g2, %g2, %o2 + and %l7, 0x87, %l7 + addxc %g3, %g3, %o3 + xor %l7, %o2, %o2 + + movxtod %o2, %f12 + movxtod %o3, %f14 + bshuffle %f12, %f12, %f12 + bshuffle %f14, %f14, %f14 + + xor %g4, %o0, %o0 ! ^= rk[0] + xor %g5, %o1, %o1 + movxtod %o0, %f0 + movxtod %o1, %f2 + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + + call _${alg}${bits}_${dir}crypt_1x + add $inp, 16, $inp + + fxor %f12, %f0, %f0 ! ^= tweak[0] + fxor %f14, %f2, %f2 + + std %f0, [%fp + $::bias-16] + std %f2, [%fp + $::bias-8] + + srl $ileft, 3, $ileft + add %fp, $::bias-16, %l7 + add $inp, $ileft, $inp ! original $inp+$len&-15 + add $out, $ooff, $out ! original $out+$len&-15 + mov 0, $ileft + add $out, 16, $out + nop ! align + +.L${bits}_xts_${dir}stealing: + ldub [$inp + $ileft], %o0 + ldub [%l7 + $ileft], %o1 + dec $rem + stb %o0, [%l7 + $ileft] + stb %o1, [$out + $ileft] + brnz $rem, .L${bits}_xts_${dir}stealing + inc $ileft + + mov %l7, $inp + sub $out, 16, $out + mov 0, $ileft + sub $out, $ooff, $out + ba .L${bits}_xts_${dir}loop ! one more time + mov 1, $len ! $rem is 0 +___ +$code.=<<___; + ret + restore +.type ${alg}${bits}_t4_xts_${dir}crypt,#function +.size ${alg}${bits}_t4_xts_${dir}crypt,.-${alg}${bits}_t4_xts_${dir}crypt +___ +} + +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my ($ref,$opf); +my %visopf = ( "faligndata" => 0x048, + "bshuffle" => 0x04c, + "fnot2" => 0x066, + "fxor" => 0x06c, + "fsrc2" => 0x078 ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +sub unvis3 { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my ($ref,$opf); +my %visopf = ( "addxc" => 0x011, + "addxccc" => 0x013, + "umulxhi" => 0x016, + "alignaddr" => 0x018, + "bmask" => 0x019, + "alignaddrl" => 0x01a ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%([goli])([0-9])/); + $_=$bias{$1}+$2; + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +sub unaes_round { # 4-argument instructions +my ($mnemonic,$rs1,$rs2,$rs3,$rd)=@_; +my ($ref,$opf); +my %aesopf = ( "aes_eround01" => 0, + "aes_eround23" => 1, + "aes_dround01" => 2, + "aes_dround23" => 3, + "aes_eround01_l"=> 4, + "aes_eround23_l"=> 5, + "aes_dround01_l"=> 6, + "aes_dround23_l"=> 7, + "aes_kexpand1" => 8 ); + + $ref = "$mnemonic\t$rs1,$rs2,$rs3,$rd"; + + if (defined($opf=$aesopf{$mnemonic})) { + $rs3 = ($rs3 =~ /%f([0-6]*[02468])/) ? (($1|$1>>5)&31) : $rs3; + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 2<<30|$rd<<25|0x19<<19|$rs1<<14|$rs3<<9|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +sub unaes_kexpand { # 3-argument instructions +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my ($ref,$opf); +my %aesopf = ( "aes_kexpand0" => 0x130, + "aes_kexpand2" => 0x131 ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if (defined($opf=$aesopf{$mnemonic})) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 2<<30|$rd<<25|0x36<<19|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +sub uncamellia_f { # 4-argument instructions +my ($mnemonic,$rs1,$rs2,$rs3,$rd)=@_; +my ($ref,$opf); + + $ref = "$mnemonic\t$rs1,$rs2,$rs3,$rd"; + + if (1) { + $rs3 = ($rs3 =~ /%f([0-6]*[02468])/) ? (($1|$1>>5)&31) : $rs3; + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 2<<30|$rd<<25|0x19<<19|$rs1<<14|$rs3<<9|0xc<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +sub uncamellia3 { # 3-argument instructions +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my ($ref,$opf); +my %cmllopf = ( "camellia_fl" => 0x13c, + "camellia_fli" => 0x13d ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if (defined($opf=$cmllopf{$mnemonic})) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 2<<30|$rd<<25|0x36<<19|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +sub unmovxtox { # 2-argument instructions +my ($mnemonic,$rs,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24, "f" => 0 ); +my ($ref,$opf); +my %movxopf = ( "movdtox" => 0x110, + "movstouw" => 0x111, + "movstosw" => 0x113, + "movxtod" => 0x118, + "movwtos" => 0x119 ); + + $ref = "$mnemonic\t$rs,$rd"; + + if (defined($opf=$movxopf{$mnemonic})) { + foreach ($rs,$rd) { + return $ref if (!/%([fgoli])([0-9]{1,2})/); + $_=$bias{$1}+$2; + if ($2>=32) { + return $ref if ($2&1); + # re-encode for upper double register addressing + $_=($2|$2>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 2<<30|$rd<<25|0x36<<19|$opf<<5|$rs, + $ref; + } else { + return $ref; + } +} + +sub undes { +my ($mnemonic)=shift; +my @args=@_; +my ($ref,$opf); +my %desopf = ( "des_round" => 0b1001, + "des_ip" => 0b100110100, + "des_iip" => 0b100110101, + "des_kexpand" => 0b100110110 ); + + $ref = "$mnemonic\t".join(",",@_); + + if (defined($opf=$desopf{$mnemonic})) { # 4-arg + if ($mnemonic eq "des_round") { + foreach (@args[0..3]) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + return sprintf ".word\t0x%08x !%s", + 2<<30|0b011001<<19|$opf<<5|$args[0]<<14|$args[1]|$args[2]<<9|$args[3]<<25, + $ref; + } elsif ($mnemonic eq "des_kexpand") { # 3-arg + foreach (@args[0..2]) { + return $ref if (!/(%f)?([0-9]{1,2})/); + $_=$2; + if ($2>=32) { + return $ref if ($2&1); + # re-encode for upper double register addressing + $_=($2|$2>>5)&31; + } + } + return sprintf ".word\t0x%08x !%s", + 2<<30|0b110110<<19|$opf<<5|$args[0]<<14|$args[1]|$args[2]<<25, + $ref; + } else { # 2-arg + foreach (@args[0..1]) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($2&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + return sprintf ".word\t0x%08x !%s", + 2<<30|0b110110<<19|$opf<<5|$args[0]<<14|$args[1]<<25, + $ref; + } + } else { + return $ref; + } +} + +sub emit_assembler { + foreach (split("\n",$::code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/\b(f[a-z]+2[sd]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})\s*$/$1\t%f0,$2,$3/go; + + s/\b(aes_[edk][^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*([%fx0-9]+),\s*(%f[0-9]{1,2})/ + &unaes_round($1,$2,$3,$4,$5) + /geo or + s/\b(aes_kexpand[02])\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ + &unaes_kexpand($1,$2,$3,$4) + /geo or + s/\b(camellia_f)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*([%fx0-9]+),\s*(%f[0-9]{1,2})/ + &uncamellia_f($1,$2,$3,$4,$5) + /geo or + s/\b(camellia_[^s]+)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ + &uncamellia3($1,$2,$3,$4) + /geo or + s/\b(des_\w+)\s+(%f[0-9]{1,2}),\s*([%fx0-9]+)(?:,\s*(%f[0-9]{1,2})(?:,\s*(%f[0-9]{1,2}))?)?/ + &undes($1,$2,$3,$4,$5) + /geo or + s/\b(mov[ds]to\w+)\s+(%f[0-9]{1,2}),\s*(%[goli][0-7])/ + &unmovxtox($1,$2,$3) + /geo or + s/\b(mov[xw]to[ds])\s+(%[goli][0-7]),\s*(%f[0-9]{1,2})/ + &unmovxtox($1,$2,$3) + /geo or + s/\b([fb][^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ + &unvis($1,$2,$3,$4) + /geo or + s/\b(umulxhi|bmask|addxc[c]{0,2}|alignaddr[l]*)\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ + &unvis3($1,$2,$3,$4) + /geo; + + print $_,"\n"; + } +} + +1; diff --git a/crypto/perlasm/x86_64-xlate.pl b/crypto/perlasm/x86_64-xlate.pl index 56d9b64b6fb3..9c70b8c2c6e9 100755 --- a/crypto/perlasm/x86_64-xlate.pl +++ b/crypto/perlasm/x86_64-xlate.pl @@ -121,7 +121,7 @@ my %globals; $self->{sz} = ""; } elsif ($self->{op} =~ /^v/) { # VEX $self->{sz} = ""; - } elsif ($self->{op} =~ /movq/ && $line =~ /%xmm/) { + } elsif ($self->{op} =~ /mov[dq]/ && $line =~ /%xmm/) { $self->{sz} = ""; } elsif ($self->{op} =~ /([a-z]{3,})([qlwb])$/) { $self->{op} = $1; @@ -250,8 +250,13 @@ my %globals; # in $self->{label}, new gas requires sign extension... use integer; $self->{label} =~ s/(?{label} =~ s/([0-9]+\s*[\*\/\%]\s*[0-9]+)/eval($1)/eg; - $self->{label} =~ s/([0-9]+)/$1<<32>>32/eg; + $self->{label} =~ s/\b([0-9]+\s*[\*\/\%]\s*[0-9]+)\b/eval($1)/eg; + $self->{label} =~ s/\b([0-9]+)\b/$1<<32>>32/eg; + + if (!$self->{label} && $self->{index} && $self->{scale}==1 && + $self->{base} =~ /(rbp|r13)/) { + $self->{base} = $self->{index}; $self->{index} = $1; + } if ($gas) { $self->{label} =~ s/^___imp_/__imp__/ if ($flavour eq "mingw64"); @@ -265,14 +270,20 @@ my %globals; sprintf "%s%s(%%%s)", $self->{asterisk},$self->{label},$self->{base}; } } else { - %szmap = ( b=>"BYTE$PTR", w=>"WORD$PTR", l=>"DWORD$PTR", - q=>"QWORD$PTR",o=>"OWORD$PTR",x=>"XMMWORD$PTR" ); + %szmap = ( b=>"BYTE$PTR", w=>"WORD$PTR", + l=>"DWORD$PTR", d=>"DWORD$PTR", + q=>"QWORD$PTR", o=>"OWORD$PTR", + x=>"XMMWORD$PTR", y=>"YMMWORD$PTR", z=>"ZMMWORD$PTR" ); $self->{label} =~ s/\./\$/g; $self->{label} =~ s/(?{label} = "($self->{label})" if ($self->{label} =~ /[\*\+\-\/]/); - $sz="q" if ($self->{asterisk} || opcode->mnemonic() eq "movq"); - $sz="l" if (opcode->mnemonic() eq "movd"); + + ($self->{asterisk}) && ($sz="q") || + (opcode->mnemonic() =~ /^v?mov([qd])$/) && ($sz=$1) || + (opcode->mnemonic() =~ /^v?pinsr([qdwb])$/) && ($sz=$1) || + (opcode->mnemonic() =~ /^vpbroadcast([qdwb])$/) && ($sz=$1) || + (opcode->mnemonic() =~ /^vinsert[fi]128$/) && ($sz="x"); if (defined($self->{index})) { sprintf "%s[%s%s*%d%s]",$szmap{$sz}, @@ -412,7 +423,7 @@ my %globals; } sub out { my $self = shift; - if ($nasm && opcode->mnemonic()=~m/^j/) { + if ($nasm && opcode->mnemonic()=~m/^j(?![re]cxz)/) { "NEAR ".$self->{value}; } else { $self->{value}; @@ -530,7 +541,7 @@ my %globals; $v="$current_segment\tENDS\n" if ($current_segment); $current_segment = ".text\$"; $v.="$current_segment\tSEGMENT "; - $v.=$masm>=$masmref ? "ALIGN(64)" : "PAGE"; + $v.=$masm>=$masmref ? "ALIGN(256)" : "PAGE"; $v.=" 'CODE'"; } $self->{value} = $v; @@ -772,10 +783,64 @@ my $rdrand = sub { } }; +my $rdseed = sub { + if (shift =~ /%[er](\w+)/) { + my @opcode=(); + my $dst=$1; + if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; } + rex(\@opcode,0,$1,8); + push @opcode,0x0f,0xc7,0xf8|($dst&7); + @opcode; + } else { + (); + } +}; + +sub rxb { + local *opcode=shift; + my ($dst,$src1,$src2,$rxb)=@_; + + $rxb|=0x7<<5; + $rxb&=~(0x04<<5) if($dst>=8); + $rxb&=~(0x01<<5) if($src1>=8); + $rxb&=~(0x02<<5) if($src2>=8); + push @opcode,$rxb; +} + +my $vprotd = sub { + if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x8f); + rxb(\@opcode,$3,$2,-1,0x08); + push @opcode,0x78,0xc2; + push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + my $c=$1; + push @opcode,$c=~/^0/?oct($c):$c; + @opcode; + } else { + (); + } +}; + +my $vprotq = sub { + if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x8f); + rxb(\@opcode,$3,$2,-1,0x08); + push @opcode,0x78,0xc3; + push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + my $c=$1; + push @opcode,$c=~/^0/?oct($c):$c; + @opcode; + } else { + (); + } +}; + if ($nasm) { print <<___; default rel %define XMMWORD +%define YMMWORD +%define ZMMWORD ___ } elsif ($masm) { print <<___; @@ -789,6 +854,7 @@ while($line=<>) { $line =~ s|[#!].*$||; # get rid of asm-style comments... $line =~ s|/\*.*\*/||; # ... and C-style comments... $line =~ s|^\s+||; # ... and skip white spaces in beginning + $line =~ s|\s+$||; # ... and at the end undef $label; undef $opcode; @@ -837,6 +903,8 @@ while($line=<>) { my $arg = $_->out(); # $insn.=$sz compensates for movq, pinsrw, ... if ($arg =~ /^xmm[0-9]+$/) { $insn.=$sz; $sz="x" if(!$sz); last; } + if ($arg =~ /^ymm[0-9]+$/) { $insn.=$sz; $sz="y" if(!$sz); last; } + if ($arg =~ /^zmm[0-9]+$/) { $insn.=$sz; $sz="z" if(!$sz); last; } if ($arg =~ /^mm[0-9]+$/) { $insn.=$sz; $sz="q" if(!$sz); last; } } @args = reverse(@args); diff --git a/crypto/perlasm/x86asm.pl b/crypto/perlasm/x86asm.pl index eb543db2f66e..cae156ae63ce 100644 --- a/crypto/perlasm/x86asm.pl +++ b/crypto/perlasm/x86asm.pl @@ -131,6 +131,40 @@ sub ::rdrand { &::generic("rdrand",@_); } } +sub ::rdseed +{ my ($dst)=@_; + if ($dst =~ /(e[a-dsd][ixp])/) + { &::data_byte(0x0f,0xc7,0xf8|$regrm{$dst}); } + else + { &::generic("rdrand",@_); } +} + +sub rxb { + local *opcode=shift; + my ($dst,$src1,$src2,$rxb)=@_; + + $rxb|=0x7<<5; + $rxb&=~(0x04<<5) if($dst>=8); + $rxb&=~(0x01<<5) if($src1>=8); + $rxb&=~(0x02<<5) if($src2>=8); + push @opcode,$rxb; +} + +sub ::vprotd +{ my $args=join(',',@_); + if ($args =~ /xmm([0-7]),xmm([0-7]),([x0-9a-f]+)/) + { my @opcode=(0x8f); + rxb(\@opcode,$1,$2,-1,0x08); + push @opcode,0x78,0xc2; + push @opcode,0xc0|($2&7)|(($1&7)<<3); # ModR/M + my $c=$3; + push @opcode,$c=~/^0/?oct($c):$c; + &::data_byte(@opcode); + } + else + { &::generic("vprotd",@_); } +} + # label management $lbdecor="L"; # local label decoration, set by package $label="000"; @@ -221,6 +255,8 @@ sub ::asm_init $elf=$cpp=$coff=$aout=$macosx=$win32=$netware=$mwerks=$android=0; if (($type eq "elf")) { $elf=1; require "x86gas.pl"; } + elsif (($type eq "elf-1")) + { $elf=-1; require "x86gas.pl"; } elsif (($type eq "a\.out")) { $aout=1; require "x86gas.pl"; } elsif (($type eq "coff" or $type eq "gaswin")) @@ -257,4 +293,6 @@ EOF &file($filename); } +sub ::hidden {} + 1; diff --git a/crypto/perlasm/x86gas.pl b/crypto/perlasm/x86gas.pl index 682a3a3163e2..63b2301fd1f0 100755 --- a/crypto/perlasm/x86gas.pl +++ b/crypto/perlasm/x86gas.pl @@ -70,6 +70,8 @@ sub ::DWP { my($addr,$reg1,$reg2,$idx)=@_; my $ret=""; + if (!defined($idx) && 1*$reg2) { $idx=$reg2; $reg2=$reg1; undef $reg1; } + $addr =~ s/^\s+//; # prepend global references with optional underscore $addr =~ s/^([^\+\-0-9][^\+\-]*)/&::islabel($1) or "$nmdecor$1"/ige; @@ -157,7 +159,7 @@ sub ::file_end } } if (grep {/\b${nmdecor}OPENSSL_ia32cap_P\b/i} @out) { - my $tmp=".comm\t${nmdecor}OPENSSL_ia32cap_P,8"; + my $tmp=".comm\t${nmdecor}OPENSSL_ia32cap_P,16"; if ($::macosx) { push (@out,"$tmp,2\n"); } elsif ($::elf) { push (@out,"$tmp,4\n"); } else { push (@out,"$tmp\n"); } @@ -170,10 +172,9 @@ sub ::data_short{ push(@out,".value\t".join(',',@_)."\n"); } sub ::data_word { push(@out,".long\t".join(',',@_)."\n"); } sub ::align -{ my $val=$_[0],$p2,$i; +{ my $val=$_[0]; if ($::aout) - { for ($p2=0;$val!=0;$val>>=1) { $p2++; } - $val=$p2-1; + { $val=int(log($val)/log(2)); $val.=",0x90"; } push(@out,".align\t$val\n"); @@ -195,6 +196,8 @@ sub ::picmeup &::mov($dst,&::DWP("$indirect-$reflabel",$base)); $non_lazy_ptr{"$nmdecor$sym"}=$indirect; } + elsif ($sym eq "OPENSSL_ia32cap_P" && $::elf>0) + { &::lea($dst,&::DWP("$sym-$reflabel",$base)); } else { &::lea($dst,&::DWP("_GLOBAL_OFFSET_TABLE_+[.-$reflabel]", $base)); @@ -250,4 +253,6 @@ ___ sub ::dataseg { push(@out,".data\n"); } +*::hidden = sub { push(@out,".hidden\t$nmdecor$_[0]\n"); } if ($::elf); + 1; diff --git a/crypto/perlasm/x86masm.pl b/crypto/perlasm/x86masm.pl index f937d07c87df..1741342c3af3 100755 --- a/crypto/perlasm/x86masm.pl +++ b/crypto/perlasm/x86masm.pl @@ -39,6 +39,8 @@ sub get_mem { my($size,$addr,$reg1,$reg2,$idx)=@_; my($post,$ret); + if (!defined($idx) && 1*$reg2) { $idx=$reg2; $reg2=$reg1; undef $reg1; } + $ret .= "$size PTR " if ($size ne ""); $addr =~ s/^\s+//; @@ -133,7 +135,7 @@ ___ if (grep {/\b${nmdecor}OPENSSL_ia32cap_P\b/i} @out) { my $comm=<<___; .bss SEGMENT 'BSS' -COMM ${nmdecor}OPENSSL_ia32cap_P:QWORD +COMM ${nmdecor}OPENSSL_ia32cap_P:DWORD:4 .bss ENDS ___ # comment out OPENSSL_ia32cap_P declarations diff --git a/crypto/perlasm/x86nasm.pl b/crypto/perlasm/x86nasm.pl index ca2511c9eb91..5d92f6092ac9 100644 --- a/crypto/perlasm/x86nasm.pl +++ b/crypto/perlasm/x86nasm.pl @@ -36,6 +36,8 @@ sub get_mem { my($size,$addr,$reg1,$reg2,$idx)=@_; my($post,$ret); + if (!defined($idx) && 1*$reg2) { $idx=$reg2; $reg2=$reg1; undef $reg1; } + if ($size ne "") { $ret .= "$size"; $ret .= " PTR" if ($::mwerks); @@ -117,7 +119,7 @@ sub ::file_end { if (grep {/\b${nmdecor}OPENSSL_ia32cap_P\b/i} @out) { my $comm=<<___; ${drdecor}segment .bss -${drdecor}common ${nmdecor}OPENSSL_ia32cap_P 8 +${drdecor}common ${nmdecor}OPENSSL_ia32cap_P 16 ___ # comment out OPENSSL_ia32cap_P declarations grep {s/(^extern\s+${nmdecor}OPENSSL_ia32cap_P)/\;$1/} @out; diff --git a/crypto/pkcs12/p12_decr.c b/crypto/pkcs12/p12_decr.c index d46eae3c66a6..b40ea10ccbee 100644 --- a/crypto/pkcs12/p12_decr.c +++ b/crypto/pkcs12/p12_decr.c @@ -171,28 +171,32 @@ ASN1_OCTET_STRING *PKCS12_item_i2d_encrypt(X509_ALGOR *algor, const char *pass, int passlen, void *obj, int zbuf) { - ASN1_OCTET_STRING *oct; + ASN1_OCTET_STRING *oct = NULL; unsigned char *in = NULL; int inlen; if (!(oct = M_ASN1_OCTET_STRING_new())) { PKCS12err(PKCS12_F_PKCS12_ITEM_I2D_ENCRYPT, ERR_R_MALLOC_FAILURE); - return NULL; + goto err; } inlen = ASN1_item_i2d(obj, &in, it); if (!in) { PKCS12err(PKCS12_F_PKCS12_ITEM_I2D_ENCRYPT, PKCS12_R_ENCODE_ERROR); - return NULL; + goto err; } if (!PKCS12_pbe_crypt(algor, pass, passlen, in, inlen, &oct->data, &oct->length, 1)) { PKCS12err(PKCS12_F_PKCS12_ITEM_I2D_ENCRYPT, PKCS12_R_ENCRYPT_ERROR); OPENSSL_free(in); - return NULL; + goto err; } if (zbuf) OPENSSL_cleanse(in, inlen); OPENSSL_free(in); return oct; + err: + if (oct) + ASN1_OCTET_STRING_free(oct); + return NULL; } IMPLEMENT_PKCS12_STACK_OF(PKCS7) diff --git a/crypto/pkcs12/p12_p8e.c b/crypto/pkcs12/p12_p8e.c index d970f0544c5e..861a087f80d7 100644 --- a/crypto/pkcs12/p12_p8e.c +++ b/crypto/pkcs12/p12_p8e.c @@ -76,8 +76,12 @@ X509_SIG *PKCS8_encrypt(int pbe_nid, const EVP_CIPHER *cipher, if (pbe_nid == -1) pbe = PKCS5_pbe2_set(cipher, iter, salt, saltlen); - else + else if (EVP_PBE_find(EVP_PBE_TYPE_PRF, pbe_nid, NULL, NULL, 0)) + pbe = PKCS5_pbe2_set_iv(cipher, iter, salt, saltlen, NULL, pbe_nid); + else { + ERR_clear_error(); pbe = PKCS5_pbe_set(pbe_nid, iter, salt, saltlen); + } if (!pbe) { PKCS12err(PKCS12_F_PKCS8_ENCRYPT, ERR_R_ASN1_LIB); goto err; diff --git a/crypto/ppc_arch.h b/crypto/ppc_arch.h new file mode 100644 index 000000000000..b50ec996a5a3 --- /dev/null +++ b/crypto/ppc_arch.h @@ -0,0 +1,10 @@ +#ifndef __PPC_ARCH_H__ +# define __PPC_ARCH_H__ + +extern unsigned int OPENSSL_ppccap_P; + +# define PPC_FPU64 (1<<0) +# define PPC_ALTIVEC (1<<1) +# define PPC_CRYPTO207 (1<<2) + +#endif diff --git a/crypto/ppccap.c b/crypto/ppccap.c index 5242294310b1..2b7f704cd82a 100644 --- a/crypto/ppccap.c +++ b/crypto/ppccap.c @@ -4,13 +4,15 @@ #include #include #include +#if defined(__linux) || defined(_AIX) +# include +#endif #include #include -#define PPC_FPU64 (1<<0) -#define PPC_ALTIVEC (1<<1) +#include "ppc_arch.h" -static int OPENSSL_ppccap_P = 0; +unsigned int OPENSSL_ppccap_P = 0; static sigset_t all_masked; @@ -25,7 +27,7 @@ int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, int num); if (sizeof(size_t) == 4) { -# if (defined(__APPLE__) && defined(__MACH__)) +# if 1 || (defined(__APPLE__) && defined(__MACH__)) if (num >= 8 && (num & 3) == 0 && (OPENSSL_ppccap_P & PPC_FPU64)) return bn_mul_mont_fpu64(rp, ap, bp, np, n0, num); # else @@ -55,6 +57,22 @@ int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, } #endif +void sha256_block_p8(void *ctx, const void *inp, size_t len); +void sha256_block_ppc(void *ctx, const void *inp, size_t len); +void sha256_block_data_order(void *ctx, const void *inp, size_t len) +{ + OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) : + sha256_block_ppc(ctx, inp, len); +} + +void sha512_block_p8(void *ctx, const void *inp, size_t len); +void sha512_block_ppc(void *ctx, const void *inp, size_t len); +void sha512_block_data_order(void *ctx, const void *inp, size_t len) +{ + OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) : + sha512_block_ppc(ctx, inp, len); +} + static sigjmp_buf ill_jmp; static void ill_handler(int sig) { @@ -63,6 +81,7 @@ static void ill_handler(int sig) void OPENSSL_ppc64_probe(void); void OPENSSL_altivec_probe(void); +void OPENSSL_crypto207_probe(void); void OPENSSL_cpuid_setup(void) { @@ -93,12 +112,15 @@ void OPENSSL_cpuid_setup(void) OPENSSL_ppccap_P = 0; #if defined(_AIX) - if (sizeof(size_t) == 4 + if (sizeof(size_t) == 4) { + struct utsname uts; # if defined(_SC_AIX_KERNEL_BITMODE) - && sysconf(_SC_AIX_KERNEL_BITMODE) != 64 + if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64) + return; # endif - ) - return; + if (uname(&uts) != 0 || atoi(uts.version) < 6) + return; + } #endif memset(&ill_act, 0, sizeof(ill_act)); @@ -109,10 +131,14 @@ void OPENSSL_cpuid_setup(void) sigaction(SIGILL, &ill_act, &ill_oact); if (sizeof(size_t) == 4) { - if (sigsetjmp(ill_jmp, 1) == 0) { - OPENSSL_ppc64_probe(); - OPENSSL_ppccap_P |= PPC_FPU64; - } +#ifdef __linux + struct utsname uts; + if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0) +#endif + if (sigsetjmp(ill_jmp, 1) == 0) { + OPENSSL_ppc64_probe(); + OPENSSL_ppccap_P |= PPC_FPU64; + } } else { /* * Wanted code detecting POWER6 CPU and setting PPC_FPU64 @@ -122,6 +148,10 @@ void OPENSSL_cpuid_setup(void) if (sigsetjmp(ill_jmp, 1) == 0) { OPENSSL_altivec_probe(); OPENSSL_ppccap_P |= PPC_ALTIVEC; + if (sigsetjmp(ill_jmp, 1) == 0) { + OPENSSL_crypto207_probe(); + OPENSSL_ppccap_P |= PPC_CRYPTO207; + } } sigaction(SIGILL, &ill_oact, NULL); diff --git a/crypto/ppccpuid.pl b/crypto/ppccpuid.pl index 4ba736a1d1bd..8d800fe7d36f 100755 --- a/crypto/ppccpuid.pl +++ b/crypto/ppccpuid.pl @@ -31,6 +31,7 @@ $code=<<___; blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size .OPENSSL_ppc64_probe,.-.OPENSSL_ppc64_probe .globl .OPENSSL_altivec_probe .align 4 @@ -39,6 +40,17 @@ $code=<<___; blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size .OPENSSL_altivec_probe,.-..OPENSSL_altivec_probe + +.globl .OPENSSL_crypto207_probe +.align 4 +.OPENSSL_crypto207_probe: + lvx_u v0,0,r1 + vcipher v0,v0,v0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +.size .OPENSSL_crypto207_probe,.-.OPENSSL_crypto207_probe .globl .OPENSSL_wipe_cpu .align 4 @@ -71,6 +83,7 @@ $code=<<___; blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size .OPENSSL_wipe_cpu,.-.OPENSSL_wipe_cpu .globl .OPENSSL_atomic_add .align 4 @@ -84,6 +97,7 @@ Ladd: lwarx r5,0,r3 .long 0 .byte 0,12,0x14,0,0,0,2,0 .long 0 +.size .OPENSSL_atomic_add,.-.OPENSSL_atomic_add .globl .OPENSSL_rdtsc .align 4 @@ -93,6 +107,7 @@ Ladd: lwarx r5,0,r3 blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size .OPENSSL_rdtsc,.-.OPENSSL_rdtsc .globl .OPENSSL_cleanse .align 4 @@ -125,6 +140,7 @@ Laligned: .long 0 .byte 0,12,0x14,0,0,0,2,0 .long 0 +.size .OPENSSL_cleanse,.-.OPENSSL_cleanse ___ $code =~ s/\`([^\`]*)\`/eval $1/gem; diff --git a/crypto/rc4/Makefile b/crypto/rc4/Makefile index f3d6e0dc0405..7434ff737e66 100644 --- a/crypto/rc4/Makefile +++ b/crypto/rc4/Makefile @@ -42,7 +42,7 @@ lib: $(LIBOBJ) @touch lib rc4-586.s: asm/rc4-586.pl ../perlasm/x86asm.pl - $(PERL) asm/rc4-586.pl $(PERLASM_SCHEME) $(CFLAGS) > $@ + $(PERL) asm/rc4-586.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@ rc4-x86_64.s: asm/rc4-x86_64.pl $(PERL) asm/rc4-x86_64.pl $(PERLASM_SCHEME) > $@ @@ -66,7 +66,7 @@ rc4-ia64.s: rc4-ia64.S rc4-%.s: asm/rc4-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ files: - $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO + $(PERL) $(TOP)/util/files.pl "RC4_ENC=$(RC4_ENC)" Makefile >> $(TOP)/MINFO links: @$(PERL) $(TOP)/util/mklink.pl ../../include/openssl $(EXHEADER) diff --git a/crypto/rc4/asm/rc4-586.pl b/crypto/rc4/asm/rc4-586.pl index 5c9ac6ad286e..1d55d551e9bb 100644 --- a/crypto/rc4/asm/rc4-586.pl +++ b/crypto/rc4/asm/rc4-586.pl @@ -60,7 +60,7 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; push(@INC,"${dir}","${dir}../../perlasm"); require "x86asm.pl"; -&asm_init($ARGV[0],"rc4-586.pl"); +&asm_init($ARGV[0],"rc4-586.pl",$x86only = $ARGV[$#ARGV] eq "386"); $xx="eax"; $yy="ebx"; @@ -184,8 +184,11 @@ if ($alt=0) { &and ($ty,-4); # how many 4-byte chunks? &jz (&label("loop1")); - &test ($ty,-8); &mov (&wparam(3),$out); # $out as accumulator in these loops + if ($x86only) { + &jmp (&label("go4loop4")); + } else { + &test ($ty,-8); &jz (&label("go4loop4")); &picmeup($out,"OPENSSL_ia32cap_P"); @@ -228,6 +231,7 @@ if ($alt=0) { &cmp ($inp,&wparam(1)); # compare to input+len &je (&label("done")); &jmp (&label("loop1")); + } &set_label("go4loop4",16); &lea ($ty,&DWP(-4,$inp,$ty)); diff --git a/crypto/rc4/rc4_enc.c b/crypto/rc4/rc4_enc.c index 6ebd54d46cad..0f0a2487a7f7 100644 --- a/crypto/rc4/rc4_enc.c +++ b/crypto/rc4/rc4_enc.c @@ -79,7 +79,7 @@ void RC4(RC4_KEY *key, size_t len, const unsigned char *indata, y = key->y; d = key->data; -#if defined(RC4_CHUNK) +#if defined(RC4_CHUNK) && !defined(PEDANTIC) /*- * The original reason for implementing this(*) was the fact that * pre-21164a Alpha CPUs don't have byte load/store instructions diff --git a/crypto/rc5/rc5_locl.h b/crypto/rc5/rc5_locl.h index 1e83f19fecb7..ee757e647765 100644 --- a/crypto/rc5/rc5_locl.h +++ b/crypto/rc5/rc5_locl.h @@ -146,9 +146,12 @@ *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ *((c)++)=(unsigned char)(((l) )&0xff)) -#if (defined(OPENSSL_SYS_WIN32) && defined(_MSC_VER)) || defined(__ICC) +#if (defined(OPENSSL_SYS_WIN32) && defined(_MSC_VER)) # define ROTATE_l32(a,n) _lrotl(a,n) # define ROTATE_r32(a,n) _lrotr(a,n) +#elif defined(__ICC) +# define ROTATE_l32(a,n) _rotl(a,n) +# define ROTATE_r32(a,n) _rotr(a,n) #elif defined(__GNUC__) && __GNUC__>=2 && !defined(__STRICT_ANSI__) && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) && !defined(PEDANTIC) # if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) # define ROTATE_l32(a,n) ({ register unsigned int ret; \ diff --git a/crypto/rsa/Makefile b/crypto/rsa/Makefile index 79c7c421bc09..e292e84db3c9 100644 --- a/crypto/rsa/Makefile +++ b/crypto/rsa/Makefile @@ -228,19 +228,20 @@ rsa_pk1.o: ../cryptlib.h rsa_pk1.c rsa_pmeth.o: ../../e_os.h ../../include/openssl/asn1.h rsa_pmeth.o: ../../include/openssl/asn1t.h ../../include/openssl/bio.h rsa_pmeth.o: ../../include/openssl/bn.h ../../include/openssl/buffer.h -rsa_pmeth.o: ../../include/openssl/cms.h ../../include/openssl/crypto.h -rsa_pmeth.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h -rsa_pmeth.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h -rsa_pmeth.o: ../../include/openssl/err.h ../../include/openssl/evp.h -rsa_pmeth.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h -rsa_pmeth.o: ../../include/openssl/objects.h +rsa_pmeth.o: ../../include/openssl/cms.h ../../include/openssl/conf.h +rsa_pmeth.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h +rsa_pmeth.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h +rsa_pmeth.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h +rsa_pmeth.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h +rsa_pmeth.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h rsa_pmeth.o: ../../include/openssl/opensslconf.h rsa_pmeth.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h rsa_pmeth.o: ../../include/openssl/pkcs7.h ../../include/openssl/rsa.h rsa_pmeth.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h rsa_pmeth.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h rsa_pmeth.o: ../../include/openssl/x509.h ../../include/openssl/x509_vfy.h -rsa_pmeth.o: ../cryptlib.h ../evp/evp_locl.h rsa_locl.h rsa_pmeth.c +rsa_pmeth.o: ../../include/openssl/x509v3.h ../cryptlib.h ../evp/evp_locl.h +rsa_pmeth.o: rsa_locl.h rsa_pmeth.c rsa_prn.o: ../../e_os.h ../../include/openssl/asn1.h rsa_prn.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h rsa_prn.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h index a8b59a9559a9..d2ee37406e3c 100644 --- a/crypto/rsa/rsa.h +++ b/crypto/rsa/rsa.h @@ -262,13 +262,31 @@ struct rsa_st { EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP, 0, pubexp) # define EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md) \ - EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_SIG, \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, \ + EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT, \ EVP_PKEY_CTRL_RSA_MGF1_MD, 0, (void *)md) +# define EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \ + EVP_PKEY_CTRL_RSA_OAEP_MD, 0, (void *)md) + # define EVP_PKEY_CTX_get_rsa_mgf1_md(ctx, pmd) \ - EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_SIG, \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, \ + EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT, \ EVP_PKEY_CTRL_GET_RSA_MGF1_MD, 0, (void *)pmd) +# define EVP_PKEY_CTX_get_rsa_oaep_md(ctx, pmd) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \ + EVP_PKEY_CTRL_GET_RSA_OAEP_MD, 0, (void *)pmd) + +# define EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, l, llen) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \ + EVP_PKEY_CTRL_RSA_OAEP_LABEL, llen, (void *)l) + +# define EVP_PKEY_CTX_get0_rsa_oaep_label(ctx, l) \ + EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \ + EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL, 0, (void *)l) + # define EVP_PKEY_CTRL_RSA_PADDING (EVP_PKEY_ALG_CTRL + 1) # define EVP_PKEY_CTRL_RSA_PSS_SALTLEN (EVP_PKEY_ALG_CTRL + 2) @@ -280,6 +298,12 @@ struct rsa_st { # define EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN (EVP_PKEY_ALG_CTRL + 7) # define EVP_PKEY_CTRL_GET_RSA_MGF1_MD (EVP_PKEY_ALG_CTRL + 8) +# define EVP_PKEY_CTRL_RSA_OAEP_MD (EVP_PKEY_ALG_CTRL + 9) +# define EVP_PKEY_CTRL_RSA_OAEP_LABEL (EVP_PKEY_ALG_CTRL + 10) + +# define EVP_PKEY_CTRL_GET_RSA_OAEP_MD (EVP_PKEY_ALG_CTRL + 11) +# define EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL (EVP_PKEY_ALG_CTRL + 12) + # define RSA_PKCS1_PADDING 1 # define RSA_SSLV23_PADDING 2 # define RSA_NO_PADDING 3 @@ -347,6 +371,14 @@ typedef struct rsa_pss_params_st { DECLARE_ASN1_FUNCTIONS(RSA_PSS_PARAMS) +typedef struct rsa_oaep_params_st { + X509_ALGOR *hashFunc; + X509_ALGOR *maskGenFunc; + X509_ALGOR *pSourceFunc; +} RSA_OAEP_PARAMS; + +DECLARE_ASN1_FUNCTIONS(RSA_OAEP_PARAMS) + # ifndef OPENSSL_NO_FP_API int RSA_print_fp(FILE *fp, const RSA *r, int offset); # endif @@ -414,6 +446,15 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen, int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen, const unsigned char *f, int fl, int rsa_len, const unsigned char *p, int pl); +int RSA_padding_add_PKCS1_OAEP_mgf1(unsigned char *to, int tlen, + const unsigned char *from, int flen, + const unsigned char *param, int plen, + const EVP_MD *md, const EVP_MD *mgf1md); +int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen, + const unsigned char *from, int flen, + int num, const unsigned char *param, + int plen, const EVP_MD *md, + const EVP_MD *mgf1md); int RSA_padding_add_SSLv23(unsigned char *to, int tlen, const unsigned char *f, int fl); int RSA_padding_check_SSLv23(unsigned char *to, int tlen, @@ -494,8 +535,10 @@ void ERR_load_RSA_strings(void); # define RSA_F_PKEY_RSA_SIGN 142 # define RSA_F_PKEY_RSA_VERIFY 154 # define RSA_F_PKEY_RSA_VERIFYRECOVER 141 +# define RSA_F_RSA_ALGOR_TO_MD 157 # define RSA_F_RSA_BUILTIN_KEYGEN 129 # define RSA_F_RSA_CHECK_KEY 123 +# define RSA_F_RSA_CMS_DECRYPT 158 # define RSA_F_RSA_EAY_PRIVATE_DECRYPT 101 # define RSA_F_RSA_EAY_PRIVATE_ENCRYPT 102 # define RSA_F_RSA_EAY_PUBLIC_DECRYPT 103 @@ -504,6 +547,7 @@ void ERR_load_RSA_strings(void); # define RSA_F_RSA_GENERATE_KEY_EX 155 # define RSA_F_RSA_ITEM_VERIFY 156 # define RSA_F_RSA_MEMORY_LOCK 130 +# define RSA_F_RSA_MGF1_TO_MD 159 # define RSA_F_RSA_NEW_METHOD 106 # define RSA_F_RSA_NULL 124 # define RSA_F_RSA_NULL_MOD_EXP 131 @@ -513,6 +557,7 @@ void ERR_load_RSA_strings(void); # define RSA_F_RSA_NULL_PUBLIC_ENCRYPT 135 # define RSA_F_RSA_PADDING_ADD_NONE 107 # define RSA_F_RSA_PADDING_ADD_PKCS1_OAEP 121 +# define RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1 160 # define RSA_F_RSA_PADDING_ADD_PKCS1_PSS 125 # define RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1 148 # define RSA_F_RSA_PADDING_ADD_PKCS1_TYPE_1 108 @@ -521,6 +566,7 @@ void ERR_load_RSA_strings(void); # define RSA_F_RSA_PADDING_ADD_X931 127 # define RSA_F_RSA_PADDING_CHECK_NONE 111 # define RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP 122 +# define RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP_MGF1 161 # define RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_1 112 # define RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_2 113 # define RSA_F_RSA_PADDING_CHECK_SSLV23 114 @@ -531,6 +577,7 @@ void ERR_load_RSA_strings(void); # define RSA_F_RSA_PRIVATE_ENCRYPT 151 # define RSA_F_RSA_PRIV_DECODE 137 # define RSA_F_RSA_PRIV_ENCODE 138 +# define RSA_F_RSA_PSS_TO_CTX 162 # define RSA_F_RSA_PUBLIC_DECRYPT 152 # define RSA_F_RSA_PUBLIC_ENCRYPT 153 # define RSA_F_RSA_PUB_DECODE 139 @@ -556,17 +603,21 @@ void ERR_load_RSA_strings(void); # define RSA_R_DATA_TOO_LARGE_FOR_MODULUS 132 # define RSA_R_DATA_TOO_SMALL 111 # define RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE 122 +# define RSA_R_DIGEST_DOES_NOT_MATCH 166 # define RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY 112 # define RSA_R_DMP1_NOT_CONGRUENT_TO_D 124 # define RSA_R_DMQ1_NOT_CONGRUENT_TO_D 125 # define RSA_R_D_E_NOT_CONGRUENT_TO_1 123 # define RSA_R_FIRST_OCTET_INVALID 133 # define RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE 144 +# define RSA_R_INVALID_DIGEST 160 # define RSA_R_INVALID_DIGEST_LENGTH 143 # define RSA_R_INVALID_HEADER 137 # define RSA_R_INVALID_KEYBITS 145 +# define RSA_R_INVALID_LABEL 161 # define RSA_R_INVALID_MESSAGE_LENGTH 131 # define RSA_R_INVALID_MGF1_MD 156 +# define RSA_R_INVALID_OAEP_PARAMETERS 162 # define RSA_R_INVALID_PADDING 138 # define RSA_R_INVALID_PADDING_MODE 141 # define RSA_R_INVALID_PSS_PARAMETERS 149 @@ -595,9 +646,12 @@ void ERR_load_RSA_strings(void); # define RSA_R_SSLV3_ROLLBACK_ATTACK 115 # define RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD 116 # define RSA_R_UNKNOWN_ALGORITHM_TYPE 117 +# define RSA_R_UNKNOWN_DIGEST 163 # define RSA_R_UNKNOWN_MASK_DIGEST 151 # define RSA_R_UNKNOWN_PADDING_TYPE 118 # define RSA_R_UNKNOWN_PSS_DIGEST 152 +# define RSA_R_UNSUPPORTED_ENCRYPTION_TYPE 164 +# define RSA_R_UNSUPPORTED_LABEL_SOURCE 165 # define RSA_R_UNSUPPORTED_MASK_ALGORITHM 153 # define RSA_R_UNSUPPORTED_MASK_PARAMETER 154 # define RSA_R_UNSUPPORTED_SIGNATURE_TYPE 155 diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c index 93e071de75e2..ca3922e6c298 100644 --- a/crypto/rsa/rsa_ameth.c +++ b/crypto/rsa/rsa_ameth.c @@ -68,6 +68,11 @@ #endif #include "asn1_locl.h" +static int rsa_cms_sign(CMS_SignerInfo *si); +static int rsa_cms_verify(CMS_SignerInfo *si); +static int rsa_cms_decrypt(CMS_RecipientInfo *ri); +static int rsa_cms_encrypt(CMS_RecipientInfo *ri); + static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) { unsigned char *penc = NULL; @@ -258,6 +263,23 @@ static int rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent, return do_rsa_print(bp, pkey->pkey.rsa, indent, 1); } +/* Given an MGF1 Algorithm ID decode to an Algorithm Identifier */ +static X509_ALGOR *rsa_mgf1_decode(X509_ALGOR *alg) +{ + const unsigned char *p; + int plen; + if (alg == NULL) + return NULL; + if (OBJ_obj2nid(alg->algorithm) != NID_mgf1) + return NULL; + if (alg->parameter->type != V_ASN1_SEQUENCE) + return NULL; + + p = alg->parameter->value.sequence->data; + plen = alg->parameter->value.sequence->length; + return d2i_X509_ALGOR(NULL, &p, plen); +} + static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg, X509_ALGOR **pmaskHash) { @@ -276,15 +298,7 @@ static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg, if (!pss) return NULL; - if (pss->maskGenAlgorithm) { - ASN1_TYPE *param = pss->maskGenAlgorithm->parameter; - if (OBJ_obj2nid(pss->maskGenAlgorithm->algorithm) == NID_mgf1 - && param->type == V_ASN1_SEQUENCE) { - p = param->value.sequence->data; - plen = param->value.sequence->length; - *pmaskHash = d2i_X509_ALGOR(NULL, &p, plen); - } - } + *pmaskHash = rsa_mgf1_decode(pss->maskGenAlgorithm); return pss; } @@ -401,17 +415,25 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2) #ifndef OPENSSL_NO_CMS case ASN1_PKEY_CTRL_CMS_SIGN: if (arg1 == 0) - CMS_SignerInfo_get0_algs(arg2, NULL, NULL, NULL, &alg); + return rsa_cms_sign(arg2); + else if (arg1 == 1) + return rsa_cms_verify(arg2); break; case ASN1_PKEY_CTRL_CMS_ENVELOPE: if (arg1 == 0) - CMS_RecipientInfo_ktri_get0_algs(arg2, NULL, NULL, &alg); + return rsa_cms_encrypt(arg2); + else if (arg1 == 1) + return rsa_cms_decrypt(arg2); break; + + case ASN1_PKEY_CTRL_CMS_RI_TYPE: + *(int *)arg2 = CMS_RECIPINFO_TRANS; + return 1; #endif case ASN1_PKEY_CTRL_DEFAULT_MD_NID: - *(int *)arg2 = NID_sha1; + *(int *)arg2 = NID_sha256; return 1; default: @@ -426,59 +448,166 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2) } +/* allocate and set algorithm ID from EVP_MD, default SHA1 */ +static int rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md) +{ + if (EVP_MD_type(md) == NID_sha1) + return 1; + *palg = X509_ALGOR_new(); + if (!*palg) + return 0; + X509_ALGOR_set_md(*palg, md); + return 1; +} + +/* Allocate and set MGF1 algorithm ID from EVP_MD */ +static int rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md) +{ + X509_ALGOR *algtmp = NULL; + ASN1_STRING *stmp = NULL; + *palg = NULL; + if (EVP_MD_type(mgf1md) == NID_sha1) + return 1; + /* need to embed algorithm ID inside another */ + if (!rsa_md_to_algor(&algtmp, mgf1md)) + goto err; + if (!ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp)) + goto err; + *palg = X509_ALGOR_new(); + if (!*palg) + goto err; + X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp); + stmp = NULL; + err: + if (stmp) + ASN1_STRING_free(stmp); + if (algtmp) + X509_ALGOR_free(algtmp); + if (*palg) + return 1; + return 0; +} + +/* convert algorithm ID to EVP_MD, default SHA1 */ +static const EVP_MD *rsa_algor_to_md(X509_ALGOR *alg) +{ + const EVP_MD *md; + if (!alg) + return EVP_sha1(); + md = EVP_get_digestbyobj(alg->algorithm); + if (md == NULL) + RSAerr(RSA_F_RSA_ALGOR_TO_MD, RSA_R_UNKNOWN_DIGEST); + return md; +} + +/* convert MGF1 algorithm ID to EVP_MD, default SHA1 */ +static const EVP_MD *rsa_mgf1_to_md(X509_ALGOR *alg, X509_ALGOR *maskHash) +{ + const EVP_MD *md; + if (!alg) + return EVP_sha1(); + /* Check mask and lookup mask hash algorithm */ + if (OBJ_obj2nid(alg->algorithm) != NID_mgf1) { + RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_ALGORITHM); + return NULL; + } + if (!maskHash) { + RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_PARAMETER); + return NULL; + } + md = EVP_get_digestbyobj(maskHash->algorithm); + if (md == NULL) { + RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNKNOWN_MASK_DIGEST); + return NULL; + } + return md; +} + /* - * Customised RSA item verification routine. This is called when a signature - * is encountered requiring special handling. We currently only handle PSS. + * Convert EVP_PKEY_CTX is PSS mode into corresponding algorithm parameter, + * suitable for setting an AlgorithmIdentifier. */ -static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, - X509_ALGOR *sigalg, ASN1_BIT_STRING *sig, - EVP_PKEY *pkey) +static ASN1_STRING *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx) +{ + const EVP_MD *sigmd, *mgf1md; + RSA_PSS_PARAMS *pss = NULL; + ASN1_STRING *os = NULL; + EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx); + int saltlen, rv = 0; + if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0) + goto err; + if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0) + goto err; + if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen)) + goto err; + if (saltlen == -1) + saltlen = EVP_MD_size(sigmd); + else if (saltlen == -2) { + saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2; + if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0) + saltlen--; + } + pss = RSA_PSS_PARAMS_new(); + if (!pss) + goto err; + if (saltlen != 20) { + pss->saltLength = ASN1_INTEGER_new(); + if (!pss->saltLength) + goto err; + if (!ASN1_INTEGER_set(pss->saltLength, saltlen)) + goto err; + } + if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd)) + goto err; + if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md)) + goto err; + /* Finally create string with pss parameter encoding. */ + if (!ASN1_item_pack(pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS), &os)) + goto err; + rv = 1; + err: + if (pss) + RSA_PSS_PARAMS_free(pss); + if (rv) + return os; + if (os) + ASN1_STRING_free(os); + return NULL; +} + +/* + * From PSS AlgorithmIdentifier set public key parameters. If pkey isn't NULL + * then the EVP_MD_CTX is setup and initalised. If it is NULL parameters are + * passed to pkctx instead. + */ + +static int rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx, + X509_ALGOR *sigalg, EVP_PKEY *pkey) { int rv = -1; int saltlen; const EVP_MD *mgf1md = NULL, *md = NULL; RSA_PSS_PARAMS *pss; X509_ALGOR *maskHash; - EVP_PKEY_CTX *pkctx; /* Sanity check: make sure it is PSS */ if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNSUPPORTED_SIGNATURE_TYPE); + RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_UNSUPPORTED_SIGNATURE_TYPE); return -1; } /* Decode PSS parameters */ pss = rsa_pss_decode(sigalg, &maskHash); if (pss == NULL) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_INVALID_PSS_PARAMETERS); + RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_PSS_PARAMETERS); goto err; } - /* Check mask and lookup mask hash algorithm */ - if (pss->maskGenAlgorithm) { - if (OBJ_obj2nid(pss->maskGenAlgorithm->algorithm) != NID_mgf1) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNSUPPORTED_MASK_ALGORITHM); - goto err; - } - if (!maskHash) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNSUPPORTED_MASK_PARAMETER); - goto err; - } - mgf1md = EVP_get_digestbyobj(maskHash->algorithm); - if (mgf1md == NULL) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNKNOWN_MASK_DIGEST); - goto err; - } - } else - mgf1md = EVP_sha1(); - - if (pss->hashAlgorithm) { - md = EVP_get_digestbyobj(pss->hashAlgorithm->algorithm); - if (md == NULL) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNKNOWN_PSS_DIGEST); - goto err; - } - } else - md = EVP_sha1(); + mgf1md = rsa_mgf1_to_md(pss->maskGenAlgorithm, maskHash); + if (!mgf1md) + goto err; + md = rsa_algor_to_md(pss->hashAlgorithm); + if (!md) + goto err; if (pss->saltLength) { saltlen = ASN1_INTEGER_get(pss->saltLength); @@ -488,7 +617,7 @@ static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, * routines will trap other invalid values anyway. */ if (saltlen < 0) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_INVALID_SALT_LENGTH); + RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_SALT_LENGTH); goto err; } } else @@ -499,14 +628,24 @@ static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, * PKCS#1 says we should reject any other value anyway. */ if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) { - RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_INVALID_TRAILER); + RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_TRAILER); goto err; } /* We have all parameters now set up context */ - if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey)) - goto err; + if (pkey) { + if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey)) + goto err; + } else { + const EVP_MD *checkmd; + if (EVP_PKEY_CTX_get_signature_md(pkctx, &checkmd) <= 0) + goto err; + if (EVP_MD_type(md) != EVP_MD_type(checkmd)) { + RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_DIGEST_DOES_NOT_MATCH); + goto err; + } + } if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0) goto err; @@ -517,7 +656,7 @@ static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0) goto err; /* Carry on */ - rv = 2; + rv = 1; err: RSA_PSS_PARAMS_free(pss); @@ -526,6 +665,71 @@ static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, return rv; } +static int rsa_cms_verify(CMS_SignerInfo *si) +{ + int nid, nid2; + X509_ALGOR *alg; + EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si); + CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg); + nid = OBJ_obj2nid(alg->algorithm); + if (nid == NID_rsaEncryption) + return 1; + if (nid == NID_rsassaPss) + return rsa_pss_to_ctx(NULL, pkctx, alg, NULL); + /* Workaround for some implementation that use a signature OID */ + if (OBJ_find_sigid_algs(nid, NULL, &nid2)) { + if (nid2 == NID_rsaEncryption) + return 1; + } + return 0; +} + +/* + * Customised RSA item verification routine. This is called when a signature + * is encountered requiring special handling. We currently only handle PSS. + */ + +static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, + X509_ALGOR *sigalg, ASN1_BIT_STRING *sig, + EVP_PKEY *pkey) +{ + /* Sanity check: make sure it is PSS */ + if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) { + RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNSUPPORTED_SIGNATURE_TYPE); + return -1; + } + if (rsa_pss_to_ctx(ctx, NULL, sigalg, pkey) > 0) { + /* Carry on */ + return 2; + } + return -1; +} + +static int rsa_cms_sign(CMS_SignerInfo *si) +{ + int pad_mode = RSA_PKCS1_PADDING; + X509_ALGOR *alg; + EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si); + ASN1_STRING *os = NULL; + CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg); + if (pkctx) { + if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0) + return 0; + } + if (pad_mode == RSA_PKCS1_PADDING) { + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0); + return 1; + } + /* We don't support it */ + if (pad_mode != RSA_PKCS1_PSS_PADDING) + return 0; + os = rsa_ctx_to_pss(pkctx); + if (!os) + return 0; + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsassaPss), V_ASN1_SEQUENCE, os); + return 1; +} + static int rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *sig) @@ -537,78 +741,184 @@ static int rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn, if (pad_mode == RSA_PKCS1_PADDING) return 2; if (pad_mode == RSA_PKCS1_PSS_PADDING) { - const EVP_MD *sigmd, *mgf1md; - RSA_PSS_PARAMS *pss = NULL; - X509_ALGOR *mgf1alg = NULL; - ASN1_STRING *os1 = NULL, *os2 = NULL; - EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx); - int saltlen, rv = 0; - sigmd = EVP_MD_CTX_md(ctx); - if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0) - goto err; - if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen)) - goto err; - if (saltlen == -1) - saltlen = EVP_MD_size(sigmd); - else if (saltlen == -2) { - saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2; - if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0) - saltlen--; - } - pss = RSA_PSS_PARAMS_new(); - if (!pss) - goto err; - if (saltlen != 20) { - pss->saltLength = ASN1_INTEGER_new(); - if (!pss->saltLength) - goto err; - if (!ASN1_INTEGER_set(pss->saltLength, saltlen)) - goto err; - } - if (EVP_MD_type(sigmd) != NID_sha1) { - pss->hashAlgorithm = X509_ALGOR_new(); - if (!pss->hashAlgorithm) - goto err; - X509_ALGOR_set_md(pss->hashAlgorithm, sigmd); - } - if (EVP_MD_type(mgf1md) != NID_sha1) { - ASN1_STRING *stmp = NULL; - /* need to embed algorithm ID inside another */ - mgf1alg = X509_ALGOR_new(); - X509_ALGOR_set_md(mgf1alg, mgf1md); - if (!ASN1_item_pack(mgf1alg, ASN1_ITEM_rptr(X509_ALGOR), &stmp)) - goto err; - pss->maskGenAlgorithm = X509_ALGOR_new(); - if (!pss->maskGenAlgorithm) - goto err; - X509_ALGOR_set0(pss->maskGenAlgorithm, - OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp); - } - /* Finally create string with pss parameter encoding. */ - if (!ASN1_item_pack(pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS), &os1)) - goto err; + ASN1_STRING *os1 = NULL; + os1 = rsa_ctx_to_pss(pkctx); + if (!os1) + return 0; + /* Duplicate parameters if we have to */ if (alg2) { - os2 = ASN1_STRING_dup(os1); - if (!os2) - goto err; + ASN1_STRING *os2 = ASN1_STRING_dup(os1); + if (!os2) { + ASN1_STRING_free(os1); + return 0; + } X509_ALGOR_set0(alg2, OBJ_nid2obj(NID_rsassaPss), V_ASN1_SEQUENCE, os2); } X509_ALGOR_set0(alg1, OBJ_nid2obj(NID_rsassaPss), V_ASN1_SEQUENCE, os1); - os1 = os2 = NULL; - rv = 3; + return 3; + } + return 2; +} + +static RSA_OAEP_PARAMS *rsa_oaep_decode(const X509_ALGOR *alg, + X509_ALGOR **pmaskHash) +{ + const unsigned char *p; + int plen; + RSA_OAEP_PARAMS *pss; + + *pmaskHash = NULL; + + if (!alg->parameter || alg->parameter->type != V_ASN1_SEQUENCE) + return NULL; + p = alg->parameter->value.sequence->data; + plen = alg->parameter->value.sequence->length; + pss = d2i_RSA_OAEP_PARAMS(NULL, &p, plen); + + if (!pss) + return NULL; + + *pmaskHash = rsa_mgf1_decode(pss->maskGenFunc); + + return pss; +} + +static int rsa_cms_decrypt(CMS_RecipientInfo *ri) +{ + EVP_PKEY_CTX *pkctx; + X509_ALGOR *cmsalg; + int nid; + int rv = -1; + unsigned char *label = NULL; + int labellen = 0; + const EVP_MD *mgf1md = NULL, *md = NULL; + RSA_OAEP_PARAMS *oaep; + X509_ALGOR *maskHash; + pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri); + if (!pkctx) + return 0; + if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg)) + return -1; + nid = OBJ_obj2nid(cmsalg->algorithm); + if (nid == NID_rsaEncryption) + return 1; + if (nid != NID_rsaesOaep) { + RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_ENCRYPTION_TYPE); + return -1; + } + /* Decode OAEP parameters */ + oaep = rsa_oaep_decode(cmsalg, &maskHash); + + if (oaep == NULL) { + RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_OAEP_PARAMETERS); + goto err; + } + + mgf1md = rsa_mgf1_to_md(oaep->maskGenFunc, maskHash); + if (!mgf1md) + goto err; + md = rsa_algor_to_md(oaep->hashFunc); + if (!md) + goto err; + + if (oaep->pSourceFunc) { + X509_ALGOR *plab = oaep->pSourceFunc; + if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) { + RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_LABEL_SOURCE); + goto err; + } + if (plab->parameter->type != V_ASN1_OCTET_STRING) { + RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_LABEL); + goto err; + } + + label = plab->parameter->value.octet_string->data; + /* Stop label being freed when OAEP parameters are freed */ + plab->parameter->value.octet_string->data = NULL; + labellen = plab->parameter->value.octet_string->length; + } + + if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0) + goto err; + if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0) + goto err; + if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0) + goto err; + if (EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0) + goto err; + /* Carry on */ + rv = 1; + err: - if (mgf1alg) - X509_ALGOR_free(mgf1alg); - if (pss) - RSA_PSS_PARAMS_free(pss); - if (os1) - ASN1_STRING_free(os1); - return rv; + RSA_OAEP_PARAMS_free(oaep); + if (maskHash) + X509_ALGOR_free(maskHash); + return rv; +} +static int rsa_cms_encrypt(CMS_RecipientInfo *ri) +{ + const EVP_MD *md, *mgf1md; + RSA_OAEP_PARAMS *oaep = NULL; + ASN1_STRING *os = NULL; + X509_ALGOR *alg; + EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri); + int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen; + unsigned char *label; + CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg); + if (pkctx) { + if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0) + return 0; } - return 2; + if (pad_mode == RSA_PKCS1_PADDING) { + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0); + return 1; + } + /* Not supported */ + if (pad_mode != RSA_PKCS1_OAEP_PADDING) + return 0; + if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0) + goto err; + if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0) + goto err; + labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label); + if (labellen < 0) + goto err; + oaep = RSA_OAEP_PARAMS_new(); + if (!oaep) + goto err; + if (!rsa_md_to_algor(&oaep->hashFunc, md)) + goto err; + if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md)) + goto err; + if (labellen > 0) { + ASN1_OCTET_STRING *los = ASN1_OCTET_STRING_new(); + oaep->pSourceFunc = X509_ALGOR_new(); + if (!oaep->pSourceFunc) + goto err; + if (!los) + goto err; + if (!ASN1_OCTET_STRING_set(los, label, labellen)) { + ASN1_OCTET_STRING_free(los); + goto err; + } + X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified), + V_ASN1_OCTET_STRING, los); + } + /* create string with pss parameter encoding. */ + if (!ASN1_item_pack(oaep, ASN1_ITEM_rptr(RSA_OAEP_PARAMS), &os)) + goto err; + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os); + os = NULL; + rv = 1; + err: + if (oaep) + RSA_OAEP_PARAMS_free(oaep); + if (os) + ASN1_STRING_free(os); + return rv; } const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = { diff --git a/crypto/rsa/rsa_asn1.c b/crypto/rsa/rsa_asn1.c index 3d82c1d0c715..aff8b583fa72 100644 --- a/crypto/rsa/rsa_asn1.c +++ b/crypto/rsa/rsa_asn1.c @@ -108,6 +108,14 @@ ASN1_SEQUENCE(RSA_PSS_PARAMS) = { IMPLEMENT_ASN1_FUNCTIONS(RSA_PSS_PARAMS) +ASN1_SEQUENCE(RSA_OAEP_PARAMS) = { + ASN1_EXP_OPT(RSA_OAEP_PARAMS, hashFunc, X509_ALGOR, 0), + ASN1_EXP_OPT(RSA_OAEP_PARAMS, maskGenFunc, X509_ALGOR, 1), + ASN1_EXP_OPT(RSA_OAEP_PARAMS, pSourceFunc, X509_ALGOR, 2), +} ASN1_SEQUENCE_END(RSA_OAEP_PARAMS) + +IMPLEMENT_ASN1_FUNCTIONS(RSA_OAEP_PARAMS) + IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(RSA, RSAPrivateKey, RSAPrivateKey) IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(RSA, RSAPublicKey, RSAPublicKey) diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c index 25b3fa743d41..0bab05efcfca 100644 --- a/crypto/rsa/rsa_err.c +++ b/crypto/rsa/rsa_err.c @@ -1,6 +1,6 @@ /* crypto/rsa/rsa_err.c */ /* ==================================================================== - * Copyright (c) 1999-2011 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2014 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -80,8 +80,10 @@ static ERR_STRING_DATA RSA_str_functs[] = { {ERR_FUNC(RSA_F_PKEY_RSA_SIGN), "PKEY_RSA_SIGN"}, {ERR_FUNC(RSA_F_PKEY_RSA_VERIFY), "PKEY_RSA_VERIFY"}, {ERR_FUNC(RSA_F_PKEY_RSA_VERIFYRECOVER), "PKEY_RSA_VERIFYRECOVER"}, + {ERR_FUNC(RSA_F_RSA_ALGOR_TO_MD), "RSA_ALGOR_TO_MD"}, {ERR_FUNC(RSA_F_RSA_BUILTIN_KEYGEN), "RSA_BUILTIN_KEYGEN"}, {ERR_FUNC(RSA_F_RSA_CHECK_KEY), "RSA_check_key"}, + {ERR_FUNC(RSA_F_RSA_CMS_DECRYPT), "RSA_CMS_DECRYPT"}, {ERR_FUNC(RSA_F_RSA_EAY_PRIVATE_DECRYPT), "RSA_EAY_PRIVATE_DECRYPT"}, {ERR_FUNC(RSA_F_RSA_EAY_PRIVATE_ENCRYPT), "RSA_EAY_PRIVATE_ENCRYPT"}, {ERR_FUNC(RSA_F_RSA_EAY_PUBLIC_DECRYPT), "RSA_EAY_PUBLIC_DECRYPT"}, @@ -90,6 +92,7 @@ static ERR_STRING_DATA RSA_str_functs[] = { {ERR_FUNC(RSA_F_RSA_GENERATE_KEY_EX), "RSA_generate_key_ex"}, {ERR_FUNC(RSA_F_RSA_ITEM_VERIFY), "RSA_ITEM_VERIFY"}, {ERR_FUNC(RSA_F_RSA_MEMORY_LOCK), "RSA_memory_lock"}, + {ERR_FUNC(RSA_F_RSA_MGF1_TO_MD), "RSA_MGF1_TO_MD"}, {ERR_FUNC(RSA_F_RSA_NEW_METHOD), "RSA_new_method"}, {ERR_FUNC(RSA_F_RSA_NULL), "RSA_NULL"}, {ERR_FUNC(RSA_F_RSA_NULL_MOD_EXP), "RSA_NULL_MOD_EXP"}, @@ -100,6 +103,8 @@ static ERR_STRING_DATA RSA_str_functs[] = { {ERR_FUNC(RSA_F_RSA_PADDING_ADD_NONE), "RSA_padding_add_none"}, {ERR_FUNC(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP), "RSA_padding_add_PKCS1_OAEP"}, + {ERR_FUNC(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1), + "RSA_padding_add_PKCS1_OAEP_mgf1"}, {ERR_FUNC(RSA_F_RSA_PADDING_ADD_PKCS1_PSS), "RSA_padding_add_PKCS1_PSS"}, {ERR_FUNC(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1), "RSA_padding_add_PKCS1_PSS_mgf1"}, @@ -112,6 +117,8 @@ static ERR_STRING_DATA RSA_str_functs[] = { {ERR_FUNC(RSA_F_RSA_PADDING_CHECK_NONE), "RSA_padding_check_none"}, {ERR_FUNC(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP), "RSA_padding_check_PKCS1_OAEP"}, + {ERR_FUNC(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP_MGF1), + "RSA_padding_check_PKCS1_OAEP_mgf1"}, {ERR_FUNC(RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_1), "RSA_padding_check_PKCS1_type_1"}, {ERR_FUNC(RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_2), @@ -124,6 +131,7 @@ static ERR_STRING_DATA RSA_str_functs[] = { {ERR_FUNC(RSA_F_RSA_PRIVATE_ENCRYPT), "RSA_private_encrypt"}, {ERR_FUNC(RSA_F_RSA_PRIV_DECODE), "RSA_PRIV_DECODE"}, {ERR_FUNC(RSA_F_RSA_PRIV_ENCODE), "RSA_PRIV_ENCODE"}, + {ERR_FUNC(RSA_F_RSA_PSS_TO_CTX), "RSA_PSS_TO_CTX"}, {ERR_FUNC(RSA_F_RSA_PUBLIC_DECRYPT), "RSA_public_decrypt"}, {ERR_FUNC(RSA_F_RSA_PUBLIC_ENCRYPT), "RSA_public_encrypt"}, {ERR_FUNC(RSA_F_RSA_PUB_DECODE), "RSA_PUB_DECODE"}, @@ -157,6 +165,7 @@ static ERR_STRING_DATA RSA_str_reasons[] = { {ERR_REASON(RSA_R_DATA_TOO_SMALL), "data too small"}, {ERR_REASON(RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE), "data too small for key size"}, + {ERR_REASON(RSA_R_DIGEST_DOES_NOT_MATCH), "digest does not match"}, {ERR_REASON(RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY), "digest too big for rsa key"}, {ERR_REASON(RSA_R_DMP1_NOT_CONGRUENT_TO_D), "dmp1 not congruent to d"}, @@ -165,11 +174,14 @@ static ERR_STRING_DATA RSA_str_reasons[] = { {ERR_REASON(RSA_R_FIRST_OCTET_INVALID), "first octet invalid"}, {ERR_REASON(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE), "illegal or unsupported padding mode"}, + {ERR_REASON(RSA_R_INVALID_DIGEST), "invalid digest"}, {ERR_REASON(RSA_R_INVALID_DIGEST_LENGTH), "invalid digest length"}, {ERR_REASON(RSA_R_INVALID_HEADER), "invalid header"}, {ERR_REASON(RSA_R_INVALID_KEYBITS), "invalid keybits"}, + {ERR_REASON(RSA_R_INVALID_LABEL), "invalid label"}, {ERR_REASON(RSA_R_INVALID_MESSAGE_LENGTH), "invalid message length"}, {ERR_REASON(RSA_R_INVALID_MGF1_MD), "invalid mgf1 md"}, + {ERR_REASON(RSA_R_INVALID_OAEP_PARAMETERS), "invalid oaep parameters"}, {ERR_REASON(RSA_R_INVALID_PADDING), "invalid padding"}, {ERR_REASON(RSA_R_INVALID_PADDING_MODE), "invalid padding mode"}, {ERR_REASON(RSA_R_INVALID_PSS_PARAMETERS), "invalid pss parameters"}, @@ -203,9 +215,13 @@ static ERR_STRING_DATA RSA_str_reasons[] = { {ERR_REASON(RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD), "the asn1 object identifier is not known for this md"}, {ERR_REASON(RSA_R_UNKNOWN_ALGORITHM_TYPE), "unknown algorithm type"}, + {ERR_REASON(RSA_R_UNKNOWN_DIGEST), "unknown digest"}, {ERR_REASON(RSA_R_UNKNOWN_MASK_DIGEST), "unknown mask digest"}, {ERR_REASON(RSA_R_UNKNOWN_PADDING_TYPE), "unknown padding type"}, {ERR_REASON(RSA_R_UNKNOWN_PSS_DIGEST), "unknown pss digest"}, + {ERR_REASON(RSA_R_UNSUPPORTED_ENCRYPTION_TYPE), + "unsupported encryption type"}, + {ERR_REASON(RSA_R_UNSUPPORTED_LABEL_SOURCE), "unsupported label source"}, {ERR_REASON(RSA_R_UNSUPPORTED_MASK_ALGORITHM), "unsupported mask algorithm"}, {ERR_REASON(RSA_R_UNSUPPORTED_MASK_PARAMETER), diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c index 499835f8143d..9c2a943cf778 100644 --- a/crypto/rsa/rsa_oaep.c +++ b/crypto/rsa/rsa_oaep.c @@ -28,39 +28,53 @@ # include # include -static int MGF1(unsigned char *mask, long len, - const unsigned char *seed, long seedlen); - int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen, const unsigned char *from, int flen, const unsigned char *param, int plen) +{ + return RSA_padding_add_PKCS1_OAEP_mgf1(to, tlen, from, flen, + param, plen, NULL, NULL); +} + +int RSA_padding_add_PKCS1_OAEP_mgf1(unsigned char *to, int tlen, + const unsigned char *from, int flen, + const unsigned char *param, int plen, + const EVP_MD *md, const EVP_MD *mgf1md) { int i, emlen = tlen - 1; unsigned char *db, *seed; - unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH]; + unsigned char *dbmask, seedmask[EVP_MAX_MD_SIZE]; + int mdlen; + + if (md == NULL) + md = EVP_sha1(); + if (mgf1md == NULL) + mgf1md = md; - if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1) { - RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, + mdlen = EVP_MD_size(md); + + if (flen > emlen - 2 * mdlen - 1) { + RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE); return 0; } - if (emlen < 2 * SHA_DIGEST_LENGTH + 1) { - RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_KEY_SIZE_TOO_SMALL); + if (emlen < 2 * mdlen + 1) { + RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, + RSA_R_KEY_SIZE_TOO_SMALL); return 0; } to[0] = 0; seed = to + 1; - db = to + SHA_DIGEST_LENGTH + 1; + db = to + mdlen + 1; - if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL)) + if (!EVP_Digest((void *)param, plen, db, NULL, md, NULL)) return 0; - memset(db + SHA_DIGEST_LENGTH, 0, - emlen - flen - 2 * SHA_DIGEST_LENGTH - 1); - db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01; - memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, (unsigned int)flen); - if (RAND_bytes(seed, SHA_DIGEST_LENGTH) <= 0) + memset(db + mdlen, 0, emlen - flen - 2 * mdlen - 1); + db[emlen - flen - mdlen - 1] = 0x01; + memcpy(db + emlen - flen - mdlen, from, (unsigned int)flen); + if (RAND_bytes(seed, mdlen) <= 0) return 0; # ifdef PKCS_TESTVECT memcpy(seed, @@ -68,20 +82,20 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen, 20); # endif - dbmask = OPENSSL_malloc(emlen - SHA_DIGEST_LENGTH); + dbmask = OPENSSL_malloc(emlen - mdlen); if (dbmask == NULL) { - RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE); + RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, ERR_R_MALLOC_FAILURE); return 0; } - if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH) < 0) + if (PKCS1_MGF1(dbmask, emlen - mdlen, seed, mdlen, mgf1md) < 0) return 0; - for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++) + for (i = 0; i < emlen - mdlen; i++) db[i] ^= dbmask[i]; - if (MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH) < 0) + if (PKCS1_MGF1(seedmask, mdlen, db, emlen - mdlen, mgf1md) < 0) return 0; - for (i = 0; i < SHA_DIGEST_LENGTH; i++) + for (i = 0; i < mdlen; i++) seed[i] ^= seedmask[i]; OPENSSL_free(dbmask); @@ -91,6 +105,16 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen, int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen, const unsigned char *from, int flen, int num, const unsigned char *param, int plen) +{ + return RSA_padding_check_PKCS1_OAEP_mgf1(to, tlen, from, flen, num, + param, plen, NULL, NULL); +} + +int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen, + const unsigned char *from, int flen, + int num, const unsigned char *param, + int plen, const EVP_MD *md, + const EVP_MD *mgf1md) { int i, dblen, mlen = -1, one_index = 0, msg_index; unsigned int good, found_one_byte; @@ -101,26 +125,33 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen, */ unsigned char *db = NULL, *em = NULL, seed[EVP_MAX_MD_SIZE], phash[EVP_MAX_MD_SIZE]; + int mdlen; + + if (md == NULL) + md = EVP_sha1(); + if (mgf1md == NULL) + mgf1md = md; + + mdlen = EVP_MD_size(md); if (tlen <= 0 || flen <= 0) return -1; - /* * |num| is the length of the modulus; |flen| is the length of the * encoded message. Therefore, for any |from| that was obtained by * decrypting a ciphertext, we must have |flen| <= |num|. Similarly, - * num < 2 * SHA_DIGEST_LENGTH + 2 must hold for the modulus - * irrespective of the ciphertext, see PKCS #1 v2.2, section 7.1.2. + * num < 2 * mdlen + 2 must hold for the modulus irrespective of + * the ciphertext, see PKCS #1 v2.2, section 7.1.2. * This does not leak any side-channel information. */ - if (num < flen || num < 2 * SHA_DIGEST_LENGTH + 2) + if (num < flen || num < 2 * mdlen + 2) goto decoding_err; - dblen = num - SHA_DIGEST_LENGTH - 1; + dblen = num - mdlen - 1; db = OPENSSL_malloc(dblen); em = OPENSSL_malloc(num); if (db == NULL || em == NULL) { - RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, ERR_R_MALLOC_FAILURE); + RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP_MGF1, ERR_R_MALLOC_FAILURE); goto cleanup; } @@ -143,26 +174,25 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen, good = constant_time_is_zero(em[0]); maskedseed = em + 1; - maskeddb = em + 1 + SHA_DIGEST_LENGTH; + maskeddb = em + 1 + mdlen; - if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen)) + if (PKCS1_MGF1(seed, mdlen, maskeddb, dblen, mgf1md)) goto cleanup; - for (i = 0; i < SHA_DIGEST_LENGTH; i++) + for (i = 0; i < mdlen; i++) seed[i] ^= maskedseed[i]; - if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH)) + if (PKCS1_MGF1(db, dblen, seed, mdlen, mgf1md)) goto cleanup; for (i = 0; i < dblen; i++) db[i] ^= maskeddb[i]; - if (!EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL)) + if (!EVP_Digest((void *)param, plen, phash, NULL, md, NULL)) goto cleanup; - good &= - constant_time_is_zero(CRYPTO_memcmp(db, phash, SHA_DIGEST_LENGTH)); + good &= constant_time_is_zero(CRYPTO_memcmp(db, phash, mdlen)); found_one_byte = 0; - for (i = SHA_DIGEST_LENGTH; i < dblen; i++) { + for (i = mdlen; i < dblen; i++) { /* * Padding consists of a number of 0-bytes, followed by a 1. */ @@ -188,7 +218,7 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen, mlen = dblen - msg_index; if (tlen < mlen) { - RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE); + RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP_MGF1, RSA_R_DATA_TOO_LARGE); mlen = -1; } else { memcpy(to, db + msg_index, mlen); @@ -200,7 +230,8 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen, * To avoid chosen ciphertext attacks, the error message should not * reveal which kind of decoding error happened. */ - RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR); + RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP_MGF1, + RSA_R_OAEP_DECODING_ERROR); cleanup: if (db != NULL) OPENSSL_free(db); @@ -249,9 +280,4 @@ int PKCS1_MGF1(unsigned char *mask, long len, return rv; } -static int MGF1(unsigned char *mask, long len, const unsigned char *seed, - long seedlen) -{ - return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1()); -} #endif diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c index 6a7c67cdb86c..203635595f4c 100644 --- a/crypto/rsa/rsa_pmeth.c +++ b/crypto/rsa/rsa_pmeth.c @@ -64,6 +64,7 @@ #include #include #include +#include #ifndef OPENSSL_NO_CMS # include #endif @@ -87,10 +88,13 @@ typedef struct { const EVP_MD *md; /* message digest for MGF1 */ const EVP_MD *mgf1md; - /* PSS/OAEP salt length */ + /* PSS salt length */ int saltlen; /* Temp buffer */ unsigned char *tbuf; + /* OAEP label */ + unsigned char *oaep_label; + size_t oaep_labellen; } RSA_PKEY_CTX; static int pkey_rsa_init(EVP_PKEY_CTX *ctx) @@ -108,6 +112,9 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx) rctx->saltlen = -2; + rctx->oaep_label = NULL; + rctx->oaep_labellen = 0; + ctx->data = rctx; ctx->keygen_info = rctx->gentmp; ctx->keygen_info_count = 2; @@ -130,6 +137,15 @@ static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src) } dctx->pad_mode = sctx->pad_mode; dctx->md = sctx->md; + dctx->mgf1md = sctx->mgf1md; + if (sctx->oaep_label) { + if (dctx->oaep_label) + OPENSSL_free(dctx->oaep_label); + dctx->oaep_label = BUF_memdup(sctx->oaep_label, sctx->oaep_labellen); + if (!dctx->oaep_label) + return 0; + dctx->oaep_labellen = sctx->oaep_labellen; + } return 1; } @@ -151,6 +167,8 @@ static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx) BN_free(rctx->pub_exp); if (rctx->tbuf) OPENSSL_free(rctx->tbuf); + if (rctx->oaep_label) + OPENSSL_free(rctx->oaep_label); OPENSSL_free(rctx); } } @@ -173,10 +191,18 @@ static int pkey_fips_check_ctx(EVP_PKEY_CTX *ctx) rv = 0; if (!(rsa->meth->flags & RSA_FLAG_FIPS_METHOD) && rv) return -1; - if (rctx->md && !(rctx->md->flags & EVP_MD_FLAG_FIPS)) - return rv; - if (rctx->mgf1md && !(rctx->mgf1md->flags & EVP_MD_FLAG_FIPS)) - return rv; + if (rctx->md) { + const EVP_MD *fmd; + fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->md)); + if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS)) + return rv; + } + if (rctx->mgf1md && !(rctx->mgf1md->flags & EVP_MD_FLAG_FIPS)) { + const EVP_MD *fmd; + fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->mgf1md)); + if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS)) + return rv; + } return 1; } #endif @@ -388,8 +414,21 @@ static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, { int ret; RSA_PKEY_CTX *rctx = ctx->data; - ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa, - rctx->pad_mode); + if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) { + int klen = RSA_size(ctx->pkey->pkey.rsa); + if (!setup_tbuf(rctx, ctx)) + return -1; + if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen, + in, inlen, + rctx->oaep_label, + rctx->oaep_labellen, + rctx->md, rctx->mgf1md)) + return -1; + ret = RSA_public_encrypt(klen, rctx->tbuf, out, + ctx->pkey->pkey.rsa, RSA_NO_PADDING); + } else + ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa, + rctx->pad_mode); if (ret < 0) return ret; *outlen = ret; @@ -402,8 +441,26 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, { int ret; RSA_PKEY_CTX *rctx = ctx->data; - ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa, - rctx->pad_mode); + if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) { + int i; + if (!setup_tbuf(rctx, ctx)) + return -1; + ret = RSA_private_decrypt(inlen, in, rctx->tbuf, + ctx->pkey->pkey.rsa, RSA_NO_PADDING); + if (ret <= 0) + return ret; + for (i = 0; i < ret; i++) { + if (rctx->tbuf[i]) + break; + } + ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf + i, + ret - i, ret, + rctx->oaep_label, + rctx->oaep_labellen, + rctx->md, rctx->mgf1md); + } else + ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa, + rctx->pad_mode); if (ret < 0) return ret; *outlen = ret; @@ -490,18 +547,36 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP: if (!p2) return -2; + BN_free(rctx->pub_exp); rctx->pub_exp = p2; return 1; + case EVP_PKEY_CTRL_RSA_OAEP_MD: + case EVP_PKEY_CTRL_GET_RSA_OAEP_MD: + if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) { + RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE); + return -2; + } + if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD) + *(const EVP_MD **)p2 = rctx->md; + else + rctx->md = p2; + return 1; + case EVP_PKEY_CTRL_MD: if (!check_padding_md(p2, rctx->pad_mode)) return 0; rctx->md = p2; return 1; + case EVP_PKEY_CTRL_GET_MD: + *(const EVP_MD **)p2 = rctx->md; + return 1; + case EVP_PKEY_CTRL_RSA_MGF1_MD: case EVP_PKEY_CTRL_GET_RSA_MGF1_MD: - if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) { + if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING + && rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) { RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD); return -2; } @@ -514,6 +589,30 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) rctx->mgf1md = p2; return 1; + case EVP_PKEY_CTRL_RSA_OAEP_LABEL: + if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) { + RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE); + return -2; + } + if (rctx->oaep_label) + OPENSSL_free(rctx->oaep_label); + if (p2 && p1 > 0) { + rctx->oaep_label = p2; + rctx->oaep_labellen = p1; + } else { + rctx->oaep_label = NULL; + rctx->oaep_labellen = 0; + } + return 1; + + case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL: + if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) { + RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE); + return -2; + } + *(unsigned char **)p2 = rctx->oaep_label; + return rctx->oaep_labellen; + case EVP_PKEY_CTRL_DIGESTINIT: case EVP_PKEY_CTRL_PKCS7_ENCRYPT: case EVP_PKEY_CTRL_PKCS7_DECRYPT: @@ -521,16 +620,6 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) return 1; #ifndef OPENSSL_NO_CMS case EVP_PKEY_CTRL_CMS_DECRYPT: - { - X509_ALGOR *alg = NULL; - ASN1_OBJECT *encalg = NULL; - if (p2) - CMS_RecipientInfo_ktri_get0_algs(p2, NULL, NULL, &alg); - if (alg) - X509_ALGOR_get0(&encalg, NULL, NULL, alg); - if (encalg && OBJ_obj2nid(encalg) == NID_rsaesOaep) - rctx->pad_mode = RSA_PKCS1_OAEP_PADDING; - } case EVP_PKEY_CTRL_CMS_ENCRYPT: case EVP_PKEY_CTRL_CMS_SIGN: return 1; @@ -599,6 +688,36 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, return ret; } + if (!strcmp(type, "rsa_mgf1_md")) { + const EVP_MD *md; + if (!(md = EVP_get_digestbyname(value))) { + RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST); + return 0; + } + return EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md); + } + + if (!strcmp(type, "rsa_oaep_md")) { + const EVP_MD *md; + if (!(md = EVP_get_digestbyname(value))) { + RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST); + return 0; + } + return EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md); + } + if (!strcmp(type, "rsa_oaep_label")) { + unsigned char *lab; + long lablen; + int ret; + lab = string_to_hex(value, &lablen); + if (!lab) + return 0; + ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen); + if (ret <= 0) + OPENSSL_free(lab); + return ret; + } + return -2; } diff --git a/crypto/rsa/rsa_sign.c b/crypto/rsa/rsa_sign.c index bc91da2c1fe9..19461c6364d4 100644 --- a/crypto/rsa/rsa_sign.c +++ b/crypto/rsa/rsa_sign.c @@ -261,19 +261,8 @@ int int_rsa_verify(int dtype, const unsigned char *m, OBJ_nid2ln(dtype)); #endif if (sigtype != dtype) { - if (((dtype == NID_md5) && - (sigtype == NID_md5WithRSAEncryption)) || - ((dtype == NID_md2) && - (sigtype == NID_md2WithRSAEncryption))) { - /* ok, we will let it through */ -#if !defined(OPENSSL_NO_STDIO) && !defined(OPENSSL_SYS_WIN16) - fprintf(stderr, - "signature has problems, re-make with post SSLeay045\n"); -#endif - } else { - RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_ALGORITHM_MISMATCH); - goto err; - } + RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_ALGORITHM_MISMATCH); + goto err; } if (rm) { const EVP_MD *md; diff --git a/crypto/sha/Makefile b/crypto/sha/Makefile index ceb8094b0cba..de6cdde58a9c 100644 --- a/crypto/sha/Makefile +++ b/crypto/sha/Makefile @@ -60,21 +60,25 @@ sha256-armv4.S: asm/sha256-armv4.pl $(PERL) $< $(PERLASM_SCHEME) $@ sha1-alpha.s: asm/sha1-alpha.pl - (preproc=/tmp/$$$$.$@; trap "rm $$preproc" INT; \ + (preproc=$$$$.$@.S; trap "rm $$preproc" INT; \ $(PERL) asm/sha1-alpha.pl > $$preproc && \ - $(CC) -E $$preproc > $@ && rm $$preproc) + $(CC) -E -P $$preproc > $@ && rm $$preproc) # Solaris make has to be explicitly told sha1-x86_64.s: asm/sha1-x86_64.pl; $(PERL) asm/sha1-x86_64.pl $(PERLASM_SCHEME) > $@ +sha1-mb-x86_64.s: asm/sha1-mb-x86_64.pl; $(PERL) asm/sha1-mb-x86_64.pl $(PERLASM_SCHEME) > $@ sha256-x86_64.s:asm/sha512-x86_64.pl; $(PERL) asm/sha512-x86_64.pl $(PERLASM_SCHEME) $@ +sha256-mb-x86_64.s: asm/sha256-mb-x86_64.pl; $(PERL) asm/sha256-mb-x86_64.pl $(PERLASM_SCHEME) > $@ sha512-x86_64.s:asm/sha512-x86_64.pl; $(PERL) asm/sha512-x86_64.pl $(PERLASM_SCHEME) $@ -sha1-sparcv9.s: asm/sha1-sparcv9.pl; $(PERL) asm/sha1-sparcv9.pl $@ $(CFLAGS) -sha256-sparcv9.s:asm/sha512-sparcv9.pl; $(PERL) asm/sha512-sparcv9.pl $@ $(CFLAGS) -sha512-sparcv9.s:asm/sha512-sparcv9.pl; $(PERL) asm/sha512-sparcv9.pl $@ $(CFLAGS) +sha1-sparcv9.S: asm/sha1-sparcv9.pl; $(PERL) asm/sha1-sparcv9.pl $@ $(CFLAGS) +sha256-sparcv9.S:asm/sha512-sparcv9.pl; $(PERL) asm/sha512-sparcv9.pl $@ $(CFLAGS) +sha512-sparcv9.S:asm/sha512-sparcv9.pl; $(PERL) asm/sha512-sparcv9.pl $@ $(CFLAGS) sha1-ppc.s: asm/sha1-ppc.pl; $(PERL) asm/sha1-ppc.pl $(PERLASM_SCHEME) $@ sha256-ppc.s: asm/sha512-ppc.pl; $(PERL) asm/sha512-ppc.pl $(PERLASM_SCHEME) $@ sha512-ppc.s: asm/sha512-ppc.pl; $(PERL) asm/sha512-ppc.pl $(PERLASM_SCHEME) $@ +sha256p8-ppc.s: asm/sha512p8-ppc.pl; $(PERL) asm/sha512p8-ppc.pl $(PERLASM_SCHEME) $@ +sha512p8-ppc.s: asm/sha512p8-ppc.pl; $(PERL) asm/sha512p8-ppc.pl $(PERLASM_SCHEME) $@ sha1-parisc.s: asm/sha1-parisc.pl; $(PERL) asm/sha1-parisc.pl $(PERLASM_SCHEME) $@ sha256-parisc.s:asm/sha512-parisc.pl; $(PERL) asm/sha512-parisc.pl $(PERLASM_SCHEME) $@ @@ -92,6 +96,9 @@ sha512-%.S: asm/sha512-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ sha1-armv4-large.o: sha1-armv4-large.S sha256-armv4.o: sha256-armv4.S sha512-armv4.o: sha512-armv4.S +sha1-armv8.o: sha1-armv8.S +sha256-armv8.o: sha256-armv8.S +sha512-armv8.o: sha512-armv8.S files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO diff --git a/crypto/sha/asm/sha1-586.pl b/crypto/sha/asm/sha1-586.pl index 1084d227fe06..4895eb3ddf85 100644 --- a/crypto/sha/asm/sha1-586.pl +++ b/crypto/sha/asm/sha1-586.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# [Re]written by Andy Polyakov for the OpenSSL +# [Re]written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -79,6 +79,10 @@ # strongly, it's probably more appropriate to discuss possibility of # using vector rotate XOP on AMD... +# March 2014. +# +# Add support for Intel SHA Extensions. + ###################################################################### # Current performance is summarized in following table. Numbers are # CPU clock cycles spent to process single byte (less is better). @@ -88,13 +92,20 @@ # PIII 11.5 - # P4 10.6 - # AMD K8 7.1 - -# Core2 7.3 6.1/+20% - -# Atom 12.5 9.5(*)/+32% - -# Westmere 7.3 5.6/+30% - -# Sandy Bridge 8.8 6.2/+40% 5.1(**)/+70% +# Core2 7.3 6.0/+22% - +# Westmere 7.3 5.5/+33% - +# Sandy Bridge 8.8 6.2/+40% 5.1(**)/+73% +# Ivy Bridge 7.2 4.8/+51% 4.7(**)/+53% +# Haswell 6.5 4.3/+51% 4.1(**)/+58% +# Bulldozer 11.6 6.0/+92% +# VIA Nano 10.6 7.5/+41% +# Atom 12.5 9.3(*)/+35% +# Silvermont 14.5 9.9(*)/+46% # # (*) Loop is 1056 instructions long and expected result is ~8.25. -# It remains mystery [to me] why ILP is limited to 1.7. +# The discrepancy is because of front-end limitations, so +# called MS-ROM penalties, and on Silvermont even rotate's +# limited parallelism. # # (**) As per above comment, the result is for AVX *plus* sh[rl]d. @@ -116,6 +127,15 @@ $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" && `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ && $1>=2.03); # first version supporting AVX +$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" && + `ml 2>&1` =~ /Version ([0-9]+)\./ && + $1>=10); # first version supporting AVX + +$ymm=1 if ($xmm && !$ymm && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/ && + $2>=3.0); # first version supporting AVX + +$shaext=$xmm; ### set to zero if compiling for 1.0.1 + &external_label("OPENSSL_ia32cap_P") if ($xmm); @@ -295,6 +315,7 @@ if ($alt) { &function_begin("sha1_block_data_order"); if ($xmm) { + &static_label("shaext_shortcut") if ($shaext); &static_label("ssse3_shortcut"); &static_label("avx_shortcut") if ($ymm); &static_label("K_XX_XX"); @@ -309,8 +330,13 @@ if ($xmm) { &mov ($D,&DWP(4,$T)); &test ($D,1<<9); # check SSSE3 bit &jz (&label("x86")); + &mov ($C,&DWP(8,$T)); &test ($A,1<<24); # check FXSR bit &jz (&label("x86")); + if ($shaext) { + &test ($C,1<<29); # check SHA bit + &jnz (&label("shaext_shortcut")); + } if ($ymm) { &and ($D,1<<28); # mask AVX bit &and ($A,1<<30); # mask "Intel CPU" bit @@ -389,6 +415,117 @@ if ($xmm) { &function_end("sha1_block_data_order"); if ($xmm) { +if ($shaext) { +###################################################################### +# Intel SHA Extensions implementation of SHA1 update function. +# +my ($ctx,$inp,$num)=("edi","esi","ecx"); +my ($ABCD,$E,$E_,$BSWAP)=map("xmm$_",(0..3)); +my @MSG=map("xmm$_",(4..7)); + +sub sha1rnds4 { + my ($dst,$src,$imm)=@_; + if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) + { &data_byte(0x0f,0x3a,0xcc,0xc0|($1<<3)|$2,$imm); } +} +sub sha1op38 { + my ($opcodelet,$dst,$src)=@_; + if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) + { &data_byte(0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2); } +} +sub sha1nexte { sha1op38(0xc8,@_); } +sub sha1msg1 { sha1op38(0xc9,@_); } +sub sha1msg2 { sha1op38(0xca,@_); } + +&function_begin("_sha1_block_data_order_shaext"); + &call (&label("pic_point")); # make it PIC! + &set_label("pic_point"); + &blindpop($tmp1); + &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); +&set_label("shaext_shortcut"); + &mov ($ctx,&wparam(0)); + &mov ("ebx","esp"); + &mov ($inp,&wparam(1)); + &mov ($num,&wparam(2)); + &sub ("esp",32); + + &movdqu ($ABCD,&QWP(0,$ctx)); + &movd ($E,&DWP(16,$ctx)); + &and ("esp",-32); + &movdqa ($BSWAP,&QWP(0x50,$tmp1)); # byte-n-word swap + + &movdqu (@MSG[0],&QWP(0,$inp)); + &pshufd ($ABCD,$ABCD,0b00011011); # flip word order + &movdqu (@MSG[1],&QWP(0x10,$inp)); + &pshufd ($E,$E,0b00011011); # flip word order + &movdqu (@MSG[2],&QWP(0x20,$inp)); + &pshufb (@MSG[0],$BSWAP); + &movdqu (@MSG[3],&QWP(0x30,$inp)); + &pshufb (@MSG[1],$BSWAP); + &pshufb (@MSG[2],$BSWAP); + &pshufb (@MSG[3],$BSWAP); + &jmp (&label("loop_shaext")); + +&set_label("loop_shaext",16); + &dec ($num); + &lea ("eax",&DWP(0x40,$inp)); + &movdqa (&QWP(0,"esp"),$E); # offload $E + &paddd ($E,@MSG[0]); + &cmovne ($inp,"eax"); + &movdqa (&QWP(16,"esp"),$ABCD); # offload $ABCD + +for($i=0;$i<20-4;$i+=2) { + &sha1msg1 (@MSG[0],@MSG[1]); + &movdqa ($E_,$ABCD); + &sha1rnds4 ($ABCD,$E,int($i/5)); # 0-3... + &sha1nexte ($E_,@MSG[1]); + &pxor (@MSG[0],@MSG[2]); + &sha1msg1 (@MSG[1],@MSG[2]); + &sha1msg2 (@MSG[0],@MSG[3]); + + &movdqa ($E,$ABCD); + &sha1rnds4 ($ABCD,$E_,int(($i+1)/5)); + &sha1nexte ($E,@MSG[2]); + &pxor (@MSG[1],@MSG[3]); + &sha1msg2 (@MSG[1],@MSG[0]); + + push(@MSG,shift(@MSG)); push(@MSG,shift(@MSG)); +} + &movdqu (@MSG[0],&QWP(0,$inp)); + &movdqa ($E_,$ABCD); + &sha1rnds4 ($ABCD,$E,3); # 64-67 + &sha1nexte ($E_,@MSG[1]); + &movdqu (@MSG[1],&QWP(0x10,$inp)); + &pshufb (@MSG[0],$BSWAP); + + &movdqa ($E,$ABCD); + &sha1rnds4 ($ABCD,$E_,3); # 68-71 + &sha1nexte ($E,@MSG[2]); + &movdqu (@MSG[2],&QWP(0x20,$inp)); + &pshufb (@MSG[1],$BSWAP); + + &movdqa ($E_,$ABCD); + &sha1rnds4 ($ABCD,$E,3); # 72-75 + &sha1nexte ($E_,@MSG[3]); + &movdqu (@MSG[3],&QWP(0x30,$inp)); + &pshufb (@MSG[2],$BSWAP); + + &movdqa ($E,$ABCD); + &sha1rnds4 ($ABCD,$E_,3); # 76-79 + &movdqa ($E_,&QWP(0,"esp")); + &pshufb (@MSG[3],$BSWAP); + &sha1nexte ($E,$E_); + &paddd ($ABCD,&QWP(16,"esp")); + + &jnz (&label("loop_shaext")); + + &pshufd ($ABCD,$ABCD,0b00011011); + &pshufd ($E,$E,0b00011011); + &movdqu (&QWP(0,$ctx),$ABCD) + &movd (&DWP(16,$ctx),$E); + &mov ("esp","ebx"); +&function_end("_sha1_block_data_order_shaext"); +} ###################################################################### # The SSSE3 implementation. # @@ -416,6 +553,7 @@ my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 my @V=($A,$B,$C,$D,$E); my $j=0; # hash round +my $rx=0; my @T=($T,$tmp1); my $inp; @@ -501,8 +639,11 @@ my $_ror=sub { &ror(@_) }; &movdqa (&QWP(0+16,"esp"),@X[-3&7]); &psubd (@X[-3&7],@X[3]); &movdqa (&QWP(0+32,"esp"),@X[-2&7]); + &mov (@T[1],$C); &psubd (@X[-2&7],@X[3]); - &movdqa (@X[0],@X[-3&7]); + &xor (@T[1],$D); + &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]); + &and (@T[0],@T[1]); &jmp (&label("loop")); ###################################################################### @@ -528,76 +669,77 @@ sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 my @insns = (&$body,&$body,&$body,&$body); # 40 instructions my ($a,$b,$c,$d,$e); + eval(shift(@insns)); # ror eval(shift(@insns)); eval(shift(@insns)); - &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]" + &punpcklqdq(@X[0],@X[-3&7]); # compose "X[-14]" in "X[0]", was &palignr(@X[0],@X[-4&7],8); &movdqa (@X[2],@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); &paddd (@X[3],@X[-1&7]); &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer - eval(shift(@insns)); + eval(shift(@insns)); # rol eval(shift(@insns)); &psrldq (@X[2],4); # "X[-3]", 3 dwords eval(shift(@insns)); eval(shift(@insns)); &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]" eval(shift(@insns)); - eval(shift(@insns)); + eval(shift(@insns)); # ror &pxor (@X[2],@X[-2&7]); # "X[-3]"^"X[-8]" eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - eval(shift(@insns)); &pxor (@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]" eval(shift(@insns)); - eval(shift(@insns)); + eval(shift(@insns)); # rol &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); &movdqa (@X[4],@X[0]); - &movdqa (@X[2],@X[0]); - eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); # ror + &movdqa (@X[2],@X[0]); eval(shift(@insns)); &pslldq (@X[4],12); # "X[0]"<<96, extract one dword &paddd (@X[0],@X[0]); eval(shift(@insns)); eval(shift(@insns)); - eval(shift(@insns)); - eval(shift(@insns)); &psrld (@X[2],31); eval(shift(@insns)); - eval(shift(@insns)); + eval(shift(@insns)); # rol &movdqa (@X[3],@X[4]); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); &psrld (@X[4],30); - &por (@X[0],@X[2]); # "X[0]"<<<=1 eval(shift(@insns)); + eval(shift(@insns)); # ror + &por (@X[0],@X[2]); # "X[0]"<<<=1 eval(shift(@insns)); &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer eval(shift(@insns)); eval(shift(@insns)); &pslld (@X[3],2); - &pxor (@X[0],@X[4]); - eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); # rol + &pxor (@X[0],@X[4]); &movdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX eval(shift(@insns)); eval(shift(@insns)); &pxor (@X[0],@X[3]); # "X[0]"^=("X[0]"<<96)<<<2 - &movdqa (@X[1],@X[-2&7]) if ($Xi<7); + &pshufd (@X[1],@X[-3&7],0xee) if ($Xi<7); # was &movdqa (@X[1],@X[-2&7]) + &pshufd (@X[3],@X[-1&7],0xee) if ($Xi==7); eval(shift(@insns)); eval(shift(@insns)); @@ -609,13 +751,12 @@ sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 sub Xupdate_ssse3_32_79() { use integer; my $body = shift; - my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions my ($a,$b,$c,$d,$e); - &movdqa (@X[2],@X[-1&7]) if ($Xi==8); eval(shift(@insns)); # body_20_39 &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" - &palignr(@X[2],@X[-2&7],8); # compose "X[-6]" + &punpcklqdq(@X[2],@X[-1&7]); # compose "X[-6]", was &palignr(@X[2],@X[-2&7],8) eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); # rol @@ -624,13 +765,14 @@ sub Xupdate_ssse3_32_79() &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)) if (@insns[0] =~ /_rol/); if ($Xi%5) { &movdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX... } else { # ... or load next one &movdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp")); } - &paddd (@X[3],@X[-1&7]); eval(shift(@insns)); # ror + &paddd (@X[3],@X[-1&7]); eval(shift(@insns)); &pxor (@X[0],@X[2]); # "X[0]"^="X[-6]" @@ -645,6 +787,7 @@ sub Xupdate_ssse3_32_79() eval(shift(@insns)); eval(shift(@insns)); # ror eval(shift(@insns)); + eval(shift(@insns)) if (@insns[0] =~ /_rol/); &pslld (@X[0],2); eval(shift(@insns)); # body_20_39 @@ -656,6 +799,8 @@ sub Xupdate_ssse3_32_79() eval(shift(@insns)); eval(shift(@insns)); # ror eval(shift(@insns)); + eval(shift(@insns)) if (@insns[1] =~ /_rol/); + eval(shift(@insns)) if (@insns[0] =~ /_rol/); &por (@X[0],@X[2]); # "X[0]"<<<=2 eval(shift(@insns)); # body_20_39 @@ -666,7 +811,7 @@ sub Xupdate_ssse3_32_79() eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); # ror - &movdqa (@X[3],@X[0]) if ($Xi<19); + &pshufd (@X[3],@X[-1],0xee) if ($Xi<19); # was &movdqa (@X[3],@X[0]) eval(shift(@insns)); foreach (@insns) { eval; } # remaining instructions @@ -680,6 +825,12 @@ sub Xuplast_ssse3_80() my @insns = (&$body,&$body,&$body,&$body); # 32 instructions my ($a,$b,$c,$d,$e); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); eval(shift(@insns)); &paddd (@X[3],@X[-1&7]); eval(shift(@insns)); @@ -717,9 +868,16 @@ sub Xloop_ssse3() eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); &pshufb (@X[($Xi-3)&7],@X[2]); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); &paddd (@X[($Xi-4)&7],@X[3]); eval(shift(@insns)); eval(shift(@insns)); @@ -728,6 +886,8 @@ sub Xloop_ssse3() &movdqa (&QWP(0+16*$Xi,"esp"),@X[($Xi-4)&7]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); &psubd (@X[($Xi-4)&7],@X[3]); foreach (@insns) { eval; } @@ -743,51 +903,124 @@ sub Xtail_ssse3() foreach (@insns) { eval; } } -sub body_00_19 () { +sub body_00_19 () { # ((c^d)&b)^d + # on start @T[0]=(c^d)&b + return &body_20_39() if ($rx==19); $rx++; ( '($a,$b,$c,$d,$e)=@V;'. - '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer - '&xor ($c,$d);', + '&$_ror ($b,$j?7:2);', # $b>>>2 + '&xor (@T[0],$d);', '&mov (@T[1],$a);', # $b in next round + + '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer + '&xor ($b,$c);', # $c^$d for next round + '&$_rol ($a,5);', - '&and (@T[0],$c);', # ($b&($c^$d)) - '&xor ($c,$d);', # restore $c - '&xor (@T[0],$d);', - '&add ($e,$a);', - '&$_ror ($b,$j?7:2);', # $b>>>2 - '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&add ($e,@T[0]);', + '&and (@T[1],$b);', # ($b&($c^$d)) for next round + + '&xor ($b,$c);', # restore $b + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); } -sub body_20_39 () { +sub body_20_39 () { # b^d^c + # on entry @T[0]=b^d + return &body_40_59() if ($rx==39); $rx++; ( '($a,$b,$c,$d,$e)=@V;'. - '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer - '&xor (@T[0],$d);', # ($b^$d) + '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer + '&xor (@T[0],$d) if($j==19);'. + '&xor (@T[0],$c) if($j> 19);', # ($b^$d^$c) '&mov (@T[1],$a);', # $b in next round + '&$_rol ($a,5);', - '&xor (@T[0],$c);', # ($b^$d^$c) - '&add ($e,$a);', + '&add ($e,@T[0]);', + '&xor (@T[1],$c) if ($j< 79);', # $b^$d for next round + '&$_ror ($b,7);', # $b>>>2 - '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); } -sub body_40_59 () { +sub body_40_59 () { # ((b^c)&(c^d))^c + # on entry @T[0]=(b^c), (c^=d) + $rx++; ( '($a,$b,$c,$d,$e)=@V;'. - '&mov (@T[1],$c);', - '&xor ($c,$d);', - '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer - '&and (@T[1],$d);', - '&and (@T[0],$c);', # ($b&($c^$d)) + '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer + '&and (@T[0],$c) if ($j>=40);', # (b^c)&(c^d) + '&xor ($c,$d) if ($j>=40);', # restore $c + '&$_ror ($b,7);', # $b>>>2 - '&add ($e,@T[1]);', - '&mov (@T[1],$a);', # $b in next round + '&mov (@T[1],$a);', # $b for next round + '&xor (@T[0],$c);', + '&$_rol ($a,5);', '&add ($e,@T[0]);', - '&xor ($c,$d);', # restore $c - '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&xor (@T[1],$c) if ($j==59);'. + '&xor (@T[1],$b) if ($j< 59);', # b^c for next round + + '&xor ($b,$c) if ($j< 59);', # c^d for next round + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} +###### +sub bodyx_00_19 () { # ((c^d)&b)^d + # on start @T[0]=(b&c)^(~b&d), $e+=X[]+K + return &bodyx_20_39() if ($rx==19); $rx++; + ( + '($a,$b,$c,$d,$e)=@V;'. + + '&rorx ($b,$b,2) if ($j==0);'. # $b>>>2 + '&rorx ($b,@T[1],7) if ($j!=0);', # $b>>>2 + '&lea ($e,&DWP(0,$e,@T[0]));', + '&rorx (@T[0],$a,5);', + + '&andn (@T[1],$a,$c);', + '&and ($a,$b)', + '&add ($d,&DWP(4*(($j+1)&15),"esp"));', # X[]+K xfer + + '&xor (@T[1],$a)', + '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} + +sub bodyx_20_39 () { # b^d^c + # on start $b=b^c^d + return &bodyx_40_59() if ($rx==39); $rx++; + ( + '($a,$b,$c,$d,$e)=@V;'. + + '&add ($e,($j==19?@T[0]:$b))', + '&rorx ($b,@T[1],7);', # $b>>>2 + '&rorx (@T[0],$a,5);', + + '&xor ($a,$b) if ($j<79);', + '&add ($d,&DWP(4*(($j+1)&15),"esp")) if ($j<79);', # X[]+K xfer + '&xor ($a,$c) if ($j<79);', + '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} + +sub bodyx_40_59 () { # ((b^c)&(c^d))^c + # on start $b=((b^c)&(c^d))^c + return &bodyx_20_39() if ($rx==59); $rx++; + ( + '($a,$b,$c,$d,$e)=@V;'. + + '&rorx (@T[0],$a,5)', + '&lea ($e,&DWP(0,$e,$b))', + '&rorx ($b,@T[1],7)', # $b>>>2 + '&add ($d,&DWP(4*(($j+1)&15),"esp"))', # X[]+K xfer + + '&mov (@T[1],$c)', + '&xor ($a,$b)', # b^c for next round + '&xor (@T[1],$b)', # c^d for next round + + '&and ($a,@T[1])', + '&add ($e,@T[0])', + '&xor ($a,$b)' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); } @@ -825,10 +1058,14 @@ sub body_40_59 () { &mov (&DWP(4,@T[1]),@T[0]); &add ($E,&DWP(16,@T[1])); &mov (&DWP(8,@T[1]),$C); - &mov ($B,@T[0]); + &mov ($B,$C); &mov (&DWP(12,@T[1]),$D); + &xor ($B,$D); &mov (&DWP(16,@T[1]),$E); - &movdqa (@X[0],@X[-3&7]); + &mov (@T[1],@T[0]); + &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]); + &and (@T[0],$B); + &mov ($B,$T[1]); &jmp (&label("loop")); @@ -853,6 +1090,8 @@ sub body_40_59 () { &function_end("_sha1_block_data_order_ssse3"); +$rx=0; # reset + if ($ymm) { my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 @@ -940,8 +1179,11 @@ my $_ror=sub { &shrd(@_[0],@_) }; &vpaddd (@X[1],@X[-3&7],@X[3]); &vpaddd (@X[2],@X[-2&7],@X[3]); &vmovdqa(&QWP(0,"esp"),@X[0]); # X[]+K xfer to IALU + &mov (@T[1],$C); &vmovdqa(&QWP(0+16,"esp"),@X[1]); + &xor (@T[1],$D); &vmovdqa(&QWP(0+32,"esp"),@X[2]); + &and (@T[0],@T[1]); &jmp (&label("loop")); sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 @@ -1025,7 +1267,7 @@ sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 sub Xupdate_avx_32_79() { use integer; my $body = shift; - my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions my ($a,$b,$c,$d,$e); &vpalignr(@X[2],@X[-1&7],@X[-2&7],8); # compose "X[-6]" @@ -1188,10 +1430,14 @@ sub Xtail_avx() &add ($D,&DWP(12,@T[1])); &mov (&DWP(4,@T[1]),@T[0]); &add ($E,&DWP(16,@T[1])); + &mov ($B,$C); &mov (&DWP(8,@T[1]),$C); - &mov ($B,@T[0]); + &xor ($B,$D); &mov (&DWP(12,@T[1]),$D); &mov (&DWP(16,@T[1]),$E); + &mov (@T[1],@T[0]); + &and (@T[0],$B); + &mov ($B,@T[1]); &jmp (&label("loop")); @@ -1223,6 +1469,7 @@ sub Xtail_avx() &data_word(0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc); # K_40_59 &data_word(0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6); # K_60_79 &data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # pbswap mask +&data_byte(0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0); } &asciz("SHA1 block transform for x86, CRYPTOGAMS by "); diff --git a/crypto/sha/asm/sha1-armv4-large.pl b/crypto/sha/asm/sha1-armv4-large.pl index 33da3e0e3c0d..b2c30322c351 100755 --- a/crypto/sha/asm/sha1-armv4-large.pl +++ b/crypto/sha/asm/sha1-armv4-large.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -52,6 +52,20 @@ # Profiler-assisted and platform-specific optimization resulted in 10% # improvement on Cortex A8 core and 12.2 cycles per byte. +# September 2013. +# +# Add NEON implementation (see sha1-586.pl for background info). On +# Cortex A8 it was measured to process one byte in 6.7 cycles or >80% +# faster than integer-only code. Because [fully unrolled] NEON code +# is ~2.5x larger and there are some redundant instructions executed +# when processing last block, improvement is not as big for smallest +# blocks, only ~30%. Snapdragon S4 is a tad faster, 6.4 cycles per +# byte, which is also >80% faster than integer-only code. + +# May 2014. +# +# Add ARMv8 code path performing at 2.35 cpb on Apple A7. + while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; @@ -153,12 +167,22 @@ $code=<<___; #include "arm_arch.h" .text +.code 32 .global sha1_block_data_order .type sha1_block_data_order,%function -.align 2 +.align 5 sha1_block_data_order: +#if __ARM_MAX_ARCH__>=7 + sub r3,pc,#8 @ sha1_block_data_order + ldr r12,.LOPENSSL_armcap + ldr r12,[r3,r12] @ OPENSSL_armcap_P + tst r12,#ARMV8_SHA1 + bne .LARMv8 + tst r12,#ARMV7_NEON + bne .LNEON +#endif stmdb sp!,{r4-r12,lr} add $len,$inp,$len,lsl#6 @ $len to point at the end of $inp ldmia $ctx,{$a,$b,$c,$d,$e} @@ -233,16 +257,427 @@ $code.=<<___; moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif -.align 2 +.size sha1_block_data_order,.-sha1_block_data_order + +.align 5 .LK_00_19: .word 0x5a827999 .LK_20_39: .word 0x6ed9eba1 .LK_40_59: .word 0x8f1bbcdc .LK_60_79: .word 0xca62c1d6 -.size sha1_block_data_order,.-sha1_block_data_order -.asciz "SHA1 block transform for ARMv4, CRYPTOGAMS by " -.align 2 +#if __ARM_MAX_ARCH__>=7 +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-sha1_block_data_order +#endif +.asciz "SHA1 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by " +.align 5 +___ +##################################################################### +# NEON stuff +# +{{{ +my @V=($a,$b,$c,$d,$e); +my ($K_XX_XX,$Ki,$t0,$t1,$Xfer,$saved_sp)=map("r$_",(8..12,14)); +my $Xi=4; +my @X=map("q$_",(8..11,0..3)); +my @Tx=("q12","q13"); +my ($K,$zero)=("q14","q15"); +my $j=0; + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +sub body_00_19 () { + ( + '($a,$b,$c,$d,$e)=@V;'. # '$code.="@ $j\n";'. + '&bic ($t0,$d,$b)', + '&add ($e,$e,$Ki)', # e+=X[i]+K + '&and ($t1,$c,$b)', + '&ldr ($Ki,sprintf "[sp,#%d]",4*(($j+1)&15))', + '&add ($e,$e,$a,"ror#27")', # e+=ROR(A,27) + '&eor ($t1,$t1,$t0)', # F_00_19 + '&mov ($b,$b,"ror#2")', # b=ROR(b,2) + '&add ($e,$e,$t1);'. # e+=F_00_19 + '$j++; unshift(@V,pop(@V));' + ) +} +sub body_20_39 () { + ( + '($a,$b,$c,$d,$e)=@V;'. # '$code.="@ $j\n";'. + '&eor ($t0,$b,$d)', + '&add ($e,$e,$Ki)', # e+=X[i]+K + '&ldr ($Ki,sprintf "[sp,#%d]",4*(($j+1)&15)) if ($j<79)', + '&eor ($t1,$t0,$c)', # F_20_39 + '&add ($e,$e,$a,"ror#27")', # e+=ROR(A,27) + '&mov ($b,$b,"ror#2")', # b=ROR(b,2) + '&add ($e,$e,$t1);'. # e+=F_20_39 + '$j++; unshift(@V,pop(@V));' + ) +} +sub body_40_59 () { + ( + '($a,$b,$c,$d,$e)=@V;'. # '$code.="@ $j\n";'. + '&add ($e,$e,$Ki)', # e+=X[i]+K + '&and ($t0,$c,$d)', + '&ldr ($Ki,sprintf "[sp,#%d]",4*(($j+1)&15))', + '&add ($e,$e,$a,"ror#27")', # e+=ROR(A,27) + '&eor ($t1,$c,$d)', + '&add ($e,$e,$t0)', + '&and ($t1,$t1,$b)', + '&mov ($b,$b,"ror#2")', # b=ROR(b,2) + '&add ($e,$e,$t1);'. # e+=F_40_59 + '$j++; unshift(@V,pop(@V));' + ) +} + +sub Xupdate_16_31 () +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e); + + &vext_8 (@X[0],@X[-4&7],@X[-3&7],8); # compose "X[-14]" in "X[0]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@Tx[1],@X[-1&7],$K); + eval(shift(@insns)); + &vld1_32 ("{$K\[]}","[$K_XX_XX,:32]!") if ($Xi%5==0); + eval(shift(@insns)); + &vext_8 (@Tx[0],@X[-1&7],$zero,4); # "X[-3]", 3 words + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &veor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" + eval(shift(@insns)); + eval(shift(@insns)); + &veor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + &veor (@Tx[0],@Tx[0],@X[0]); # "X[0]"^="X[-3]"^"X[-8] + eval(shift(@insns)); + eval(shift(@insns)); + &vst1_32 ("{@Tx[1]}","[$Xfer,:128]!"); # X[]+K xfer + &sub ($Xfer,$Xfer,64) if ($Xi%4==0); + eval(shift(@insns)); + eval(shift(@insns)); + &vext_8 (@Tx[1],$zero,@Tx[0],4); # "X[0]"<<96, extract one dword + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@X[0],@Tx[0],@Tx[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vsri_32 (@X[0],@Tx[0],31); # "X[0]"<<<=1 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 (@Tx[0],@Tx[1],30); + eval(shift(@insns)); + eval(shift(@insns)); + &vshl_u32 (@Tx[1],@Tx[1],2); + eval(shift(@insns)); + eval(shift(@insns)); + &veor (@X[0],@X[0],@Tx[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor (@X[0],@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2 + + foreach (@insns) { eval; } # remaining instructions [if any] + + $Xi++; push(@X,shift(@X)); # "rotate" X[] +} + +sub Xupdate_32_79 () +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e); + + &vext_8 (@Tx[0],@X[-2&7],@X[-1&7],8); # compose "X[-6]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &veor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" + eval(shift(@insns)); + eval(shift(@insns)); + &veor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@Tx[1],@X[-1&7],$K); + eval(shift(@insns)); + &vld1_32 ("{$K\[]}","[$K_XX_XX,:32]!") if ($Xi%5==0); + eval(shift(@insns)); + &veor (@Tx[0],@Tx[0],@X[0]); # "X[-6]"^="X[0]" + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 (@X[0],@Tx[0],30); + eval(shift(@insns)); + eval(shift(@insns)); + &vst1_32 ("{@Tx[1]}","[$Xfer,:128]!"); # X[]+K xfer + &sub ($Xfer,$Xfer,64) if ($Xi%4==0); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 (@X[0],@Tx[0],2); # "X[0]"="X[-6]"<<<2 + + foreach (@insns) { eval; } # remaining instructions [if any] + + $Xi++; push(@X,shift(@X)); # "rotate" X[] +} + +sub Xuplast_80 () +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e); + + &vadd_i32 (@Tx[1],@X[-1&7],$K); + eval(shift(@insns)); + eval(shift(@insns)); + &vst1_32 ("{@Tx[1]}","[$Xfer,:128]!"); + &sub ($Xfer,$Xfer,64); + + &teq ($inp,$len); + &sub ($K_XX_XX,$K_XX_XX,16); # rewind $K_XX_XX + &subeq ($inp,$inp,64); # reload last block to avoid SEGV + &vld1_8 ("{@X[-4&7]-@X[-3&7]}","[$inp]!"); + eval(shift(@insns)); + eval(shift(@insns)); + &vld1_8 ("{@X[-2&7]-@X[-1&7]}","[$inp]!"); + eval(shift(@insns)); + eval(shift(@insns)); + &vld1_32 ("{$K\[]}","[$K_XX_XX,:32]!"); # load K_00_19 + eval(shift(@insns)); + eval(shift(@insns)); + &vrev32_8 (@X[-4&7],@X[-4&7]); + + foreach (@insns) { eval; } # remaining instructions + + $Xi=0; +} + +sub Xloop() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e); + + &vrev32_8 (@X[($Xi-3)&7],@X[($Xi-3)&7]); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@X[$Xi&7],@X[($Xi-4)&7],$K); + eval(shift(@insns)); + eval(shift(@insns)); + &vst1_32 ("{@X[$Xi&7]}","[$Xfer,:128]!");# X[]+K xfer to IALU + + foreach (@insns) { eval; } + + $Xi++; +} + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.type sha1_block_data_order_neon,%function +.align 4 +sha1_block_data_order_neon: +.LNEON: + stmdb sp!,{r4-r12,lr} + add $len,$inp,$len,lsl#6 @ $len to point at the end of $inp + @ dmb @ errata #451034 on early Cortex A8 + @ vstmdb sp!,{d8-d15} @ ABI specification says so + mov $saved_sp,sp + sub sp,sp,#64 @ alloca + adr $K_XX_XX,.LK_00_19 + bic sp,sp,#15 @ align for 128-bit stores + + ldmia $ctx,{$a,$b,$c,$d,$e} @ load context + mov $Xfer,sp + + vld1.8 {@X[-4&7]-@X[-3&7]},[$inp]! @ handles unaligned + veor $zero,$zero,$zero + vld1.8 {@X[-2&7]-@X[-1&7]},[$inp]! + vld1.32 {${K}\[]},[$K_XX_XX,:32]! @ load K_00_19 + vrev32.8 @X[-4&7],@X[-4&7] @ yes, even on + vrev32.8 @X[-3&7],@X[-3&7] @ big-endian... + vrev32.8 @X[-2&7],@X[-2&7] + vadd.i32 @X[0],@X[-4&7],$K + vrev32.8 @X[-1&7],@X[-1&7] + vadd.i32 @X[1],@X[-3&7],$K + vst1.32 {@X[0]},[$Xfer,:128]! + vadd.i32 @X[2],@X[-2&7],$K + vst1.32 {@X[1]},[$Xfer,:128]! + vst1.32 {@X[2]},[$Xfer,:128]! + ldr $Ki,[sp] @ big RAW stall + +.Loop_neon: +___ + &Xupdate_16_31(\&body_00_19); + &Xupdate_16_31(\&body_00_19); + &Xupdate_16_31(\&body_00_19); + &Xupdate_16_31(\&body_00_19); + &Xupdate_32_79(\&body_00_19); + &Xupdate_32_79(\&body_20_39); + &Xupdate_32_79(\&body_20_39); + &Xupdate_32_79(\&body_20_39); + &Xupdate_32_79(\&body_20_39); + &Xupdate_32_79(\&body_20_39); + &Xupdate_32_79(\&body_40_59); + &Xupdate_32_79(\&body_40_59); + &Xupdate_32_79(\&body_40_59); + &Xupdate_32_79(\&body_40_59); + &Xupdate_32_79(\&body_40_59); + &Xupdate_32_79(\&body_20_39); + &Xuplast_80(\&body_20_39); + &Xloop(\&body_20_39); + &Xloop(\&body_20_39); + &Xloop(\&body_20_39); +$code.=<<___; + ldmia $ctx,{$Ki,$t0,$t1,$Xfer} @ accumulate context + add $a,$a,$Ki + ldr $Ki,[$ctx,#16] + add $b,$b,$t0 + add $c,$c,$t1 + add $d,$d,$Xfer + moveq sp,$saved_sp + add $e,$e,$Ki + ldrne $Ki,[sp] + stmia $ctx,{$a,$b,$c,$d,$e} + addne $Xfer,sp,#3*16 + bne .Loop_neon + + @ vldmia sp!,{d8-d15} + ldmia sp!,{r4-r12,pc} +.size sha1_block_data_order_neon,.-sha1_block_data_order_neon +#endif +___ +}}} +##################################################################### +# ARMv8 stuff +# +{{{ +my ($ABCD,$E,$E0,$E1)=map("q$_",(0..3)); +my @MSG=map("q$_",(4..7)); +my @Kxx=map("q$_",(8..11)); +my ($W0,$W1,$ABCD_SAVE)=map("q$_",(12..14)); + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 +.type sha1_block_data_order_armv8,%function +.align 5 +sha1_block_data_order_armv8: +.LARMv8: + vstmdb sp!,{d8-d15} @ ABI specification says so + + veor $E,$E,$E + adr r3,.LK_00_19 + vld1.32 {$ABCD},[$ctx]! + vld1.32 {$E\[0]},[$ctx] + sub $ctx,$ctx,#16 + vld1.32 {@Kxx[0]\[]},[r3,:32]! + vld1.32 {@Kxx[1]\[]},[r3,:32]! + vld1.32 {@Kxx[2]\[]},[r3,:32]! + vld1.32 {@Kxx[3]\[]},[r3,:32] + +.Loop_v8: + vld1.8 {@MSG[0]-@MSG[1]},[$inp]! + vld1.8 {@MSG[2]-@MSG[3]},[$inp]! + vrev32.8 @MSG[0],@MSG[0] + vrev32.8 @MSG[1],@MSG[1] + + vadd.i32 $W0,@Kxx[0],@MSG[0] + vrev32.8 @MSG[2],@MSG[2] + vmov $ABCD_SAVE,$ABCD @ offload + subs $len,$len,#1 + + vadd.i32 $W1,@Kxx[0],@MSG[1] + vrev32.8 @MSG[3],@MSG[3] + sha1h $E1,$ABCD @ 0 + sha1c $ABCD,$E,$W0 + vadd.i32 $W0,@Kxx[$j],@MSG[2] + sha1su0 @MSG[0],@MSG[1],@MSG[2] +___ +for ($j=0,$i=1;$i<20-3;$i++) { +my $f=("c","p","m","p")[$i/5]; +$code.=<<___; + sha1h $E0,$ABCD @ $i + sha1$f $ABCD,$E1,$W1 + vadd.i32 $W1,@Kxx[$j],@MSG[3] + sha1su1 @MSG[0],@MSG[3] +___ +$code.=<<___ if ($i<20-4); + sha1su0 @MSG[1],@MSG[2],@MSG[3] ___ + ($E0,$E1)=($E1,$E0); ($W0,$W1)=($W1,$W0); + push(@MSG,shift(@MSG)); $j++ if ((($i+3)%5)==0); +} +$code.=<<___; + sha1h $E0,$ABCD @ $i + sha1p $ABCD,$E1,$W1 + vadd.i32 $W1,@Kxx[$j],@MSG[3] + + sha1h $E1,$ABCD @ 18 + sha1p $ABCD,$E0,$W0 + + sha1h $E0,$ABCD @ 19 + sha1p $ABCD,$E1,$W1 + + vadd.i32 $E,$E,$E0 + vadd.i32 $ABCD,$ABCD,$ABCD_SAVE + bne .Loop_v8 + + vst1.32 {$ABCD},[$ctx]! + vst1.32 {$E\[0]},[$ctx] + + vldmia sp!,{d8-d15} + ret @ bx lr +.size sha1_block_data_order_armv8,.-sha1_block_data_order_armv8 +#endif +___ +}}} +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 +.comm OPENSSL_armcap_P,4,4 +#endif +___ + +{ my %opcode = ( + "sha1c" => 0xf2000c40, "sha1p" => 0xf2100c40, + "sha1m" => 0xf2200c40, "sha1su0" => 0xf2300c40, + "sha1h" => 0xf3b902c0, "sha1su1" => 0xf3ba0380 ); + + sub unsha1 { + my ($mnemonic,$arg)=@_; + + if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) { + my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19) + |(($2&7)<<17)|(($2&8)<<4) + |(($3&7)<<1) |(($3&8)<<2); + # since ARMv7 instructions are always encoded little-endian. + # correct solution is to use .inst directive, but older + # assemblers don't implement it:-( + sprintf ".byte\t0x%02x,0x%02x,0x%02x,0x%02x\t@ %s %s", + $word&0xff,($word>>8)&0xff, + ($word>>16)&0xff,($word>>24)&0xff, + $mnemonic,$arg; + } + } +} + +foreach (split($/,$code)) { + s/{q([0-9]+)\[\]}/sprintf "{d%d[],d%d[]}",2*$1,2*$1+1/eo or + s/{q([0-9]+)\[0\]}/sprintf "{d%d[0]}",2*$1/eo; + + s/\b(sha1\w+)\s+(q.*)/unsha1($1,$2)/geo; + + s/\bret\b/bx lr/o or + s/\bbx\s+lr\b/.word\t0xe12fff1e/o; # make it possible to compile with -march=armv4 + + print $_,$/; +} -$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 -print $code; close STDOUT; # enforce flush diff --git a/crypto/sha/asm/sha1-armv8.pl b/crypto/sha/asm/sha1-armv8.pl new file mode 100755 index 000000000000..c04432a54394 --- /dev/null +++ b/crypto/sha/asm/sha1-armv8.pl @@ -0,0 +1,338 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# SHA1 for ARMv8. +# +# Performance in cycles per processed byte and improvement coefficient +# over code generated with "default" compiler: +# +# hardware-assisted software(*) +# Apple A7 2.31 4.13 (+14%) +# Cortex-A53 2.24 8.03 (+97%) +# Cortex-A57 2.35 7.88 (+74%) +# Denver 2.13 3.97 (+0%)(**) +# X-Gene 8.80 (+200%) +# +# (*) Software results are presented mostly for reference purposes. +# (**) Keep in mind that Denver relies on binary translation, which +# optimizes compiler output at run-time. + +$flavour = shift; +open STDOUT,">".shift; + +($ctx,$inp,$num)=("x0","x1","x2"); +@Xw=map("w$_",(3..17,19)); +@Xx=map("x$_",(3..17,19)); +@V=($A,$B,$C,$D,$E)=map("w$_",(20..24)); +($t0,$t1,$t2,$K)=map("w$_",(25..28)); + + +sub BODY_00_19 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=($i+2)&15; + +$code.=<<___ if ($i<15 && !($i&1)); + lsr @Xx[$i+1],@Xx[$i],#32 +___ +$code.=<<___ if ($i<14 && !($i&1)); + ldr @Xx[$i+2],[$inp,#`($i+2)*4-64`] +___ +$code.=<<___ if ($i<14 && ($i&1)); +#ifdef __ARMEB__ + ror @Xx[$i+1],@Xx[$i+1],#32 +#else + rev32 @Xx[$i+1],@Xx[$i+1] +#endif +___ +$code.=<<___ if ($i<14); + bic $t0,$d,$b + and $t1,$c,$b + ror $t2,$a,#27 + add $d,$d,$K // future e+=K + orr $t0,$t0,$t1 + add $e,$e,$t2 // e+=rot(a,5) + ror $b,$b,#2 + add $d,$d,@Xw[($i+1)&15] // future e+=X[i] + add $e,$e,$t0 // e+=F(b,c,d) +___ +$code.=<<___ if ($i==19); + movz $K,#0xeba1 + movk $K,#0x6ed9,lsl#16 +___ +$code.=<<___ if ($i>=14); + eor @Xw[$j],@Xw[$j],@Xw[($j+2)&15] + bic $t0,$d,$b + and $t1,$c,$b + ror $t2,$a,#27 + eor @Xw[$j],@Xw[$j],@Xw[($j+8)&15] + add $d,$d,$K // future e+=K + orr $t0,$t0,$t1 + add $e,$e,$t2 // e+=rot(a,5) + eor @Xw[$j],@Xw[$j],@Xw[($j+13)&15] + ror $b,$b,#2 + add $d,$d,@Xw[($i+1)&15] // future e+=X[i] + add $e,$e,$t0 // e+=F(b,c,d) + ror @Xw[$j],@Xw[$j],#31 +___ +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=($i+2)&15; + +$code.=<<___ if ($i==59); + movz $K,#0xc1d6 + movk $K,#0xca62,lsl#16 +___ +$code.=<<___; + orr $t0,$b,$c + and $t1,$b,$c + eor @Xw[$j],@Xw[$j],@Xw[($j+2)&15] + ror $t2,$a,#27 + and $t0,$t0,$d + add $d,$d,$K // future e+=K + eor @Xw[$j],@Xw[$j],@Xw[($j+8)&15] + add $e,$e,$t2 // e+=rot(a,5) + orr $t0,$t0,$t1 + ror $b,$b,#2 + eor @Xw[$j],@Xw[$j],@Xw[($j+13)&15] + add $d,$d,@Xw[($i+1)&15] // future e+=X[i] + add $e,$e,$t0 // e+=F(b,c,d) + ror @Xw[$j],@Xw[$j],#31 +___ +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=($i+2)&15; + +$code.=<<___ if ($i==39); + movz $K,#0xbcdc + movk $K,#0x8f1b,lsl#16 +___ +$code.=<<___ if ($i<78); + eor @Xw[$j],@Xw[$j],@Xw[($j+2)&15] + eor $t0,$d,$b + ror $t2,$a,#27 + add $d,$d,$K // future e+=K + eor @Xw[$j],@Xw[$j],@Xw[($j+8)&15] + eor $t0,$t0,$c + add $e,$e,$t2 // e+=rot(a,5) + ror $b,$b,#2 + eor @Xw[$j],@Xw[$j],@Xw[($j+13)&15] + add $d,$d,@Xw[($i+1)&15] // future e+=X[i] + add $e,$e,$t0 // e+=F(b,c,d) + ror @Xw[$j],@Xw[$j],#31 +___ +$code.=<<___ if ($i==78); + ldp @Xw[1],@Xw[2],[$ctx] + eor $t0,$d,$b + ror $t2,$a,#27 + add $d,$d,$K // future e+=K + eor $t0,$t0,$c + add $e,$e,$t2 // e+=rot(a,5) + ror $b,$b,#2 + add $d,$d,@Xw[($i+1)&15] // future e+=X[i] + add $e,$e,$t0 // e+=F(b,c,d) +___ +$code.=<<___ if ($i==79); + ldp @Xw[3],@Xw[4],[$ctx,#8] + eor $t0,$d,$b + ror $t2,$a,#27 + eor $t0,$t0,$c + add $e,$e,$t2 // e+=rot(a,5) + ror $b,$b,#2 + ldr @Xw[5],[$ctx,#16] + add $e,$e,$t0 // e+=F(b,c,d) +___ +} + +$code.=<<___; +#include "arm_arch.h" + +.text + +.globl sha1_block_data_order +.type sha1_block_data_order,%function +.align 6 +sha1_block_data_order: + ldr x16,.LOPENSSL_armcap_P + adr x17,.LOPENSSL_armcap_P + add x16,x16,x17 + ldr w16,[x16] + tst w16,#ARMV8_SHA1 + b.ne .Lv8_entry + + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp $A,$B,[$ctx] + ldp $C,$D,[$ctx,#8] + ldr $E,[$ctx,#16] + +.Loop: + ldr @Xx[0],[$inp],#64 + movz $K,#0x7999 + sub $num,$num,#1 + movk $K,#0x5a82,lsl#16 +#ifdef __ARMEB__ + ror $Xx[0],@Xx[0],#32 +#else + rev32 @Xx[0],@Xx[0] +#endif + add $E,$E,$K // warm it up + add $E,$E,@Xw[0] +___ +for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } +for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + add $B,$B,@Xw[2] + add $C,$C,@Xw[3] + add $A,$A,@Xw[1] + add $D,$D,@Xw[4] + add $E,$E,@Xw[5] + stp $A,$B,[$ctx] + stp $C,$D,[$ctx,#8] + str $E,[$ctx,#16] + cbnz $num,.Loop + + ldp x19,x20,[sp,#16] + ldp x21,x22,[sp,#32] + ldp x23,x24,[sp,#48] + ldp x25,x26,[sp,#64] + ldp x27,x28,[sp,#80] + ldr x29,[sp],#96 + ret +.size sha1_block_data_order,.-sha1_block_data_order +___ +{{{ +my ($ABCD,$E,$E0,$E1)=map("v$_.16b",(0..3)); +my @MSG=map("v$_.16b",(4..7)); +my @Kxx=map("v$_.4s",(16..19)); +my ($W0,$W1)=("v20.4s","v21.4s"); +my $ABCD_SAVE="v22.16b"; + +$code.=<<___; +.type sha1_block_armv8,%function +.align 6 +sha1_block_armv8: +.Lv8_entry: + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + adr x4,.Lconst + eor $E,$E,$E + ld1.32 {$ABCD},[$ctx],#16 + ld1.32 {$E}[0],[$ctx] + sub $ctx,$ctx,#16 + ld1.32 {@Kxx[0]-@Kxx[3]},[x4] + +.Loop_hw: + ld1 {@MSG[0]-@MSG[3]},[$inp],#64 + sub $num,$num,#1 + rev32 @MSG[0],@MSG[0] + rev32 @MSG[1],@MSG[1] + + add.i32 $W0,@Kxx[0],@MSG[0] + rev32 @MSG[2],@MSG[2] + orr $ABCD_SAVE,$ABCD,$ABCD // offload + + add.i32 $W1,@Kxx[0],@MSG[1] + rev32 @MSG[3],@MSG[3] + sha1h $E1,$ABCD + sha1c $ABCD,$E,$W0 // 0 + add.i32 $W0,@Kxx[$j],@MSG[2] + sha1su0 @MSG[0],@MSG[1],@MSG[2] +___ +for ($j=0,$i=1;$i<20-3;$i++) { +my $f=("c","p","m","p")[$i/5]; +$code.=<<___; + sha1h $E0,$ABCD // $i + sha1$f $ABCD,$E1,$W1 + add.i32 $W1,@Kxx[$j],@MSG[3] + sha1su1 @MSG[0],@MSG[3] +___ +$code.=<<___ if ($i<20-4); + sha1su0 @MSG[1],@MSG[2],@MSG[3] +___ + ($E0,$E1)=($E1,$E0); ($W0,$W1)=($W1,$W0); + push(@MSG,shift(@MSG)); $j++ if ((($i+3)%5)==0); +} +$code.=<<___; + sha1h $E0,$ABCD // $i + sha1p $ABCD,$E1,$W1 + add.i32 $W1,@Kxx[$j],@MSG[3] + + sha1h $E1,$ABCD // 18 + sha1p $ABCD,$E0,$W0 + + sha1h $E0,$ABCD // 19 + sha1p $ABCD,$E1,$W1 + + add.i32 $E,$E,$E0 + add.i32 $ABCD,$ABCD,$ABCD_SAVE + + cbnz $num,.Loop_hw + + st1.32 {$ABCD},[$ctx],#16 + st1.32 {$E}[0],[$ctx] + + ldr x29,[sp],#16 + ret +.size sha1_block_armv8,.-sha1_block_armv8 +.align 6 +.Lconst: +.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 //K_00_19 +.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 //K_20_39 +.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc //K_40_59 +.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 //K_60_79 +.LOPENSSL_armcap_P: +.quad OPENSSL_armcap_P-. +.asciz "SHA1 block transform for ARMv8, CRYPTOGAMS by " +.align 2 +.comm OPENSSL_armcap_P,4,4 +___ +}}} + +{ my %opcode = ( + "sha1c" => 0x5e000000, "sha1p" => 0x5e001000, + "sha1m" => 0x5e002000, "sha1su0" => 0x5e003000, + "sha1h" => 0x5e280800, "sha1su1" => 0x5e281800 ); + + sub unsha1 { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o + && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5)|($3<<16), + $mnemonic,$arg; + } +} + +foreach(split("\n",$code)) { + + s/\`([^\`]*)\`/eval($1)/geo; + + s/\b(sha1\w+)\s+([qv].*)/unsha1($1,$2)/geo; + + s/\.\w?32\b//o and s/\.16b/\.4s/go; + m/(ld|st)1[^\[]+\[0\]/o and s/\.4s/\.s/go; + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/sha/asm/sha1-mb-x86_64.pl b/crypto/sha/asm/sha1-mb-x86_64.pl new file mode 100755 index 000000000000..a8ee075eaaa0 --- /dev/null +++ b/crypto/sha/asm/sha1-mb-x86_64.pl @@ -0,0 +1,1574 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# Multi-buffer SHA1 procedure processes n buffers in parallel by +# placing buffer data to designated lane of SIMD register. n is +# naturally limited to 4 on pre-AVX2 processors and to 8 on +# AVX2-capable processors such as Haswell. +# +# this +aesni(i) sha1 aesni-sha1 gain(iv) +# ------------------------------------------------------------------- +# Westmere(ii) 10.7/n +1.28=3.96(n=4) 5.30 6.66 +68% +# Atom(ii) 18.1/n +3.93=8.46(n=4) 9.37 12.8 +51% +# Sandy Bridge (8.16 +5.15=13.3)/n 4.99 5.98 +80% +# Ivy Bridge (8.08 +5.14=13.2)/n 4.60 5.54 +68% +# Haswell(iii) (8.96 +5.00=14.0)/n 3.57 4.55 +160% +# Bulldozer (9.76 +5.76=15.5)/n 5.95 6.37 +64% +# +# (i) multi-block CBC encrypt with 128-bit key; +# (ii) (HASH+AES)/n does not apply to Westmere for n>3 and Atom, +# because of lower AES-NI instruction throughput; +# (iii) "this" is for n=8, when we gather twice as much data, result +# for n=4 is 8.00+4.44=12.4; +# (iv) presented improvement coefficients are asymptotic limits and +# in real-life application are somewhat lower, e.g. for 2KB +# fragments they range from 30% to 100% (on Haswell); + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +$avx=0; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +# void sha1_multi_block ( +# struct { unsigned int A[8]; +# unsigned int B[8]; +# unsigned int C[8]; +# unsigned int D[8]; +# unsigned int E[8]; } *ctx, +# struct { void *ptr; int blocks; } inp[8], +# int num); /* 1 or 2 */ +# +$ctx="%rdi"; # 1st arg +$inp="%rsi"; # 2nd arg +$num="%edx"; +@ptr=map("%r$_",(8..11)); +$Tbl="%rbp"; + +@V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4)); +($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9)); +@Xi=map("%xmm$_",(10..14)); +$K="%xmm15"; + +if (1) { + # Atom-specific optimization aiming to eliminate pshufb with high + # registers [and thus get rid of 48 cycles accumulated penalty] + @Xi=map("%xmm$_",(0..4)); + ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9)); + @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14)); +} + +$REG_SZ=16; + +sub Xi_off { +my $off = shift; + + $off %= 16; $off *= $REG_SZ; + $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)"; +} + +sub BODY_00_19 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +my $k=$i+2; + +# Loads are performed 2+3/4 iterations in advance. 3/4 means that out +# of 4 words you would expect to be loaded per given iteration one is +# spilled to next iteration. In other words indices in four input +# streams are distributed as following: +# +# $i==0: 0,0,0,0,1,1,1,1,2,2,2, +# $i==1: 2,3,3,3, +# $i==2: 3,4,4,4, +# ... +# $i==13: 14,15,15,15, +# $i==14: 15 +# +# Then at $i==15 Xupdate is applied one iteration in advance... +$code.=<<___ if ($i==0); + movd (@ptr[0]),@Xi[0] + lea `16*4`(@ptr[0]),@ptr[0] + movd (@ptr[1]),@Xi[2] # borrow @Xi[2] + lea `16*4`(@ptr[1]),@ptr[1] + movd (@ptr[2]),@Xi[3] # borrow @Xi[3] + lea `16*4`(@ptr[2]),@ptr[2] + movd (@ptr[3]),@Xi[4] # borrow @Xi[4] + lea `16*4`(@ptr[3]),@ptr[3] + punpckldq @Xi[3],@Xi[0] + movd `4*$j-16*4`(@ptr[0]),@Xi[1] + punpckldq @Xi[4],@Xi[2] + movd `4*$j-16*4`(@ptr[1]),$t3 + punpckldq @Xi[2],@Xi[0] + movd `4*$j-16*4`(@ptr[2]),$t2 + pshufb $tx,@Xi[0] +___ +$code.=<<___ if ($i<14); # just load input + movd `4*$j-16*4`(@ptr[3]),$t1 + punpckldq $t2,@Xi[1] + movdqa $a,$t2 + paddd $K,$e # e+=K_00_19 + punpckldq $t1,$t3 + movdqa $b,$t1 + movdqa $b,$t0 + pslld \$5,$t2 + pandn $d,$t1 + pand $c,$t0 + punpckldq $t3,@Xi[1] + movdqa $a,$t3 + + movdqa @Xi[0],`&Xi_off($i)` + paddd @Xi[0],$e # e+=X[i] + movd `4*$k-16*4`(@ptr[0]),@Xi[2] + psrld \$27,$t3 + pxor $t1,$t0 # Ch(b,c,d) + movdqa $b,$t1 + + por $t3,$t2 # rol(a,5) + movd `4*$k-16*4`(@ptr[1]),$t3 + pslld \$30,$t1 + paddd $t0,$e # e+=Ch(b,c,d) + + psrld \$2,$b + paddd $t2,$e # e+=rol(a,5) + pshufb $tx,@Xi[1] + movd `4*$k-16*4`(@ptr[2]),$t2 + por $t1,$b # b=rol(b,30) +___ +$code.=<<___ if ($i==14); # just load input + movd `4*$j-16*4`(@ptr[3]),$t1 + punpckldq $t2,@Xi[1] + movdqa $a,$t2 + paddd $K,$e # e+=K_00_19 + punpckldq $t1,$t3 + movdqa $b,$t1 + movdqa $b,$t0 + pslld \$5,$t2 + prefetcht0 63(@ptr[0]) + pandn $d,$t1 + pand $c,$t0 + punpckldq $t3,@Xi[1] + movdqa $a,$t3 + + movdqa @Xi[0],`&Xi_off($i)` + paddd @Xi[0],$e # e+=X[i] + psrld \$27,$t3 + pxor $t1,$t0 # Ch(b,c,d) + movdqa $b,$t1 + prefetcht0 63(@ptr[1]) + + por $t3,$t2 # rol(a,5) + pslld \$30,$t1 + paddd $t0,$e # e+=Ch(b,c,d) + prefetcht0 63(@ptr[2]) + + psrld \$2,$b + paddd $t2,$e # e+=rol(a,5) + pshufb $tx,@Xi[1] + prefetcht0 63(@ptr[3]) + por $t1,$b # b=rol(b,30) +___ +$code.=<<___ if ($i>=13 && $i<15); + movdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]" +___ +$code.=<<___ if ($i>=15); # apply Xupdate + pxor @Xi[-2],@Xi[1] # "X[13]" + movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" + + movdqa $a,$t2 + pxor `&Xi_off($j+8)`,@Xi[1] + paddd $K,$e # e+=K_00_19 + movdqa $b,$t1 + pslld \$5,$t2 + pxor @Xi[3],@Xi[1] + movdqa $b,$t0 + pandn $d,$t1 + movdqa @Xi[1],$tx + pand $c,$t0 + movdqa $a,$t3 + psrld \$31,$tx + paddd @Xi[1],@Xi[1] + + movdqa @Xi[0],`&Xi_off($i)` + paddd @Xi[0],$e # e+=X[i] + psrld \$27,$t3 + pxor $t1,$t0 # Ch(b,c,d) + + movdqa $b,$t1 + por $t3,$t2 # rol(a,5) + pslld \$30,$t1 + paddd $t0,$e # e+=Ch(b,c,d) + + psrld \$2,$b + paddd $t2,$e # e+=rol(a,5) + por $tx,@Xi[1] # rol \$1,@Xi[1] + por $t1,$b # b=rol(b,30) +___ +push(@Xi,shift(@Xi)); +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; + +$code.=<<___ if ($i<79); + pxor @Xi[-2],@Xi[1] # "X[13]" + movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" + + movdqa $a,$t2 + movdqa $d,$t0 + pxor `&Xi_off($j+8)`,@Xi[1] + paddd $K,$e # e+=K_20_39 + pslld \$5,$t2 + pxor $b,$t0 + + movdqa $a,$t3 +___ +$code.=<<___ if ($i<72); + movdqa @Xi[0],`&Xi_off($i)` +___ +$code.=<<___ if ($i<79); + paddd @Xi[0],$e # e+=X[i] + pxor @Xi[3],@Xi[1] + psrld \$27,$t3 + pxor $c,$t0 # Parity(b,c,d) + movdqa $b,$t1 + + pslld \$30,$t1 + movdqa @Xi[1],$tx + por $t3,$t2 # rol(a,5) + psrld \$31,$tx + paddd $t0,$e # e+=Parity(b,c,d) + paddd @Xi[1],@Xi[1] + + psrld \$2,$b + paddd $t2,$e # e+=rol(a,5) + por $tx,@Xi[1] # rol(@Xi[1],1) + por $t1,$b # b=rol(b,30) +___ +$code.=<<___ if ($i==79); + movdqa $a,$t2 + paddd $K,$e # e+=K_20_39 + movdqa $d,$t0 + pslld \$5,$t2 + pxor $b,$t0 + + movdqa $a,$t3 + paddd @Xi[0],$e # e+=X[i] + psrld \$27,$t3 + movdqa $b,$t1 + pxor $c,$t0 # Parity(b,c,d) + + pslld \$30,$t1 + por $t3,$t2 # rol(a,5) + paddd $t0,$e # e+=Parity(b,c,d) + + psrld \$2,$b + paddd $t2,$e # e+=rol(a,5) + por $t1,$b # b=rol(b,30) +___ +push(@Xi,shift(@Xi)); +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; + +$code.=<<___; + pxor @Xi[-2],@Xi[1] # "X[13]" + movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" + + movdqa $a,$t2 + movdqa $d,$t1 + pxor `&Xi_off($j+8)`,@Xi[1] + pxor @Xi[3],@Xi[1] + paddd $K,$e # e+=K_40_59 + pslld \$5,$t2 + movdqa $a,$t3 + pand $c,$t1 + + movdqa $d,$t0 + movdqa @Xi[1],$tx + psrld \$27,$t3 + paddd $t1,$e + pxor $c,$t0 + + movdqa @Xi[0],`&Xi_off($i)` + paddd @Xi[0],$e # e+=X[i] + por $t3,$t2 # rol(a,5) + psrld \$31,$tx + pand $b,$t0 + movdqa $b,$t1 + + pslld \$30,$t1 + paddd @Xi[1],@Xi[1] + paddd $t0,$e # e+=Maj(b,d,c) + + psrld \$2,$b + paddd $t2,$e # e+=rol(a,5) + por $tx,@Xi[1] # rol(@X[1],1) + por $t1,$b # b=rol(b,30) +___ +push(@Xi,shift(@Xi)); +} + +$code.=<<___; +.text + +.extern OPENSSL_ia32cap_P + +.globl sha1_multi_block +.type sha1_multi_block,\@function,3 +.align 32 +sha1_multi_block: + mov OPENSSL_ia32cap_P+4(%rip),%rcx + bt \$61,%rcx # check SHA bit + jc _shaext_shortcut +___ +$code.=<<___ if ($avx); + test \$`1<<28`,%ecx + jnz _avx_shortcut +___ +$code.=<<___; + mov %rsp,%rax + push %rbx + push %rbp +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,-0x78(%rax) + movaps %xmm11,-0x68(%rax) + movaps %xmm12,-0x58(%rax) + movaps %xmm13,-0x48(%rax) + movaps %xmm14,-0x38(%rax) + movaps %xmm15,-0x28(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`,%rsp + and \$-256,%rsp + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody: + lea K_XX_XX(%rip),$Tbl + lea `$REG_SZ*16`(%rsp),%rbx + +.Loop_grande: + mov $num,`$REG_SZ*17+8`(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldone + + movdqu 0x00($ctx),$A # load context + lea 128(%rsp),%rax + movdqu 0x20($ctx),$B + movdqu 0x40($ctx),$C + movdqu 0x60($ctx),$D + movdqu 0x80($ctx),$E + movdqa 0x60($Tbl),$tx # pbswap_mask + movdqa -0x20($Tbl),$K # K_00_19 + jmp .Loop + +.align 32 +.Loop: +___ +for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } +$code.=" movdqa 0x00($Tbl),$K\n"; # K_20_39 +for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=" movdqa 0x20($Tbl),$K\n"; # K_40_59 +for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +$code.=" movdqa 0x40($Tbl),$K\n"; # K_60_79 +for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + movdqa (%rbx),@Xi[0] # pull counters + mov \$1,%ecx + cmp 4*0(%rbx),%ecx # examinte counters + pxor $t2,$t2 + cmovge $Tbl,@ptr[0] # cancel input + cmp 4*1(%rbx),%ecx + movdqa @Xi[0],@Xi[1] + cmovge $Tbl,@ptr[1] + cmp 4*2(%rbx),%ecx + pcmpgtd $t2,@Xi[1] # mask value + cmovge $Tbl,@ptr[2] + cmp 4*3(%rbx),%ecx + paddd @Xi[1],@Xi[0] # counters-- + cmovge $Tbl,@ptr[3] + + movdqu 0x00($ctx),$t0 + pand @Xi[1],$A + movdqu 0x20($ctx),$t1 + pand @Xi[1],$B + paddd $t0,$A + movdqu 0x40($ctx),$t2 + pand @Xi[1],$C + paddd $t1,$B + movdqu 0x60($ctx),$t3 + pand @Xi[1],$D + paddd $t2,$C + movdqu 0x80($ctx),$tx + pand @Xi[1],$E + movdqu $A,0x00($ctx) + paddd $t3,$D + movdqu $B,0x20($ctx) + paddd $tx,$E + movdqu $C,0x40($ctx) + movdqu $D,0x60($ctx) + movdqu $E,0x80($ctx) + + movdqa @Xi[0],(%rbx) # save counters + movdqa 0x60($Tbl),$tx # pbswap_mask + movdqa -0x20($Tbl),$K # K_00_19 + dec $num + jnz .Loop + + mov `$REG_SZ*17+8`(%rsp),$num + lea $REG_SZ($ctx),$ctx + lea `16*$REG_SZ/4`($inp),$inp + dec $num + jnz .Loop_grande + +.Ldone: + mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp +___ +$code.=<<___ if ($win64); + movaps -0xb8(%rax),%xmm6 + movaps -0xa8(%rax),%xmm7 + movaps -0x98(%rax),%xmm8 + movaps -0x88(%rax),%xmm9 + movaps -0x78(%rax),%xmm10 + movaps -0x68(%rax),%xmm11 + movaps -0x58(%rax),%xmm12 + movaps -0x48(%rax),%xmm13 + movaps -0x38(%rax),%xmm14 + movaps -0x28(%rax),%xmm15 +___ +$code.=<<___; + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue: + ret +.size sha1_multi_block,.-sha1_multi_block +___ + {{{ +my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10)); +my @MSG0=map("%xmm$_",(4..7)); +my @MSG1=map("%xmm$_",(11..14)); + +$code.=<<___; +.type sha1_multi_block_shaext,\@function,3 +.align 32 +sha1_multi_block_shaext: +_shaext_shortcut: + mov %rsp,%rax + push %rbx + push %rbp +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,-0x78(%rax) + movaps %xmm11,-0x68(%rax) + movaps %xmm12,-0x58(%rax) + movaps %xmm13,-0x48(%rax) + movaps %xmm14,-0x38(%rax) + movaps %xmm15,-0x28(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`,%rsp + shl \$1,$num # we process pair at a time + and \$-256,%rsp + lea 0x40($ctx),$ctx # size optimization + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody_shaext: + lea `$REG_SZ*16`(%rsp),%rbx + movdqa K_XX_XX+0x80(%rip),$BSWAP # byte-n-word swap + +.Loop_grande_shaext: + mov $num,`$REG_SZ*17+8`(%rsp) # orignal $num + xor $num,$num +___ +for($i=0;$i<2;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle %rsp,@ptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldone_shaext + + movq 0x00-0x40($ctx),$ABCD0 # a1.a0 + movq 0x20-0x40($ctx),@MSG0[0]# b1.b0 + movq 0x40-0x40($ctx),@MSG0[1]# c1.c0 + movq 0x60-0x40($ctx),@MSG0[2]# d1.d0 + movq 0x80-0x40($ctx),@MSG0[3]# e1.e0 + + punpckldq @MSG0[0],$ABCD0 # b1.a1.b0.a0 + punpckldq @MSG0[2],@MSG0[1] # d1.c1.d0.c0 + + movdqa $ABCD0,$ABCD1 + punpcklqdq @MSG0[1],$ABCD0 # d0.c0.b0.a0 + punpckhqdq @MSG0[1],$ABCD1 # d1.c1.b1.a1 + + pshufd \$0b00111111,@MSG0[3],$E0 + pshufd \$0b01111111,@MSG0[3],$E1 + pshufd \$0b00011011,$ABCD0,$ABCD0 + pshufd \$0b00011011,$ABCD1,$ABCD1 + jmp .Loop_shaext + +.align 32 +.Loop_shaext: + movdqu 0x00(@ptr[0]),@MSG0[0] + movdqu 0x00(@ptr[1]),@MSG1[0] + movdqu 0x10(@ptr[0]),@MSG0[1] + movdqu 0x10(@ptr[1]),@MSG1[1] + movdqu 0x20(@ptr[0]),@MSG0[2] + pshufb $BSWAP,@MSG0[0] + movdqu 0x20(@ptr[1]),@MSG1[2] + pshufb $BSWAP,@MSG1[0] + movdqu 0x30(@ptr[0]),@MSG0[3] + lea 0x40(@ptr[0]),@ptr[0] + pshufb $BSWAP,@MSG0[1] + movdqu 0x30(@ptr[1]),@MSG1[3] + lea 0x40(@ptr[1]),@ptr[1] + pshufb $BSWAP,@MSG1[1] + + movdqa $E0,0x50(%rsp) # offload + paddd @MSG0[0],$E0 + movdqa $E1,0x70(%rsp) + paddd @MSG1[0],$E1 + movdqa $ABCD0,0x40(%rsp) # offload + movdqa $ABCD0,$E0_ + movdqa $ABCD1,0x60(%rsp) + movdqa $ABCD1,$E1_ + sha1rnds4 \$0,$E0,$ABCD0 # 0-3 + sha1nexte @MSG0[1],$E0_ + sha1rnds4 \$0,$E1,$ABCD1 # 0-3 + sha1nexte @MSG1[1],$E1_ + pshufb $BSWAP,@MSG0[2] + prefetcht0 127(@ptr[0]) + sha1msg1 @MSG0[1],@MSG0[0] + pshufb $BSWAP,@MSG1[2] + prefetcht0 127(@ptr[1]) + sha1msg1 @MSG1[1],@MSG1[0] + + pshufb $BSWAP,@MSG0[3] + movdqa $ABCD0,$E0 + pshufb $BSWAP,@MSG1[3] + movdqa $ABCD1,$E1 + sha1rnds4 \$0,$E0_,$ABCD0 # 4-7 + sha1nexte @MSG0[2],$E0 + sha1rnds4 \$0,$E1_,$ABCD1 # 4-7 + sha1nexte @MSG1[2],$E1 + pxor @MSG0[2],@MSG0[0] + sha1msg1 @MSG0[2],@MSG0[1] + pxor @MSG1[2],@MSG1[0] + sha1msg1 @MSG1[2],@MSG1[1] +___ +for($i=2;$i<20-4;$i++) { +$code.=<<___; + movdqa $ABCD0,$E0_ + movdqa $ABCD1,$E1_ + sha1rnds4 \$`int($i/5)`,$E0,$ABCD0 # 8-11 + sha1nexte @MSG0[3],$E0_ + sha1rnds4 \$`int($i/5)`,$E1,$ABCD1 # 8-11 + sha1nexte @MSG1[3],$E1_ + sha1msg2 @MSG0[3],@MSG0[0] + sha1msg2 @MSG1[3],@MSG1[0] + pxor @MSG0[3],@MSG0[1] + sha1msg1 @MSG0[3],@MSG0[2] + pxor @MSG1[3],@MSG1[1] + sha1msg1 @MSG1[3],@MSG1[2] +___ + ($E0,$E0_)=($E0_,$E0); ($E1,$E1_)=($E1_,$E1); + push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1)); +} +$code.=<<___; + movdqa $ABCD0,$E0_ + movdqa $ABCD1,$E1_ + sha1rnds4 \$3,$E0,$ABCD0 # 64-67 + sha1nexte @MSG0[3],$E0_ + sha1rnds4 \$3,$E1,$ABCD1 # 64-67 + sha1nexte @MSG1[3],$E1_ + sha1msg2 @MSG0[3],@MSG0[0] + sha1msg2 @MSG1[3],@MSG1[0] + pxor @MSG0[3],@MSG0[1] + pxor @MSG1[3],@MSG1[1] + + mov \$1,%ecx + pxor @MSG0[2],@MSG0[2] # zero + cmp 4*0(%rbx),%ecx # examine counters + cmovge %rsp,@ptr[0] # cancel input + + movdqa $ABCD0,$E0 + movdqa $ABCD1,$E1 + sha1rnds4 \$3,$E0_,$ABCD0 # 68-71 + sha1nexte @MSG0[0],$E0 + sha1rnds4 \$3,$E1_,$ABCD1 # 68-71 + sha1nexte @MSG1[0],$E1 + sha1msg2 @MSG0[0],@MSG0[1] + sha1msg2 @MSG1[0],@MSG1[1] + + cmp 4*1(%rbx),%ecx + cmovge %rsp,@ptr[1] + movq (%rbx),@MSG0[0] # pull counters + + movdqa $ABCD0,$E0_ + movdqa $ABCD1,$E1_ + sha1rnds4 \$3,$E0,$ABCD0 # 72-75 + sha1nexte @MSG0[1],$E0_ + sha1rnds4 \$3,$E1,$ABCD1 # 72-75 + sha1nexte @MSG1[1],$E1_ + + pshufd \$0x00,@MSG0[0],@MSG1[2] + pshufd \$0x55,@MSG0[0],@MSG1[3] + movdqa @MSG0[0],@MSG0[1] + pcmpgtd @MSG0[2],@MSG1[2] + pcmpgtd @MSG0[2],@MSG1[3] + + movdqa $ABCD0,$E0 + movdqa $ABCD1,$E1 + sha1rnds4 \$3,$E0_,$ABCD0 # 76-79 + sha1nexte $MSG0[2],$E0 + sha1rnds4 \$3,$E1_,$ABCD1 # 76-79 + sha1nexte $MSG0[2],$E1 + + pcmpgtd @MSG0[2],@MSG0[1] # counter mask + pand @MSG1[2],$ABCD0 + pand @MSG1[2],$E0 + pand @MSG1[3],$ABCD1 + pand @MSG1[3],$E1 + paddd @MSG0[1],@MSG0[0] # counters-- + + paddd 0x40(%rsp),$ABCD0 + paddd 0x50(%rsp),$E0 + paddd 0x60(%rsp),$ABCD1 + paddd 0x70(%rsp),$E1 + + movq @MSG0[0],(%rbx) # save counters + dec $num + jnz .Loop_shaext + + mov `$REG_SZ*17+8`(%rsp),$num + + pshufd \$0b00011011,$ABCD0,$ABCD0 + pshufd \$0b00011011,$ABCD1,$ABCD1 + + movdqa $ABCD0,@MSG0[0] + punpckldq $ABCD1,$ABCD0 # b1.b0.a1.a0 + punpckhdq $ABCD1,@MSG0[0] # d1.d0.c1.c0 + punpckhdq $E1,$E0 # e1.e0.xx.xx + movq $ABCD0,0x00-0x40($ctx) # a1.a0 + psrldq \$8,$ABCD0 + movq @MSG0[0],0x40-0x40($ctx)# c1.c0 + psrldq \$8,@MSG0[0] + movq $ABCD0,0x20-0x40($ctx) # b1.b0 + psrldq \$8,$E0 + movq @MSG0[0],0x60-0x40($ctx)# d1.d0 + movq $E0,0x80-0x40($ctx) # e1.e0 + + lea `$REG_SZ/2`($ctx),$ctx + lea `16*2`($inp),$inp + dec $num + jnz .Loop_grande_shaext + +.Ldone_shaext: + #mov `$REG_SZ*17`(%rsp),%rax # original %rsp +___ +$code.=<<___ if ($win64); + movaps -0xb8(%rax),%xmm6 + movaps -0xa8(%rax),%xmm7 + movaps -0x98(%rax),%xmm8 + movaps -0x88(%rax),%xmm9 + movaps -0x78(%rax),%xmm10 + movaps -0x68(%rax),%xmm11 + movaps -0x58(%rax),%xmm12 + movaps -0x48(%rax),%xmm13 + movaps -0x38(%rax),%xmm14 + movaps -0x28(%rax),%xmm15 +___ +$code.=<<___; + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue_shaext: + ret +.size sha1_multi_block_shaext,.-sha1_multi_block_shaext +___ + }}} + + if ($avx) {{{ +sub BODY_00_19_avx { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +my $k=$i+2; +my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128"; +my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4]; + +$code.=<<___ if ($i==0 && $REG_SZ==16); + vmovd (@ptr[0]),@Xi[0] + lea `16*4`(@ptr[0]),@ptr[0] + vmovd (@ptr[1]),@Xi[2] # borrow Xi[2] + lea `16*4`(@ptr[1]),@ptr[1] + vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0] + lea `16*4`(@ptr[2]),@ptr[2] + vpinsrd \$1,(@ptr[3]),@Xi[2],@Xi[2] + lea `16*4`(@ptr[3]),@ptr[3] + vmovd `4*$j-16*4`(@ptr[0]),@Xi[1] + vpunpckldq @Xi[2],@Xi[0],@Xi[0] + vmovd `4*$j-16*4`($ptr_n),$t3 + vpshufb $tx,@Xi[0],@Xi[0] +___ +$code.=<<___ if ($i<15 && $REG_SZ==16); # just load input + vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1] + vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3 +___ +$code.=<<___ if ($i==0 && $REG_SZ==32); + vmovd (@ptr[0]),@Xi[0] + lea `16*4`(@ptr[0]),@ptr[0] + vmovd (@ptr[4]),@Xi[2] # borrow Xi[2] + lea `16*4`(@ptr[4]),@ptr[4] + vmovd (@ptr[1]),$t2 + lea `16*4`(@ptr[1]),@ptr[1] + vmovd (@ptr[5]),$t1 + lea `16*4`(@ptr[5]),@ptr[5] + vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0] + lea `16*4`(@ptr[2]),@ptr[2] + vpinsrd \$1,(@ptr[6]),@Xi[2],@Xi[2] + lea `16*4`(@ptr[6]),@ptr[6] + vpinsrd \$1,(@ptr[3]),$t2,$t2 + lea `16*4`(@ptr[3]),@ptr[3] + vpunpckldq $t2,@Xi[0],@Xi[0] + vpinsrd \$1,(@ptr[7]),$t1,$t1 + lea `16*4`(@ptr[7]),@ptr[7] + vpunpckldq $t1,@Xi[2],@Xi[2] + vmovd `4*$j-16*4`(@ptr[0]),@Xi[1] + vinserti128 @Xi[2],@Xi[0],@Xi[0] + vmovd `4*$j-16*4`($ptr_n),$t3 + vpshufb $tx,@Xi[0],@Xi[0] +___ +$code.=<<___ if ($i<15 && $REG_SZ==32); # just load input + vmovd `4*$j-16*4`(@ptr[1]),$t2 + vmovd `4*$j-16*4`(@ptr[5]),$t1 + vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1] + vpinsrd \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3 + vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2 + vpunpckldq $t2,@Xi[1],@Xi[1] + vpinsrd \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1 + vpunpckldq $t1,$t3,$t3 +___ +$code.=<<___ if ($i<14); + vpaddd $K,$e,$e # e+=K_00_19 + vpslld \$5,$a,$t2 + vpandn $d,$b,$t1 + vpand $c,$b,$t0 + + vmovdqa @Xi[0],`&Xi_off($i)` + vpaddd @Xi[0],$e,$e # e+=X[i] + $vpack $t3,@Xi[1],@Xi[1] + vpsrld \$27,$a,$t3 + vpxor $t1,$t0,$t0 # Ch(b,c,d) + vmovd `4*$k-16*4`(@ptr[0]),@Xi[2] + + vpslld \$30,$b,$t1 + vpor $t3,$t2,$t2 # rol(a,5) + vmovd `4*$k-16*4`($ptr_n),$t3 + vpaddd $t0,$e,$e # e+=Ch(b,c,d) + + vpsrld \$2,$b,$b + vpaddd $t2,$e,$e # e+=rol(a,5) + vpshufb $tx,@Xi[1],@Xi[1] + vpor $t1,$b,$b # b=rol(b,30) +___ +$code.=<<___ if ($i==14); + vpaddd $K,$e,$e # e+=K_00_19 + prefetcht0 63(@ptr[0]) + vpslld \$5,$a,$t2 + vpandn $d,$b,$t1 + vpand $c,$b,$t0 + + vmovdqa @Xi[0],`&Xi_off($i)` + vpaddd @Xi[0],$e,$e # e+=X[i] + $vpack $t3,@Xi[1],@Xi[1] + vpsrld \$27,$a,$t3 + prefetcht0 63(@ptr[1]) + vpxor $t1,$t0,$t0 # Ch(b,c,d) + + vpslld \$30,$b,$t1 + vpor $t3,$t2,$t2 # rol(a,5) + prefetcht0 63(@ptr[2]) + vpaddd $t0,$e,$e # e+=Ch(b,c,d) + + vpsrld \$2,$b,$b + vpaddd $t2,$e,$e # e+=rol(a,5) + prefetcht0 63(@ptr[3]) + vpshufb $tx,@Xi[1],@Xi[1] + vpor $t1,$b,$b # b=rol(b,30) +___ +$code.=<<___ if ($i>=13 && $i<15); + vmovdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]" +___ +$code.=<<___ if ($i>=15); # apply Xupdate + vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]" + vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" + + vpaddd $K,$e,$e # e+=K_00_19 + vpslld \$5,$a,$t2 + vpandn $d,$b,$t1 + `"prefetcht0 63(@ptr[4])" if ($i==15 && $REG_SZ==32)` + vpand $c,$b,$t0 + + vmovdqa @Xi[0],`&Xi_off($i)` + vpaddd @Xi[0],$e,$e # e+=X[i] + vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1] + vpsrld \$27,$a,$t3 + vpxor $t1,$t0,$t0 # Ch(b,c,d) + vpxor @Xi[3],@Xi[1],@Xi[1] + `"prefetcht0 63(@ptr[5])" if ($i==15 && $REG_SZ==32)` + + vpslld \$30,$b,$t1 + vpor $t3,$t2,$t2 # rol(a,5) + vpaddd $t0,$e,$e # e+=Ch(b,c,d) + `"prefetcht0 63(@ptr[6])" if ($i==15 && $REG_SZ==32)` + vpsrld \$31,@Xi[1],$tx + vpaddd @Xi[1],@Xi[1],@Xi[1] + + vpsrld \$2,$b,$b + `"prefetcht0 63(@ptr[7])" if ($i==15 && $REG_SZ==32)` + vpaddd $t2,$e,$e # e+=rol(a,5) + vpor $tx,@Xi[1],@Xi[1] # rol \$1,@Xi[1] + vpor $t1,$b,$b # b=rol(b,30) +___ +push(@Xi,shift(@Xi)); +} + +sub BODY_20_39_avx { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; + +$code.=<<___ if ($i<79); + vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]" + vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" + + vpslld \$5,$a,$t2 + vpaddd $K,$e,$e # e+=K_20_39 + vpxor $b,$d,$t0 +___ +$code.=<<___ if ($i<72); + vmovdqa @Xi[0],`&Xi_off($i)` +___ +$code.=<<___ if ($i<79); + vpaddd @Xi[0],$e,$e # e+=X[i] + vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1] + vpsrld \$27,$a,$t3 + vpxor $c,$t0,$t0 # Parity(b,c,d) + vpxor @Xi[3],@Xi[1],@Xi[1] + + vpslld \$30,$b,$t1 + vpor $t3,$t2,$t2 # rol(a,5) + vpaddd $t0,$e,$e # e+=Parity(b,c,d) + vpsrld \$31,@Xi[1],$tx + vpaddd @Xi[1],@Xi[1],@Xi[1] + + vpsrld \$2,$b,$b + vpaddd $t2,$e,$e # e+=rol(a,5) + vpor $tx,@Xi[1],@Xi[1] # rol(@Xi[1],1) + vpor $t1,$b,$b # b=rol(b,30) +___ +$code.=<<___ if ($i==79); + vpslld \$5,$a,$t2 + vpaddd $K,$e,$e # e+=K_20_39 + vpxor $b,$d,$t0 + + vpsrld \$27,$a,$t3 + vpaddd @Xi[0],$e,$e # e+=X[i] + vpxor $c,$t0,$t0 # Parity(b,c,d) + + vpslld \$30,$b,$t1 + vpor $t3,$t2,$t2 # rol(a,5) + vpaddd $t0,$e,$e # e+=Parity(b,c,d) + + vpsrld \$2,$b,$b + vpaddd $t2,$e,$e # e+=rol(a,5) + vpor $t1,$b,$b # b=rol(b,30) +___ +push(@Xi,shift(@Xi)); +} + +sub BODY_40_59_avx { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; + +$code.=<<___; + vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]" + vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" + + vpaddd $K,$e,$e # e+=K_40_59 + vpslld \$5,$a,$t2 + vpand $c,$d,$t1 + vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1] + + vpaddd $t1,$e,$e + vpsrld \$27,$a,$t3 + vpxor $c,$d,$t0 + vpxor @Xi[3],@Xi[1],@Xi[1] + + vmovdqu @Xi[0],`&Xi_off($i)` + vpaddd @Xi[0],$e,$e # e+=X[i] + vpor $t3,$t2,$t2 # rol(a,5) + vpsrld \$31,@Xi[1],$tx + vpand $b,$t0,$t0 + vpaddd @Xi[1],@Xi[1],@Xi[1] + + vpslld \$30,$b,$t1 + vpaddd $t0,$e,$e # e+=Maj(b,d,c) + + vpsrld \$2,$b,$b + vpaddd $t2,$e,$e # e+=rol(a,5) + vpor $tx,@Xi[1],@Xi[1] # rol(@X[1],1) + vpor $t1,$b,$b # b=rol(b,30) +___ +push(@Xi,shift(@Xi)); +} + +$code.=<<___; +.type sha1_multi_block_avx,\@function,3 +.align 32 +sha1_multi_block_avx: +_avx_shortcut: +___ +$code.=<<___ if ($avx>1); + shr \$32,%rcx + cmp \$2,$num + jb .Lavx + test \$`1<<5`,%ecx + jnz _avx2_shortcut + jmp .Lavx +.align 32 +.Lavx: +___ +$code.=<<___; + mov %rsp,%rax + push %rbx + push %rbp +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,-0x78(%rax) + movaps %xmm11,-0x68(%rax) + movaps %xmm12,-0x58(%rax) + movaps %xmm13,-0x48(%rax) + movaps %xmm14,-0x38(%rax) + movaps %xmm15,-0x28(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`, %rsp + and \$-256,%rsp + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody_avx: + lea K_XX_XX(%rip),$Tbl + lea `$REG_SZ*16`(%rsp),%rbx + + vzeroupper +.Loop_grande_avx: + mov $num,`$REG_SZ*17+8`(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldone_avx + + vmovdqu 0x00($ctx),$A # load context + lea 128(%rsp),%rax + vmovdqu 0x20($ctx),$B + vmovdqu 0x40($ctx),$C + vmovdqu 0x60($ctx),$D + vmovdqu 0x80($ctx),$E + vmovdqu 0x60($Tbl),$tx # pbswap_mask + jmp .Loop_avx + +.align 32 +.Loop_avx: +___ +$code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19 +for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); } +$code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39 +for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } +$code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59 +for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); } +$code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79 +for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + mov \$1,%ecx +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + cmp `4*$i`(%rbx),%ecx # examine counters + cmovge $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + vmovdqu (%rbx),$t0 # pull counters + vpxor $t2,$t2,$t2 + vmovdqa $t0,$t1 + vpcmpgtd $t2,$t1,$t1 # mask value + vpaddd $t1,$t0,$t0 # counters-- + + vpand $t1,$A,$A + vpand $t1,$B,$B + vpaddd 0x00($ctx),$A,$A + vpand $t1,$C,$C + vpaddd 0x20($ctx),$B,$B + vpand $t1,$D,$D + vpaddd 0x40($ctx),$C,$C + vpand $t1,$E,$E + vpaddd 0x60($ctx),$D,$D + vpaddd 0x80($ctx),$E,$E + vmovdqu $A,0x00($ctx) + vmovdqu $B,0x20($ctx) + vmovdqu $C,0x40($ctx) + vmovdqu $D,0x60($ctx) + vmovdqu $E,0x80($ctx) + + vmovdqu $t0,(%rbx) # save counters + vmovdqu 0x60($Tbl),$tx # pbswap_mask + dec $num + jnz .Loop_avx + + mov `$REG_SZ*17+8`(%rsp),$num + lea $REG_SZ($ctx),$ctx + lea `16*$REG_SZ/4`($inp),$inp + dec $num + jnz .Loop_grande_avx + +.Ldone_avx: + mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xb8(%rax),%xmm6 + movaps -0xa8(%rax),%xmm7 + movaps -0x98(%rax),%xmm8 + movaps -0x88(%rax),%xmm9 + movaps -0x78(%rax),%xmm10 + movaps -0x68(%rax),%xmm11 + movaps -0x58(%rax),%xmm12 + movaps -0x48(%rax),%xmm13 + movaps -0x38(%rax),%xmm14 + movaps -0x28(%rax),%xmm15 +___ +$code.=<<___; + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue_avx: + ret +.size sha1_multi_block_avx,.-sha1_multi_block_avx +___ + + if ($avx>1) { +$code =~ s/\`([^\`]*)\`/eval $1/gem; + +$REG_SZ=32; + +@ptr=map("%r$_",(12..15,8..11)); + +@V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4)); +($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9)); +@Xi=map("%ymm$_",(10..14)); +$K="%ymm15"; + +$code.=<<___; +.type sha1_multi_block_avx2,\@function,3 +.align 32 +sha1_multi_block_avx2: +_avx2_shortcut: + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,0x40(%rsp) + movaps %xmm11,0x50(%rsp) + movaps %xmm12,-0x78(%rax) + movaps %xmm13,-0x68(%rax) + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`, %rsp + and \$-256,%rsp + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody_avx2: + lea K_XX_XX(%rip),$Tbl + shr \$1,$num + + vzeroupper +.Loop_grande_avx2: + mov $num,`$REG_SZ*17+8`(%rsp) # original $num + xor $num,$num + lea `$REG_SZ*16`(%rsp),%rbx +___ +for($i=0;$i<8;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + vmovdqu 0x00($ctx),$A # load context + lea 128(%rsp),%rax + vmovdqu 0x20($ctx),$B + lea 256+128(%rsp),%rbx + vmovdqu 0x40($ctx),$C + vmovdqu 0x60($ctx),$D + vmovdqu 0x80($ctx),$E + vmovdqu 0x60($Tbl),$tx # pbswap_mask + jmp .Loop_avx2 + +.align 32 +.Loop_avx2: +___ +$code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19 +for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); } +$code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39 +for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } +$code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59 +for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); } +$code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79 +for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + mov \$1,%ecx + lea `$REG_SZ*16`(%rsp),%rbx +___ +for($i=0;$i<8;$i++) { + $code.=<<___; + cmp `4*$i`(%rbx),%ecx # examine counters + cmovge $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + vmovdqu (%rbx),$t0 # pull counters + vpxor $t2,$t2,$t2 + vmovdqa $t0,$t1 + vpcmpgtd $t2,$t1,$t1 # mask value + vpaddd $t1,$t0,$t0 # counters-- + + vpand $t1,$A,$A + vpand $t1,$B,$B + vpaddd 0x00($ctx),$A,$A + vpand $t1,$C,$C + vpaddd 0x20($ctx),$B,$B + vpand $t1,$D,$D + vpaddd 0x40($ctx),$C,$C + vpand $t1,$E,$E + vpaddd 0x60($ctx),$D,$D + vpaddd 0x80($ctx),$E,$E + vmovdqu $A,0x00($ctx) + vmovdqu $B,0x20($ctx) + vmovdqu $C,0x40($ctx) + vmovdqu $D,0x60($ctx) + vmovdqu $E,0x80($ctx) + + vmovdqu $t0,(%rbx) # save counters + lea 256+128(%rsp),%rbx + vmovdqu 0x60($Tbl),$tx # pbswap_mask + dec $num + jnz .Loop_avx2 + + #mov `$REG_SZ*17+8`(%rsp),$num + #lea $REG_SZ($ctx),$ctx + #lea `16*$REG_SZ/4`($inp),$inp + #dec $num + #jnz .Loop_grande_avx2 + +.Ldone_avx2: + mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue_avx2: + ret +.size sha1_multi_block_avx2,.-sha1_multi_block_avx2 +___ + } }}} +$code.=<<___; + +.align 256 + .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 + .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 +K_XX_XX: + .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39 + .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39 + .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 + .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 + .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 + .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap + .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 + .asciz "SHA1 multi-block transform for x86_64, CRYPTOGAMS by " +___ + +if ($win64) { +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # end of prologue label + cmp %r10,%rbx # context->Rip<.Lbody + jb .Lin_prologue + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lin_prologue + + mov `16*17`(%rax),%rax # pull saved stack pointer + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + + lea -24-10*16(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lin_prologue: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler +___ +$code.=<<___ if ($avx>1); +.type avx2_handler,\@abi-omnipotent +.align 16 +avx2_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # end of prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lin_prologue + + mov `32*17`($context),%rax # pull saved stack pointer + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore cotnext->R12 + mov %r13,224($context) # restore cotnext->R13 + mov %r14,232($context) # restore cotnext->R14 + mov %r15,240($context) # restore cotnext->R15 + + lea -56-10*16(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + + jmp .Lin_prologue +.size avx2_handler,.-avx2_handler +___ +$code.=<<___; +.section .pdata +.align 4 + .rva .LSEH_begin_sha1_multi_block + .rva .LSEH_end_sha1_multi_block + .rva .LSEH_info_sha1_multi_block + .rva .LSEH_begin_sha1_multi_block_shaext + .rva .LSEH_end_sha1_multi_block_shaext + .rva .LSEH_info_sha1_multi_block_shaext +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_sha1_multi_block_avx + .rva .LSEH_end_sha1_multi_block_avx + .rva .LSEH_info_sha1_multi_block_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_sha1_multi_block_avx2 + .rva .LSEH_end_sha1_multi_block_avx2 + .rva .LSEH_info_sha1_multi_block_avx2 +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_sha1_multi_block: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbody,.Lepilogue # HandlerData[] +.LSEH_info_sha1_multi_block_shaext: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbody_shaext,.Lepilogue_shaext # HandlerData[] +___ +$code.=<<___ if ($avx); +.LSEH_info_sha1_multi_block_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbody_avx,.Lepilogue_avx # HandlerData[] +___ +$code.=<<___ if ($avx>1); +.LSEH_info_sha1_multi_block_avx2: + .byte 9,0,0,0 + .rva avx2_handler + .rva .Lbody_avx2,.Lepilogue_avx2 # HandlerData[] +___ +} +#################################################################### + +sub rex { + local *opcode=shift; + my ($dst,$src)=@_; + my $rex=0; + + $rex|=0x04 if ($dst>=8); + $rex|=0x01 if ($src>=8); + unshift @opcode,$rex|0x40 if ($rex); +} + +sub sha1rnds4 { + if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x3a,0xcc); + rex(\@opcode,$3,$2); + push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + my $c=$1; + push @opcode,$c=~/^0/?oct($c):$c; + return ".byte\t".join(',',@opcode); + } else { + return "sha1rnds4\t".@_[0]; + } +} + +sub sha1op38 { + my $instr = shift; + my %opcodelet = ( + "sha1nexte" => 0xc8, + "sha1msg1" => 0xc9, + "sha1msg2" => 0xca ); + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x38); + rex(\@opcode,$2,$1); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + + s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or + s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo or + + s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go or + s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go or + s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go; + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/sha/asm/sha1-mips.pl b/crypto/sha/asm/sha1-mips.pl index 197bc6b50e95..340849389993 100755 --- a/crypto/sha/asm/sha1-mips.pl +++ b/crypto/sha/asm/sha1-mips.pl @@ -15,6 +15,10 @@ # compatible subroutine. There is room for minor optimization on # little-endian platforms... +# September 2012. +# +# Add MIPS32r2 code (>25% less instructions). + ###################################################################### # There is a number of MIPS ABI in use, O32 and N32/64 are most # widely used. Then there is a new contender: NUBI. It appears that if @@ -42,7 +46,7 @@ # ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); # ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); # -$flavour = shift; # supported flavours are o32,n32,64,nubi32,nubi64 +$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64 if ($flavour =~ /64|n32/i) { $PTR_ADD="dadd"; # incidentally works even on n32 @@ -95,6 +99,10 @@ sub BODY_00_14 { my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; $code.=<<___ if (!$big_endian); +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + wsbh @X[$i],@X[$i] # byte swap($i) + rotr @X[$i],@X[$i],16 +#else srl $t0,@X[$i],24 # byte swap($i) srl $t1,@X[$i],8 andi $t2,@X[$i],0xFF00 @@ -104,8 +112,22 @@ $code.=<<___ if (!$big_endian); or @X[$i],$t0 or $t1,$t2 or @X[$i],$t1 +#endif ___ $code.=<<___; +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + addu $e,$K # $i + xor $t0,$c,$d + rotr $t1,$a,27 + lwl @X[$j],$j*4+$MSB($inp) + and $t0,$b + addu $e,$t1 + lwr @X[$j],$j*4+$LSB($inp) + xor $t0,$d + addu $e,@X[$i] + rotr $b,$b,2 + addu $e,$t0 +#else lwl @X[$j],$j*4+$MSB($inp) sll $t0,$a,5 # $i addu $e,$K @@ -121,6 +143,7 @@ $code.=<<___; addu $e,@X[$i] or $b,$t2 addu $e,$t0 +#endif ___ } @@ -129,6 +152,10 @@ my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; $code.=<<___ if (!$big_endian && $i==15); +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + wsbh @X[$i],@X[$i] # byte swap($i) + rotr @X[$i],@X[$i],16 +#else srl $t0,@X[$i],24 # byte swap($i) srl $t1,@X[$i],8 andi $t2,@X[$i],0xFF00 @@ -138,8 +165,24 @@ $code.=<<___ if (!$big_endian && $i==15); or @X[$i],$t0 or @X[$i],$t1 or @X[$i],$t2 +#endif ___ $code.=<<___; +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + addu $e,$K # $i + xor @X[$j%16],@X[($j+2)%16] + xor $t0,$c,$d + rotr $t1,$a,27 + xor @X[$j%16],@X[($j+8)%16] + and $t0,$b + addu $e,$t1 + xor @X[$j%16],@X[($j+13)%16] + xor $t0,$d + addu $e,@X[$i%16] + rotr @X[$j%16],@X[$j%16],31 + rotr $b,$b,2 + addu $e,$t0 +#else xor @X[$j%16],@X[($j+2)%16] sll $t0,$a,5 # $i addu $e,$K @@ -159,6 +202,7 @@ $code.=<<___; addu $e,@X[$i%16] or $b,$t2 addu $e,$t0 +#endif ___ } @@ -166,6 +210,20 @@ sub BODY_20_39 { my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; $code.=<<___ if ($i<79); +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + xor @X[$j%16],@X[($j+2)%16] + addu $e,$K # $i + rotr $t1,$a,27 + xor @X[$j%16],@X[($j+8)%16] + xor $t0,$c,$d + addu $e,$t1 + xor @X[$j%16],@X[($j+13)%16] + xor $t0,$b + addu $e,@X[$i%16] + rotr @X[$j%16],@X[$j%16],31 + rotr $b,$b,2 + addu $e,$t0 +#else xor @X[$j%16],@X[($j+2)%16] sll $t0,$a,5 # $i addu $e,$K @@ -184,8 +242,24 @@ $code.=<<___ if ($i<79); or @X[$j%16],$t1 or $b,$t2 addu $e,$t0 +#endif ___ $code.=<<___ if ($i==79); +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + lw @X[0],0($ctx) + addu $e,$K # $i + lw @X[1],4($ctx) + rotr $t1,$a,27 + lw @X[2],8($ctx) + xor $t0,$c,$d + addu $e,$t1 + lw @X[3],12($ctx) + xor $t0,$b + addu $e,@X[$i%16] + lw @X[4],16($ctx) + rotr $b,$b,2 + addu $e,$t0 +#else lw @X[0],0($ctx) sll $t0,$a,5 # $i addu $e,$K @@ -203,6 +277,7 @@ $code.=<<___ if ($i==79); addu $e,@X[$i%16] or $b,$t2 addu $e,$t0 +#endif ___ } @@ -210,6 +285,22 @@ sub BODY_40_59 { my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; $code.=<<___ if ($i<79); +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + addu $e,$K # $i + and $t0,$c,$d + xor @X[$j%16],@X[($j+2)%16] + rotr $t1,$a,27 + addu $e,$t0 + xor @X[$j%16],@X[($j+8)%16] + xor $t0,$c,$d + addu $e,$t1 + xor @X[$j%16],@X[($j+13)%16] + and $t0,$b + addu $e,@X[$i%16] + rotr @X[$j%16],@X[$j%16],31 + rotr $b,$b,2 + addu $e,$t0 +#else xor @X[$j%16],@X[($j+2)%16] sll $t0,$a,5 # $i addu $e,$K @@ -230,6 +321,7 @@ $code.=<<___ if ($i<79); addu $e,@X[$i%16] or $b,$t2 addu $e,$t0 +#endif ___ } @@ -241,6 +333,10 @@ $code=<<___; # include #endif +#if defined(__mips_smartmips) && !defined(_MIPS_ARCH_MIPS32R2) +#define _MIPS_ARCH_MIPS32R2 +#endif + .text .set noat diff --git a/crypto/sha/asm/sha1-ppc.pl b/crypto/sha/asm/sha1-ppc.pl index 2140dd2f8dd6..df5989610c4c 100755 --- a/crypto/sha/asm/sha1-ppc.pl +++ b/crypto/sha/asm/sha1-ppc.pl @@ -9,8 +9,7 @@ # I let hardware handle unaligned input(*), except on page boundaries # (see below for details). Otherwise straightforward implementation -# with X vector in register bank. The module is big-endian [which is -# not big deal as there're no little-endian targets left around]. +# with X vector in register bank. # # (*) this means that this module is inappropriate for PPC403? Does # anybody know if pre-POWER3 can sustain unaligned load? @@ -38,6 +37,10 @@ if ($flavour =~ /64/) { $PUSH ="stw"; } else { die "nonsense $flavour"; } +# Define endianess based on flavour +# i.e.: linux64le +$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0; + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or @@ -68,14 +71,28 @@ $T ="r12"; @X=("r16","r17","r18","r19","r20","r21","r22","r23", "r24","r25","r26","r27","r28","r29","r30","r31"); +sub loadbe { +my ($dst, $src, $temp_reg) = @_; +$code.=<<___ if (!$LITTLE_ENDIAN); + lwz $dst,$src +___ +$code.=<<___ if ($LITTLE_ENDIAN); + lwz $temp_reg,$src + rotlwi $dst,$temp_reg,8 + rlwimi $dst,$temp_reg,24,0,7 + rlwimi $dst,$temp_reg,24,16,23 +___ +} + sub BODY_00_19 { my ($i,$a,$b,$c,$d,$e,$f)=@_; my $j=$i+1; -$code.=<<___ if ($i==0); - lwz @X[$i],`$i*4`($inp) -___ + + # Since the last value of $f is discarded, we can use + # it as a temp reg to swap byte-order when needed. + loadbe("@X[$i]","`$i*4`($inp)",$f) if ($i==0); + loadbe("@X[$j]","`$j*4`($inp)",$f) if ($i<15); $code.=<<___ if ($i<15); - lwz @X[$j],`$j*4`($inp) add $f,$K,$e rotlwi $e,$a,5 add $f,$f,@X[$i] @@ -108,31 +125,31 @@ my ($i,$a,$b,$c,$d,$e,$f)=@_; my $j=$i+1; $code.=<<___ if ($i<79); add $f,$K,$e + xor $t0,$b,$d rotlwi $e,$a,5 xor @X[$j%16],@X[$j%16],@X[($j+2)%16] add $f,$f,@X[$i%16] - xor $t0,$b,$c + xor $t0,$t0,$c xor @X[$j%16],@X[$j%16],@X[($j+8)%16] - add $f,$f,$e + add $f,$f,$t0 rotlwi $b,$b,30 - xor $t0,$t0,$d xor @X[$j%16],@X[$j%16],@X[($j+13)%16] - add $f,$f,$t0 + add $f,$f,$e rotlwi @X[$j%16],@X[$j%16],1 ___ $code.=<<___ if ($i==79); add $f,$K,$e + xor $t0,$b,$d rotlwi $e,$a,5 lwz r16,0($ctx) add $f,$f,@X[$i%16] - xor $t0,$b,$c + xor $t0,$t0,$c lwz r17,4($ctx) - add $f,$f,$e + add $f,$f,$t0 rotlwi $b,$b,30 lwz r18,8($ctx) - xor $t0,$t0,$d lwz r19,12($ctx) - add $f,$f,$t0 + add $f,$f,$e lwz r20,16($ctx) ___ } @@ -316,6 +333,7 @@ $code.=<<___; blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size .sha1_block_data_order,.-.sha1_block_data_order ___ $code.=<<___; .asciz "SHA1 block transform for PPC, CRYPTOGAMS by " diff --git a/crypto/sha/asm/sha1-sparcv9.pl b/crypto/sha/asm/sha1-sparcv9.pl index 5c161cecd696..b5efcde5c139 100755 --- a/crypto/sha/asm/sha1-sparcv9.pl +++ b/crypto/sha/asm/sha1-sparcv9.pl @@ -5,6 +5,8 @@ # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. +# +# Hardware SPARC T4 support by David S. Miller . # ==================================================================== # Performance improvement is not really impressive on pre-T1 CPU: +8% @@ -18,10 +20,10 @@ # ensure scalability on UltraSPARC T1, or rather to avoid decay when # amount of active threads exceeds the number of physical cores. -$bits=32; -for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } -if ($bits==64) { $bias=2047; $frame=192; } -else { $bias=0; $frame=112; } +# SPARC T4 SHA1 hardware achieves 3.72 cycles per byte, which is 3.1x +# faster than software. Multi-process benchmark saturates at 11x +# single-process result on 8-core processor, or ~9GBps per 2.85GHz +# socket. $output=shift; open STDOUT,">$output"; @@ -178,17 +180,102 @@ $code.=<<___; ___ } -$code.=<<___ if ($bits==64); +$code.=<<___; +#include "sparc_arch.h" + +#ifdef __arch64__ .register %g2,#scratch .register %g3,#scratch -___ -$code.=<<___; +#endif + .section ".text",#alloc,#execinstr +#ifdef __PIC__ +SPARC_PIC_THUNK(%g1) +#endif + .align 32 .globl sha1_block_data_order sha1_block_data_order: - save %sp,-$frame,%sp + SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5) + ld [%g1+4],%g1 ! OPENSSL_sparcv9cap_P[1] + + andcc %g1, CFR_SHA1, %g0 + be .Lsoftware + nop + + ld [%o0 + 0x00], %f0 ! load context + ld [%o0 + 0x04], %f1 + ld [%o0 + 0x08], %f2 + andcc %o1, 0x7, %g0 + ld [%o0 + 0x0c], %f3 + bne,pn %icc, .Lhwunaligned + ld [%o0 + 0x10], %f4 + +.Lhw_loop: + ldd [%o1 + 0x00], %f8 + ldd [%o1 + 0x08], %f10 + ldd [%o1 + 0x10], %f12 + ldd [%o1 + 0x18], %f14 + ldd [%o1 + 0x20], %f16 + ldd [%o1 + 0x28], %f18 + ldd [%o1 + 0x30], %f20 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x38], %f22 + add %o1, 0x40, %o1 + prefetch [%o1 + 63], 20 + + .word 0x81b02820 ! SHA1 + + bne,pt SIZE_T_CC, .Lhw_loop + nop + +.Lhwfinish: + st %f0, [%o0 + 0x00] ! store context + st %f1, [%o0 + 0x04] + st %f2, [%o0 + 0x08] + st %f3, [%o0 + 0x0c] + retl + st %f4, [%o0 + 0x10] + +.align 8 +.Lhwunaligned: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f10 +.Lhwunaligned_loop: + ldd [%o1 + 0x08], %f12 + ldd [%o1 + 0x10], %f14 + ldd [%o1 + 0x18], %f16 + ldd [%o1 + 0x20], %f18 + ldd [%o1 + 0x28], %f20 + ldd [%o1 + 0x30], %f22 + ldd [%o1 + 0x38], %f24 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x40], %f26 + add %o1, 0x40, %o1 + prefetch [%o1 + 63], 20 + + faligndata %f10, %f12, %f8 + faligndata %f12, %f14, %f10 + faligndata %f14, %f16, %f12 + faligndata %f16, %f18, %f14 + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + + .word 0x81b02820 ! SHA1 + + bne,pt SIZE_T_CC, .Lhwunaligned_loop + for %f26, %f26, %f10 ! %f10=%f26 + + ba .Lhwfinish + nop + +.align 16 +.Lsoftware: + save %sp,-STACK_FRAME,%sp sllx $len,6,$len add $inp,$len,$len @@ -268,7 +355,7 @@ $code.=<<___; add $E,@X[4],$E st $E,[$ctx+16] - bne `$bits==64?"%xcc":"%icc"`,.Lloop + bne SIZE_T_CC,.Lloop andn $inp,7,$tmp0 ret @@ -279,6 +366,62 @@ $code.=<<___; .align 4 ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -print $code; +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my $ref,$opf; +my %visopf = ( "faligndata" => 0x048, + "for" => 0x07c ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} +sub unalignaddr { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my $ref="$mnemonic\t$rs1,$rs2,$rd"; + + foreach ($rs1,$rs2,$rd) { + if (/%([goli])([0-7])/) { $_=$bias{$1}+$2; } + else { return $ref; } + } + return sprintf ".word\t0x%08x !%s", + 0x81b00300|$rd<<25|$rs1<<14|$rs2, + $ref; +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/\b(f[^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ + &unvis($1,$2,$3,$4) + /ge; + s/\b(alignaddr)\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ + &unalignaddr($1,$2,$3,$4) + /ge; + + print $_,"\n"; +} + close STDOUT; diff --git a/crypto/sha/asm/sha1-x86_64.pl b/crypto/sha/asm/sha1-x86_64.pl index f15c7ec39b20..9bb6b498190f 100755 --- a/crypto/sha/asm/sha1-x86_64.pl +++ b/crypto/sha/asm/sha1-x86_64.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -49,17 +49,37 @@ # # Add AVX code path. See sha1-586.pl for further information. +# May 2013. +# +# Add AVX2+BMI code path. Initial attempt (utilizing BMI instructions +# and loading pair of consecutive blocks to 256-bit %ymm registers) +# did not provide impressive performance improvement till a crucial +# hint regarding the number of Xupdate iterations to pre-compute in +# advance was provided by Ilya Albrekht of Intel Corp. + +# March 2014. +# +# Add support for Intel SHA Extensions. + ###################################################################### # Current performance is summarized in following table. Numbers are # CPU clock cycles spent to process single byte (less is better). # -# x86_64 SSSE3 AVX -# P4 9.8 - -# Opteron 6.6 - -# Core2 6.7 6.1/+10% - -# Atom 11.0 9.7/+13% - -# Westmere 7.1 5.6/+27% - -# Sandy Bridge 7.9 6.3/+25% 5.2/+51% +# x86_64 SSSE3 AVX[2] +# P4 9.05 - +# Opteron 6.26 - +# Core2 6.55 6.05/+8% - +# Westmere 6.73 5.30/+27% - +# Sandy Bridge 7.70 6.10/+26% 4.99/+54% +# Ivy Bridge 6.06 4.67/+30% 4.60/+32% +# Haswell 5.45 4.15/+31% 3.57/+53% +# Bulldozer 9.11 5.95/+53% +# VIA Nano 9.32 7.15/+30% +# Atom 10.3 9.17/+12% +# Silvermont 13.1(*) 9.37/+40% +# +# (*) obviously suboptimal result, nothing was done about it, +# because SSSE3 code is compiled unconditionally; $flavour = shift; $output = shift; @@ -72,15 +92,27 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or die "can't locate x86_64-xlate.pl"; -$avx=1 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` - =~ /GNU assembler version ([2-9]\.[0-9]+)/ && - $1>=2.19); -$avx=1 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && - `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ && - $1>=2.09); -$avx=1 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && - `ml64 2>&1` =~ /Version ([0-9]+)\./ && - $1>=10); +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([2-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +$shaext=1; ### set to zero if compiling for 1.0.1 +$avx=1 if (!$shaext && $avx); open OUT,"| \"$^X\" $xlate $flavour $output"; *STDOUT=*OUT; @@ -97,7 +129,7 @@ $num="%r10"; $t0="%eax"; $t1="%ebx"; $t2="%ecx"; -@xi=("%edx","%ebp"); +@xi=("%edx","%ebp","%r14d"); $A="%esi"; $B="%edi"; $C="%r11d"; @@ -112,42 +144,40 @@ my $j=$i+1; $code.=<<___ if ($i==0); mov `4*$i`($inp),$xi[0] bswap $xi[0] - mov $xi[0],`4*$i`(%rsp) ___ $code.=<<___ if ($i<15); - mov $c,$t0 mov `4*$j`($inp),$xi[1] + mov $d,$t0 + mov $xi[0],`4*$i`(%rsp) mov $a,$t2 - xor $d,$t0 bswap $xi[1] + xor $c,$t0 rol \$5,$t2 - lea 0x5a827999($xi[0],$e),$e and $b,$t0 - mov $xi[1],`4*$j`(%rsp) + lea 0x5a827999($xi[0],$e),$e add $t2,$e xor $d,$t0 rol \$30,$b add $t0,$e ___ $code.=<<___ if ($i>=15); - mov `4*($j%16)`(%rsp),$xi[1] - mov $c,$t0 + xor `4*($j%16)`(%rsp),$xi[1] + mov $d,$t0 + mov $xi[0],`4*($i%16)`(%rsp) mov $a,$t2 xor `4*(($j+2)%16)`(%rsp),$xi[1] - xor $d,$t0 + xor $c,$t0 rol \$5,$t2 xor `4*(($j+8)%16)`(%rsp),$xi[1] and $b,$t0 lea 0x5a827999($xi[0],$e),$e - xor `4*(($j+13)%16)`(%rsp),$xi[1] + rol \$30,$b xor $d,$t0 - rol \$1,$xi[1] add $t2,$e - rol \$30,$b - mov $xi[1],`4*($j%16)`(%rsp) + rol \$1,$xi[1] add $t0,$e ___ -unshift(@xi,pop(@xi)); +push(@xi,shift(@xi)); } sub BODY_20_39 { @@ -155,62 +185,58 @@ my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; my $K=($i<40)?0x6ed9eba1:0xca62c1d6; $code.=<<___ if ($i<79); - mov `4*($j%16)`(%rsp),$xi[1] - mov $c,$t0 + xor `4*($j%16)`(%rsp),$xi[1] + mov $b,$t0 + `"mov $xi[0],".4*($i%16)."(%rsp)" if ($i<72)` mov $a,$t2 xor `4*(($j+2)%16)`(%rsp),$xi[1] - xor $b,$t0 + xor $d,$t0 rol \$5,$t2 - lea $K($xi[0],$e),$e xor `4*(($j+8)%16)`(%rsp),$xi[1] - xor $d,$t0 + lea $K($xi[0],$e),$e + xor $c,$t0 add $t2,$e - xor `4*(($j+13)%16)`(%rsp),$xi[1] rol \$30,$b add $t0,$e rol \$1,$xi[1] ___ -$code.=<<___ if ($i<76); - mov $xi[1],`4*($j%16)`(%rsp) -___ $code.=<<___ if ($i==79); - mov $c,$t0 + mov $b,$t0 mov $a,$t2 - xor $b,$t0 + xor $d,$t0 lea $K($xi[0],$e),$e rol \$5,$t2 - xor $d,$t0 + xor $c,$t0 add $t2,$e rol \$30,$b add $t0,$e ___ -unshift(@xi,pop(@xi)); +push(@xi,shift(@xi)); } sub BODY_40_59 { my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; $code.=<<___; - mov `4*($j%16)`(%rsp),$xi[1] - mov $c,$t0 - mov $c,$t1 + xor `4*($j%16)`(%rsp),$xi[1] + mov $d,$t0 + mov $xi[0],`4*($i%16)`(%rsp) + mov $d,$t1 xor `4*(($j+2)%16)`(%rsp),$xi[1] - and $d,$t0 + and $c,$t0 mov $a,$t2 xor `4*(($j+8)%16)`(%rsp),$xi[1] - xor $d,$t1 lea 0x8f1bbcdc($xi[0],$e),$e + xor $c,$t1 rol \$5,$t2 - xor `4*(($j+13)%16)`(%rsp),$xi[1] add $t0,$e - and $b,$t1 rol \$1,$xi[1] - add $t1,$e - rol \$30,$b - mov $xi[1],`4*($j%16)`(%rsp) + and $b,$t1 add $t2,$e + rol \$30,$b + add $t1,$e ___ -unshift(@xi,pop(@xi)); +push(@xi,shift(@xi)); } $code.=<<___; @@ -223,9 +249,19 @@ $code.=<<___; sha1_block_data_order: mov OPENSSL_ia32cap_P+0(%rip),%r9d mov OPENSSL_ia32cap_P+4(%rip),%r8d + mov OPENSSL_ia32cap_P+8(%rip),%r10d test \$`1<<9`,%r8d # check SSSE3 bit jz .Lialu ___ +$code.=<<___ if ($shaext); + test \$`1<<29`,%r10d # check SHA bit + jnz _shaext_shortcut +___ +$code.=<<___ if ($avx>1); + and \$`1<<3|1<<5|1<<8`,%r10d # check AVX2+BMI1+BMI2 + cmp \$`1<<3|1<<5|1<<8`,%r10d + je _avx2_shortcut +___ $code.=<<___ if ($avx); and \$`1<<28`,%r8d # mask AVX bit and \$`1<<30`,%r9d # mask "Intel CPU" bit @@ -238,17 +274,18 @@ $code.=<<___; .align 16 .Lialu: + mov %rsp,%rax push %rbx push %rbp push %r12 push %r13 - mov %rsp,%r11 + push %r14 mov %rdi,$ctx # reassigned argument sub \$`8+16*4`,%rsp mov %rsi,$inp # reassigned argument and \$-64,%rsp mov %rdx,$num # reassigned argument - mov %r11,`16*4`(%rsp) + mov %rax,`16*4`(%rsp) .Lprologue: mov 0($ctx),$A @@ -282,53 +319,187 @@ $code.=<<___; jnz .Lloop mov `16*4`(%rsp),%rsi - mov (%rsi),%r13 - mov 8(%rsi),%r12 - mov 16(%rsi),%rbp - mov 24(%rsi),%rbx - lea 32(%rsi),%rsp + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp .Lepilogue: ret .size sha1_block_data_order,.-sha1_block_data_order ___ +if ($shaext) {{{ +###################################################################### +# Intel SHA Extensions implementation of SHA1 update function. +# +my ($ctx,$inp,$num)=("%rdi","%rsi","%rdx"); +my ($ABCD,$E,$E_,$BSWAP,$ABCD_SAVE,$E_SAVE)=map("%xmm$_",(0..3,8,9)); +my @MSG=map("%xmm$_",(4..7)); + +$code.=<<___; +.type sha1_block_data_order_shaext,\@function,3 +.align 32 +sha1_block_data_order_shaext: +_shaext_shortcut: +___ +$code.=<<___ if ($win64); + lea `-8-4*16`(%rsp),%rsp + movaps %xmm6,-8-4*16(%rax) + movaps %xmm7,-8-3*16(%rax) + movaps %xmm8,-8-2*16(%rax) + movaps %xmm9,-8-1*16(%rax) +.Lprologue_shaext: +___ +$code.=<<___; + movdqu ($ctx),$ABCD + movd 16($ctx),$E + movdqa K_XX_XX+0xa0(%rip),$BSWAP # byte-n-word swap + + movdqu ($inp),@MSG[0] + pshufd \$0b00011011,$ABCD,$ABCD # flip word order + movdqu 0x10($inp),@MSG[1] + pshufd \$0b00011011,$E,$E # flip word order + movdqu 0x20($inp),@MSG[2] + pshufb $BSWAP,@MSG[0] + movdqu 0x30($inp),@MSG[3] + pshufb $BSWAP,@MSG[1] + pshufb $BSWAP,@MSG[2] + movdqa $E,$E_SAVE # offload $E + pshufb $BSWAP,@MSG[3] + jmp .Loop_shaext + +.align 16 +.Loop_shaext: + dec $num + lea 0x40($inp),%rax # next input block + paddd @MSG[0],$E + cmovne %rax,$inp + movdqa $ABCD,$ABCD_SAVE # offload $ABCD +___ +for($i=0;$i<20-4;$i+=2) { +$code.=<<___; + sha1msg1 @MSG[1],@MSG[0] + movdqa $ABCD,$E_ + sha1rnds4 \$`int($i/5)`,$E,$ABCD # 0-3... + sha1nexte @MSG[1],$E_ + pxor @MSG[2],@MSG[0] + sha1msg1 @MSG[2],@MSG[1] + sha1msg2 @MSG[3],@MSG[0] + + movdqa $ABCD,$E + sha1rnds4 \$`int(($i+1)/5)`,$E_,$ABCD + sha1nexte @MSG[2],$E + pxor @MSG[3],@MSG[1] + sha1msg2 @MSG[0],@MSG[1] +___ + push(@MSG,shift(@MSG)); push(@MSG,shift(@MSG)); +} +$code.=<<___; + movdqu ($inp),@MSG[0] + movdqa $ABCD,$E_ + sha1rnds4 \$3,$E,$ABCD # 64-67 + sha1nexte @MSG[1],$E_ + movdqu 0x10($inp),@MSG[1] + pshufb $BSWAP,@MSG[0] + + movdqa $ABCD,$E + sha1rnds4 \$3,$E_,$ABCD # 68-71 + sha1nexte @MSG[2],$E + movdqu 0x20($inp),@MSG[2] + pshufb $BSWAP,@MSG[1] + + movdqa $ABCD,$E_ + sha1rnds4 \$3,$E,$ABCD # 72-75 + sha1nexte @MSG[3],$E_ + movdqu 0x30($inp),@MSG[3] + pshufb $BSWAP,@MSG[2] + + movdqa $ABCD,$E + sha1rnds4 \$3,$E_,$ABCD # 76-79 + sha1nexte $E_SAVE,$E + pshufb $BSWAP,@MSG[3] + + paddd $ABCD_SAVE,$ABCD + movdqa $E,$E_SAVE # offload $E + + jnz .Loop_shaext + + pshufd \$0b00011011,$ABCD,$ABCD + pshufd \$0b00011011,$E,$E + movdqu $ABCD,($ctx) + movd $E,16($ctx) +___ +$code.=<<___ if ($win64); + movaps -8-4*16(%rax),%xmm6 + movaps -8-3*16(%rax),%xmm7 + movaps -8-2*16(%rax),%xmm8 + movaps -8-1*16(%rax),%xmm9 + mov %rax,%rsp +.Lepilogue_shaext: +___ +$code.=<<___; + ret +.size sha1_block_data_order_shaext,.-sha1_block_data_order_shaext +___ +}}} {{{ my $Xi=4; my @X=map("%xmm$_",(4..7,0..3)); my @Tx=map("%xmm$_",(8..10)); +my $Kx="%xmm11"; my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization my @T=("%esi","%edi"); my $j=0; +my $rx=0; my $K_XX_XX="%r11"; my $_rol=sub { &rol(@_) }; my $_ror=sub { &ror(@_) }; +{ my $sn; +sub align32() { + ++$sn; +$code.=<<___; + jmp .Lalign32_$sn # see "Decoded ICache" in manual +.align 32 +.Lalign32_$sn: +___ +} +} + $code.=<<___; .type sha1_block_data_order_ssse3,\@function,3 .align 16 sha1_block_data_order_ssse3: _ssse3_shortcut: + mov %rsp,%rax push %rbx push %rbp push %r12 - lea `-64-($win64?5*16:0)`(%rsp),%rsp + push %r13 # redundant, done to share Win64 SE handler + push %r14 + lea `-64-($win64?6*16:0)`(%rsp),%rsp ___ $code.=<<___ if ($win64); - movaps %xmm6,64+0(%rsp) - movaps %xmm7,64+16(%rsp) - movaps %xmm8,64+32(%rsp) - movaps %xmm9,64+48(%rsp) - movaps %xmm10,64+64(%rsp) + movaps %xmm6,-40-6*16(%rax) + movaps %xmm7,-40-5*16(%rax) + movaps %xmm8,-40-4*16(%rax) + movaps %xmm9,-40-3*16(%rax) + movaps %xmm10,-40-2*16(%rax) + movaps %xmm11,-40-1*16(%rax) .Lprologue_ssse3: ___ $code.=<<___; + mov %rax,%r14 # original %rsp + and \$-64,%rsp mov %rdi,$ctx # reassigned argument mov %rsi,$inp # reassigned argument mov %rdx,$num # reassigned argument shl \$6,$num add $inp,$num - lea K_XX_XX(%rip),$K_XX_XX + lea K_XX_XX+64(%rip),$K_XX_XX mov 0($ctx),$A # load context mov 4($ctx),$B @@ -336,19 +507,22 @@ $code.=<<___; mov 12($ctx),$D mov $B,@T[0] # magic seed mov 16($ctx),$E + mov $C,@T[1] + xor $D,@T[1] + and @T[1],@T[0] movdqa 64($K_XX_XX),@X[2] # pbswap mask - movdqa 0($K_XX_XX),@Tx[1] # K_00_19 + movdqa -64($K_XX_XX),@Tx[1] # K_00_19 movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] movdqu 16($inp),@X[-3&7] movdqu 32($inp),@X[-2&7] movdqu 48($inp),@X[-1&7] pshufb @X[2],@X[-4&7] # byte swap - add \$64,$inp pshufb @X[2],@X[-3&7] pshufb @X[2],@X[-2&7] - pshufb @X[2],@X[-1&7] + add \$64,$inp paddd @Tx[1],@X[-4&7] # add K_00_19 + pshufb @X[2],@X[-1&7] paddd @Tx[1],@X[-3&7] paddd @Tx[1],@X[-2&7] movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU @@ -373,61 +547,61 @@ sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 my @insns = (&$body,&$body,&$body,&$body); # 40 instructions my ($a,$b,$c,$d,$e); - &movdqa (@X[0],@X[-3&7]); - eval(shift(@insns)); + eval(shift(@insns)); # ror + &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]); eval(shift(@insns)); &movdqa (@Tx[0],@X[-1&7]); - &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]" + &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); - &paddd (@Tx[1],@X[-1&7]); + &punpcklqdq(@X[0],@X[-3&7]); # compose "X[-14]" in "X[0]", was &palignr(@X[0],@X[-4&7],8); eval(shift(@insns)); + eval(shift(@insns)); # rol eval(shift(@insns)); &psrldq (@Tx[0],4); # "X[-3]", 3 dwords eval(shift(@insns)); eval(shift(@insns)); + &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]" eval(shift(@insns)); - eval(shift(@insns)); - + eval(shift(@insns)); # ror &pxor (@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]" eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - eval(shift(@insns)); &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]" eval(shift(@insns)); - eval(shift(@insns)); + eval(shift(@insns)); # rol &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); &movdqa (@Tx[2],@X[0]); - &movdqa (@Tx[0],@X[0]); - eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); # ror + &movdqa (@Tx[0],@X[0]); eval(shift(@insns)); &pslldq (@Tx[2],12); # "X[0]"<<96, extract one dword &paddd (@X[0],@X[0]); eval(shift(@insns)); eval(shift(@insns)); - eval(shift(@insns)); - eval(shift(@insns)); &psrld (@Tx[0],31); eval(shift(@insns)); + eval(shift(@insns)); # rol eval(shift(@insns)); &movdqa (@Tx[1],@Tx[2]); eval(shift(@insns)); eval(shift(@insns)); &psrld (@Tx[2],30); - &por (@X[0],@Tx[0]); # "X[0]"<<<=1 eval(shift(@insns)); + eval(shift(@insns)); # ror + &por (@X[0],@Tx[0]); # "X[0]"<<<=1 eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); @@ -435,12 +609,13 @@ sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 &pslld (@Tx[1],2); &pxor (@X[0],@Tx[2]); eval(shift(@insns)); - eval(shift(@insns)); - &movdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX + &movdqa (@Tx[2],eval(2*16*(($Xi)/5)-64)."($K_XX_XX)"); # K_XX_XX + eval(shift(@insns)); # rol eval(shift(@insns)); eval(shift(@insns)); &pxor (@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2 + &pshufd (@Tx[1],@X[-1&7],0xee) if ($Xi==7); # was &movdqa (@Tx[0],@X[-1&7]) in Xupdate_ssse3_32_79 foreach (@insns) { eval; } # remaining instructions [if any] @@ -451,27 +626,30 @@ sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 sub Xupdate_ssse3_32_79() { use integer; my $body = shift; - my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions my ($a,$b,$c,$d,$e); - &movdqa (@Tx[0],@X[-1&7]) if ($Xi==8); - eval(shift(@insns)); # body_20_39 + eval(shift(@insns)) if ($Xi==8); &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" - &palignr(@Tx[0],@X[-2&7],8); # compose "X[-6]" + eval(shift(@insns)) if ($Xi==8); + eval(shift(@insns)); # body_20_39 eval(shift(@insns)); + eval(shift(@insns)) if (@insns[1] =~ /_ror/); + eval(shift(@insns)) if (@insns[0] =~ /_ror/); + &punpcklqdq(@Tx[0],@X[-1&7]); # compose "X[-6]", was &palignr(@Tx[0],@X[-2&7],8); eval(shift(@insns)); eval(shift(@insns)); # rol &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]" eval(shift(@insns)); - eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/); + eval(shift(@insns)); if ($Xi%5) { &movdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX... } else { # ... or load next one - &movdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)"); + &movdqa (@Tx[2],eval(2*16*($Xi/5)-64)."($K_XX_XX)"); } - &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); # ror + &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-6]" @@ -479,29 +657,31 @@ sub Xupdate_ssse3_32_79() eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); # rol + eval(shift(@insns)) if (@insns[0] =~ /_ror/); &movdqa (@Tx[0],@X[0]); - &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); + &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU eval(shift(@insns)); # ror eval(shift(@insns)); + eval(shift(@insns)); # body_20_39 &pslld (@X[0],2); - eval(shift(@insns)); # body_20_39 eval(shift(@insns)); - &psrld (@Tx[0],30); eval(shift(@insns)); - eval(shift(@insns)); # rol + &psrld (@Tx[0],30); + eval(shift(@insns)) if (@insns[0] =~ /_rol/);# rol eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); # ror - eval(shift(@insns)); &por (@X[0],@Tx[0]); # "X[0]"<<<=2 - eval(shift(@insns)); # body_20_39 eval(shift(@insns)); - &movdqa (@Tx[1],@X[0]) if ($Xi<19); + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)) if (@insns[1] =~ /_rol/); + eval(shift(@insns)) if (@insns[0] =~ /_rol/); + &pshufd(@Tx[1],@X[-1&7],0xee) if ($Xi<19); # was &movdqa (@Tx[1],@X[0]) eval(shift(@insns)); eval(shift(@insns)); # rol eval(shift(@insns)); @@ -522,9 +702,10 @@ sub Xuplast_ssse3_80() my ($a,$b,$c,$d,$e); eval(shift(@insns)); - &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@Tx[1],@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); @@ -538,7 +719,7 @@ sub Xuplast_ssse3_80() unshift(@Tx,pop(@Tx)); &movdqa (@X[2],"64($K_XX_XX)"); # pbswap mask - &movdqa (@Tx[1],"0($K_XX_XX)"); # K_00_19 + &movdqa (@Tx[1],"-64($K_XX_XX)"); # K_00_19 &movdqu (@X[-4&7],"0($inp)"); # load input &movdqu (@X[-3&7],"16($inp)"); &movdqu (@X[-2&7],"32($inp)"); @@ -557,9 +738,12 @@ sub Xloop_ssse3() eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); &pshufb (@X[($Xi-3)&7],@X[2]); eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); &paddd (@X[($Xi-4)&7],@Tx[1]); eval(shift(@insns)); eval(shift(@insns)); @@ -568,6 +752,8 @@ sub Xloop_ssse3() &movdqa (eval(16*$Xi)."(%rsp)",@X[($Xi-4)&7]); # X[]+K xfer to IALU eval(shift(@insns)); eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); &psubd (@X[($Xi-4)&7],@Tx[1]); foreach (@insns) { eval; } @@ -583,51 +769,66 @@ sub Xtail_ssse3() foreach (@insns) { eval; } } -sub body_00_19 () { +sub body_00_19 () { # ((c^d)&b)^d + # on start @T[0]=(c^d)&b + return &body_20_39() if ($rx==19); $rx++; ( '($a,$b,$c,$d,$e)=@V;'. - '&add ($e,eval(4*($j&15))."(%rsp)");', # X[]+K xfer - '&xor ($c,$d);', - '&mov (@T[1],$a);', # $b in next round - '&$_rol ($a,5);', - '&and (@T[0],$c);', # ($b&($c^$d)) - '&xor ($c,$d);', # restore $c - '&xor (@T[0],$d);', - '&add ($e,$a);', - '&$_ror ($b,$j?7:2);', # $b>>>2 - '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&$_ror ($b,$j?7:2)', # $b>>>2 + '&xor (@T[0],$d)', + '&mov (@T[1],$a)', # $b for next round + + '&add ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer + '&xor ($b,$c)', # $c^$d for next round + + '&$_rol ($a,5)', + '&add ($e,@T[0])', + '&and (@T[1],$b)', # ($b&($c^$d)) for next round + + '&xor ($b,$c)', # restore $b + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); } -sub body_20_39 () { +sub body_20_39 () { # b^d^c + # on entry @T[0]=b^d + return &body_40_59() if ($rx==39); $rx++; ( '($a,$b,$c,$d,$e)=@V;'. - '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer - '&xor (@T[0],$d);', # ($b^$d) - '&mov (@T[1],$a);', # $b in next round - '&$_rol ($a,5);', - '&xor (@T[0],$c);', # ($b^$d^$c) - '&add ($e,$a);', - '&$_ror ($b,7);', # $b>>>2 - '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&add ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer + '&xor (@T[0],$d) if($j==19);'. + '&xor (@T[0],$c) if($j> 19)', # ($b^$d^$c) + '&mov (@T[1],$a)', # $b for next round + + '&$_rol ($a,5)', + '&add ($e,@T[0])', + '&xor (@T[1],$c) if ($j< 79)', # $b^$d for next round + + '&$_ror ($b,7)', # $b>>>2 + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); } -sub body_40_59 () { +sub body_40_59 () { # ((b^c)&(c^d))^c + # on entry @T[0]=(b^c), (c^=d) + $rx++; ( '($a,$b,$c,$d,$e)=@V;'. - '&mov (@T[1],$c);', - '&xor ($c,$d);', - '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer - '&and (@T[1],$d);', - '&and (@T[0],$c);', # ($b&($c^$d)) - '&$_ror ($b,7);', # $b>>>2 - '&add ($e,@T[1]);', - '&mov (@T[1],$a);', # $b in next round - '&$_rol ($a,5);', - '&add ($e,@T[0]);', - '&xor ($c,$d);', # restore $c - '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + '&add ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer + '&and (@T[0],$c) if ($j>=40)', # (b^c)&(c^d) + '&xor ($c,$d) if ($j>=40)', # restore $c + + '&$_ror ($b,7)', # $b>>>2 + '&mov (@T[1],$a)', # $b for next round + '&xor (@T[0],$c)', + + '&$_rol ($a,5)', + '&add ($e,@T[0])', + '&xor (@T[1],$c) if ($j==59);'. + '&xor (@T[1],$b) if ($j< 59)', # b^c for next round + + '&xor ($b,$c) if ($j< 59)', # c^d for next round + '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' ); } $code.=<<___; @@ -668,8 +869,11 @@ $code.=<<___; mov @T[0],4($ctx) mov @T[0],$B # magic seed mov $C,8($ctx) + mov $C,@T[1] mov $D,12($ctx) + xor $D,@T[1] mov $E,16($ctx) + and @T[1],@T[0] jmp .Loop_ssse3 .align 16 @@ -694,31 +898,34 @@ $code.=<<___; mov $E,16($ctx) ___ $code.=<<___ if ($win64); - movaps 64+0(%rsp),%xmm6 - movaps 64+16(%rsp),%xmm7 - movaps 64+32(%rsp),%xmm8 - movaps 64+48(%rsp),%xmm9 - movaps 64+64(%rsp),%xmm10 + movaps -40-6*16(%r14),%xmm6 + movaps -40-5*16(%r14),%xmm7 + movaps -40-4*16(%r14),%xmm8 + movaps -40-3*16(%r14),%xmm9 + movaps -40-2*16(%r14),%xmm10 + movaps -40-1*16(%r14),%xmm11 ___ $code.=<<___; - lea `64+($win64?5*16:0)`(%rsp),%rsi - mov 0(%rsi),%r12 - mov 8(%rsi),%rbp - mov 16(%rsi),%rbx - lea 24(%rsi),%rsp + lea (%r14),%rsi + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp .Lepilogue_ssse3: ret .size sha1_block_data_order_ssse3,.-sha1_block_data_order_ssse3 ___ if ($avx) { -my $Xi=4; -my @X=map("%xmm$_",(4..7,0..3)); -my @Tx=map("%xmm$_",(8..10)); -my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization -my @T=("%esi","%edi"); -my $j=0; -my $K_XX_XX="%r11"; +$Xi=4; # reset variables +@X=map("%xmm$_",(4..7,0..3)); +@Tx=map("%xmm$_",(8..10)); +$j=0; +$rx=0; + +my $done_avx_label=".Ldone_avx"; my $_rol=sub { &shld(@_[0],@_) }; my $_ror=sub { &shrd(@_[0],@_) }; @@ -728,28 +935,34 @@ $code.=<<___; .align 16 sha1_block_data_order_avx: _avx_shortcut: + mov %rsp,%rax push %rbx push %rbp push %r12 - lea `-64-($win64?5*16:0)`(%rsp),%rsp + push %r13 # redundant, done to share Win64 SE handler + push %r14 + lea `-64-($win64?6*16:0)`(%rsp),%rsp + vzeroupper ___ $code.=<<___ if ($win64); - movaps %xmm6,64+0(%rsp) - movaps %xmm7,64+16(%rsp) - movaps %xmm8,64+32(%rsp) - movaps %xmm9,64+48(%rsp) - movaps %xmm10,64+64(%rsp) + vmovaps %xmm6,-40-6*16(%rax) + vmovaps %xmm7,-40-5*16(%rax) + vmovaps %xmm8,-40-4*16(%rax) + vmovaps %xmm9,-40-3*16(%rax) + vmovaps %xmm10,-40-2*16(%rax) + vmovaps %xmm11,-40-1*16(%rax) .Lprologue_avx: ___ $code.=<<___; + mov %rax,%r14 # original %rsp + and \$-64,%rsp mov %rdi,$ctx # reassigned argument mov %rsi,$inp # reassigned argument mov %rdx,$num # reassigned argument - vzeroupper shl \$6,$num add $inp,$num - lea K_XX_XX(%rip),$K_XX_XX + lea K_XX_XX+64(%rip),$K_XX_XX mov 0($ctx),$A # load context mov 4($ctx),$B @@ -757,9 +970,12 @@ $code.=<<___; mov 12($ctx),$D mov $B,@T[0] # magic seed mov 16($ctx),$E + mov $C,@T[1] + xor $D,@T[1] + and @T[1],@T[0] vmovdqa 64($K_XX_XX),@X[2] # pbswap mask - vmovdqa 0($K_XX_XX),@Tx[1] # K_00_19 + vmovdqa -64($K_XX_XX),$Kx # K_00_19 vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] vmovdqu 16($inp),@X[-3&7] vmovdqu 32($inp),@X[-2&7] @@ -769,9 +985,9 @@ $code.=<<___; vpshufb @X[2],@X[-3&7],@X[-3&7] vpshufb @X[2],@X[-2&7],@X[-2&7] vpshufb @X[2],@X[-1&7],@X[-1&7] - vpaddd @Tx[1],@X[-4&7],@X[0] # add K_00_19 - vpaddd @Tx[1],@X[-3&7],@X[1] - vpaddd @Tx[1],@X[-2&7],@X[2] + vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19 + vpaddd $Kx,@X[-3&7],@X[1] + vpaddd $Kx,@X[-2&7],@X[2] vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU vmovdqa @X[1],16(%rsp) vmovdqa @X[2],32(%rsp) @@ -790,10 +1006,10 @@ sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 eval(shift(@insns)); eval(shift(@insns)); - &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + &vpaddd (@Tx[1],$Kx,@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); - &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords + &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords eval(shift(@insns)); eval(shift(@insns)); &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" @@ -843,7 +1059,7 @@ sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2 eval(shift(@insns)); eval(shift(@insns)); - &vmovdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX + &vmovdqa ($Kx,eval(2*16*(($Xi)/5)-64)."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX eval(shift(@insns)); eval(shift(@insns)); @@ -851,13 +1067,12 @@ sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 foreach (@insns) { eval; } # remaining instructions [if any] $Xi++; push(@X,shift(@X)); # "rotate" X[] - push(@Tx,shift(@Tx)); } sub Xupdate_avx_32_79() { use integer; my $body = shift; - my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions my ($a,$b,$c,$d,$e); &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8); # compose "X[-6]" @@ -870,12 +1085,8 @@ sub Xupdate_avx_32_79() &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" eval(shift(@insns)); eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/); - if ($Xi%5) { - &vmovdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX... - } else { # ... or load next one - &vmovdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)"); - } - &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + &vpaddd (@Tx[1],$Kx,@X[-1&7]); + &vmovdqa ($Kx,eval(2*16*($Xi/5)-64)."($K_XX_XX)") if ($Xi%5==0); eval(shift(@insns)); # ror eval(shift(@insns)); @@ -905,7 +1116,6 @@ sub Xupdate_avx_32_79() &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2 eval(shift(@insns)); # body_20_39 eval(shift(@insns)); - &vmovdqa (@Tx[1],@X[0]) if ($Xi<19); eval(shift(@insns)); eval(shift(@insns)); # rol eval(shift(@insns)); @@ -916,7 +1126,6 @@ sub Xupdate_avx_32_79() foreach (@insns) { eval; } # remaining instructions $Xi++; push(@X,shift(@X)); # "rotate" X[] - push(@Tx,shift(@Tx)); } sub Xuplast_avx_80() @@ -926,23 +1135,21 @@ sub Xuplast_avx_80() my ($a,$b,$c,$d,$e); eval(shift(@insns)); - &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + &vpaddd (@Tx[1],$Kx,@X[-1&7]); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); - &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU + &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU foreach (@insns) { eval; } # remaining instructions &cmp ($inp,$num); - &je (".Ldone_avx"); - - unshift(@Tx,pop(@Tx)); + &je ($done_avx_label); &vmovdqa(@X[2],"64($K_XX_XX)"); # pbswap mask - &vmovdqa(@Tx[1],"0($K_XX_XX)"); # K_00_19 + &vmovdqa($Kx,"-64($K_XX_XX)"); # K_00_19 &vmovdqu(@X[-4&7],"0($inp)"); # load input &vmovdqu(@X[-3&7],"16($inp)"); &vmovdqu(@X[-2&7],"32($inp)"); @@ -964,7 +1171,7 @@ sub Xloop_avx() &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]); eval(shift(@insns)); eval(shift(@insns)); - &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@Tx[1]); + &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],$Kx); eval(shift(@insns)); eval(shift(@insns)); eval(shift(@insns)); @@ -1024,12 +1231,15 @@ $code.=<<___; mov @T[0],4($ctx) mov @T[0],$B # magic seed mov $C,8($ctx) + mov $C,@T[1] mov $D,12($ctx) + xor $D,@T[1] mov $E,16($ctx) + and @T[1],@T[0] jmp .Loop_avx .align 16 -.Ldone_avx: +$done_avx_label: ___ $j=$saved_j; @V=@saved_V; @@ -1052,31 +1262,520 @@ $code.=<<___; mov $E,16($ctx) ___ $code.=<<___ if ($win64); - movaps 64+0(%rsp),%xmm6 - movaps 64+16(%rsp),%xmm7 - movaps 64+32(%rsp),%xmm8 - movaps 64+48(%rsp),%xmm9 - movaps 64+64(%rsp),%xmm10 + movaps -40-6*16(%r14),%xmm6 + movaps -40-5*16(%r14),%xmm7 + movaps -40-4*16(%r14),%xmm8 + movaps -40-3*16(%r14),%xmm9 + movaps -40-2*16(%r14),%xmm10 + movaps -40-1*16(%r14),%xmm11 ___ $code.=<<___; - lea `64+($win64?5*16:0)`(%rsp),%rsi - mov 0(%rsi),%r12 - mov 8(%rsi),%rbp - mov 16(%rsi),%rbx - lea 24(%rsi),%rsp + lea (%r14),%rsi + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp .Lepilogue_avx: ret .size sha1_block_data_order_avx,.-sha1_block_data_order_avx ___ + +if ($avx>1) { +use integer; +$Xi=4; # reset variables +@X=map("%ymm$_",(4..7,0..3)); +@Tx=map("%ymm$_",(8..10)); +$Kx="%ymm11"; +$j=0; + +my @ROTX=("%eax","%ebp","%ebx","%ecx","%edx","%esi"); +my ($a5,$t0)=("%r12d","%edi"); + +my ($A,$F,$B,$C,$D,$E)=@ROTX; +my $rx=0; +my $frame="%r13"; + +$code.=<<___; +.type sha1_block_data_order_avx2,\@function,3 +.align 16 +sha1_block_data_order_avx2: +_avx2_shortcut: + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + vzeroupper +___ +$code.=<<___ if ($win64); + lea -6*16(%rsp),%rsp + vmovaps %xmm6,-40-6*16(%rax) + vmovaps %xmm7,-40-5*16(%rax) + vmovaps %xmm8,-40-4*16(%rax) + vmovaps %xmm9,-40-3*16(%rax) + vmovaps %xmm10,-40-2*16(%rax) + vmovaps %xmm11,-40-1*16(%rax) +.Lprologue_avx2: +___ +$code.=<<___; + mov %rax,%r14 # original %rsp + mov %rdi,$ctx # reassigned argument + mov %rsi,$inp # reassigned argument + mov %rdx,$num # reassigned argument + + lea -640(%rsp),%rsp + shl \$6,$num + lea 64($inp),$frame + and \$-128,%rsp + add $inp,$num + lea K_XX_XX+64(%rip),$K_XX_XX + + mov 0($ctx),$A # load context + cmp $num,$frame + cmovae $inp,$frame # next or same block + mov 4($ctx),$F + mov 8($ctx),$C + mov 12($ctx),$D + mov 16($ctx),$E + vmovdqu 64($K_XX_XX),@X[2] # pbswap mask + + vmovdqu ($inp),%xmm0 + vmovdqu 16($inp),%xmm1 + vmovdqu 32($inp),%xmm2 + vmovdqu 48($inp),%xmm3 + lea 64($inp),$inp + vinserti128 \$1,($frame),@X[-4&7],@X[-4&7] + vinserti128 \$1,16($frame),@X[-3&7],@X[-3&7] + vpshufb @X[2],@X[-4&7],@X[-4&7] + vinserti128 \$1,32($frame),@X[-2&7],@X[-2&7] + vpshufb @X[2],@X[-3&7],@X[-3&7] + vinserti128 \$1,48($frame),@X[-1&7],@X[-1&7] + vpshufb @X[2],@X[-2&7],@X[-2&7] + vmovdqu -64($K_XX_XX),$Kx # K_00_19 + vpshufb @X[2],@X[-1&7],@X[-1&7] + + vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19 + vpaddd $Kx,@X[-3&7],@X[1] + vmovdqu @X[0],0(%rsp) # X[]+K xfer to IALU + vpaddd $Kx,@X[-2&7],@X[2] + vmovdqu @X[1],32(%rsp) + vpaddd $Kx,@X[-1&7],@X[3] + vmovdqu @X[2],64(%rsp) + vmovdqu @X[3],96(%rsp) +___ +for (;$Xi<8;$Xi++) { # Xupdate_avx2_16_31 + use integer; + + &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]" + &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords + &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" + &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]" + &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]" + &vpsrld (@Tx[0],@X[0],31); + &vmovdqu($Kx,eval(2*16*(($Xi)/5)-64)."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX + &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword + &vpaddd (@X[0],@X[0],@X[0]); + &vpsrld (@Tx[1],@Tx[2],30); + &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1 + &vpslld (@Tx[2],@Tx[2],2); + &vpxor (@X[0],@X[0],@Tx[1]); + &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2 + &vpaddd (@Tx[1],@X[0],$Kx); + &vmovdqu("32*$Xi(%rsp)",@Tx[1]); # X[]+K xfer to IALU + + push(@X,shift(@X)); # "rotate" X[] +} +$code.=<<___; + lea 128(%rsp),$frame + jmp .Loop_avx2 +.align 32 +.Loop_avx2: + rorx \$2,$F,$B + andn $D,$F,$t0 + and $C,$F + xor $t0,$F +___ +sub bodyx_00_19 () { # 8 instructions, 3 cycles critical path + # at start $f=(b&c)^(~b&d), $b>>>=2 + return &bodyx_20_39() if ($rx==19); $rx++; + ( + '($a,$f,$b,$c,$d,$e)=@ROTX;'. + + '&add ($e,((32*($j/4)+4*($j%4))%256-128)."($frame)");'. # e+=X[i]+K + '&lea ($frame,"256($frame)") if ($j%32==31);', + '&andn ($t0,$a,$c)', # ~b&d for next round + + '&add ($e,$f)', # e+=(b&c)^(~b&d) + '&rorx ($a5,$a,27)', # a<<<5 + '&rorx ($f,$a,2)', # b>>>2 for next round + '&and ($a,$b)', # b&c for next round + + '&add ($e,$a5)', # e+=a<<<5 + '&xor ($a,$t0);'. # f=(b&c)^(~b&d) for next round + + 'unshift(@ROTX,pop(@ROTX)); $j++;' + ) +} + +sub bodyx_20_39 () { # 7 instructions, 2 cycles critical path + # on entry $f=b^c^d, $b>>>=2 + return &bodyx_40_59() if ($rx==39); $rx++; + ( + '($a,$f,$b,$c,$d,$e)=@ROTX;'. + + '&add ($e,((32*($j/4)+4*($j%4))%256-128)."($frame)");'. # e+=X[i]+K + '&lea ($frame,"256($frame)") if ($j%32==31);', + + '&lea ($e,"($e,$f)")', # e+=b^c^d + '&rorx ($a5,$a,27)', # a<<<5 + '&rorx ($f,$a,2) if ($j<79)', # b>>>2 in next round + '&xor ($a,$b) if ($j<79)', # b^c for next round + + '&add ($e,$a5)', # e+=a<<<5 + '&xor ($a,$c) if ($j<79);'. # f=b^c^d for next round + + 'unshift(@ROTX,pop(@ROTX)); $j++;' + ) +} + +sub bodyx_40_59 () { # 10 instructions, 3 cycles critical path + # on entry $f=((b^c)&(c^d)), $b>>>=2 + $rx++; + ( + '($a,$f,$b,$c,$d,$e)=@ROTX;'. + + '&add ($e,((32*($j/4)+4*($j%4))%256-128)."($frame)");'. # e+=X[i]+K + '&lea ($frame,"256($frame)") if ($j%32==31);', + '&xor ($f,$c) if ($j>39)', # (b^c)&(c^d)^c + '&mov ($t0,$b) if ($j<59)', # count on zero latency + '&xor ($t0,$c) if ($j<59)', # c^d for next round + + '&lea ($e,"($e,$f)")', # e+=(b^c)&(c^d)^c + '&rorx ($a5,$a,27)', # a<<<5 + '&rorx ($f,$a,2)', # b>>>2 in next round + '&xor ($a,$b)', # b^c for next round + + '&add ($e,$a5)', # e+=a<<<5 + '&and ($a,$t0) if ($j< 59);'. # f=(b^c)&(c^d) for next round + '&xor ($a,$c) if ($j==59);'. # f=b^c^d for next round + + 'unshift(@ROTX,pop(@ROTX)); $j++;' + ) +} + +sub Xupdate_avx2_16_31() # recall that $Xi starts wtih 4 +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body,&$body); # 35 instructions + my ($a,$b,$c,$d,$e); + + &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" + &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrld (@Tx[0],@X[0],31); + &vmovdqu($Kx,eval(2*16*(($Xi)/5)-64)."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword + &vpaddd (@X[0],@X[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrld (@Tx[1],@Tx[2],30); + &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1 + eval(shift(@insns)); + eval(shift(@insns)); + + &vpslld (@Tx[2],@Tx[2],2); + &vpxor (@X[0],@X[0],@Tx[1]); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpaddd (@Tx[1],@X[0],$Kx); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vmovdqu(eval(32*($Xi))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU + + foreach (@insns) { eval; } # remaining instructions [if any] + + $Xi++; + push(@X,shift(@X)); # "rotate" X[] +} + +sub Xupdate_avx2_32_79() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body,&$body); # 35 to 50 instructions + my ($a,$b,$c,$d,$e); + + &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8); # compose "X[-6]" + &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" + &vmovdqu($Kx,eval(2*16*($Xi/5)-64)."($K_XX_XX)") if ($Xi%5==0); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-6]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrld (@Tx[0],@X[0],30); + &vpslld (@X[0],@X[0],2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + #&vpslld (@X[0],@X[0],2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpaddd (@Tx[1],@X[0],$Kx); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vmovdqu("32*$Xi(%rsp)",@Tx[1]); # X[]+K xfer to IALU + + foreach (@insns) { eval; } # remaining instructions + + $Xi++; + push(@X,shift(@X)); # "rotate" X[] +} + +sub Xloop_avx2() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + foreach (@insns) { eval; } +} + + &align32(); + &Xupdate_avx2_32_79(\&bodyx_00_19); + &Xupdate_avx2_32_79(\&bodyx_00_19); + &Xupdate_avx2_32_79(\&bodyx_00_19); + &Xupdate_avx2_32_79(\&bodyx_00_19); + + &Xupdate_avx2_32_79(\&bodyx_20_39); + &Xupdate_avx2_32_79(\&bodyx_20_39); + &Xupdate_avx2_32_79(\&bodyx_20_39); + &Xupdate_avx2_32_79(\&bodyx_20_39); + + &align32(); + &Xupdate_avx2_32_79(\&bodyx_40_59); + &Xupdate_avx2_32_79(\&bodyx_40_59); + &Xupdate_avx2_32_79(\&bodyx_40_59); + &Xupdate_avx2_32_79(\&bodyx_40_59); + + &Xloop_avx2(\&bodyx_20_39); + &Xloop_avx2(\&bodyx_20_39); + &Xloop_avx2(\&bodyx_20_39); + &Xloop_avx2(\&bodyx_20_39); + +$code.=<<___; + lea 128($inp),$frame + lea 128($inp),%rdi # borrow $t0 + cmp $num,$frame + cmovae $inp,$frame # next or previous block + + # output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + add 0($ctx),@ROTX[0] # update context + add 4($ctx),@ROTX[1] + add 8($ctx),@ROTX[3] + mov @ROTX[0],0($ctx) + add 12($ctx),@ROTX[4] + mov @ROTX[1],4($ctx) + mov @ROTX[0],$A # A=d + add 16($ctx),@ROTX[5] + mov @ROTX[3],$a5 + mov @ROTX[3],8($ctx) + mov @ROTX[4],$D # D=b + #xchg @ROTX[5],$F # F=c, C=f + mov @ROTX[4],12($ctx) + mov @ROTX[1],$F # F=e + mov @ROTX[5],16($ctx) + #mov $F,16($ctx) + mov @ROTX[5],$E # E=c + mov $a5,$C # C=f + #xchg $F,$E # E=c, F=e + + cmp $num,$inp + je .Ldone_avx2 +___ + +$Xi=4; # reset variables +@X=map("%ymm$_",(4..7,0..3)); + +$code.=<<___; + vmovdqu 64($K_XX_XX),@X[2] # pbswap mask + cmp $num,%rdi # borrowed $t0 + ja .Last_avx2 + + vmovdqu -64(%rdi),%xmm0 # low part of @X[-4&7] + vmovdqu -48(%rdi),%xmm1 + vmovdqu -32(%rdi),%xmm2 + vmovdqu -16(%rdi),%xmm3 + vinserti128 \$1,0($frame),@X[-4&7],@X[-4&7] + vinserti128 \$1,16($frame),@X[-3&7],@X[-3&7] + vinserti128 \$1,32($frame),@X[-2&7],@X[-2&7] + vinserti128 \$1,48($frame),@X[-1&7],@X[-1&7] + jmp .Last_avx2 + +.align 32 +.Last_avx2: + lea 128+16(%rsp),$frame + rorx \$2,$F,$B + andn $D,$F,$t0 + and $C,$F + xor $t0,$F + sub \$-128,$inp +___ + $rx=$j=0; @ROTX=($A,$F,$B,$C,$D,$E); + + &Xloop_avx2 (\&bodyx_00_19); + &Xloop_avx2 (\&bodyx_00_19); + &Xloop_avx2 (\&bodyx_00_19); + &Xloop_avx2 (\&bodyx_00_19); + + &Xloop_avx2 (\&bodyx_20_39); + &vmovdqu ($Kx,"-64($K_XX_XX)"); # K_00_19 + &vpshufb (@X[-4&7],@X[-4&7],@X[2]); # byte swap + &Xloop_avx2 (\&bodyx_20_39); + &vpshufb (@X[-3&7],@X[-3&7],@X[2]); + &vpaddd (@Tx[0],@X[-4&7],$Kx); # add K_00_19 + &Xloop_avx2 (\&bodyx_20_39); + &vmovdqu ("0(%rsp)",@Tx[0]); + &vpshufb (@X[-2&7],@X[-2&7],@X[2]); + &vpaddd (@Tx[1],@X[-3&7],$Kx); + &Xloop_avx2 (\&bodyx_20_39); + &vmovdqu ("32(%rsp)",@Tx[1]); + &vpshufb (@X[-1&7],@X[-1&7],@X[2]); + &vpaddd (@X[2],@X[-2&7],$Kx); + + &Xloop_avx2 (\&bodyx_40_59); + &align32 (); + &vmovdqu ("64(%rsp)",@X[2]); + &vpaddd (@X[3],@X[-1&7],$Kx); + &Xloop_avx2 (\&bodyx_40_59); + &vmovdqu ("96(%rsp)",@X[3]); + &Xloop_avx2 (\&bodyx_40_59); + &Xupdate_avx2_16_31(\&bodyx_40_59); + + &Xupdate_avx2_16_31(\&bodyx_20_39); + &Xupdate_avx2_16_31(\&bodyx_20_39); + &Xupdate_avx2_16_31(\&bodyx_20_39); + &Xloop_avx2 (\&bodyx_20_39); + +$code.=<<___; + lea 128(%rsp),$frame + + # output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + add 0($ctx),@ROTX[0] # update context + add 4($ctx),@ROTX[1] + add 8($ctx),@ROTX[3] + mov @ROTX[0],0($ctx) + add 12($ctx),@ROTX[4] + mov @ROTX[1],4($ctx) + mov @ROTX[0],$A # A=d + add 16($ctx),@ROTX[5] + mov @ROTX[3],$a5 + mov @ROTX[3],8($ctx) + mov @ROTX[4],$D # D=b + #xchg @ROTX[5],$F # F=c, C=f + mov @ROTX[4],12($ctx) + mov @ROTX[1],$F # F=e + mov @ROTX[5],16($ctx) + #mov $F,16($ctx) + mov @ROTX[5],$E # E=c + mov $a5,$C # C=f + #xchg $F,$E # E=c, F=e + + cmp $num,$inp + jbe .Loop_avx2 + +.Ldone_avx2: + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -40-6*16(%r14),%xmm6 + movaps -40-5*16(%r14),%xmm7 + movaps -40-4*16(%r14),%xmm8 + movaps -40-3*16(%r14),%xmm9 + movaps -40-2*16(%r14),%xmm10 + movaps -40-1*16(%r14),%xmm11 +___ +$code.=<<___; + lea (%r14),%rsi + mov -40(%rsi),%r14 + mov -32(%rsi),%r13 + mov -24(%rsi),%r12 + mov -16(%rsi),%rbp + mov -8(%rsi),%rbx + lea (%rsi),%rsp +.Lepilogue_avx2: + ret +.size sha1_block_data_order_avx2,.-sha1_block_data_order_avx2 +___ +} } $code.=<<___; .align 64 K_XX_XX: .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 +.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 +.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39 .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39 .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 +.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 +.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask +.byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 ___ }}} $code.=<<___; @@ -1122,20 +1821,58 @@ se_handler: jae .Lcommon_seh_tail mov `16*4`(%rax),%rax # pull saved stack pointer - lea 32(%rax),%rax mov -8(%rax),%rbx mov -16(%rax),%rbp mov -24(%rax),%r12 mov -32(%rax),%r13 + mov -40(%rax),%r14 mov %rbx,144($context) # restore context->Rbx mov %rbp,160($context) # restore context->Rbp mov %r12,216($context) # restore context->R12 mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 jmp .Lcommon_seh_tail .size se_handler,.-se_handler +___ + +$code.=<<___ if ($shaext); +.type shaext_handler,\@abi-omnipotent +.align 16 +shaext_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + lea .Lprologue_shaext(%rip),%r10 + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + lea .Lepilogue_shaext(%rip),%r10 + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea -8-4*16(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$8,%ecx + .long 0xa548f3fc # cld; rep movsq + jmp .Lcommon_seh_tail +.size shaext_handler,.-shaext_handler +___ + +$code.=<<___; .type ssse3_handler,\@abi-omnipotent .align 16 ssse3_handler: @@ -1168,18 +1905,23 @@ ssse3_handler: cmp %r10,%rbx # context->Rip>=epilogue label jae .Lcommon_seh_tail - lea 64(%rax),%rsi + mov 232($context),%rax # pull context->R14 + + lea -40-6*16(%rax),%rsi lea 512($context),%rdi # &context.Xmm6 - mov \$10,%ecx + mov \$12,%ecx .long 0xa548f3fc # cld; rep movsq - lea `24+64+5*16`(%rax),%rax # adjust stack pointer mov -8(%rax),%rbx mov -16(%rax),%rbp mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 mov %rbx,144($context) # restore context->Rbx mov %rbp,160($context) # restore context->Rbp mov %r12,216($context) # restore cotnext->R12 + mov %r13,224($context) # restore cotnext->R13 + mov %r14,232($context) # restore cotnext->R14 .Lcommon_seh_tail: mov 8(%rax),%rdi @@ -1226,6 +1968,13 @@ ssse3_handler: .rva .LSEH_begin_sha1_block_data_order .rva .LSEH_end_sha1_block_data_order .rva .LSEH_info_sha1_block_data_order +___ +$code.=<<___ if ($shaext); + .rva .LSEH_begin_sha1_block_data_order_shaext + .rva .LSEH_end_sha1_block_data_order_shaext + .rva .LSEH_info_sha1_block_data_order_shaext +___ +$code.=<<___; .rva .LSEH_begin_sha1_block_data_order_ssse3 .rva .LSEH_end_sha1_block_data_order_ssse3 .rva .LSEH_info_sha1_block_data_order_ssse3 @@ -1235,12 +1984,24 @@ $code.=<<___ if ($avx); .rva .LSEH_end_sha1_block_data_order_avx .rva .LSEH_info_sha1_block_data_order_avx ___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_sha1_block_data_order_avx2 + .rva .LSEH_end_sha1_block_data_order_avx2 + .rva .LSEH_info_sha1_block_data_order_avx2 +___ $code.=<<___; .section .xdata .align 8 .LSEH_info_sha1_block_data_order: .byte 9,0,0,0 .rva se_handler +___ +$code.=<<___ if ($shaext); +.LSEH_info_sha1_block_data_order_shaext: + .byte 9,0,0,0 + .rva shaext_handler +___ +$code.=<<___; .LSEH_info_sha1_block_data_order_ssse3: .byte 9,0,0,0 .rva ssse3_handler @@ -1252,10 +2013,55 @@ $code.=<<___ if ($avx); .rva ssse3_handler .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[] ___ +$code.=<<___ if ($avx>1); +.LSEH_info_sha1_block_data_order_avx2: + .byte 9,0,0,0 + .rva ssse3_handler + .rva .Lprologue_avx2,.Lepilogue_avx2 # HandlerData[] +___ } #################################################################### -$code =~ s/\`([^\`]*)\`/eval $1/gem; -print $code; +sub sha1rnds4 { + if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-7]),\s*%xmm([0-7])/) { + my @opcode=(0x0f,0x3a,0xcc); + push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M + my $c=$1; + push @opcode,$c=~/^0/?oct($c):$c; + return ".byte\t".join(',',@opcode); + } else { + return "sha1rnds4\t".@_[0]; + } +} + +sub sha1op38 { + my $instr = shift; + my %opcodelet = ( + "sha1nexte" => 0xc8, + "sha1msg1" => 0xc9, + "sha1msg2" => 0xca ); + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x38); + my $rex=0; + $rex|=0x04 if ($2>=8); + $rex|=0x01 if ($1>=8); + unshift @opcode,0x40|$rex if ($rex); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or + s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo; + + print $_,"\n"; +} close STDOUT; diff --git a/crypto/sha/asm/sha256-586.pl b/crypto/sha/asm/sha256-586.pl index 928ec53123bf..6462e45ba75b 100755 --- a/crypto/sha/asm/sha256-586.pl +++ b/crypto/sha/asm/sha256-586.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -9,20 +9,55 @@ # # SHA256 block transform for x86. September 2007. # -# Performance in clock cycles per processed byte (less is better): +# Performance improvement over compiler generated code varies from +# 10% to 40% [see below]. Not very impressive on some µ-archs, but +# it's 5 times smaller and optimizies amount of writes. # -# Pentium PIII P4 AMD K8 Core2 -# gcc 46 36 41 27 26 -# icc 57 33 38 25 23 -# x86 asm 40 30 33 20 18 -# x86_64 asm(*) - - 21 16 16 +# May 2012. # -# (*) x86_64 assembler performance is presented for reference -# purposes. +# Optimization including two of Pavel Semjanov's ideas, alternative +# Maj and full unroll, resulted in ~20-25% improvement on most CPUs, +# ~7% on Pentium, ~40% on Atom. As fully unrolled loop body is almost +# 15x larger, 8KB vs. 560B, it's fired only for longer inputs. But not +# on P4, where it kills performance, nor Sandy Bridge, where folded +# loop is approximately as fast... # -# Performance improvement over compiler generated code varies from -# 10% to 40% [see above]. Not very impressive on some µ-archs, but -# it's 5 times smaller and optimizies amount of writes. +# June 2012. +# +# Add AMD XOP-specific code path, >30% improvement on Bulldozer over +# May version, >60% over original. Add AVX+shrd code path, >25% +# improvement on Sandy Bridge over May version, 60% over original. +# +# May 2013. +# +# Replace AMD XOP code path with SSSE3 to cover more processors. +# (Biggest improvement coefficient is on upcoming Atom Silvermont, +# not shown.) Add AVX+BMI code path. +# +# March 2014. +# +# Add support for Intel SHA Extensions. +# +# Performance in clock cycles per processed byte (less is better): +# +# gcc icc x86 asm(*) SIMD x86_64 asm(**) +# Pentium 46 57 40/38 - - +# PIII 36 33 27/24 - - +# P4 41 38 28 - 17.3 +# AMD K8 27 25 19/15.5 - 14.9 +# Core2 26 23 18/15.6 14.3 13.8 +# Westmere 27 - 19/15.7 13.4 12.3 +# Sandy Bridge 25 - 15.9 12.4 11.6 +# Ivy Bridge 24 - 15.0 11.4 10.3 +# Haswell 22 - 13.9 9.46 7.80 +# Bulldozer 36 - 27/22 17.0 13.6 +# VIA Nano 36 - 25/22 16.8 16.5 +# Atom 50 - 30/25 21.9 18.9 +# Silvermont 40 - 34/31 22.9 20.6 +# +# (*) numbers after slash are for unrolled loop, where applicable; +# (**) x86_64 assembly performance is presented for reference +# purposes, results are best-available; $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; push(@INC,"${dir}","${dir}../../perlasm"); @@ -30,72 +65,122 @@ require "x86asm.pl"; &asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386"); +$xmm=$avx=0; +for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); } + +if ($xmm && `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if ($xmm && !$avx && $ARGV[0] eq "win32n" && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.03) + ($1>=2.10); +} + +if ($xmm && !$avx && $ARGV[0] eq "win32" && + `ml 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if ($xmm && !$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +$shaext=$xmm; ### set to zero if compiling for 1.0.1 + +$unroll_after = 64*4; # If pre-evicted from L1P cache first spin of + # fully unrolled loop was measured to run about + # 3-4x slower. If slowdown coefficient is N and + # unrolled loop is m times faster, then you break + # even at (N-1)/(m-1) blocks. Then it needs to be + # adjusted for probability of code being evicted, + # code size/cache size=1/4. Typical m is 1.15... + $A="eax"; $E="edx"; $T="ebx"; -$Aoff=&DWP(0,"esp"); -$Boff=&DWP(4,"esp"); -$Coff=&DWP(8,"esp"); -$Doff=&DWP(12,"esp"); -$Eoff=&DWP(16,"esp"); -$Foff=&DWP(20,"esp"); -$Goff=&DWP(24,"esp"); -$Hoff=&DWP(28,"esp"); -$Xoff=&DWP(32,"esp"); +$Aoff=&DWP(4,"esp"); +$Boff=&DWP(8,"esp"); +$Coff=&DWP(12,"esp"); +$Doff=&DWP(16,"esp"); +$Eoff=&DWP(20,"esp"); +$Foff=&DWP(24,"esp"); +$Goff=&DWP(28,"esp"); +$Hoff=&DWP(32,"esp"); +$Xoff=&DWP(36,"esp"); $K256="ebp"; +sub BODY_16_63() { + &mov ($T,"ecx"); # "ecx" is preloaded + &mov ("esi",&DWP(4*(9+15+16-14),"esp")); + &ror ("ecx",18-7); + &mov ("edi","esi"); + &ror ("esi",19-17); + &xor ("ecx",$T); + &shr ($T,3); + &ror ("ecx",7); + &xor ("esi","edi"); + &xor ($T,"ecx"); # T = sigma0(X[-15]) + &ror ("esi",17); + &add ($T,&DWP(4*(9+15+16),"esp")); # T += X[-16] + &shr ("edi",10); + &add ($T,&DWP(4*(9+15+16-9),"esp")); # T += X[-7] + #&xor ("edi","esi") # sigma1(X[-2]) + # &add ($T,"edi"); # T += sigma1(X[-2]) + # &mov (&DWP(4*(9+15),"esp"),$T); # save X[0] + + &BODY_00_15(1); +} sub BODY_00_15() { my $in_16_63=shift; &mov ("ecx",$E); - &add ($T,"edi") if ($in_16_63); # T += sigma1(X[-2]) - &ror ("ecx",25-11); + &xor ("edi","esi") if ($in_16_63); # sigma1(X[-2]) &mov ("esi",$Foff); + &ror ("ecx",25-11); + &add ($T,"edi") if ($in_16_63); # T += sigma1(X[-2]) + &mov ("edi",$Goff); &xor ("ecx",$E); + &xor ("esi","edi"); + &mov ($T,&DWP(4*(9+15),"esp")) if (!$in_16_63); + &mov (&DWP(4*(9+15),"esp"),$T) if ($in_16_63); # save X[0] &ror ("ecx",11-6); - &mov (&DWP(4*(8+15),"esp"),$T) if ($in_16_63); # save X[0] - &xor ("ecx",$E); - &ror ("ecx",6); # Sigma1(e) - &mov ("edi",$Goff); - &add ($T,"ecx"); # T += Sigma1(e) - - &xor ("esi","edi"); - &mov ($Eoff,$E); # modulo-scheduled + &and ("esi",$E); + &mov ($Eoff,$E); # modulo-scheduled + &xor ($E,"ecx"); + &add ($T,$Hoff); # T += h + &xor ("esi","edi"); # Ch(e,f,g) + &ror ($E,6); # Sigma1(e) &mov ("ecx",$A); - &and ("esi",$E); - &mov ($E,$Doff); # e becomes d, which is e in next iteration - &xor ("esi","edi"); # Ch(e,f,g) - &mov ("edi",$A); - &add ($T,"esi"); # T += Ch(e,f,g) + &add ($T,"esi"); # T += Ch(e,f,g) &ror ("ecx",22-13); - &add ($T,$Hoff); # T += h + &add ($T,$E); # T += Sigma1(e) + &mov ("edi",$Boff); &xor ("ecx",$A); + &mov ($Aoff,$A); # modulo-scheduled + &lea ("esp",&DWP(-4,"esp")); &ror ("ecx",13-2); - &mov ("esi",$Boff); - &xor ("ecx",$A); - &ror ("ecx",2); # Sigma0(a) - &add ($E,$T); # d += T - &mov ("edi",$Coff); - - &add ($T,"ecx"); # T += Sigma0(a) - &mov ($Aoff,$A); # modulo-scheduled - - &mov ("ecx",$A); - &sub ("esp",4); - &or ($A,"esi"); # a becomes h, which is a in next iteration - &and ("ecx","esi"); - &and ($A,"edi"); &mov ("esi",&DWP(0,$K256)); - &or ($A,"ecx"); # h=Maj(a,b,c) + &xor ("ecx",$A); + &mov ($E,$Eoff); # e in next iteration, d in this one + &xor ($A,"edi"); # a ^= b + &ror ("ecx",2); # Sigma0(a) + &add ($T,"esi"); # T+= K[i] + &mov (&DWP(0,"esp"),$A); # (b^c) in next round + &add ($E,$T); # d += T + &and ($A,&DWP(4,"esp")); # a &= (b^c) + &add ($T,"ecx"); # T += Sigma0(a) + &xor ($A,"edi"); # h = Maj(a,b,c) = Ch(a^b,c,b) + &mov ("ecx",&DWP(4*(9+15+16-1),"esp")) if ($in_16_63); # preload T &add ($K256,4); - &add ($A,$T); # h += T - &mov ($T,&DWP(4*(8+15+16-1),"esp")) if ($in_16_63); # preload T - &add ($E,"esi"); # d += K256[i] - &add ($A,"esi"); # h += K256[i] + &add ($A,$T); # h += T } +&external_label("OPENSSL_ia32cap_P") if (!$i386); + &function_begin("sha256_block_data_order"); &mov ("esi",wparam(0)); # ctx &mov ("edi",wparam(1)); # inp @@ -116,26 +201,59 @@ sub BODY_00_15() { &mov (&DWP(4,"esp"),"edi"); # inp &mov (&DWP(8,"esp"),"eax"); # inp+num*128 &mov (&DWP(12,"esp"),"ebx"); # saved sp + if (!$i386 && $xmm) { + &picmeup("edx","OPENSSL_ia32cap_P",$K256,&label("K256")); + &mov ("ecx",&DWP(0,"edx")); + &mov ("ebx",&DWP(4,"edx")); + &test ("ecx",1<<20); # check for P4 + &jnz (&label("loop")); + &mov ("edx",&DWP(8,"edx")) if ($xmm); + &test ("ecx",1<<24); # check for FXSR + &jz ($unroll_after?&label("no_xmm"):&label("loop")); + &and ("ecx",1<<30); # mask "Intel CPU" bit + &and ("ebx",1<<28|1<<9); # mask AVX and SSSE3 bits + &test ("edx",1<<29) if ($shaext); # check for SHA + &jnz (&label("shaext")) if ($shaext); + &or ("ecx","ebx"); + &and ("ecx",1<<28|1<<30); + &cmp ("ecx",1<<28|1<<30); + if ($xmm) { + &je (&label("AVX")) if ($avx); + &test ("ebx",1<<9); # check for SSSE3 + &jnz (&label("SSSE3")); + } else { + &je (&label("loop_shrd")); + } + if ($unroll_after) { +&set_label("no_xmm"); + &sub ("eax","edi"); + &cmp ("eax",$unroll_after); + &jae (&label("unrolled")); + } } + &jmp (&label("loop")); + +sub COMPACT_LOOP() { +my $suffix=shift; -&set_label("loop",16); +&set_label("loop$suffix",$suffix?32:16); # copy input block to stack reversing byte and dword order for($i=0;$i<4;$i++) { &mov ("eax",&DWP($i*16+0,"edi")); &mov ("ebx",&DWP($i*16+4,"edi")); &mov ("ecx",&DWP($i*16+8,"edi")); - &mov ("edx",&DWP($i*16+12,"edi")); &bswap ("eax"); + &mov ("edx",&DWP($i*16+12,"edi")); &bswap ("ebx"); - &bswap ("ecx"); - &bswap ("edx"); &push ("eax"); + &bswap ("ecx"); &push ("ebx"); + &bswap ("edx"); &push ("ecx"); &push ("edx"); } &add ("edi",64); - &sub ("esp",4*8); # place for A,B,C,D,E,F,G,H - &mov (&DWP(4*(8+16)+4,"esp"),"edi"); + &lea ("esp",&DWP(-4*9,"esp"));# place for A,B,C,D,E,F,G,H + &mov (&DWP(4*(9+16)+4,"esp"),"edi"); # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack &mov ($A,&DWP(0,"esi")); @@ -144,8 +262,10 @@ sub BODY_00_15() { &mov ("edi",&DWP(12,"esi")); # &mov ($Aoff,$A); &mov ($Boff,"ebx"); + &xor ("ebx","ecx"); &mov ($Coff,"ecx"); &mov ($Doff,"edi"); + &mov (&DWP(0,"esp"),"ebx"); # magic &mov ($E,&DWP(16,"esi")); &mov ("ebx",&DWP(20,"esi")); &mov ("ecx",&DWP(24,"esi")); @@ -155,59 +275,41 @@ sub BODY_00_15() { &mov ($Goff,"ecx"); &mov ($Hoff,"edi"); -&set_label("00_15",16); - &mov ($T,&DWP(4*(8+15),"esp")); +&set_label("00_15$suffix",16); &BODY_00_15(); &cmp ("esi",0xc19bf174); - &jne (&label("00_15")); - - &mov ($T,&DWP(4*(8+15+16-1),"esp")); # preloaded in BODY_00_15(1) -&set_label("16_63",16); - &mov ("esi",$T); - &mov ("ecx",&DWP(4*(8+15+16-14),"esp")); - &ror ("esi",18-7); - &mov ("edi","ecx"); - &xor ("esi",$T); - &ror ("esi",7); - &shr ($T,3); - - &ror ("edi",19-17); - &xor ($T,"esi"); # T = sigma0(X[-15]) - &xor ("edi","ecx"); - &ror ("edi",17); - &shr ("ecx",10); - &add ($T,&DWP(4*(8+15+16),"esp")); # T += X[-16] - &xor ("edi","ecx"); # sigma1(X[-2]) - - &add ($T,&DWP(4*(8+15+16-9),"esp")); # T += X[-7] - # &add ($T,"edi"); # T += sigma1(X[-2]) - # &mov (&DWP(4*(8+15),"esp"),$T); # save X[0] + &jne (&label("00_15$suffix")); - &BODY_00_15(1); + &mov ("ecx",&DWP(4*(9+15+16-1),"esp")); # preloaded in BODY_00_15(1) + &jmp (&label("16_63$suffix")); + +&set_label("16_63$suffix",16); + + &BODY_16_63(); &cmp ("esi",0xc67178f2); - &jne (&label("16_63")); + &jne (&label("16_63$suffix")); - &mov ("esi",&DWP(4*(8+16+64)+0,"esp"));#ctx + &mov ("esi",&DWP(4*(9+16+64)+0,"esp"));#ctx # &mov ($A,$Aoff); &mov ("ebx",$Boff); - &mov ("ecx",$Coff); - &mov ("edi",$Doff); + # &mov ("edi",$Coff); + &mov ("ecx",$Doff); &add ($A,&DWP(0,"esi")); &add ("ebx",&DWP(4,"esi")); - &add ("ecx",&DWP(8,"esi")); - &add ("edi",&DWP(12,"esi")); + &add ("edi",&DWP(8,"esi")); + &add ("ecx",&DWP(12,"esi")); &mov (&DWP(0,"esi"),$A); &mov (&DWP(4,"esi"),"ebx"); - &mov (&DWP(8,"esi"),"ecx"); - &mov (&DWP(12,"esi"),"edi"); + &mov (&DWP(8,"esi"),"edi"); + &mov (&DWP(12,"esi"),"ecx"); # &mov ($E,$Eoff); &mov ("eax",$Foff); &mov ("ebx",$Goff); &mov ("ecx",$Hoff); - &mov ("edi",&DWP(4*(8+16+64)+4,"esp"));#inp + &mov ("edi",&DWP(4*(9+16+64)+4,"esp"));#inp &add ($E,&DWP(16,"esi")); &add ("eax",&DWP(20,"esi")); &add ("ebx",&DWP(24,"esi")); @@ -217,33 +319,963 @@ sub BODY_00_15() { &mov (&DWP(24,"esi"),"ebx"); &mov (&DWP(28,"esi"),"ecx"); - &add ("esp",4*(8+16+64)); # destroy frame + &lea ("esp",&DWP(4*(9+16+64),"esp"));# destroy frame &sub ($K256,4*64); # rewind K &cmp ("edi",&DWP(8,"esp")); # are we done yet? - &jb (&label("loop")); - + &jb (&label("loop$suffix")); +} + &COMPACT_LOOP(); + &mov ("esp",&DWP(12,"esp")); # restore sp +&function_end_A(); + if (!$i386 && !$xmm) { + # ~20% improvement on Sandy Bridge + local *ror = sub { &shrd(@_[0],@_) }; + &COMPACT_LOOP("_shrd"); &mov ("esp",&DWP(12,"esp")); # restore sp &function_end_A(); + } &set_label("K256",64); # Yes! I keep it in the code segment! - &data_word(0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5); - &data_word(0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5); - &data_word(0xd807aa98,0x12835b01,0x243185be,0x550c7dc3); - &data_word(0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174); - &data_word(0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc); - &data_word(0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da); - &data_word(0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7); - &data_word(0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967); - &data_word(0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13); - &data_word(0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85); - &data_word(0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3); - &data_word(0xd192e819,0xd6990624,0xf40e3585,0x106aa070); - &data_word(0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5); - &data_word(0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3); - &data_word(0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208); - &data_word(0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2); -&function_end_B("sha256_block_data_order"); +@K256=( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5, + 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3, + 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc, + 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7, + 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13, + 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3, + 0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5, + 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208, + 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 ); +&data_word(@K256); +&data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # byte swap mask &asciz("SHA256 block transform for x86, CRYPTOGAMS by "); +($a,$b,$c,$d,$e,$f,$g,$h)=(0..7); # offsets +sub off { &DWP(4*(((shift)-$i)&7),"esp"); } + +if (!$i386 && $unroll_after) { +my @AH=($A,$K256); + +&set_label("unrolled",16); + &lea ("esp",&DWP(-96,"esp")); + # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack + &mov ($AH[0],&DWP(0,"esi")); + &mov ($AH[1],&DWP(4,"esi")); + &mov ("ecx",&DWP(8,"esi")); + &mov ("ebx",&DWP(12,"esi")); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"ecx"); # magic + &mov (&DWP(8,"esp"),"ecx"); + &mov (&DWP(12,"esp"),"ebx"); + &mov ($E,&DWP(16,"esi")); + &mov ("ebx",&DWP(20,"esi")); + &mov ("ecx",&DWP(24,"esi")); + &mov ("esi",&DWP(28,"esi")); + #&mov (&DWP(16,"esp"),$E); + &mov (&DWP(20,"esp"),"ebx"); + &mov (&DWP(24,"esp"),"ecx"); + &mov (&DWP(28,"esp"),"esi"); + &jmp (&label("grand_loop")); + +&set_label("grand_loop",16); + # copy input block to stack reversing byte order + for($i=0;$i<5;$i++) { + &mov ("ebx",&DWP(12*$i+0,"edi")); + &mov ("ecx",&DWP(12*$i+4,"edi")); + &bswap ("ebx"); + &mov ("esi",&DWP(12*$i+8,"edi")); + &bswap ("ecx"); + &mov (&DWP(32+12*$i+0,"esp"),"ebx"); + &bswap ("esi"); + &mov (&DWP(32+12*$i+4,"esp"),"ecx"); + &mov (&DWP(32+12*$i+8,"esp"),"esi"); + } + &mov ("ebx",&DWP($i*12,"edi")); + &add ("edi",64); + &bswap ("ebx"); + &mov (&DWP(96+4,"esp"),"edi"); + &mov (&DWP(32+12*$i,"esp"),"ebx"); + + my ($t1,$t2) = ("ecx","esi"); + + for ($i=0;$i<64;$i++) { + + if ($i>=16) { + &mov ($T,$t1); # $t1 is preloaded + # &mov ($t2,&DWP(32+4*(($i+14)&15),"esp")); + &ror ($t1,18-7); + &mov ("edi",$t2); + &ror ($t2,19-17); + &xor ($t1,$T); + &shr ($T,3); + &ror ($t1,7); + &xor ($t2,"edi"); + &xor ($T,$t1); # T = sigma0(X[-15]) + &ror ($t2,17); + &add ($T,&DWP(32+4*($i&15),"esp")); # T += X[-16] + &shr ("edi",10); + &add ($T,&DWP(32+4*(($i+9)&15),"esp")); # T += X[-7] + #&xor ("edi",$t2) # sigma1(X[-2]) + # &add ($T,"edi"); # T += sigma1(X[-2]) + # &mov (&DWP(4*(9+15),"esp"),$T); # save X[0] + } + &mov ($t1,$E); + &xor ("edi",$t2) if ($i>=16); # sigma1(X[-2]) + &mov ($t2,&off($f)); + &ror ($E,25-11); + &add ($T,"edi") if ($i>=16); # T += sigma1(X[-2]) + &mov ("edi",&off($g)); + &xor ($E,$t1); + &mov ($T,&DWP(32+4*($i&15),"esp")) if ($i<16); # X[i] + &mov (&DWP(32+4*($i&15),"esp"),$T) if ($i>=16 && $i<62); # save X[0] + &xor ($t2,"edi"); + &ror ($E,11-6); + &and ($t2,$t1); + &mov (&off($e),$t1); # save $E, modulo-scheduled + &xor ($E,$t1); + &add ($T,&off($h)); # T += h + &xor ("edi",$t2); # Ch(e,f,g) + &ror ($E,6); # Sigma1(e) + &mov ($t1,$AH[0]); + &add ($T,"edi"); # T += Ch(e,f,g) + + &ror ($t1,22-13); + &mov ($t2,$AH[0]); + &mov ("edi",&off($b)); + &xor ($t1,$AH[0]); + &mov (&off($a),$AH[0]); # save $A, modulo-scheduled + &xor ($AH[0],"edi"); # a ^= b, (b^c) in next round + &ror ($t1,13-2); + &and ($AH[1],$AH[0]); # (b^c) &= (a^b) + &lea ($E,&DWP(@K256[$i],$T,$E)); # T += Sigma1(1)+K[i] + &xor ($t1,$t2); + &xor ($AH[1],"edi"); # h = Maj(a,b,c) = Ch(a^b,c,b) + &mov ($t2,&DWP(32+4*(($i+2)&15),"esp")) if ($i>=15 && $i<63); + &ror ($t1,2); # Sigma0(a) + + &add ($AH[1],$E); # h += T + &add ($E,&off($d)); # d += T + &add ($AH[1],$t1); # h += Sigma0(a) + &mov ($t1,&DWP(32+4*(($i+15)&15),"esp")) if ($i>=15 && $i<63); + + @AH = reverse(@AH); # rotate(a,h) + ($t1,$t2) = ($t2,$t1); # rotate(t1,t2) + } + &mov ("esi",&DWP(96,"esp")); #ctx + #&mov ($AH[0],&DWP(0,"esp")); + &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp")); + #&mov ("edi", &DWP(8,"esp")); + &mov ("ecx",&DWP(12,"esp")); + &add ($AH[0],&DWP(0,"esi")); + &add ($AH[1],&DWP(4,"esi")); + &add ("edi",&DWP(8,"esi")); + &add ("ecx",&DWP(12,"esi")); + &mov (&DWP(0,"esi"),$AH[0]); + &mov (&DWP(4,"esi"),$AH[1]); + &mov (&DWP(8,"esi"),"edi"); + &mov (&DWP(12,"esi"),"ecx"); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"edi"); # magic + &mov (&DWP(8,"esp"),"edi"); + &mov (&DWP(12,"esp"),"ecx"); + #&mov ($E,&DWP(16,"esp")); + &mov ("edi",&DWP(20,"esp")); + &mov ("ebx",&DWP(24,"esp")); + &mov ("ecx",&DWP(28,"esp")); + &add ($E,&DWP(16,"esi")); + &add ("edi",&DWP(20,"esi")); + &add ("ebx",&DWP(24,"esi")); + &add ("ecx",&DWP(28,"esi")); + &mov (&DWP(16,"esi"),$E); + &mov (&DWP(20,"esi"),"edi"); + &mov (&DWP(24,"esi"),"ebx"); + &mov (&DWP(28,"esi"),"ecx"); + #&mov (&DWP(16,"esp"),$E); + &mov (&DWP(20,"esp"),"edi"); + &mov ("edi",&DWP(96+4,"esp")); # inp + &mov (&DWP(24,"esp"),"ebx"); + &mov (&DWP(28,"esp"),"ecx"); + + &cmp ("edi",&DWP(96+8,"esp")); # are we done yet? + &jb (&label("grand_loop")); + + &mov ("esp",&DWP(96+12,"esp")); # restore sp +&function_end_A(); +} + if (!$i386 && $xmm) {{{ +if ($shaext) { +###################################################################### +# Intel SHA Extensions implementation of SHA256 update function. +# +my ($ctx,$inp,$end)=("esi","edi","eax"); +my ($Wi,$ABEF,$CDGH,$TMP)=map("xmm$_",(0..2,7)); +my @MSG=map("xmm$_",(3..6)); + +sub sha256op38 { + my ($opcodelet,$dst,$src)=@_; + if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) + { &data_byte(0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2); } +} +sub sha256rnds2 { sha256op38(0xcb,@_); } +sub sha256msg1 { sha256op38(0xcc,@_); } +sub sha256msg2 { sha256op38(0xcd,@_); } + +&set_label("shaext",32); + &sub ("esp",32); + + &movdqu ($ABEF,&QWP(0,$ctx)); # DCBA + &lea ($K256,&DWP(0x80,$K256)); + &movdqu ($CDGH,&QWP(16,$ctx)); # HGFE + &movdqa ($TMP,&QWP(0x100-0x80,$K256)); # byte swap mask + + &pshufd ($Wi,$ABEF,0x1b); # ABCD + &pshufd ($ABEF,$ABEF,0xb1); # CDAB + &pshufd ($CDGH,$CDGH,0x1b); # EFGH + &palignr ($ABEF,$CDGH,8); # ABEF + &punpcklqdq ($CDGH,$Wi); # CDGH + &jmp (&label("loop_shaext")); + +&set_label("loop_shaext",16); + &movdqu (@MSG[0],&QWP(0,$inp)); + &movdqu (@MSG[1],&QWP(0x10,$inp)); + &movdqu (@MSG[2],&QWP(0x20,$inp)); + &pshufb (@MSG[0],$TMP); + &movdqu (@MSG[3],&QWP(0x30,$inp)); + &movdqa (&QWP(16,"esp"),$CDGH); # offload + + &movdqa ($Wi,&QWP(0*16-0x80,$K256)); + &paddd ($Wi,@MSG[0]); + &pshufb (@MSG[1],$TMP); + &sha256rnds2 ($CDGH,$ABEF); # 0-3 + &pshufd ($Wi,$Wi,0x0e); + &nop (); + &movdqa (&QWP(0,"esp"),$ABEF); # offload + &sha256rnds2 ($ABEF,$CDGH); + + &movdqa ($Wi,&QWP(1*16-0x80,$K256)); + &paddd ($Wi,@MSG[1]); + &pshufb (@MSG[2],$TMP); + &sha256rnds2 ($CDGH,$ABEF); # 4-7 + &pshufd ($Wi,$Wi,0x0e); + &lea ($inp,&DWP(0x40,$inp)); + &sha256msg1 (@MSG[0],@MSG[1]); + &sha256rnds2 ($ABEF,$CDGH); + + &movdqa ($Wi,&QWP(2*16-0x80,$K256)); + &paddd ($Wi,@MSG[2]); + &pshufb (@MSG[3],$TMP); + &sha256rnds2 ($CDGH,$ABEF); # 8-11 + &pshufd ($Wi,$Wi,0x0e); + &movdqa ($TMP,@MSG[3]); + &palignr ($TMP,@MSG[2],4); + &nop (); + &paddd (@MSG[0],$TMP); + &sha256msg1 (@MSG[1],@MSG[2]); + &sha256rnds2 ($ABEF,$CDGH); + + &movdqa ($Wi,&QWP(3*16-0x80,$K256)); + &paddd ($Wi,@MSG[3]); + &sha256msg2 (@MSG[0],@MSG[3]); + &sha256rnds2 ($CDGH,$ABEF); # 12-15 + &pshufd ($Wi,$Wi,0x0e); + &movdqa ($TMP,@MSG[0]); + &palignr ($TMP,@MSG[3],4); + &nop (); + &paddd (@MSG[1],$TMP); + &sha256msg1 (@MSG[2],@MSG[3]); + &sha256rnds2 ($ABEF,$CDGH); + +for($i=4;$i<16-3;$i++) { + &movdqa ($Wi,&QWP($i*16-0x80,$K256)); + &paddd ($Wi,@MSG[0]); + &sha256msg2 (@MSG[1],@MSG[0]); + &sha256rnds2 ($CDGH,$ABEF); # 16-19... + &pshufd ($Wi,$Wi,0x0e); + &movdqa ($TMP,@MSG[1]); + &palignr ($TMP,@MSG[0],4); + &nop (); + &paddd (@MSG[2],$TMP); + &sha256msg1 (@MSG[3],@MSG[0]); + &sha256rnds2 ($ABEF,$CDGH); + + push(@MSG,shift(@MSG)); +} + &movdqa ($Wi,&QWP(13*16-0x80,$K256)); + &paddd ($Wi,@MSG[0]); + &sha256msg2 (@MSG[1],@MSG[0]); + &sha256rnds2 ($CDGH,$ABEF); # 52-55 + &pshufd ($Wi,$Wi,0x0e); + &movdqa ($TMP,@MSG[1]) + &palignr ($TMP,@MSG[0],4); + &sha256rnds2 ($ABEF,$CDGH); + &paddd (@MSG[2],$TMP); + + &movdqa ($Wi,&QWP(14*16-0x80,$K256)); + &paddd ($Wi,@MSG[1]); + &sha256rnds2 ($CDGH,$ABEF); # 56-59 + &pshufd ($Wi,$Wi,0x0e); + &sha256msg2 (@MSG[2],@MSG[1]); + &movdqa ($TMP,&QWP(0x100-0x80,$K256)); # byte swap mask + &sha256rnds2 ($ABEF,$CDGH); + + &movdqa ($Wi,&QWP(15*16-0x80,$K256)); + &paddd ($Wi,@MSG[2]); + &nop (); + &sha256rnds2 ($CDGH,$ABEF); # 60-63 + &pshufd ($Wi,$Wi,0x0e); + &cmp ($end,$inp); + &nop (); + &sha256rnds2 ($ABEF,$CDGH); + + &paddd ($CDGH,&QWP(16,"esp")); + &paddd ($ABEF,&QWP(0,"esp")); + &jnz (&label("loop_shaext")); + + &pshufd ($CDGH,$CDGH,0xb1); # DCHG + &pshufd ($TMP,$ABEF,0x1b); # FEBA + &pshufd ($ABEF,$ABEF,0xb1); # BAFE + &punpckhqdq ($ABEF,$CDGH); # DCBA + &palignr ($CDGH,$TMP,8); # HGFE + + &mov ("esp",&DWP(32+12,"esp")); + &movdqu (&QWP(0,$ctx),$ABEF); + &movdqu (&QWP(16,$ctx),$CDGH); +&function_end_A(); +} + +my @X = map("xmm$_",(0..3)); +my ($t0,$t1,$t2,$t3) = map("xmm$_",(4..7)); +my @AH = ($A,$T); + +&set_label("SSSE3",32); + &lea ("esp",&DWP(-96,"esp")); + # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack + &mov ($AH[0],&DWP(0,"esi")); + &mov ($AH[1],&DWP(4,"esi")); + &mov ("ecx",&DWP(8,"esi")); + &mov ("edi",&DWP(12,"esi")); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"ecx"); # magic + &mov (&DWP(8,"esp"),"ecx"); + &mov (&DWP(12,"esp"),"edi"); + &mov ($E,&DWP(16,"esi")); + &mov ("edi",&DWP(20,"esi")); + &mov ("ecx",&DWP(24,"esi")); + &mov ("esi",&DWP(28,"esi")); + #&mov (&DWP(16,"esp"),$E); + &mov (&DWP(20,"esp"),"edi"); + &mov ("edi",&DWP(96+4,"esp")); # inp + &mov (&DWP(24,"esp"),"ecx"); + &mov (&DWP(28,"esp"),"esi"); + &movdqa ($t3,&QWP(256,$K256)); + &jmp (&label("grand_ssse3")); + +&set_label("grand_ssse3",16); + # load input, reverse byte order, add K256[0..15], save to stack + &movdqu (@X[0],&QWP(0,"edi")); + &movdqu (@X[1],&QWP(16,"edi")); + &movdqu (@X[2],&QWP(32,"edi")); + &movdqu (@X[3],&QWP(48,"edi")); + &add ("edi",64); + &pshufb (@X[0],$t3); + &mov (&DWP(96+4,"esp"),"edi"); + &pshufb (@X[1],$t3); + &movdqa ($t0,&QWP(0,$K256)); + &pshufb (@X[2],$t3); + &movdqa ($t1,&QWP(16,$K256)); + &paddd ($t0,@X[0]); + &pshufb (@X[3],$t3); + &movdqa ($t2,&QWP(32,$K256)); + &paddd ($t1,@X[1]); + &movdqa ($t3,&QWP(48,$K256)); + &movdqa (&QWP(32+0,"esp"),$t0); + &paddd ($t2,@X[2]); + &movdqa (&QWP(32+16,"esp"),$t1); + &paddd ($t3,@X[3]); + &movdqa (&QWP(32+32,"esp"),$t2); + &movdqa (&QWP(32+48,"esp"),$t3); + &jmp (&label("ssse3_00_47")); + +&set_label("ssse3_00_47",16); + &add ($K256,64); + +sub SSSE3_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 120 instructions + + eval(shift(@insns)); + &movdqa ($t0,@X[1]); + eval(shift(@insns)); # @ + eval(shift(@insns)); + &movdqa ($t3,@X[3]); + eval(shift(@insns)); + eval(shift(@insns)); + &palignr ($t0,@X[0],4); # X[1..4] + eval(shift(@insns)); + eval(shift(@insns)); # @ + eval(shift(@insns)); + &palignr ($t3,@X[2],4); # X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa ($t1,$t0); + eval(shift(@insns)); # @ + eval(shift(@insns)); + &movdqa ($t2,$t0); + eval(shift(@insns)); + eval(shift(@insns)); + &psrld ($t0,3); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &paddd (@X[0],$t3); # X[0..3] += X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + &psrld ($t2,7); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + eval(shift(@insns)); + &pshufd ($t3,@X[3],0b11111010); # X[14..15] + eval(shift(@insns)); + eval(shift(@insns)); + &pslld ($t1,32-18); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t0,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + &psrld ($t2,18-7); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t0,$t1); + eval(shift(@insns)); + eval(shift(@insns)); + &pslld ($t1,18-7); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t0,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa ($t2,$t3); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t0,$t1); # sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &psrld ($t3,10); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &paddd (@X[0],$t0); # X[0..3] += sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &psrlq ($t2,17); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + &psrlq ($t2,19-17); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + &pshufd ($t3,$t3,0b10000000); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + eval(shift(@insns)); + &psrldq ($t3,8); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@X[0],$t3); # X[0..1] += sigma1(X[14..15]) + eval(shift(@insns)); # @ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + eval(shift(@insns)); + &pshufd ($t3,@X[0],0b01010000); # X[16..17] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa ($t2,$t3); + eval(shift(@insns)); # @ + &psrld ($t3,10); + eval(shift(@insns)); + &psrlq ($t2,17); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + &psrlq ($t2,19-17); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &pxor ($t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &pshufd ($t3,$t3,0b00001000); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &movdqa ($t2,&QWP(16*$j,$K256)); + eval(shift(@insns)); + eval(shift(@insns)); + &pslldq ($t3,8); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # @ + &paddd (@X[0],$t3); # X[2..3] += sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &paddd ($t2,@X[0]); + eval(shift(@insns)); # @ + + foreach (@insns) { eval; } # remaining instructions + + &movdqa (&QWP(32+16*$j,"esp"),$t2); +} + +sub body_00_15 () { + ( + '&mov ("ecx",$E);', + '&ror ($E,25-11);', + '&mov ("esi",&off($f));', + '&xor ($E,"ecx");', + '&mov ("edi",&off($g));', + '&xor ("esi","edi");', + '&ror ($E,11-6);', + '&and ("esi","ecx");', + '&mov (&off($e),"ecx");', # save $E, modulo-scheduled + '&xor ($E,"ecx");', + '&xor ("edi","esi");', # Ch(e,f,g) + '&ror ($E,6);', # T = Sigma1(e) + '&mov ("ecx",$AH[0]);', + '&add ($E,"edi");', # T += Ch(e,f,g) + '&mov ("edi",&off($b));', + '&mov ("esi",$AH[0]);', + + '&ror ("ecx",22-13);', + '&mov (&off($a),$AH[0]);', # save $A, modulo-scheduled + '&xor ("ecx",$AH[0]);', + '&xor ($AH[0],"edi");', # a ^= b, (b^c) in next round + '&add ($E,&off($h));', # T += h + '&ror ("ecx",13-2);', + '&and ($AH[1],$AH[0]);', # (b^c) &= (a^b) + '&xor ("ecx","esi");', + '&add ($E,&DWP(32+4*($i&15),"esp"));', # T += K[i]+X[i] + '&xor ($AH[1],"edi");', # h = Maj(a,b,c) = Ch(a^b,c,b) + '&ror ("ecx",2);', # Sigma0(a) + + '&add ($AH[1],$E);', # h += T + '&add ($E,&off($d));', # d += T + '&add ($AH[1],"ecx");'. # h += Sigma0(a) + + '@AH = reverse(@AH); $i++;' # rotate(a,h) + ); +} + + for ($i=0,$j=0; $j<4; $j++) { + &SSSE3_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmp (&DWP(16*$j,$K256),0x00010203); + &jne (&label("ssse3_00_47")); + + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } + + &mov ("esi",&DWP(96,"esp")); #ctx + #&mov ($AH[0],&DWP(0,"esp")); + &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp")); + #&mov ("edi", &DWP(8,"esp")); + &mov ("ecx",&DWP(12,"esp")); + &add ($AH[0],&DWP(0,"esi")); + &add ($AH[1],&DWP(4,"esi")); + &add ("edi",&DWP(8,"esi")); + &add ("ecx",&DWP(12,"esi")); + &mov (&DWP(0,"esi"),$AH[0]); + &mov (&DWP(4,"esi"),$AH[1]); + &mov (&DWP(8,"esi"),"edi"); + &mov (&DWP(12,"esi"),"ecx"); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"edi"); # magic + &mov (&DWP(8,"esp"),"edi"); + &mov (&DWP(12,"esp"),"ecx"); + #&mov ($E,&DWP(16,"esp")); + &mov ("edi",&DWP(20,"esp")); + &mov ("ecx",&DWP(24,"esp")); + &add ($E,&DWP(16,"esi")); + &add ("edi",&DWP(20,"esi")); + &add ("ecx",&DWP(24,"esi")); + &mov (&DWP(16,"esi"),$E); + &mov (&DWP(20,"esi"),"edi"); + &mov (&DWP(20,"esp"),"edi"); + &mov ("edi",&DWP(28,"esp")); + &mov (&DWP(24,"esi"),"ecx"); + #&mov (&DWP(16,"esp"),$E); + &add ("edi",&DWP(28,"esi")); + &mov (&DWP(24,"esp"),"ecx"); + &mov (&DWP(28,"esi"),"edi"); + &mov (&DWP(28,"esp"),"edi"); + &mov ("edi",&DWP(96+4,"esp")); # inp + + &movdqa ($t3,&QWP(64,$K256)); + &sub ($K256,3*64); # rewind K + &cmp ("edi",&DWP(96+8,"esp")); # are we done yet? + &jb (&label("grand_ssse3")); + + &mov ("esp",&DWP(96+12,"esp")); # restore sp +&function_end_A(); + if ($avx) { +&set_label("AVX",32); + if ($avx>1) { + &and ("edx",1<<8|1<<3); # check for BMI2+BMI1 + &cmp ("edx",1<<8|1<<3); + &je (&label("AVX_BMI")); + } + &lea ("esp",&DWP(-96,"esp")); + &vzeroall (); + # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack + &mov ($AH[0],&DWP(0,"esi")); + &mov ($AH[1],&DWP(4,"esi")); + &mov ("ecx",&DWP(8,"esi")); + &mov ("edi",&DWP(12,"esi")); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"ecx"); # magic + &mov (&DWP(8,"esp"),"ecx"); + &mov (&DWP(12,"esp"),"edi"); + &mov ($E,&DWP(16,"esi")); + &mov ("edi",&DWP(20,"esi")); + &mov ("ecx",&DWP(24,"esi")); + &mov ("esi",&DWP(28,"esi")); + #&mov (&DWP(16,"esp"),$E); + &mov (&DWP(20,"esp"),"edi"); + &mov ("edi",&DWP(96+4,"esp")); # inp + &mov (&DWP(24,"esp"),"ecx"); + &mov (&DWP(28,"esp"),"esi"); + &vmovdqa ($t3,&QWP(256,$K256)); + &jmp (&label("grand_avx")); + +&set_label("grand_avx",32); + # load input, reverse byte order, add K256[0..15], save to stack + &vmovdqu (@X[0],&QWP(0,"edi")); + &vmovdqu (@X[1],&QWP(16,"edi")); + &vmovdqu (@X[2],&QWP(32,"edi")); + &vmovdqu (@X[3],&QWP(48,"edi")); + &add ("edi",64); + &vpshufb (@X[0],@X[0],$t3); + &mov (&DWP(96+4,"esp"),"edi"); + &vpshufb (@X[1],@X[1],$t3); + &vpshufb (@X[2],@X[2],$t3); + &vpaddd ($t0,@X[0],&QWP(0,$K256)); + &vpshufb (@X[3],@X[3],$t3); + &vpaddd ($t1,@X[1],&QWP(16,$K256)); + &vpaddd ($t2,@X[2],&QWP(32,$K256)); + &vpaddd ($t3,@X[3],&QWP(48,$K256)); + &vmovdqa (&QWP(32+0,"esp"),$t0); + &vmovdqa (&QWP(32+16,"esp"),$t1); + &vmovdqa (&QWP(32+32,"esp"),$t2); + &vmovdqa (&QWP(32+48,"esp"),$t3); + &jmp (&label("avx_00_47")); + +&set_label("avx_00_47",16); + &add ($K256,64); + +sub Xupdate_AVX () { + ( + '&vpalignr ($t0,@X[1],@X[0],4);', # X[1..4] + '&vpalignr ($t3,@X[3],@X[2],4);', # X[9..12] + '&vpsrld ($t2,$t0,7);', + '&vpaddd (@X[0],@X[0],$t3);', # X[0..3] += X[9..16] + '&vpsrld ($t3,$t0,3);', + '&vpslld ($t1,$t0,14);', + '&vpxor ($t0,$t3,$t2);', + '&vpshufd ($t3,@X[3],0b11111010)',# X[14..15] + '&vpsrld ($t2,$t2,18-7);', + '&vpxor ($t0,$t0,$t1);', + '&vpslld ($t1,$t1,25-14);', + '&vpxor ($t0,$t0,$t2);', + '&vpsrld ($t2,$t3,10);', + '&vpxor ($t0,$t0,$t1);', # sigma0(X[1..4]) + '&vpsrlq ($t1,$t3,17);', + '&vpaddd (@X[0],@X[0],$t0);', # X[0..3] += sigma0(X[1..4]) + '&vpxor ($t2,$t2,$t1);', + '&vpsrlq ($t3,$t3,19);', + '&vpxor ($t2,$t2,$t3);', # sigma1(X[14..15] + '&vpshufd ($t3,$t2,0b10000100);', + '&vpsrldq ($t3,$t3,8);', + '&vpaddd (@X[0],@X[0],$t3);', # X[0..1] += sigma1(X[14..15]) + '&vpshufd ($t3,@X[0],0b01010000)',# X[16..17] + '&vpsrld ($t2,$t3,10);', + '&vpsrlq ($t1,$t3,17);', + '&vpxor ($t2,$t2,$t1);', + '&vpsrlq ($t3,$t3,19);', + '&vpxor ($t2,$t2,$t3);', # sigma1(X[16..17] + '&vpshufd ($t3,$t2,0b11101000);', + '&vpslldq ($t3,$t3,8);', + '&vpaddd (@X[0],@X[0],$t3);' # X[2..3] += sigma1(X[16..17]) + ); +} + +local *ror = sub { &shrd(@_[0],@_) }; +sub AVX_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 120 instructions +my $insn; + + foreach (Xupdate_AVX()) { # 31 instructions + eval; + eval(shift(@insns)); + eval(shift(@insns)); + eval($insn = shift(@insns)); + eval(shift(@insns)) if ($insn =~ /rorx/ && @insns[0] =~ /rorx/); + } + &vpaddd ($t2,@X[0],&QWP(16*$j,$K256)); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa (&QWP(32+16*$j,"esp"),$t2); +} + + for ($i=0,$j=0; $j<4; $j++) { + &AVX_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmp (&DWP(16*$j,$K256),0x00010203); + &jne (&label("avx_00_47")); + + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } + + &mov ("esi",&DWP(96,"esp")); #ctx + #&mov ($AH[0],&DWP(0,"esp")); + &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp")); + #&mov ("edi", &DWP(8,"esp")); + &mov ("ecx",&DWP(12,"esp")); + &add ($AH[0],&DWP(0,"esi")); + &add ($AH[1],&DWP(4,"esi")); + &add ("edi",&DWP(8,"esi")); + &add ("ecx",&DWP(12,"esi")); + &mov (&DWP(0,"esi"),$AH[0]); + &mov (&DWP(4,"esi"),$AH[1]); + &mov (&DWP(8,"esi"),"edi"); + &mov (&DWP(12,"esi"),"ecx"); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"edi"); # magic + &mov (&DWP(8,"esp"),"edi"); + &mov (&DWP(12,"esp"),"ecx"); + #&mov ($E,&DWP(16,"esp")); + &mov ("edi",&DWP(20,"esp")); + &mov ("ecx",&DWP(24,"esp")); + &add ($E,&DWP(16,"esi")); + &add ("edi",&DWP(20,"esi")); + &add ("ecx",&DWP(24,"esi")); + &mov (&DWP(16,"esi"),$E); + &mov (&DWP(20,"esi"),"edi"); + &mov (&DWP(20,"esp"),"edi"); + &mov ("edi",&DWP(28,"esp")); + &mov (&DWP(24,"esi"),"ecx"); + #&mov (&DWP(16,"esp"),$E); + &add ("edi",&DWP(28,"esi")); + &mov (&DWP(24,"esp"),"ecx"); + &mov (&DWP(28,"esi"),"edi"); + &mov (&DWP(28,"esp"),"edi"); + &mov ("edi",&DWP(96+4,"esp")); # inp + + &vmovdqa ($t3,&QWP(64,$K256)); + &sub ($K256,3*64); # rewind K + &cmp ("edi",&DWP(96+8,"esp")); # are we done yet? + &jb (&label("grand_avx")); + + &mov ("esp",&DWP(96+12,"esp")); # restore sp + &vzeroall (); +&function_end_A(); + if ($avx>1) { +sub bodyx_00_15 () { # +10% + ( + '&rorx ("ecx",$E,6)', + '&rorx ("esi",$E,11)', + '&mov (&off($e),$E)', # save $E, modulo-scheduled + '&rorx ("edi",$E,25)', + '&xor ("ecx","esi")', + '&andn ("esi",$E,&off($g))', + '&xor ("ecx","edi")', # Sigma1(e) + '&and ($E,&off($f))', + '&mov (&off($a),$AH[0]);', # save $A, modulo-scheduled + '&or ($E,"esi")', # T = Ch(e,f,g) + + '&rorx ("edi",$AH[0],2)', + '&rorx ("esi",$AH[0],13)', + '&lea ($E,&DWP(0,$E,"ecx"))', # T += Sigma1(e) + '&rorx ("ecx",$AH[0],22)', + '&xor ("esi","edi")', + '&mov ("edi",&off($b))', + '&xor ("ecx","esi")', # Sigma0(a) + + '&xor ($AH[0],"edi")', # a ^= b, (b^c) in next round + '&add ($E,&off($h))', # T += h + '&and ($AH[1],$AH[0])', # (b^c) &= (a^b) + '&add ($E,&DWP(32+4*($i&15),"esp"))', # T += K[i]+X[i] + '&xor ($AH[1],"edi")', # h = Maj(a,b,c) = Ch(a^b,c,b) + + '&add ("ecx",$E)', # h += T + '&add ($E,&off($d))', # d += T + '&lea ($AH[1],&DWP(0,$AH[1],"ecx"));'. # h += Sigma0(a) + + '@AH = reverse(@AH); $i++;' # rotate(a,h) + ); +} + +&set_label("AVX_BMI",32); + &lea ("esp",&DWP(-96,"esp")); + &vzeroall (); + # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack + &mov ($AH[0],&DWP(0,"esi")); + &mov ($AH[1],&DWP(4,"esi")); + &mov ("ecx",&DWP(8,"esi")); + &mov ("edi",&DWP(12,"esi")); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"ecx"); # magic + &mov (&DWP(8,"esp"),"ecx"); + &mov (&DWP(12,"esp"),"edi"); + &mov ($E,&DWP(16,"esi")); + &mov ("edi",&DWP(20,"esi")); + &mov ("ecx",&DWP(24,"esi")); + &mov ("esi",&DWP(28,"esi")); + #&mov (&DWP(16,"esp"),$E); + &mov (&DWP(20,"esp"),"edi"); + &mov ("edi",&DWP(96+4,"esp")); # inp + &mov (&DWP(24,"esp"),"ecx"); + &mov (&DWP(28,"esp"),"esi"); + &vmovdqa ($t3,&QWP(256,$K256)); + &jmp (&label("grand_avx_bmi")); + +&set_label("grand_avx_bmi",32); + # load input, reverse byte order, add K256[0..15], save to stack + &vmovdqu (@X[0],&QWP(0,"edi")); + &vmovdqu (@X[1],&QWP(16,"edi")); + &vmovdqu (@X[2],&QWP(32,"edi")); + &vmovdqu (@X[3],&QWP(48,"edi")); + &add ("edi",64); + &vpshufb (@X[0],@X[0],$t3); + &mov (&DWP(96+4,"esp"),"edi"); + &vpshufb (@X[1],@X[1],$t3); + &vpshufb (@X[2],@X[2],$t3); + &vpaddd ($t0,@X[0],&QWP(0,$K256)); + &vpshufb (@X[3],@X[3],$t3); + &vpaddd ($t1,@X[1],&QWP(16,$K256)); + &vpaddd ($t2,@X[2],&QWP(32,$K256)); + &vpaddd ($t3,@X[3],&QWP(48,$K256)); + &vmovdqa (&QWP(32+0,"esp"),$t0); + &vmovdqa (&QWP(32+16,"esp"),$t1); + &vmovdqa (&QWP(32+32,"esp"),$t2); + &vmovdqa (&QWP(32+48,"esp"),$t3); + &jmp (&label("avx_bmi_00_47")); + +&set_label("avx_bmi_00_47",16); + &add ($K256,64); + + for ($i=0,$j=0; $j<4; $j++) { + &AVX_00_47($j,\&bodyx_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmp (&DWP(16*$j,$K256),0x00010203); + &jne (&label("avx_bmi_00_47")); + + for ($i=0; $i<16; ) { + foreach(bodyx_00_15()) { eval; } + } + + &mov ("esi",&DWP(96,"esp")); #ctx + #&mov ($AH[0],&DWP(0,"esp")); + &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp")); + #&mov ("edi", &DWP(8,"esp")); + &mov ("ecx",&DWP(12,"esp")); + &add ($AH[0],&DWP(0,"esi")); + &add ($AH[1],&DWP(4,"esi")); + &add ("edi",&DWP(8,"esi")); + &add ("ecx",&DWP(12,"esi")); + &mov (&DWP(0,"esi"),$AH[0]); + &mov (&DWP(4,"esi"),$AH[1]); + &mov (&DWP(8,"esi"),"edi"); + &mov (&DWP(12,"esi"),"ecx"); + #&mov (&DWP(0,"esp"),$AH[0]); + &mov (&DWP(4,"esp"),$AH[1]); + &xor ($AH[1],"edi"); # magic + &mov (&DWP(8,"esp"),"edi"); + &mov (&DWP(12,"esp"),"ecx"); + #&mov ($E,&DWP(16,"esp")); + &mov ("edi",&DWP(20,"esp")); + &mov ("ecx",&DWP(24,"esp")); + &add ($E,&DWP(16,"esi")); + &add ("edi",&DWP(20,"esi")); + &add ("ecx",&DWP(24,"esi")); + &mov (&DWP(16,"esi"),$E); + &mov (&DWP(20,"esi"),"edi"); + &mov (&DWP(20,"esp"),"edi"); + &mov ("edi",&DWP(28,"esp")); + &mov (&DWP(24,"esi"),"ecx"); + #&mov (&DWP(16,"esp"),$E); + &add ("edi",&DWP(28,"esi")); + &mov (&DWP(24,"esp"),"ecx"); + &mov (&DWP(28,"esi"),"edi"); + &mov (&DWP(28,"esp"),"edi"); + &mov ("edi",&DWP(96+4,"esp")); # inp + + &vmovdqa ($t3,&QWP(64,$K256)); + &sub ($K256,3*64); # rewind K + &cmp ("edi",&DWP(96+8,"esp")); # are we done yet? + &jb (&label("grand_avx_bmi")); + + &mov ("esp",&DWP(96+12,"esp")); # restore sp + &vzeroall (); +&function_end_A(); + } + } + }}} +&function_end_B("sha256_block_data_order"); + &asm_finish(); diff --git a/crypto/sha/asm/sha256-armv4.pl b/crypto/sha/asm/sha256-armv4.pl index 9c84e8d93c30..4fee74d832d1 100755 --- a/crypto/sha/asm/sha256-armv4.pl +++ b/crypto/sha/asm/sha256-armv4.pl @@ -1,10 +1,12 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. +# +# Permission to use under GPL terms is granted. # ==================================================================== # SHA256 block procedure for ARMv4. May 2007. @@ -21,15 +23,27 @@ # February 2011. # # Profiler-assisted and platform-specific optimization resulted in 16% -# improvement on Cortex A8 core and ~17 cycles per processed byte. +# improvement on Cortex A8 core and ~15.4 cycles per processed byte. + +# September 2013. +# +# Add NEON implementation. On Cortex A8 it was measured to process one +# byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon +# S4 does it in 12.5 cycles too, but it's 50% faster than integer-only +# code (meaning that latter performs sub-optimally, nothing was done +# about it). + +# May 2014. +# +# Add ARMv8 code path performing at 2.0 cpb on Apple A7. while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} open STDOUT,">$output"; $ctx="r0"; $t0="r0"; -$inp="r1"; $t3="r1"; +$inp="r1"; $t4="r1"; $len="r2"; $t1="r2"; -$T1="r3"; +$T1="r3"; $t3="r3"; $A="r4"; $B="r5"; $C="r6"; @@ -52,80 +66,111 @@ my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; $code.=<<___ if ($i<16); #if __ARM_ARCH__>=7 - ldr $T1,[$inp],#4 + @ ldr $t1,[$inp],#4 @ $i +# if $i==15 + str $inp,[sp,#17*4] @ make room for $t4 +# endif + eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` + add $a,$a,$t2 @ h+=Maj(a,b,c) from the past + eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e) + rev $t1,$t1 #else - ldrb $T1,[$inp,#3] @ $i + @ ldrb $t1,[$inp,#3] @ $i + add $a,$a,$t2 @ h+=Maj(a,b,c) from the past ldrb $t2,[$inp,#2] - ldrb $t1,[$inp,#1] - ldrb $t0,[$inp],#4 - orr $T1,$T1,$t2,lsl#8 - orr $T1,$T1,$t1,lsl#16 - orr $T1,$T1,$t0,lsl#24 + ldrb $t0,[$inp,#1] + orr $t1,$t1,$t2,lsl#8 + ldrb $t2,[$inp],#4 + orr $t1,$t1,$t0,lsl#16 +# if $i==15 + str $inp,[sp,#17*4] @ make room for $t4 +# endif + eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` + orr $t1,$t1,$t2,lsl#24 + eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e) #endif ___ $code.=<<___; - mov $t0,$e,ror#$Sigma1[0] ldr $t2,[$Ktbl],#4 @ *K256++ - eor $t0,$t0,$e,ror#$Sigma1[1] + add $h,$h,$t1 @ h+=X[i] + str $t1,[sp,#`$i%16`*4] eor $t1,$f,$g -#if $i>=16 - add $T1,$T1,$t3 @ from BODY_16_xx -#elif __ARM_ARCH__>=7 && defined(__ARMEL__) - rev $T1,$T1 -#endif -#if $i==15 - str $inp,[sp,#17*4] @ leave room for $t3 -#endif - eor $t0,$t0,$e,ror#$Sigma1[2] @ Sigma1(e) + add $h,$h,$t0,ror#$Sigma1[0] @ h+=Sigma1(e) and $t1,$t1,$e - str $T1,[sp,#`$i%16`*4] - add $T1,$T1,$t0 + add $h,$h,$t2 @ h+=K256[i] eor $t1,$t1,$g @ Ch(e,f,g) - add $T1,$T1,$h - mov $h,$a,ror#$Sigma0[0] - add $T1,$T1,$t1 - eor $h,$h,$a,ror#$Sigma0[1] - add $T1,$T1,$t2 - eor $h,$h,$a,ror#$Sigma0[2] @ Sigma0(a) -#if $i>=15 - ldr $t3,[sp,#`($i+2)%16`*4] @ from BODY_16_xx + eor $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]` + add $h,$h,$t1 @ h+=Ch(e,f,g) +#if $i==31 + and $t2,$t2,#0xff + cmp $t2,#0xf2 @ done? +#endif +#if $i<15 +# if __ARM_ARCH__>=7 + ldr $t1,[$inp],#4 @ prefetch +# else + ldrb $t1,[$inp,#3] +# endif + eor $t2,$a,$b @ a^b, b^c in next round +#else + ldr $t1,[sp,#`($i+2)%16`*4] @ from future BODY_16_xx + eor $t2,$a,$b @ a^b, b^c in next round + ldr $t4,[sp,#`($i+15)%16`*4] @ from future BODY_16_xx #endif - orr $t0,$a,$b - and $t1,$a,$b - and $t0,$t0,$c - add $h,$h,$T1 - orr $t0,$t0,$t1 @ Maj(a,b,c) - add $d,$d,$T1 - add $h,$h,$t0 + eor $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]` @ Sigma0(a) + and $t3,$t3,$t2 @ (b^c)&=(a^b) + add $d,$d,$h @ d+=h + eor $t3,$t3,$b @ Maj(a,b,c) + add $h,$h,$t0,ror#$Sigma0[0] @ h+=Sigma0(a) + @ add $h,$h,$t3 @ h+=Maj(a,b,c) ___ + ($t2,$t3)=($t3,$t2); } sub BODY_16_XX { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; $code.=<<___; - @ ldr $t3,[sp,#`($i+1)%16`*4] @ $i - ldr $t2,[sp,#`($i+14)%16`*4] - mov $t0,$t3,ror#$sigma0[0] - ldr $T1,[sp,#`($i+0)%16`*4] - eor $t0,$t0,$t3,ror#$sigma0[1] - ldr $t1,[sp,#`($i+9)%16`*4] - eor $t0,$t0,$t3,lsr#$sigma0[2] @ sigma0(X[i+1]) - mov $t3,$t2,ror#$sigma1[0] - add $T1,$T1,$t0 - eor $t3,$t3,$t2,ror#$sigma1[1] - add $T1,$T1,$t1 - eor $t3,$t3,$t2,lsr#$sigma1[2] @ sigma1(X[i+14]) - @ add $T1,$T1,$t3 + @ ldr $t1,[sp,#`($i+1)%16`*4] @ $i + @ ldr $t4,[sp,#`($i+14)%16`*4] + mov $t0,$t1,ror#$sigma0[0] + add $a,$a,$t2 @ h+=Maj(a,b,c) from the past + mov $t2,$t4,ror#$sigma1[0] + eor $t0,$t0,$t1,ror#$sigma0[1] + eor $t2,$t2,$t4,ror#$sigma1[1] + eor $t0,$t0,$t1,lsr#$sigma0[2] @ sigma0(X[i+1]) + ldr $t1,[sp,#`($i+0)%16`*4] + eor $t2,$t2,$t4,lsr#$sigma1[2] @ sigma1(X[i+14]) + ldr $t4,[sp,#`($i+9)%16`*4] + + add $t2,$t2,$t0 + eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` @ from BODY_00_15 + add $t1,$t1,$t2 + eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e) + add $t1,$t1,$t4 @ X[i] ___ &BODY_00_15(@_); } $code=<<___; -#include "arm_arch.h" +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ 7 +#endif .text +#if __ARM_ARCH__<7 .code 32 +#else +.syntax unified +# ifdef __thumb2__ +.thumb +# else +.code 32 +# endif +#endif .type K256,%object .align 5 @@ -147,46 +192,73 @@ K256: .word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 .word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 .size K256,.-K256 +.word 0 @ terminator +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-sha256_block_data_order +#endif +.align 5 .global sha256_block_data_order .type sha256_block_data_order,%function sha256_block_data_order: +#if __ARM_ARCH__<7 sub r3,pc,#8 @ sha256_block_data_order +#else + adr r3,sha256_block_data_order +#endif +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + ldr r12,.LOPENSSL_armcap + ldr r12,[r3,r12] @ OPENSSL_armcap_P + tst r12,#ARMV8_SHA256 + bne .LARMv8 + tst r12,#ARMV7_NEON + bne .LNEON +#endif add $len,$inp,$len,lsl#6 @ len to point at the end of inp stmdb sp!,{$ctx,$inp,$len,r4-r11,lr} ldmia $ctx,{$A,$B,$C,$D,$E,$F,$G,$H} - sub $Ktbl,r3,#256 @ K256 + sub $Ktbl,r3,#256+32 @ K256 sub sp,sp,#16*4 @ alloca(X[16]) .Loop: +# if __ARM_ARCH__>=7 + ldr $t1,[$inp],#4 +# else + ldrb $t1,[$inp,#3] +# endif + eor $t3,$B,$C @ magic + eor $t2,$t2,$t2 ___ for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } $code.=".Lrounds_16_xx:\n"; for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); } $code.=<<___; - and $t2,$t2,#0xff - cmp $t2,#0xf2 +#if __ARM_ARCH__>=7 + ite eq @ Thumb2 thing, sanity check in ARM +#endif + ldreq $t3,[sp,#16*4] @ pull ctx bne .Lrounds_16_xx - ldr $T1,[sp,#16*4] @ pull ctx - ldr $t0,[$T1,#0] - ldr $t1,[$T1,#4] - ldr $t2,[$T1,#8] + add $A,$A,$t2 @ h+=Maj(a,b,c) from the past + ldr $t0,[$t3,#0] + ldr $t1,[$t3,#4] + ldr $t2,[$t3,#8] add $A,$A,$t0 - ldr $t0,[$T1,#12] + ldr $t0,[$t3,#12] add $B,$B,$t1 - ldr $t1,[$T1,#16] + ldr $t1,[$t3,#16] add $C,$C,$t2 - ldr $t2,[$T1,#20] + ldr $t2,[$t3,#20] add $D,$D,$t0 - ldr $t0,[$T1,#24] + ldr $t0,[$t3,#24] add $E,$E,$t1 - ldr $t1,[$T1,#28] + ldr $t1,[$t3,#28] add $F,$F,$t2 ldr $inp,[sp,#17*4] @ pull inp ldr $t2,[sp,#18*4] @ pull inp+len add $G,$G,$t0 add $H,$H,$t1 - stmia $T1,{$A,$B,$C,$D,$E,$F,$G,$H} + stmia $t3,{$A,$B,$C,$D,$E,$F,$G,$H} cmp $inp,$t2 sub $Ktbl,$Ktbl,#256 @ rewind Ktbl bne .Loop @@ -200,12 +272,442 @@ $code.=<<___; moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif -.size sha256_block_data_order,.-sha256_block_data_order -.asciz "SHA256 block transform for ARMv4, CRYPTOGAMS by " +.size sha256_block_data_order,.-sha256_block_data_order +___ +###################################################################### +# NEON stuff +# +{{{ +my @X=map("q$_",(0..3)); +my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25"); +my $Xfer=$t4; +my $j=0; + +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +sub Xupdate() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e,$f,$g,$h); + + &vext_8 ($T0,@X[0],@X[1],4); # X[1..4] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vext_8 ($T1,@X[2],@X[3],4); # X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T2,$T0,$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T1,$T0,$sigma0[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T2,$T0,32-$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T3,$T0,$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T1,$T1,$T2); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T3,$T0,32-$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T1,$T1,$T3); # sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T5,&Dhi(@X[3]),$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); # sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T5,&Dlo(@X[0]),$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vld1_32 ("{$T0}","[$Ktbl,:128]!"); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); # sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 ($T0,$T0,@X[0]); + while($#insns>=2) { eval(shift(@insns)); } + &vst1_32 ("{$T0}","[$Xfer,:128]!"); + eval(shift(@insns)); + eval(shift(@insns)); + + push(@X,shift(@X)); # "rotate" X[] +} + +sub Xpreload() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e,$f,$g,$h); + + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vld1_32 ("{$T0}","[$Ktbl,:128]!"); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vrev32_8 (@X[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 ($T0,$T0,@X[0]); + foreach (@insns) { eval; } # remaining instructions + &vst1_32 ("{$T0}","[$Xfer,:128]!"); + + push(@X,shift(@X)); # "rotate" X[] +} + +sub body_00_15 () { + ( + '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'. + '&add ($h,$h,$t1)', # h+=X[i]+K[i] + '&eor ($t1,$f,$g)', + '&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))', + '&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past + '&and ($t1,$t1,$e)', + '&eor ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e) + '&eor ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))', + '&eor ($t1,$t1,$g)', # Ch(e,f,g) + '&add ($h,$h,$t2,"ror#$Sigma1[0]")', # h+=Sigma1(e) + '&eor ($t2,$a,$b)', # a^b, b^c in next round + '&eor ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a) + '&add ($h,$h,$t1)', # h+=Ch(e,f,g) + '&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'. + '&ldr ($t1,"[$Ktbl]") if ($j==15);'. + '&ldr ($t1,"[sp,#64]") if ($j==31)', + '&and ($t3,$t3,$t2)', # (b^c)&=(a^b) + '&add ($d,$d,$h)', # d+=h + '&add ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a) + '&eor ($t3,$t3,$b)', # Maj(a,b,c) + '$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);' + ) +} + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.global sha256_block_data_order_neon +.type sha256_block_data_order_neon,%function +.align 4 +sha256_block_data_order_neon: +.LNEON: + stmdb sp!,{r4-r12,lr} + + sub $H,sp,#16*4+16 + adr $Ktbl,K256 + bic $H,$H,#15 @ align for 128-bit stores + mov $t2,sp + mov sp,$H @ alloca + add $len,$inp,$len,lsl#6 @ len to point at the end of inp + + vld1.8 {@X[0]},[$inp]! + vld1.8 {@X[1]},[$inp]! + vld1.8 {@X[2]},[$inp]! + vld1.8 {@X[3]},[$inp]! + vld1.32 {$T0},[$Ktbl,:128]! + vld1.32 {$T1},[$Ktbl,:128]! + vld1.32 {$T2},[$Ktbl,:128]! + vld1.32 {$T3},[$Ktbl,:128]! + vrev32.8 @X[0],@X[0] @ yes, even on + str $ctx,[sp,#64] + vrev32.8 @X[1],@X[1] @ big-endian + str $inp,[sp,#68] + mov $Xfer,sp + vrev32.8 @X[2],@X[2] + str $len,[sp,#72] + vrev32.8 @X[3],@X[3] + str $t2,[sp,#76] @ save original sp + vadd.i32 $T0,$T0,@X[0] + vadd.i32 $T1,$T1,@X[1] + vst1.32 {$T0},[$Xfer,:128]! + vadd.i32 $T2,$T2,@X[2] + vst1.32 {$T1},[$Xfer,:128]! + vadd.i32 $T3,$T3,@X[3] + vst1.32 {$T2},[$Xfer,:128]! + vst1.32 {$T3},[$Xfer,:128]! + + ldmia $ctx,{$A-$H} + sub $Xfer,$Xfer,#64 + ldr $t1,[sp,#0] + eor $t2,$t2,$t2 + eor $t3,$B,$C + b .L_00_48 + +.align 4 +.L_00_48: +___ + &Xupdate(\&body_00_15); + &Xupdate(\&body_00_15); + &Xupdate(\&body_00_15); + &Xupdate(\&body_00_15); +$code.=<<___; + teq $t1,#0 @ check for K256 terminator + ldr $t1,[sp,#0] + sub $Xfer,$Xfer,#64 + bne .L_00_48 + + ldr $inp,[sp,#68] + ldr $t0,[sp,#72] + sub $Ktbl,$Ktbl,#256 @ rewind $Ktbl + teq $inp,$t0 + it eq + subeq $inp,$inp,#64 @ avoid SEGV + vld1.8 {@X[0]},[$inp]! @ load next input block + vld1.8 {@X[1]},[$inp]! + vld1.8 {@X[2]},[$inp]! + vld1.8 {@X[3]},[$inp]! + it ne + strne $inp,[sp,#68] + mov $Xfer,sp +___ + &Xpreload(\&body_00_15); + &Xpreload(\&body_00_15); + &Xpreload(\&body_00_15); + &Xpreload(\&body_00_15); +$code.=<<___; + ldr $t0,[$t1,#0] + add $A,$A,$t2 @ h+=Maj(a,b,c) from the past + ldr $t2,[$t1,#4] + ldr $t3,[$t1,#8] + ldr $t4,[$t1,#12] + add $A,$A,$t0 @ accumulate + ldr $t0,[$t1,#16] + add $B,$B,$t2 + ldr $t2,[$t1,#20] + add $C,$C,$t3 + ldr $t3,[$t1,#24] + add $D,$D,$t4 + ldr $t4,[$t1,#28] + add $E,$E,$t0 + str $A,[$t1],#4 + add $F,$F,$t2 + str $B,[$t1],#4 + add $G,$G,$t3 + str $C,[$t1],#4 + add $H,$H,$t4 + str $D,[$t1],#4 + stmia $t1,{$E-$H} + + ittte ne + movne $Xfer,sp + ldrne $t1,[sp,#0] + eorne $t2,$t2,$t2 + ldreq sp,[sp,#76] @ restore original sp + itt ne + eorne $t3,$B,$C + bne .L_00_48 + + ldmia sp!,{r4-r12,pc} +.size sha256_block_data_order_neon,.-sha256_block_data_order_neon +#endif +___ +}}} +###################################################################### +# ARMv8 stuff +# +{{{ +my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2)); +my @MSG=map("q$_",(8..11)); +my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15)); +my $Ktbl="r3"; + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + +# ifdef __thumb2__ +# define INST(a,b,c,d) .byte c,d|0xc,a,b +# else +# define INST(a,b,c,d) .byte a,b,c,d +# endif + +.type sha256_block_data_order_armv8,%function +.align 5 +sha256_block_data_order_armv8: +.LARMv8: + vld1.32 {$ABCD,$EFGH},[$ctx] +# ifdef __thumb2__ + adr $Ktbl,.LARMv8 + sub $Ktbl,$Ktbl,#.LARMv8-K256 +# else + adrl $Ktbl,K256 +# endif + add $len,$inp,$len,lsl#6 @ len to point at the end of inp + +.Loop_v8: + vld1.8 {@MSG[0]-@MSG[1]},[$inp]! + vld1.8 {@MSG[2]-@MSG[3]},[$inp]! + vld1.32 {$W0},[$Ktbl]! + vrev32.8 @MSG[0],@MSG[0] + vrev32.8 @MSG[1],@MSG[1] + vrev32.8 @MSG[2],@MSG[2] + vrev32.8 @MSG[3],@MSG[3] + vmov $ABCD_SAVE,$ABCD @ offload + vmov $EFGH_SAVE,$EFGH + teq $inp,$len +___ +for($i=0;$i<12;$i++) { +$code.=<<___; + vld1.32 {$W1},[$Ktbl]! + vadd.i32 $W0,$W0,@MSG[0] + sha256su0 @MSG[0],@MSG[1] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + sha256su1 @MSG[0],@MSG[2],@MSG[3] +___ + ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG)); +} +$code.=<<___; + vld1.32 {$W1},[$Ktbl]! + vadd.i32 $W0,$W0,@MSG[0] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + + vld1.32 {$W0},[$Ktbl]! + vadd.i32 $W1,$W1,@MSG[1] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W1 + sha256h2 $EFGH,$abcd,$W1 + + vld1.32 {$W1},[$Ktbl] + vadd.i32 $W0,$W0,@MSG[2] + sub $Ktbl,$Ktbl,#256-16 @ rewind + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + + vadd.i32 $W1,$W1,@MSG[3] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W1 + sha256h2 $EFGH,$abcd,$W1 + + vadd.i32 $ABCD,$ABCD,$ABCD_SAVE + vadd.i32 $EFGH,$EFGH,$EFGH_SAVE + it ne + bne .Loop_v8 + + vst1.32 {$ABCD,$EFGH},[$ctx] + + ret @ bx lr +.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8 +#endif +___ +}}} +$code.=<<___; +.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by " .align 2 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.comm OPENSSL_armcap_P,4,4 +#endif ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 -print $code; +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +{ my %opcode = ( + "sha256h" => 0xf3000c40, "sha256h2" => 0xf3100c40, + "sha256su0" => 0xf3ba03c0, "sha256su1" => 0xf3200c40 ); + + sub unsha256 { + my ($mnemonic,$arg)=@_; + + if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) { + my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19) + |(($2&7)<<17)|(($2&8)<<4) + |(($3&7)<<1) |(($3&8)<<2); + # since ARMv7 instructions are always encoded little-endian. + # correct solution is to use .inst directive, but older + # assemblers don't implement it:-( + sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s", + $word&0xff,($word>>8)&0xff, + ($word>>16)&0xff,($word>>24)&0xff, + $mnemonic,$arg; + } + } +} + +foreach (split($/,$code)) { + + s/\`([^\`]*)\`/eval $1/geo; + + s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo; + + s/\bret\b/bx lr/go or + s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4 + + print $_,"\n"; +} + close STDOUT; # enforce flush diff --git a/crypto/sha/asm/sha256-mb-x86_64.pl b/crypto/sha/asm/sha256-mb-x86_64.pl new file mode 100755 index 000000000000..adf2ddccd18b --- /dev/null +++ b/crypto/sha/asm/sha256-mb-x86_64.pl @@ -0,0 +1,1560 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# Multi-buffer SHA256 procedure processes n buffers in parallel by +# placing buffer data to designated lane of SIMD register. n is +# naturally limited to 4 on pre-AVX2 processors and to 8 on +# AVX2-capable processors such as Haswell. +# +# this +aesni(i) sha256 aesni-sha256 gain(iv) +# ------------------------------------------------------------------- +# Westmere(ii) 23.3/n +1.28=7.11(n=4) 12.3 +3.75=16.1 +126% +# Atom(ii) 38.7/n +3.93=13.6(n=4) 20.8 +5.69=26.5 +95% +# Sandy Bridge (20.5 +5.15=25.7)/n 11.6 13.0 +103% +# Ivy Bridge (20.4 +5.14=25.5)/n 10.3 11.6 +82% +# Haswell(iii) (21.0 +5.00=26.0)/n 7.80 8.79 +170% +# Bulldozer (21.6 +5.76=27.4)/n 13.6 13.7 +100% +# +# (i) multi-block CBC encrypt with 128-bit key; +# (ii) (HASH+AES)/n does not apply to Westmere for n>3 and Atom, +# because of lower AES-NI instruction throughput, nor is there +# AES-NI-SHA256 stitch for these processors; +# (iii) "this" is for n=8, when we gather twice as much data, result +# for n=4 is 20.3+4.44=24.7; +# (iv) presented improvement coefficients are asymptotic limits and +# in real-life application are somewhat lower, e.g. for 2KB +# fragments they range from 75% to 130% (on Haswell); + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +$avx=0; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +# void sha256_multi_block ( +# struct { unsigned int A[8]; +# unsigned int B[8]; +# unsigned int C[8]; +# unsigned int D[8]; +# unsigned int E[8]; +# unsigned int F[8]; +# unsigned int G[8]; +# unsigned int H[8]; } *ctx, +# struct { void *ptr; int blocks; } inp[8], +# int num); /* 1 or 2 */ +# +$ctx="%rdi"; # 1st arg +$inp="%rsi"; # 2nd arg +$num="%edx"; # 3rd arg +@ptr=map("%r$_",(8..11)); +$Tbl="%rbp"; + +@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%xmm$_",(8..15)); +($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%xmm$_",(0..7)); + +$REG_SZ=16; + +sub Xi_off { +my $off = shift; + + $off %= 16; $off *= $REG_SZ; + $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)"; +} + +sub ROUND_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; + +$code.=<<___ if ($i<15); + movd `4*$i`(@ptr[0]),$Xi + movd `4*$i`(@ptr[1]),$t1 + movd `4*$i`(@ptr[2]),$t2 + movd `4*$i`(@ptr[3]),$t3 + punpckldq $t2,$Xi + punpckldq $t3,$t1 + punpckldq $t1,$Xi +___ +$code.=<<___ if ($i==15); + movd `4*$i`(@ptr[0]),$Xi + lea `16*4`(@ptr[0]),@ptr[0] + movd `4*$i`(@ptr[1]),$t1 + lea `16*4`(@ptr[1]),@ptr[1] + movd `4*$i`(@ptr[2]),$t2 + lea `16*4`(@ptr[2]),@ptr[2] + movd `4*$i`(@ptr[3]),$t3 + lea `16*4`(@ptr[3]),@ptr[3] + punpckldq $t2,$Xi + punpckldq $t3,$t1 + punpckldq $t1,$Xi +___ +$code.=<<___; + movdqa $e,$sigma + `"pshufb $Xn,$Xi" if ($i<=15 && ($i&1)==0)` + movdqa $e,$t3 + `"pshufb $Xn,$Xi" if ($i<=15 && ($i&1)==1)` + psrld \$6,$sigma + movdqa $e,$t2 + pslld \$7,$t3 + movdqa $Xi,`&Xi_off($i)` + paddd $h,$Xi # Xi+=h + + psrld \$11,$t2 + pxor $t3,$sigma + pslld \$21-7,$t3 + paddd `32*($i%8)-128`($Tbl),$Xi # Xi+=K[round] + pxor $t2,$sigma + + psrld \$25-11,$t2 + movdqa $e,$t1 + `"prefetcht0 63(@ptr[0])" if ($i==15)` + pxor $t3,$sigma + movdqa $e,$axb # borrow $axb + pslld \$26-21,$t3 + pandn $g,$t1 + pand $f,$axb + pxor $t2,$sigma + + `"prefetcht0 63(@ptr[1])" if ($i==15)` + movdqa $a,$t2 + pxor $t3,$sigma # Sigma1(e) + movdqa $a,$t3 + psrld \$2,$t2 + paddd $sigma,$Xi # Xi+=Sigma1(e) + pxor $axb,$t1 # Ch(e,f,g) + movdqa $b,$axb + movdqa $a,$sigma + pslld \$10,$t3 + pxor $a,$axb # a^b, b^c in next round + + `"prefetcht0 63(@ptr[2])" if ($i==15)` + psrld \$13,$sigma + pxor $t3,$t2 + paddd $t1,$Xi # Xi+=Ch(e,f,g) + pslld \$19-10,$t3 + pand $axb,$bxc + pxor $sigma,$t2 + + `"prefetcht0 63(@ptr[3])" if ($i==15)` + psrld \$22-13,$sigma + pxor $t3,$t2 + movdqa $b,$h + pslld \$30-19,$t3 + pxor $t2,$sigma + pxor $bxc,$h # h=Maj(a,b,c)=Ch(a^b,c,b) + paddd $Xi,$d # d+=Xi + pxor $t3,$sigma # Sigma0(a) + + paddd $Xi,$h # h+=Xi + paddd $sigma,$h # h+=Sigma0(a) +___ +$code.=<<___ if (($i%8)==7); + lea `32*8`($Tbl),$Tbl +___ + ($axb,$bxc)=($bxc,$axb); +} + +sub ROUND_16_XX { +my $i=shift; + +$code.=<<___; + movdqa `&Xi_off($i+1)`,$Xn + paddd `&Xi_off($i+9)`,$Xi # Xi+=X[i+9] + + movdqa $Xn,$sigma + movdqa $Xn,$t2 + psrld \$3,$sigma + movdqa $Xn,$t3 + + psrld \$7,$t2 + movdqa `&Xi_off($i+14)`,$t1 + pslld \$14,$t3 + pxor $t2,$sigma + psrld \$18-7,$t2 + movdqa $t1,$axb # borrow $axb + pxor $t3,$sigma + pslld \$25-14,$t3 + pxor $t2,$sigma + psrld \$10,$t1 + movdqa $axb,$t2 + + psrld \$17,$axb + pxor $t3,$sigma # sigma0(X[i+1]) + pslld \$13,$t2 + paddd $sigma,$Xi # Xi+=sigma0(e) + pxor $axb,$t1 + psrld \$19-17,$axb + pxor $t2,$t1 + pslld \$15-13,$t2 + pxor $axb,$t1 + pxor $t2,$t1 # sigma0(X[i+14]) + paddd $t1,$Xi # Xi+=sigma1(X[i+14]) +___ + &ROUND_00_15($i,@_); + ($Xi,$Xn)=($Xn,$Xi); +} + +$code.=<<___; +.text + +.extern OPENSSL_ia32cap_P + +.globl sha256_multi_block +.type sha256_multi_block,\@function,3 +.align 32 +sha256_multi_block: + mov OPENSSL_ia32cap_P+4(%rip),%rcx + bt \$61,%rcx # check SHA bit + jc _shaext_shortcut +___ +$code.=<<___ if ($avx); + test \$`1<<28`,%ecx + jnz _avx_shortcut +___ +$code.=<<___; + mov %rsp,%rax + push %rbx + push %rbp +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,-0x78(%rax) + movaps %xmm11,-0x68(%rax) + movaps %xmm12,-0x58(%rax) + movaps %xmm13,-0x48(%rax) + movaps %xmm14,-0x38(%rax) + movaps %xmm15,-0x28(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`, %rsp + and \$-256,%rsp + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody: + lea K256+128(%rip),$Tbl + lea `$REG_SZ*16`(%rsp),%rbx + lea 0x80($ctx),$ctx # size optimization + +.Loop_grande: + mov $num,`$REG_SZ*17+8`(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldone + + movdqu 0x00-0x80($ctx),$A # load context + lea 128(%rsp),%rax + movdqu 0x20-0x80($ctx),$B + movdqu 0x40-0x80($ctx),$C + movdqu 0x60-0x80($ctx),$D + movdqu 0x80-0x80($ctx),$E + movdqu 0xa0-0x80($ctx),$F + movdqu 0xc0-0x80($ctx),$G + movdqu 0xe0-0x80($ctx),$H + movdqu .Lpbswap(%rip),$Xn + jmp .Loop + +.align 32 +.Loop: + movdqa $C,$bxc + pxor $B,$bxc # magic seed +___ +for($i=0;$i<16;$i++) { &ROUND_00_15($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + movdqu `&Xi_off($i)`,$Xi + mov \$3,%ecx + jmp .Loop_16_xx +.align 32 +.Loop_16_xx: +___ +for(;$i<32;$i++) { &ROUND_16_XX($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + dec %ecx + jnz .Loop_16_xx + + mov \$1,%ecx + lea K256+128(%rip),$Tbl + + movdqa (%rbx),$sigma # pull counters + cmp 4*0(%rbx),%ecx # examine counters + pxor $t1,$t1 + cmovge $Tbl,@ptr[0] # cancel input + cmp 4*1(%rbx),%ecx + movdqa $sigma,$Xn + cmovge $Tbl,@ptr[1] + cmp 4*2(%rbx),%ecx + pcmpgtd $t1,$Xn # mask value + cmovge $Tbl,@ptr[2] + cmp 4*3(%rbx),%ecx + paddd $Xn,$sigma # counters-- + cmovge $Tbl,@ptr[3] + + movdqu 0x00-0x80($ctx),$t1 + pand $Xn,$A + movdqu 0x20-0x80($ctx),$t2 + pand $Xn,$B + movdqu 0x40-0x80($ctx),$t3 + pand $Xn,$C + movdqu 0x60-0x80($ctx),$Xi + pand $Xn,$D + paddd $t1,$A + movdqu 0x80-0x80($ctx),$t1 + pand $Xn,$E + paddd $t2,$B + movdqu 0xa0-0x80($ctx),$t2 + pand $Xn,$F + paddd $t3,$C + movdqu 0xc0-0x80($ctx),$t3 + pand $Xn,$G + paddd $Xi,$D + movdqu 0xe0-0x80($ctx),$Xi + pand $Xn,$H + paddd $t1,$E + paddd $t2,$F + movdqu $A,0x00-0x80($ctx) + paddd $t3,$G + movdqu $B,0x20-0x80($ctx) + paddd $Xi,$H + movdqu $C,0x40-0x80($ctx) + movdqu $D,0x60-0x80($ctx) + movdqu $E,0x80-0x80($ctx) + movdqu $F,0xa0-0x80($ctx) + movdqu $G,0xc0-0x80($ctx) + movdqu $H,0xe0-0x80($ctx) + + movdqa $sigma,(%rbx) # save counters + movdqa .Lpbswap(%rip),$Xn + dec $num + jnz .Loop + + mov `$REG_SZ*17+8`(%rsp),$num + lea $REG_SZ($ctx),$ctx + lea `16*$REG_SZ/4`($inp),$inp + dec $num + jnz .Loop_grande + +.Ldone: + mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp +___ +$code.=<<___ if ($win64); + movaps -0xb8(%rax),%xmm6 + movaps -0xa8(%rax),%xmm7 + movaps -0x98(%rax),%xmm8 + movaps -0x88(%rax),%xmm9 + movaps -0x78(%rax),%xmm10 + movaps -0x68(%rax),%xmm11 + movaps -0x58(%rax),%xmm12 + movaps -0x48(%rax),%xmm13 + movaps -0x38(%rax),%xmm14 + movaps -0x28(%rax),%xmm15 +___ +$code.=<<___; + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue: + ret +.size sha256_multi_block,.-sha256_multi_block +___ + {{{ +my ($Wi,$TMP0,$TMP1,$TMPx,$ABEF0,$CDGH0,$ABEF1,$CDGH1)=map("%xmm$_",(0..3,12..15)); +my @MSG0=map("%xmm$_",(4..7)); +my @MSG1=map("%xmm$_",(8..11)); + +$code.=<<___; +.type sha256_multi_block_shaext,\@function,3 +.align 32 +sha256_multi_block_shaext: +_shaext_shortcut: + mov %rsp,%rax + push %rbx + push %rbp +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,-0x78(%rax) + movaps %xmm11,-0x68(%rax) + movaps %xmm12,-0x58(%rax) + movaps %xmm13,-0x48(%rax) + movaps %xmm14,-0x38(%rax) + movaps %xmm15,-0x28(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`,%rsp + shl \$1,$num # we process pair at a time + and \$-256,%rsp + lea 0x80($ctx),$ctx # size optimization + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody_shaext: + lea `$REG_SZ*16`(%rsp),%rbx + lea K256_shaext+0x80(%rip),$Tbl + +.Loop_grande_shaext: + mov $num,`$REG_SZ*17+8`(%rsp) # orignal $num + xor $num,$num +___ +for($i=0;$i<2;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle %rsp,@ptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldone_shaext + + movq 0x00-0x80($ctx),$ABEF0 # A1.A0 + movq 0x20-0x80($ctx),@MSG0[0] # B1.B0 + movq 0x40-0x80($ctx),$CDGH0 # C1.C0 + movq 0x60-0x80($ctx),@MSG0[1] # D1.D0 + movq 0x80-0x80($ctx),@MSG1[0] # E1.E0 + movq 0xa0-0x80($ctx),@MSG1[1] # F1.F0 + movq 0xc0-0x80($ctx),@MSG1[2] # G1.G0 + movq 0xe0-0x80($ctx),@MSG1[3] # H1.H0 + + punpckldq @MSG0[0],$ABEF0 # B1.A1.B0.A0 + punpckldq @MSG0[1],$CDGH0 # D1.C1.D0.C0 + punpckldq @MSG1[1],@MSG1[0] # F1.E1.F0.E0 + punpckldq @MSG1[3],@MSG1[2] # H1.G1.H0.G0 + movdqa K256_shaext-0x10(%rip),$TMPx # byte swap + + movdqa $ABEF0,$ABEF1 + movdqa $CDGH0,$CDGH1 + punpcklqdq @MSG1[0],$ABEF0 # F0.E0.B0.A0 + punpcklqdq @MSG1[2],$CDGH0 # H0.G0.D0.C0 + punpckhqdq @MSG1[0],$ABEF1 # F1.E1.B1.A1 + punpckhqdq @MSG1[2],$CDGH1 # H1.G1.D1.C1 + + pshufd \$0b00011011,$ABEF0,$ABEF0 + pshufd \$0b00011011,$CDGH0,$CDGH0 + pshufd \$0b00011011,$ABEF1,$ABEF1 + pshufd \$0b00011011,$CDGH1,$CDGH1 + jmp .Loop_shaext + +.align 32 +.Loop_shaext: + movdqu 0x00(@ptr[0]),@MSG0[0] + movdqu 0x00(@ptr[1]),@MSG1[0] + movdqu 0x10(@ptr[0]),@MSG0[1] + movdqu 0x10(@ptr[1]),@MSG1[1] + movdqu 0x20(@ptr[0]),@MSG0[2] + pshufb $TMPx,@MSG0[0] + movdqu 0x20(@ptr[1]),@MSG1[2] + pshufb $TMPx,@MSG1[0] + movdqu 0x30(@ptr[0]),@MSG0[3] + lea 0x40(@ptr[0]),@ptr[0] + movdqu 0x30(@ptr[1]),@MSG1[3] + lea 0x40(@ptr[1]),@ptr[1] + + movdqa 0*16-0x80($Tbl),$Wi + pshufb $TMPx,@MSG0[1] + paddd @MSG0[0],$Wi + pxor $ABEF0,@MSG0[0] # black magic + movdqa $Wi,$TMP0 + movdqa 0*16-0x80($Tbl),$TMP1 + pshufb $TMPx,@MSG1[1] + paddd @MSG1[0],$TMP1 + movdqa $CDGH0,0x50(%rsp) # offload + sha256rnds2 $ABEF0,$CDGH0 # 0-3 + pxor $ABEF1,@MSG1[0] # black magic + movdqa $TMP1,$Wi + movdqa $CDGH1,0x70(%rsp) + sha256rnds2 $ABEF1,$CDGH1 # 0-3 + pshufd \$0x0e,$TMP0,$Wi + pxor $ABEF0,@MSG0[0] # black magic + movdqa $ABEF0,0x40(%rsp) # offload + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + pxor $ABEF1,@MSG1[0] # black magic + movdqa $ABEF1,0x60(%rsp) + movdqa 1*16-0x80($Tbl),$TMP0 + paddd @MSG0[1],$TMP0 + pshufb $TMPx,@MSG0[2] + sha256rnds2 $CDGH1,$ABEF1 + + movdqa $TMP0,$Wi + movdqa 1*16-0x80($Tbl),$TMP1 + paddd @MSG1[1],$TMP1 + sha256rnds2 $ABEF0,$CDGH0 # 4-7 + movdqa $TMP1,$Wi + prefetcht0 127(@ptr[0]) + pshufb $TMPx,@MSG0[3] + pshufb $TMPx,@MSG1[2] + prefetcht0 127(@ptr[1]) + sha256rnds2 $ABEF1,$CDGH1 # 4-7 + pshufd \$0x0e,$TMP0,$Wi + pshufb $TMPx,@MSG1[3] + sha256msg1 @MSG0[1],@MSG0[0] + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + movdqa 2*16-0x80($Tbl),$TMP0 + paddd @MSG0[2],$TMP0 + sha256rnds2 $CDGH1,$ABEF1 + + movdqa $TMP0,$Wi + movdqa 2*16-0x80($Tbl),$TMP1 + paddd @MSG1[2],$TMP1 + sha256rnds2 $ABEF0,$CDGH0 # 8-11 + sha256msg1 @MSG1[1],@MSG1[0] + movdqa $TMP1,$Wi + movdqa @MSG0[3],$TMPx + sha256rnds2 $ABEF1,$CDGH1 # 8-11 + pshufd \$0x0e,$TMP0,$Wi + palignr \$4,@MSG0[2],$TMPx + paddd $TMPx,@MSG0[0] + movdqa @MSG1[3],$TMPx + palignr \$4,@MSG1[2],$TMPx + sha256msg1 @MSG0[2],@MSG0[1] + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + movdqa 3*16-0x80($Tbl),$TMP0 + paddd @MSG0[3],$TMP0 + sha256rnds2 $CDGH1,$ABEF1 + sha256msg1 @MSG1[2],@MSG1[1] + + movdqa $TMP0,$Wi + movdqa 3*16-0x80($Tbl),$TMP1 + paddd $TMPx,@MSG1[0] + paddd @MSG1[3],$TMP1 + sha256msg2 @MSG0[3],@MSG0[0] + sha256rnds2 $ABEF0,$CDGH0 # 12-15 + movdqa $TMP1,$Wi + movdqa @MSG0[0],$TMPx + palignr \$4,@MSG0[3],$TMPx + sha256rnds2 $ABEF1,$CDGH1 # 12-15 + sha256msg2 @MSG1[3],@MSG1[0] + pshufd \$0x0e,$TMP0,$Wi + paddd $TMPx,@MSG0[1] + movdqa @MSG1[0],$TMPx + palignr \$4,@MSG1[3],$TMPx + sha256msg1 @MSG0[3],@MSG0[2] + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + movdqa 4*16-0x80($Tbl),$TMP0 + paddd @MSG0[0],$TMP0 + sha256rnds2 $CDGH1,$ABEF1 + sha256msg1 @MSG1[3],@MSG1[2] +___ +for($i=4;$i<16-3;$i++) { +$code.=<<___; + movdqa $TMP0,$Wi + movdqa $i*16-0x80($Tbl),$TMP1 + paddd $TMPx,@MSG1[1] + paddd @MSG1[0],$TMP1 + sha256msg2 @MSG0[0],@MSG0[1] + sha256rnds2 $ABEF0,$CDGH0 # 16-19... + movdqa $TMP1,$Wi + movdqa @MSG0[1],$TMPx + palignr \$4,@MSG0[0],$TMPx + sha256rnds2 $ABEF1,$CDGH1 # 16-19... + sha256msg2 @MSG1[0],@MSG1[1] + pshufd \$0x0e,$TMP0,$Wi + paddd $TMPx,@MSG0[2] + movdqa @MSG1[1],$TMPx + palignr \$4,@MSG1[0],$TMPx + sha256msg1 @MSG0[0],@MSG0[3] + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + movdqa `($i+1)*16`-0x80($Tbl),$TMP0 + paddd @MSG0[1],$TMP0 + sha256rnds2 $CDGH1,$ABEF1 + sha256msg1 @MSG1[0],@MSG1[3] +___ + push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1)); +} +$code.=<<___; + movdqa $TMP0,$Wi + movdqa 13*16-0x80($Tbl),$TMP1 + paddd $TMPx,@MSG1[1] + paddd @MSG1[0],$TMP1 + sha256msg2 @MSG0[0],@MSG0[1] + sha256rnds2 $ABEF0,$CDGH0 # 52-55 + movdqa $TMP1,$Wi + movdqa @MSG0[1],$TMPx + palignr \$4,@MSG0[0],$TMPx + sha256rnds2 $ABEF1,$CDGH1 # 52-55 + sha256msg2 @MSG1[0],@MSG1[1] + pshufd \$0x0e,$TMP0,$Wi + paddd $TMPx,@MSG0[2] + movdqa @MSG1[1],$TMPx + palignr \$4,@MSG1[0],$TMPx + nop + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + movdqa 14*16-0x80($Tbl),$TMP0 + paddd @MSG0[1],$TMP0 + sha256rnds2 $CDGH1,$ABEF1 + + movdqa $TMP0,$Wi + movdqa 14*16-0x80($Tbl),$TMP1 + paddd $TMPx,@MSG1[2] + paddd @MSG1[1],$TMP1 + sha256msg2 @MSG0[1],@MSG0[2] + nop + sha256rnds2 $ABEF0,$CDGH0 # 56-59 + movdqa $TMP1,$Wi + mov \$1,%ecx + pxor @MSG0[1],@MSG0[1] # zero + sha256rnds2 $ABEF1,$CDGH1 # 56-59 + sha256msg2 @MSG1[1],@MSG1[2] + pshufd \$0x0e,$TMP0,$Wi + movdqa 15*16-0x80($Tbl),$TMP0 + paddd @MSG0[2],$TMP0 + movq (%rbx),@MSG0[2] # pull counters + nop + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + movdqa 15*16-0x80($Tbl),$TMP1 + paddd @MSG1[2],$TMP1 + sha256rnds2 $CDGH1,$ABEF1 + + movdqa $TMP0,$Wi + cmp 4*0(%rbx),%ecx # examine counters + cmovge %rsp,@ptr[0] # cancel input + cmp 4*1(%rbx),%ecx + cmovge %rsp,@ptr[1] + pshufd \$0x00,@MSG0[2],@MSG1[0] + sha256rnds2 $ABEF0,$CDGH0 # 60-63 + movdqa $TMP1,$Wi + pshufd \$0x55,@MSG0[2],@MSG1[1] + movdqa @MSG0[2],@MSG1[2] + sha256rnds2 $ABEF1,$CDGH1 # 60-63 + pshufd \$0x0e,$TMP0,$Wi + pcmpgtd @MSG0[1],@MSG1[0] + pcmpgtd @MSG0[1],@MSG1[1] + sha256rnds2 $CDGH0,$ABEF0 + pshufd \$0x0e,$TMP1,$Wi + pcmpgtd @MSG0[1],@MSG1[2] # counter mask + movdqa K256_shaext-0x10(%rip),$TMPx + sha256rnds2 $CDGH1,$ABEF1 + + pand @MSG1[0],$CDGH0 + pand @MSG1[1],$CDGH1 + pand @MSG1[0],$ABEF0 + pand @MSG1[1],$ABEF1 + paddd @MSG0[2],@MSG1[2] # counters-- + + paddd 0x50(%rsp),$CDGH0 + paddd 0x70(%rsp),$CDGH1 + paddd 0x40(%rsp),$ABEF0 + paddd 0x60(%rsp),$ABEF1 + + movq @MSG1[2],(%rbx) # save counters + dec $num + jnz .Loop_shaext + + mov `$REG_SZ*17+8`(%rsp),$num + + pshufd \$0b00011011,$ABEF0,$ABEF0 + pshufd \$0b00011011,$CDGH0,$CDGH0 + pshufd \$0b00011011,$ABEF1,$ABEF1 + pshufd \$0b00011011,$CDGH1,$CDGH1 + + movdqa $ABEF0,@MSG0[0] + movdqa $CDGH0,@MSG0[1] + punpckldq $ABEF1,$ABEF0 # B1.B0.A1.A0 + punpckhdq $ABEF1,@MSG0[0] # F1.F0.E1.E0 + punpckldq $CDGH1,$CDGH0 # D1.D0.C1.C0 + punpckhdq $CDGH1,@MSG0[1] # H1.H0.G1.G0 + + movq $ABEF0,0x00-0x80($ctx) # A1.A0 + psrldq \$8,$ABEF0 + movq @MSG0[0],0x80-0x80($ctx) # E1.E0 + psrldq \$8,@MSG0[0] + movq $ABEF0,0x20-0x80($ctx) # B1.B0 + movq @MSG0[0],0xa0-0x80($ctx) # F1.F0 + + movq $CDGH0,0x40-0x80($ctx) # C1.C0 + psrldq \$8,$CDGH0 + movq @MSG0[1],0xc0-0x80($ctx) # G1.G0 + psrldq \$8,@MSG0[1] + movq $CDGH0,0x60-0x80($ctx) # D1.D0 + movq @MSG0[1],0xe0-0x80($ctx) # H1.H0 + + lea `$REG_SZ/2`($ctx),$ctx + lea `16*2`($inp),$inp + dec $num + jnz .Loop_grande_shaext + +.Ldone_shaext: + #mov `$REG_SZ*17`(%rsp),%rax # original %rsp +___ +$code.=<<___ if ($win64); + movaps -0xb8(%rax),%xmm6 + movaps -0xa8(%rax),%xmm7 + movaps -0x98(%rax),%xmm8 + movaps -0x88(%rax),%xmm9 + movaps -0x78(%rax),%xmm10 + movaps -0x68(%rax),%xmm11 + movaps -0x58(%rax),%xmm12 + movaps -0x48(%rax),%xmm13 + movaps -0x38(%rax),%xmm14 + movaps -0x28(%rax),%xmm15 +___ +$code.=<<___; + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue_shaext: + ret +.size sha256_multi_block_shaext,.-sha256_multi_block_shaext +___ + }}} + if ($avx) {{{ +sub ROUND_00_15_avx { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; + +$code.=<<___ if ($i<15 && $REG_SZ==16); + vmovd `4*$i`(@ptr[0]),$Xi + vmovd `4*$i`(@ptr[1]),$t1 + vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi + vpinsrd \$1,`4*$i`(@ptr[3]),$t1,$t1 + vpunpckldq $t1,$Xi,$Xi + vpshufb $Xn,$Xi,$Xi +___ +$code.=<<___ if ($i==15 && $REG_SZ==16); + vmovd `4*$i`(@ptr[0]),$Xi + lea `16*4`(@ptr[0]),@ptr[0] + vmovd `4*$i`(@ptr[1]),$t1 + lea `16*4`(@ptr[1]),@ptr[1] + vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi + lea `16*4`(@ptr[2]),@ptr[2] + vpinsrd \$1,`4*$i`(@ptr[3]),$t1,$t1 + lea `16*4`(@ptr[3]),@ptr[3] + vpunpckldq $t1,$Xi,$Xi + vpshufb $Xn,$Xi,$Xi +___ +$code.=<<___ if ($i<15 && $REG_SZ==32); + vmovd `4*$i`(@ptr[0]),$Xi + vmovd `4*$i`(@ptr[4]),$t1 + vmovd `4*$i`(@ptr[1]),$t2 + vmovd `4*$i`(@ptr[5]),$t3 + vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi + vpinsrd \$1,`4*$i`(@ptr[6]),$t1,$t1 + vpinsrd \$1,`4*$i`(@ptr[3]),$t2,$t2 + vpunpckldq $t2,$Xi,$Xi + vpinsrd \$1,`4*$i`(@ptr[7]),$t3,$t3 + vpunpckldq $t3,$t1,$t1 + vinserti128 $t1,$Xi,$Xi + vpshufb $Xn,$Xi,$Xi +___ +$code.=<<___ if ($i==15 && $REG_SZ==32); + vmovd `4*$i`(@ptr[0]),$Xi + lea `16*4`(@ptr[0]),@ptr[0] + vmovd `4*$i`(@ptr[4]),$t1 + lea `16*4`(@ptr[4]),@ptr[4] + vmovd `4*$i`(@ptr[1]),$t2 + lea `16*4`(@ptr[1]),@ptr[1] + vmovd `4*$i`(@ptr[5]),$t3 + lea `16*4`(@ptr[5]),@ptr[5] + vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi + lea `16*4`(@ptr[2]),@ptr[2] + vpinsrd \$1,`4*$i`(@ptr[6]),$t1,$t1 + lea `16*4`(@ptr[6]),@ptr[6] + vpinsrd \$1,`4*$i`(@ptr[3]),$t2,$t2 + lea `16*4`(@ptr[3]),@ptr[3] + vpunpckldq $t2,$Xi,$Xi + vpinsrd \$1,`4*$i`(@ptr[7]),$t3,$t3 + lea `16*4`(@ptr[7]),@ptr[7] + vpunpckldq $t3,$t1,$t1 + vinserti128 $t1,$Xi,$Xi + vpshufb $Xn,$Xi,$Xi +___ +$code.=<<___; + vpsrld \$6,$e,$sigma + vpslld \$26,$e,$t3 + vmovdqu $Xi,`&Xi_off($i)` + vpaddd $h,$Xi,$Xi # Xi+=h + + vpsrld \$11,$e,$t2 + vpxor $t3,$sigma,$sigma + vpslld \$21,$e,$t3 + vpaddd `32*($i%8)-128`($Tbl),$Xi,$Xi # Xi+=K[round] + vpxor $t2,$sigma,$sigma + + vpsrld \$25,$e,$t2 + vpxor $t3,$sigma,$sigma + `"prefetcht0 63(@ptr[0])" if ($i==15)` + vpslld \$7,$e,$t3 + vpandn $g,$e,$t1 + vpand $f,$e,$axb # borrow $axb + `"prefetcht0 63(@ptr[1])" if ($i==15)` + vpxor $t2,$sigma,$sigma + + vpsrld \$2,$a,$h # borrow $h + vpxor $t3,$sigma,$sigma # Sigma1(e) + `"prefetcht0 63(@ptr[2])" if ($i==15)` + vpslld \$30,$a,$t2 + vpxor $axb,$t1,$t1 # Ch(e,f,g) + vpxor $a,$b,$axb # a^b, b^c in next round + `"prefetcht0 63(@ptr[3])" if ($i==15)` + vpxor $t2,$h,$h + vpaddd $sigma,$Xi,$Xi # Xi+=Sigma1(e) + + vpsrld \$13,$a,$t2 + `"prefetcht0 63(@ptr[4])" if ($i==15 && $REG_SZ==32)` + vpslld \$19,$a,$t3 + vpaddd $t1,$Xi,$Xi # Xi+=Ch(e,f,g) + vpand $axb,$bxc,$bxc + `"prefetcht0 63(@ptr[5])" if ($i==15 && $REG_SZ==32)` + vpxor $t2,$h,$sigma + + vpsrld \$22,$a,$t2 + vpxor $t3,$sigma,$sigma + `"prefetcht0 63(@ptr[6])" if ($i==15 && $REG_SZ==32)` + vpslld \$10,$a,$t3 + vpxor $bxc,$b,$h # h=Maj(a,b,c)=Ch(a^b,c,b) + vpaddd $Xi,$d,$d # d+=Xi + `"prefetcht0 63(@ptr[7])" if ($i==15 && $REG_SZ==32)` + vpxor $t2,$sigma,$sigma + vpxor $t3,$sigma,$sigma # Sigma0(a) + + vpaddd $Xi,$h,$h # h+=Xi + vpaddd $sigma,$h,$h # h+=Sigma0(a) +___ +$code.=<<___ if (($i%8)==7); + add \$`32*8`,$Tbl +___ + ($axb,$bxc)=($bxc,$axb); +} + +sub ROUND_16_XX_avx { +my $i=shift; + +$code.=<<___; + vmovdqu `&Xi_off($i+1)`,$Xn + vpaddd `&Xi_off($i+9)`,$Xi,$Xi # Xi+=X[i+9] + + vpsrld \$3,$Xn,$sigma + vpsrld \$7,$Xn,$t2 + vpslld \$25,$Xn,$t3 + vpxor $t2,$sigma,$sigma + vpsrld \$18,$Xn,$t2 + vpxor $t3,$sigma,$sigma + vpslld \$14,$Xn,$t3 + vmovdqu `&Xi_off($i+14)`,$t1 + vpsrld \$10,$t1,$axb # borrow $axb + + vpxor $t2,$sigma,$sigma + vpsrld \$17,$t1,$t2 + vpxor $t3,$sigma,$sigma # sigma0(X[i+1]) + vpslld \$15,$t1,$t3 + vpaddd $sigma,$Xi,$Xi # Xi+=sigma0(e) + vpxor $t2,$axb,$sigma + vpsrld \$19,$t1,$t2 + vpxor $t3,$sigma,$sigma + vpslld \$13,$t1,$t3 + vpxor $t2,$sigma,$sigma + vpxor $t3,$sigma,$sigma # sigma0(X[i+14]) + vpaddd $sigma,$Xi,$Xi # Xi+=sigma1(X[i+14]) +___ + &ROUND_00_15_avx($i,@_); + ($Xi,$Xn)=($Xn,$Xi); +} + +$code.=<<___; +.type sha256_multi_block_avx,\@function,3 +.align 32 +sha256_multi_block_avx: +_avx_shortcut: +___ +$code.=<<___ if ($avx>1); + shr \$32,%rcx + cmp \$2,$num + jb .Lavx + test \$`1<<5`,%ecx + jnz _avx2_shortcut + jmp .Lavx +.align 32 +.Lavx: +___ +$code.=<<___; + mov %rsp,%rax + push %rbx + push %rbp +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,-0x78(%rax) + movaps %xmm11,-0x68(%rax) + movaps %xmm12,-0x58(%rax) + movaps %xmm13,-0x48(%rax) + movaps %xmm14,-0x38(%rax) + movaps %xmm15,-0x28(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`, %rsp + and \$-256,%rsp + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody_avx: + lea K256+128(%rip),$Tbl + lea `$REG_SZ*16`(%rsp),%rbx + lea 0x80($ctx),$ctx # size optimization + +.Loop_grande_avx: + mov $num,`$REG_SZ*17+8`(%rsp) # original $num + xor $num,$num +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + test $num,$num + jz .Ldone_avx + + vmovdqu 0x00-0x80($ctx),$A # load context + lea 128(%rsp),%rax + vmovdqu 0x20-0x80($ctx),$B + vmovdqu 0x40-0x80($ctx),$C + vmovdqu 0x60-0x80($ctx),$D + vmovdqu 0x80-0x80($ctx),$E + vmovdqu 0xa0-0x80($ctx),$F + vmovdqu 0xc0-0x80($ctx),$G + vmovdqu 0xe0-0x80($ctx),$H + vmovdqu .Lpbswap(%rip),$Xn + jmp .Loop_avx + +.align 32 +.Loop_avx: + vpxor $B,$C,$bxc # magic seed +___ +for($i=0;$i<16;$i++) { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + vmovdqu `&Xi_off($i)`,$Xi + mov \$3,%ecx + jmp .Loop_16_xx_avx +.align 32 +.Loop_16_xx_avx: +___ +for(;$i<32;$i++) { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + dec %ecx + jnz .Loop_16_xx_avx + + mov \$1,%ecx + lea K256+128(%rip),$Tbl +___ +for($i=0;$i<4;$i++) { + $code.=<<___; + cmp `4*$i`(%rbx),%ecx # examine counters + cmovge $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + vmovdqa (%rbx),$sigma # pull counters + vpxor $t1,$t1,$t1 + vmovdqa $sigma,$Xn + vpcmpgtd $t1,$Xn,$Xn # mask value + vpaddd $Xn,$sigma,$sigma # counters-- + + vmovdqu 0x00-0x80($ctx),$t1 + vpand $Xn,$A,$A + vmovdqu 0x20-0x80($ctx),$t2 + vpand $Xn,$B,$B + vmovdqu 0x40-0x80($ctx),$t3 + vpand $Xn,$C,$C + vmovdqu 0x60-0x80($ctx),$Xi + vpand $Xn,$D,$D + vpaddd $t1,$A,$A + vmovdqu 0x80-0x80($ctx),$t1 + vpand $Xn,$E,$E + vpaddd $t2,$B,$B + vmovdqu 0xa0-0x80($ctx),$t2 + vpand $Xn,$F,$F + vpaddd $t3,$C,$C + vmovdqu 0xc0-0x80($ctx),$t3 + vpand $Xn,$G,$G + vpaddd $Xi,$D,$D + vmovdqu 0xe0-0x80($ctx),$Xi + vpand $Xn,$H,$H + vpaddd $t1,$E,$E + vpaddd $t2,$F,$F + vmovdqu $A,0x00-0x80($ctx) + vpaddd $t3,$G,$G + vmovdqu $B,0x20-0x80($ctx) + vpaddd $Xi,$H,$H + vmovdqu $C,0x40-0x80($ctx) + vmovdqu $D,0x60-0x80($ctx) + vmovdqu $E,0x80-0x80($ctx) + vmovdqu $F,0xa0-0x80($ctx) + vmovdqu $G,0xc0-0x80($ctx) + vmovdqu $H,0xe0-0x80($ctx) + + vmovdqu $sigma,(%rbx) # save counters + vmovdqu .Lpbswap(%rip),$Xn + dec $num + jnz .Loop_avx + + mov `$REG_SZ*17+8`(%rsp),$num + lea $REG_SZ($ctx),$ctx + lea `16*$REG_SZ/4`($inp),$inp + dec $num + jnz .Loop_grande_avx + +.Ldone_avx: + mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xb8(%rax),%xmm6 + movaps -0xa8(%rax),%xmm7 + movaps -0x98(%rax),%xmm8 + movaps -0x88(%rax),%xmm9 + movaps -0x78(%rax),%xmm10 + movaps -0x68(%rax),%xmm11 + movaps -0x58(%rax),%xmm12 + movaps -0x48(%rax),%xmm13 + movaps -0x38(%rax),%xmm14 + movaps -0x28(%rax),%xmm15 +___ +$code.=<<___; + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue_avx: + ret +.size sha256_multi_block_avx,.-sha256_multi_block_avx +___ + if ($avx>1) { +$code =~ s/\`([^\`]*)\`/eval $1/gem; + +$REG_SZ=32; +@ptr=map("%r$_",(12..15,8..11)); + +@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%ymm$_",(8..15)); +($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%ymm$_",(0..7)); + +$code.=<<___; +.type sha256_multi_block_avx2,\@function,3 +.align 32 +sha256_multi_block_avx2: +_avx2_shortcut: + mov %rsp,%rax + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 +___ +$code.=<<___ if ($win64); + lea -0xa8(%rsp),%rsp + movaps %xmm6,(%rsp) + movaps %xmm7,0x10(%rsp) + movaps %xmm8,0x20(%rsp) + movaps %xmm9,0x30(%rsp) + movaps %xmm10,0x40(%rsp) + movaps %xmm11,0x50(%rsp) + movaps %xmm12,-0x78(%rax) + movaps %xmm13,-0x68(%rax) + movaps %xmm14,-0x58(%rax) + movaps %xmm15,-0x48(%rax) +___ +$code.=<<___; + sub \$`$REG_SZ*18`, %rsp + and \$-256,%rsp + mov %rax,`$REG_SZ*17`(%rsp) # original %rsp +.Lbody_avx2: + lea K256+128(%rip),$Tbl + lea 0x80($ctx),$ctx # size optimization + +.Loop_grande_avx2: + mov $num,`$REG_SZ*17+8`(%rsp) # original $num + xor $num,$num + lea `$REG_SZ*16`(%rsp),%rbx +___ +for($i=0;$i<8;$i++) { + $code.=<<___; + mov `16*$i+0`($inp),@ptr[$i] # input pointer + mov `16*$i+8`($inp),%ecx # number of blocks + cmp $num,%ecx + cmovg %ecx,$num # find maximum + test %ecx,%ecx + mov %ecx,`4*$i`(%rbx) # initialize counters + cmovle $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + vmovdqu 0x00-0x80($ctx),$A # load context + lea 128(%rsp),%rax + vmovdqu 0x20-0x80($ctx),$B + lea 256+128(%rsp),%rbx + vmovdqu 0x40-0x80($ctx),$C + vmovdqu 0x60-0x80($ctx),$D + vmovdqu 0x80-0x80($ctx),$E + vmovdqu 0xa0-0x80($ctx),$F + vmovdqu 0xc0-0x80($ctx),$G + vmovdqu 0xe0-0x80($ctx),$H + vmovdqu .Lpbswap(%rip),$Xn + jmp .Loop_avx2 + +.align 32 +.Loop_avx2: + vpxor $B,$C,$bxc # magic seed +___ +for($i=0;$i<16;$i++) { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + vmovdqu `&Xi_off($i)`,$Xi + mov \$3,%ecx + jmp .Loop_16_xx_avx2 +.align 32 +.Loop_16_xx_avx2: +___ +for(;$i<32;$i++) { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + dec %ecx + jnz .Loop_16_xx_avx2 + + mov \$1,%ecx + lea `$REG_SZ*16`(%rsp),%rbx + lea K256+128(%rip),$Tbl +___ +for($i=0;$i<8;$i++) { + $code.=<<___; + cmp `4*$i`(%rbx),%ecx # examine counters + cmovge $Tbl,@ptr[$i] # cancel input +___ +} +$code.=<<___; + vmovdqa (%rbx),$sigma # pull counters + vpxor $t1,$t1,$t1 + vmovdqa $sigma,$Xn + vpcmpgtd $t1,$Xn,$Xn # mask value + vpaddd $Xn,$sigma,$sigma # counters-- + + vmovdqu 0x00-0x80($ctx),$t1 + vpand $Xn,$A,$A + vmovdqu 0x20-0x80($ctx),$t2 + vpand $Xn,$B,$B + vmovdqu 0x40-0x80($ctx),$t3 + vpand $Xn,$C,$C + vmovdqu 0x60-0x80($ctx),$Xi + vpand $Xn,$D,$D + vpaddd $t1,$A,$A + vmovdqu 0x80-0x80($ctx),$t1 + vpand $Xn,$E,$E + vpaddd $t2,$B,$B + vmovdqu 0xa0-0x80($ctx),$t2 + vpand $Xn,$F,$F + vpaddd $t3,$C,$C + vmovdqu 0xc0-0x80($ctx),$t3 + vpand $Xn,$G,$G + vpaddd $Xi,$D,$D + vmovdqu 0xe0-0x80($ctx),$Xi + vpand $Xn,$H,$H + vpaddd $t1,$E,$E + vpaddd $t2,$F,$F + vmovdqu $A,0x00-0x80($ctx) + vpaddd $t3,$G,$G + vmovdqu $B,0x20-0x80($ctx) + vpaddd $Xi,$H,$H + vmovdqu $C,0x40-0x80($ctx) + vmovdqu $D,0x60-0x80($ctx) + vmovdqu $E,0x80-0x80($ctx) + vmovdqu $F,0xa0-0x80($ctx) + vmovdqu $G,0xc0-0x80($ctx) + vmovdqu $H,0xe0-0x80($ctx) + + vmovdqu $sigma,(%rbx) # save counters + lea 256+128(%rsp),%rbx + vmovdqu .Lpbswap(%rip),$Xn + dec $num + jnz .Loop_avx2 + + #mov `$REG_SZ*17+8`(%rsp),$num + #lea $REG_SZ($ctx),$ctx + #lea `16*$REG_SZ/4`($inp),$inp + #dec $num + #jnz .Loop_grande_avx2 + +.Ldone_avx2: + mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xd8(%rax),%xmm6 + movaps -0xc8(%rax),%xmm7 + movaps -0xb8(%rax),%xmm8 + movaps -0xa8(%rax),%xmm9 + movaps -0x98(%rax),%xmm10 + movaps -0x88(%rax),%xmm11 + movaps -0x78(%rax),%xmm12 + movaps -0x68(%rax),%xmm13 + movaps -0x58(%rax),%xmm14 + movaps -0x48(%rax),%xmm15 +___ +$code.=<<___; + mov -48(%rax),%r15 + mov -40(%rax),%r14 + mov -32(%rax),%r13 + mov -24(%rax),%r12 + mov -16(%rax),%rbp + mov -8(%rax),%rbx + lea (%rax),%rsp +.Lepilogue_avx2: + ret +.size sha256_multi_block_avx2,.-sha256_multi_block_avx2 +___ + } }}} +$code.=<<___; +.align 256 +K256: +___ +sub TABLE { + foreach (@_) { + $code.=<<___; + .long $_,$_,$_,$_ + .long $_,$_,$_,$_ +___ + } +} +&TABLE( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5, + 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3, + 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc, + 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7, + 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13, + 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3, + 0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5, + 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208, + 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 ); +$code.=<<___; +.Lpbswap: + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap +K256_shaext: + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + .asciz "SHA256 multi-block transform for x86_64, CRYPTOGAMS by " +___ + +if ($win64) { +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # end of prologue label + cmp %r10,%rbx # context->Rip<.Lbody + jb .Lin_prologue + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lin_prologue + + mov `16*17`(%rax),%rax # pull saved stack pointer + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + + lea -24-10*16(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lin_prologue: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler +___ +$code.=<<___ if ($avx>1); +.type avx2_handler,\@abi-omnipotent +.align 16 +avx2_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # end of prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lin_prologue + + mov `32*17`($context),%rax # pull saved stack pointer + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore cotnext->R12 + mov %r13,224($context) # restore cotnext->R13 + mov %r14,232($context) # restore cotnext->R14 + mov %r15,240($context) # restore cotnext->R15 + + lea -56-10*16(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + + jmp .Lin_prologue +.size avx2_handler,.-avx2_handler +___ +$code.=<<___; +.section .pdata +.align 4 + .rva .LSEH_begin_sha256_multi_block + .rva .LSEH_end_sha256_multi_block + .rva .LSEH_info_sha256_multi_block + .rva .LSEH_begin_sha256_multi_block_shaext + .rva .LSEH_end_sha256_multi_block_shaext + .rva .LSEH_info_sha256_multi_block_shaext +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_sha256_multi_block_avx + .rva .LSEH_end_sha256_multi_block_avx + .rva .LSEH_info_sha256_multi_block_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_sha256_multi_block_avx2 + .rva .LSEH_end_sha256_multi_block_avx2 + .rva .LSEH_info_sha256_multi_block_avx2 +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_sha256_multi_block: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbody,.Lepilogue # HandlerData[] +.LSEH_info_sha256_multi_block_shaext: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbody_shaext,.Lepilogue_shaext # HandlerData[] +___ +$code.=<<___ if ($avx); +.LSEH_info_sha256_multi_block_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbody_avx,.Lepilogue_avx # HandlerData[] +___ +$code.=<<___ if ($avx>1); +.LSEH_info_sha256_multi_block_avx2: + .byte 9,0,0,0 + .rva avx2_handler + .rva .Lbody_avx2,.Lepilogue_avx2 # HandlerData[] +___ +} +#################################################################### + +sub rex { + local *opcode=shift; + my ($dst,$src)=@_; + my $rex=0; + + $rex|=0x04 if ($dst>=8); + $rex|=0x01 if ($src>=8); + unshift @opcode,$rex|0x40 if ($rex); +} + +sub sha256op38 { + my $instr = shift; + my %opcodelet = ( + "sha256rnds2" => 0xcb, + "sha256msg1" => 0xcc, + "sha256msg2" => 0xcd ); + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { + my @opcode=(0x0f,0x38); + rex(\@opcode,$2,$1); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + + s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo or + + s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go or + s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or + s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go or + s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go; + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/sha/asm/sha512-586.pl b/crypto/sha/asm/sha512-586.pl index 7eab6a5b88b2..e96ec00314a4 100755 --- a/crypto/sha/asm/sha512-586.pl +++ b/crypto/sha/asm/sha512-586.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -9,17 +9,31 @@ # # SHA512 block transform for x86. September 2007. # +# May 2013. +# +# Add SSSE3 code path, 20-25% improvement [over original SSE2 code]. +# # Performance in clock cycles per processed byte (less is better): # -# Pentium PIII P4 AMD K8 Core2 -# gcc 100 75 116 54 66 -# icc 97 77 95 55 57 -# x86 asm 61 56 82 36 40 -# SSE2 asm - - 38 24 20 -# x86_64 asm(*) - - 30 10.0 10.5 +# gcc icc x86 asm SIMD(*) x86_64(**) +# Pentium 100 97 61 - - +# PIII 75 77 56 - - +# P4 116 95 82 34.6 30.8 +# AMD K8 54 55 36 20.7 9.57 +# Core2 66 57 40 15.9 9.97 +# Westmere 70 - 38 12.2 9.58 +# Sandy Bridge 58 - 35 11.9 11.2 +# Ivy Bridge 50 - 33 11.5 8.17 +# Haswell 46 - 29 11.3 7.66 +# Bulldozer 121 - 50 14.0 13.5 +# VIA Nano 91 - 52 33 14.7 +# Atom 126 - 68 48(***) 14.7 +# Silvermont 97 - 58 42(***) 17.5 # -# (*) x86_64 assembler performance is presented for reference -# purposes. +# (*) whichever best applicable. +# (**) x86_64 assembler performance is presented for reference +# purposes, the results are for integer-only code. +# (***) paddq is increadibly slow on Atom. # # IALU code-path is optimized for elder Pentiums. On vanilla Pentium # performance improvement over compiler generated code reaches ~60%, @@ -66,72 +80,77 @@ $Hsse2=&QWP(56,"esp"); $A="mm0"; # B-D and $E="mm4"; # F-H are commonly loaded to respectively mm1-mm3 and # mm5-mm7, but it's done on on-demand basis... +$BxC="mm2"; # ... except for B^C sub BODY_00_15_sse2 { - my $prefetch=shift; + my $phase=shift; - &movq ("mm5",$Fsse2); # load f - &movq ("mm6",$Gsse2); # load g - &movq ("mm7",$Hsse2); # load h + #&movq ("mm5",$Fsse2); # load f + #&movq ("mm6",$Gsse2); # load g &movq ("mm1",$E); # %mm1 is sliding right - &movq ("mm2",$E); # %mm2 is sliding left + &pxor ("mm5","mm6"); # f^=g &psrlq ("mm1",14); - &movq ($Esse2,$E); # modulo-scheduled save e - &psllq ("mm2",23); + &movq ($Esse2,$E); # modulo-scheduled save e + &pand ("mm5",$E); # f&=e + &psllq ($E,23); # $E is sliding left + &movq ($A,"mm3") if ($phase<2); + &movq (&QWP(8*9,"esp"),"mm7") # save X[i] &movq ("mm3","mm1"); # %mm3 is T1 - &psrlq ("mm1",4); - &pxor ("mm3","mm2"); - &psllq ("mm2",23); + &psrlq ("mm1",4); + &pxor ("mm5","mm6"); # Ch(e,f,g) + &pxor ("mm3",$E); + &psllq ($E,23); &pxor ("mm3","mm1"); - &psrlq ("mm1",23); - &pxor ("mm3","mm2"); - &psllq ("mm2",4); + &movq ($Asse2,$A); # modulo-scheduled save a + &paddq ("mm7","mm5"); # X[i]+=Ch(e,f,g) + &pxor ("mm3",$E); + &psrlq ("mm1",23); + &paddq ("mm7",$Hsse2); # X[i]+=h &pxor ("mm3","mm1"); - &paddq ("mm7",QWP(0,$K512)); # h+=K512[i] - &pxor ("mm3","mm2"); # T1=Sigma1_512(e) - - &pxor ("mm5","mm6"); # f^=g + &psllq ($E,4); + &paddq ("mm7",QWP(0,$K512)); # X[i]+=K512[i] + &pxor ("mm3",$E); # T1=Sigma1_512(e) + + &movq ($E,$Dsse2); # e = load d, e in next round + &paddq ("mm3","mm7"); # T1+=X[i] + &movq ("mm5",$A); # %mm5 is sliding right + &psrlq ("mm5",28); + &paddq ($E,"mm3"); # d += T1 + &movq ("mm6",$A); # %mm6 is sliding left + &movq ("mm7","mm5"); + &psllq ("mm6",25); &movq ("mm1",$Bsse2); # load b - &pand ("mm5",$E); # f&=e - &movq ("mm2",$Csse2); # load c - &pxor ("mm5","mm6"); # f^=g - &movq ($E,$Dsse2); # e = load d - &paddq ("mm3","mm5"); # T1+=Ch(e,f,g) - &movq (&QWP(0,"esp"),$A); # modulo-scheduled save a - &paddq ("mm3","mm7"); # T1+=h - - &movq ("mm5",$A); # %mm5 is sliding right - &movq ("mm6",$A); # %mm6 is sliding left - &paddq ("mm3",&QWP(8*9,"esp")); # T1+=X[0] - &psrlq ("mm5",28); - &paddq ($E,"mm3"); # e += T1 - &psllq ("mm6",25); - &movq ("mm7","mm5"); # %mm7 is T2 - &psrlq ("mm5",6); - &pxor ("mm7","mm6"); - &psllq ("mm6",5); - &pxor ("mm7","mm5"); - &psrlq ("mm5",5); - &pxor ("mm7","mm6"); - &psllq ("mm6",6); - &pxor ("mm7","mm5"); + &psrlq ("mm5",6); + &pxor ("mm7","mm6"); &sub ("esp",8); - &pxor ("mm7","mm6"); # T2=Sigma0_512(a) - - &movq ("mm5",$A); # %mm5=a - &por ($A,"mm2"); # a=a|c - &movq ("mm6",&QWP(8*(9+16-14),"esp")) if ($prefetch); - &pand ("mm5","mm2"); # %mm5=a&c - &pand ($A,"mm1"); # a=(a|c)&b - &movq ("mm2",&QWP(8*(9+16-1),"esp")) if ($prefetch); - &por ("mm5",$A); # %mm5=(a&c)|((a|c)&b) - &paddq ("mm7","mm5"); # T2+=Maj(a,b,c) - &movq ($A,"mm3"); # a=T1 - - &mov (&LB("edx"),&BP(0,$K512)); - &paddq ($A,"mm7"); # a+=T2 - &add ($K512,8); + &psllq ("mm6",5); + &pxor ("mm7","mm5"); + &pxor ($A,"mm1"); # a^b, b^c in next round + &psrlq ("mm5",5); + &pxor ("mm7","mm6"); + &pand ($BxC,$A); # (b^c)&(a^b) + &psllq ("mm6",6); + &pxor ("mm7","mm5"); + &pxor ($BxC,"mm1"); # [h=]Maj(a,b,c) + &pxor ("mm6","mm7"); # Sigma0_512(a) + &movq ("mm7",&QWP(8*(9+16-1),"esp")) if ($phase!=0); # pre-fetch + &movq ("mm5",$Fsse2) if ($phase==0); # load f + + if ($phase>1) { + &paddq ($BxC,"mm6"); # h+=Sigma0(a) + &add ($K512,8); + #&paddq ($BxC,"mm3"); # h+=T1 + + ($A,$BxC) = ($BxC,$A); # rotate registers + } else { + &paddq ("mm3",$BxC); # T1+=Maj(a,b,c) + &movq ($BxC,$A); + &add ($K512,8); + &paddq ("mm3","mm6"); # T1+=Sigma0(a) + &movq ("mm6",$Gsse2) if ($phase==0); # load g + #&movq ($A,"mm3"); # h=T1 + } } sub BODY_00_15_x86 { @@ -284,110 +303,357 @@ sub BODY_00_15_x86 { if ($sse2) { &picmeup("edx","OPENSSL_ia32cap_P",$K512,&label("K512")); - &bt (&DWP(0,"edx"),26); - &jnc (&label("loop_x86")); + &mov ("ecx",&DWP(0,"edx")); + &test ("ecx",1<<26); + &jz (&label("loop_x86")); + + &mov ("edx",&DWP(4,"edx")); # load ctx->h[0-7] &movq ($A,&QWP(0,"esi")); + &and ("ecx",1<<24); # XMM registers availability &movq ("mm1",&QWP(8,"esi")); - &movq ("mm2",&QWP(16,"esi")); + &and ("edx",1<<9); # SSSE3 bit + &movq ($BxC,&QWP(16,"esi")); + &or ("ecx","edx"); &movq ("mm3",&QWP(24,"esi")); &movq ($E,&QWP(32,"esi")); &movq ("mm5",&QWP(40,"esi")); &movq ("mm6",&QWP(48,"esi")); &movq ("mm7",&QWP(56,"esi")); + &cmp ("ecx",1<<24|1<<9); + &je (&label("SSSE3")); &sub ("esp",8*10); + &jmp (&label("loop_sse2")); &set_label("loop_sse2",16); - # &movq ($Asse2,$A); + #&movq ($Asse2,$A); &movq ($Bsse2,"mm1"); - &movq ($Csse2,"mm2"); + &movq ($Csse2,$BxC); &movq ($Dsse2,"mm3"); - # &movq ($Esse2,$E); + #&movq ($Esse2,$E); &movq ($Fsse2,"mm5"); &movq ($Gsse2,"mm6"); + &pxor ($BxC,"mm1"); # magic &movq ($Hsse2,"mm7"); + &movq ("mm3",$A); # magic - &mov ("ecx",&DWP(0,"edi")); - &mov ("edx",&DWP(4,"edi")); + &mov ("eax",&DWP(0,"edi")); + &mov ("ebx",&DWP(4,"edi")); &add ("edi",8); - &bswap ("ecx"); - &bswap ("edx"); - &mov (&DWP(8*9+4,"esp"),"ecx"); - &mov (&DWP(8*9+0,"esp"),"edx"); + &mov ("edx",15); # counter + &bswap ("eax"); + &bswap ("ebx"); + &jmp (&label("00_14_sse2")); &set_label("00_14_sse2",16); + &movd ("mm1","eax"); &mov ("eax",&DWP(0,"edi")); + &movd ("mm7","ebx"); &mov ("ebx",&DWP(4,"edi")); &add ("edi",8); &bswap ("eax"); &bswap ("ebx"); - &mov (&DWP(8*8+4,"esp"),"eax"); - &mov (&DWP(8*8+0,"esp"),"ebx"); + &punpckldq("mm7","mm1"); &BODY_00_15_sse2(); - &cmp (&LB("edx"),0x35); - &jne (&label("00_14_sse2")); + &dec ("edx"); + &jnz (&label("00_14_sse2")); + + &movd ("mm1","eax"); + &movd ("mm7","ebx"); + &punpckldq("mm7","mm1"); &BODY_00_15_sse2(1); + &pxor ($A,$A); # A is in %mm3 + &mov ("edx",32); # counter + &jmp (&label("16_79_sse2")); + &set_label("16_79_sse2",16); - #&movq ("mm2",&QWP(8*(9+16-1),"esp")); #prefetched in BODY_00_15 - #&movq ("mm6",&QWP(8*(9+16-14),"esp")); - &movq ("mm1","mm2"); + for ($j=0;$j<2;$j++) { # 2x unroll + #&movq ("mm7",&QWP(8*(9+16-1),"esp")); # prefetched in BODY_00_15 + &movq ("mm5",&QWP(8*(9+16-14),"esp")); + &movq ("mm1","mm7"); + &psrlq ("mm7",1); + &movq ("mm6","mm5"); + &psrlq ("mm5",6); + &psllq ("mm1",56); + &paddq ($A,"mm3"); # from BODY_00_15 + &movq ("mm3","mm7"); + &psrlq ("mm7",7-1); + &pxor ("mm3","mm1"); + &psllq ("mm1",63-56); + &pxor ("mm3","mm7"); + &psrlq ("mm7",8-7); + &pxor ("mm3","mm1"); + &movq ("mm1","mm5"); + &psrlq ("mm5",19-6); + &pxor ("mm7","mm3"); # sigma0 + + &psllq ("mm6",3); + &pxor ("mm1","mm5"); + &paddq ("mm7",&QWP(8*(9+16),"esp")); + &pxor ("mm1","mm6"); + &psrlq ("mm5",61-19); + &paddq ("mm7",&QWP(8*(9+16-9),"esp")); + &pxor ("mm1","mm5"); + &psllq ("mm6",45-3); + &movq ("mm5",$Fsse2); # load f + &pxor ("mm1","mm6"); # sigma1 + &movq ("mm6",$Gsse2); # load g - &psrlq ("mm2",1); - &movq ("mm7","mm6"); - &psrlq ("mm6",6); - &movq ("mm3","mm2"); + &paddq ("mm7","mm1"); # X[i] + #&movq (&QWP(8*9,"esp"),"mm7"); # moved to BODY_00_15 - &psrlq ("mm2",7-1); - &movq ("mm5","mm6"); - &psrlq ("mm6",19-6); - &pxor ("mm3","mm2"); + &BODY_00_15_sse2(2); + } + &dec ("edx"); + &jnz (&label("16_79_sse2")); - &psrlq ("mm2",8-7); - &pxor ("mm5","mm6"); - &psrlq ("mm6",61-19); - &pxor ("mm3","mm2"); + #&movq ($A,$Asse2); + &paddq ($A,"mm3"); # from BODY_00_15 + &movq ("mm1",$Bsse2); + #&movq ($BxC,$Csse2); + &movq ("mm3",$Dsse2); + #&movq ($E,$Esse2); + &movq ("mm5",$Fsse2); + &movq ("mm6",$Gsse2); + &movq ("mm7",$Hsse2); - &movq ("mm2",&QWP(8*(9+16),"esp")); + &pxor ($BxC,"mm1"); # de-magic + &paddq ($A,&QWP(0,"esi")); + &paddq ("mm1",&QWP(8,"esi")); + &paddq ($BxC,&QWP(16,"esi")); + &paddq ("mm3",&QWP(24,"esi")); + &paddq ($E,&QWP(32,"esi")); + &paddq ("mm5",&QWP(40,"esi")); + &paddq ("mm6",&QWP(48,"esi")); + &paddq ("mm7",&QWP(56,"esi")); - &psllq ("mm1",56); - &pxor ("mm5","mm6"); - &psllq ("mm7",3); - &pxor ("mm3","mm1"); + &mov ("eax",8*80); + &movq (&QWP(0,"esi"),$A); + &movq (&QWP(8,"esi"),"mm1"); + &movq (&QWP(16,"esi"),$BxC); + &movq (&QWP(24,"esi"),"mm3"); + &movq (&QWP(32,"esi"),$E); + &movq (&QWP(40,"esi"),"mm5"); + &movq (&QWP(48,"esi"),"mm6"); + &movq (&QWP(56,"esi"),"mm7"); - &paddq ("mm2",&QWP(8*(9+16-9),"esp")); + &lea ("esp",&DWP(0,"esp","eax")); # destroy frame + &sub ($K512,"eax"); # rewind K - &psllq ("mm1",63-56); - &pxor ("mm5","mm7"); - &psllq ("mm7",45-3); - &pxor ("mm3","mm1"); - &pxor ("mm5","mm7"); + &cmp ("edi",&DWP(8*10+8,"esp")); # are we done yet? + &jb (&label("loop_sse2")); - &paddq ("mm3","mm5"); - &paddq ("mm3","mm2"); - &movq (&QWP(8*9,"esp"),"mm3"); + &mov ("esp",&DWP(8*10+12,"esp")); # restore sp + &emms (); +&function_end_A(); - &BODY_00_15_sse2(1); +&set_label("SSSE3",32); +{ my ($cnt,$frame)=("ecx","edx"); + my @X=map("xmm$_",(0..7)); + my $j; + my $i=0; + + &lea ($frame,&DWP(-64,"esp")); + &sub ("esp",256); + + # fixed stack frame layout + # + # +0 A B C D E F G H # backing store + # +64 X[0]+K[i] .. X[15]+K[i] # XMM->MM xfer area + # +192 # XMM off-load ring buffer + # +256 # saved parameters + + &movdqa (@X[1],&QWP(80*8,$K512)); # byte swap mask + &movdqu (@X[0],&QWP(0,"edi")); + &pshufb (@X[0],@X[1]); + for ($j=0;$j<8;$j++) { + &movdqa (&QWP(16*(($j-1)%4),$frame),@X[3]) if ($j>4); # off-load + &movdqa (@X[3],&QWP(16*($j%8),$K512)); + &movdqa (@X[2],@X[1]) if ($j<7); # perpetuate byte swap mask + &movdqu (@X[1],&QWP(16*($j+1),"edi")) if ($j<7); # next input + &movdqa (@X[1],&QWP(16*(($j+1)%4),$frame)) if ($j==7);# restore @X[0] + &paddq (@X[3],@X[0]); + &pshufb (@X[1],@X[2]) if ($j<7); + &movdqa (&QWP(16*($j%8)-128,$frame),@X[3]); # xfer X[i]+K[i] + + push(@X,shift(@X)); # rotate(@X) + } + #&jmp (&label("loop_ssse3")); + &nop (); - &cmp (&LB("edx"),0x17); - &jne (&label("16_79_sse2")); +&set_label("loop_ssse3",32); + &movdqa (@X[2],&QWP(16*(($j+1)%4),$frame)); # pre-restore @X[1] + &movdqa (&QWP(16*(($j-1)%4),$frame),@X[3]); # off-load @X[3] + &lea ($K512,&DWP(16*8,$K512)); + + #&movq ($Asse2,$A); # off-load A-H + &movq ($Bsse2,"mm1"); + &mov ("ebx","edi"); + &movq ($Csse2,$BxC); + &lea ("edi",&DWP(128,"edi")); # advance input + &movq ($Dsse2,"mm3"); + &cmp ("edi","eax"); + #&movq ($Esse2,$E); + &movq ($Fsse2,"mm5"); + &cmovb ("ebx","edi"); + &movq ($Gsse2,"mm6"); + &mov ("ecx",4); # loop counter + &pxor ($BxC,"mm1"); # magic + &movq ($Hsse2,"mm7"); + &pxor ("mm3","mm3"); # magic + + &jmp (&label("00_47_ssse3")); + +sub BODY_00_15_ssse3 { # "phase-less" copy of BODY_00_15_sse2 + ( + '&movq ("mm1",$E)', # %mm1 is sliding right + '&movq ("mm7",&QWP(((-8*$i)%128)-128,$frame))',# X[i]+K[i] + '&pxor ("mm5","mm6")', # f^=g + '&psrlq ("mm1",14)', + '&movq (&QWP(8*($i+4)%64,"esp"),$E)', # modulo-scheduled save e + '&pand ("mm5",$E)', # f&=e + '&psllq ($E,23)', # $E is sliding left + '&paddq ($A,"mm3")', # [h+=Maj(a,b,c)] + '&movq ("mm3","mm1")', # %mm3 is T1 + '&psrlq("mm1",4)', + '&pxor ("mm5","mm6")', # Ch(e,f,g) + '&pxor ("mm3",$E)', + '&psllq($E,23)', + '&pxor ("mm3","mm1")', + '&movq (&QWP(8*$i%64,"esp"),$A)', # modulo-scheduled save a + '&paddq("mm7","mm5")', # X[i]+=Ch(e,f,g) + '&pxor ("mm3",$E)', + '&psrlq("mm1",23)', + '&paddq("mm7",&QWP(8*($i+7)%64,"esp"))', # X[i]+=h + '&pxor ("mm3","mm1")', + '&psllq($E,4)', + '&pxor ("mm3",$E)', # T1=Sigma1_512(e) + + '&movq ($E,&QWP(8*($i+3)%64,"esp"))', # e = load d, e in next round + '&paddq ("mm3","mm7")', # T1+=X[i] + '&movq ("mm5",$A)', # %mm5 is sliding right + '&psrlq("mm5",28)', + '&paddq ($E,"mm3")', # d += T1 + '&movq ("mm6",$A)', # %mm6 is sliding left + '&movq ("mm7","mm5")', + '&psllq("mm6",25)', + '&movq ("mm1",&QWP(8*($i+1)%64,"esp"))', # load b + '&psrlq("mm5",6)', + '&pxor ("mm7","mm6")', + '&psllq("mm6",5)', + '&pxor ("mm7","mm5")', + '&pxor ($A,"mm1")', # a^b, b^c in next round + '&psrlq("mm5",5)', + '&pxor ("mm7","mm6")', + '&pand ($BxC,$A)', # (b^c)&(a^b) + '&psllq("mm6",6)', + '&pxor ("mm7","mm5")', + '&pxor ($BxC,"mm1")', # [h=]Maj(a,b,c) + '&pxor ("mm6","mm7")', # Sigma0_512(a) + '&movq ("mm5",&QWP(8*($i+5-1)%64,"esp"))', # pre-load f + '&paddq ($BxC,"mm6")', # h+=Sigma0(a) + '&movq ("mm6",&QWP(8*($i+6-1)%64,"esp"))', # pre-load g + + '($A,$BxC) = ($BxC,$A); $i--;' + ); +} - # &movq ($A,$Asse2); +&set_label("00_47_ssse3",32); + + for(;$j<16;$j++) { + my ($t0,$t2,$t1)=@X[2..4]; + my @insns = (&BODY_00_15_ssse3(),&BODY_00_15_ssse3()); + + &movdqa ($t2,@X[5]); + &movdqa (@X[1],$t0); # restore @X[1] + &palignr ($t0,@X[0],8); # X[1..2] + &movdqa (&QWP(16*($j%4),$frame),@X[4]); # off-load @X[4] + &palignr ($t2,@X[4],8); # X[9..10] + + &movdqa ($t1,$t0); + &psrlq ($t0,7); + &paddq (@X[0],$t2); # X[0..1] += X[9..10] + &movdqa ($t2,$t1); + &psrlq ($t1,1); + &psllq ($t2,64-8); + &pxor ($t0,$t1); + &psrlq ($t1,8-1); + &pxor ($t0,$t2); + &psllq ($t2,8-1); + &pxor ($t0,$t1); + &movdqa ($t1,@X[7]); + &pxor ($t0,$t2); # sigma0(X[1..2]) + &movdqa ($t2,@X[7]); + &psrlq ($t1,6); + &paddq (@X[0],$t0); # X[0..1] += sigma0(X[1..2]) + + &movdqa ($t0,@X[7]); + &psrlq ($t2,19); + &psllq ($t0,64-61); + &pxor ($t1,$t2); + &psrlq ($t2,61-19); + &pxor ($t1,$t0); + &psllq ($t0,61-19); + &pxor ($t1,$t2); + &movdqa ($t2,&QWP(16*(($j+2)%4),$frame));# pre-restore @X[1] + &pxor ($t1,$t0); # sigma0(X[1..2]) + &movdqa ($t0,&QWP(16*($j%8),$K512)); + eval(shift(@insns)); + &paddq (@X[0],$t1); # X[0..1] += sigma0(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &paddq ($t0,@X[0]); + foreach(@insns) { eval; } + &movdqa (&QWP(16*($j%8)-128,$frame),$t0);# xfer X[i]+K[i] + + push(@X,shift(@X)); # rotate(@X) + } + &lea ($K512,&DWP(16*8,$K512)); + &dec ("ecx"); + &jnz (&label("00_47_ssse3")); + + &movdqa (@X[1],&QWP(0,$K512)); # byte swap mask + &lea ($K512,&DWP(-80*8,$K512)); # rewind + &movdqu (@X[0],&QWP(0,"ebx")); + &pshufb (@X[0],@X[1]); + + for ($j=0;$j<8;$j++) { # load next or same block + my @insns = (&BODY_00_15_ssse3(),&BODY_00_15_ssse3()); + + &movdqa (&QWP(16*(($j-1)%4),$frame),@X[3]) if ($j>4); # off-load + &movdqa (@X[3],&QWP(16*($j%8),$K512)); + &movdqa (@X[2],@X[1]) if ($j<7); # perpetuate byte swap mask + &movdqu (@X[1],&QWP(16*($j+1),"ebx")) if ($j<7); # next input + &movdqa (@X[1],&QWP(16*(($j+1)%4),$frame)) if ($j==7);# restore @X[0] + &paddq (@X[3],@X[0]); + &pshufb (@X[1],@X[2]) if ($j<7); + foreach(@insns) { eval; } + &movdqa (&QWP(16*($j%8)-128,$frame),@X[3]);# xfer X[i]+K[i] + + push(@X,shift(@X)); # rotate(@X) + } + + #&movq ($A,$Asse2); # load A-H &movq ("mm1",$Bsse2); - &movq ("mm2",$Csse2); + &paddq ($A,"mm3"); # from BODY_00_15 + #&movq ($BxC,$Csse2); &movq ("mm3",$Dsse2); - # &movq ($E,$Esse2); - &movq ("mm5",$Fsse2); - &movq ("mm6",$Gsse2); + #&movq ($E,$Esse2); + #&movq ("mm5",$Fsse2); + #&movq ("mm6",$Gsse2); &movq ("mm7",$Hsse2); + &pxor ($BxC,"mm1"); # de-magic &paddq ($A,&QWP(0,"esi")); &paddq ("mm1",&QWP(8,"esi")); - &paddq ("mm2",&QWP(16,"esi")); + &paddq ($BxC,&QWP(16,"esi")); &paddq ("mm3",&QWP(24,"esi")); &paddq ($E,&QWP(32,"esi")); &paddq ("mm5",&QWP(40,"esi")); @@ -396,21 +662,19 @@ if ($sse2) { &movq (&QWP(0,"esi"),$A); &movq (&QWP(8,"esi"),"mm1"); - &movq (&QWP(16,"esi"),"mm2"); + &movq (&QWP(16,"esi"),$BxC); &movq (&QWP(24,"esi"),"mm3"); &movq (&QWP(32,"esi"),$E); &movq (&QWP(40,"esi"),"mm5"); &movq (&QWP(48,"esi"),"mm6"); &movq (&QWP(56,"esi"),"mm7"); - &add ("esp",8*80); # destroy frame - &sub ($K512,8*80); # rewind K - - &cmp ("edi",&DWP(8*10+8,"esp")); # are we done yet? - &jb (&label("loop_sse2")); + &cmp ("edi","eax") # are we done yet? + &jb (&label("loop_ssse3")); + &mov ("esp",&DWP(64+12,$frame)); # restore sp &emms (); - &mov ("esp",&DWP(8*10+12,"esp")); # restore sp +} &function_end_A(); } &set_label("loop_x86",16); @@ -638,6 +902,9 @@ if ($sse2) { &data_word(0xfc657e2a,0x597f299c); # u64 &data_word(0x3ad6faec,0x5fcb6fab); # u64 &data_word(0x4a475817,0x6c44198c); # u64 + + &data_word(0x04050607,0x00010203); # byte swap + &data_word(0x0c0d0e0f,0x08090a0b); # mask &function_end_B("sha512_block_data_order"); &asciz("SHA512 block transform for x86, CRYPTOGAMS by "); diff --git a/crypto/sha/asm/sha512-armv4.pl b/crypto/sha/asm/sha512-armv4.pl index 7faf37b14790..fb7dc506aca1 100755 --- a/crypto/sha/asm/sha512-armv4.pl +++ b/crypto/sha/asm/sha512-armv4.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -26,7 +26,24 @@ # March 2011. # # Add NEON implementation. On Cortex A8 it was measured to process -# one byte in 25.5 cycles or 47% faster than integer-only code. +# one byte in 23.3 cycles or ~60% faster than integer-only code. + +# August 2012. +# +# Improve NEON performance by 12% on Snapdragon S4. In absolute +# terms it's 22.6 cycles per byte, which is disappointing result. +# Technical writers asserted that 3-way S4 pipeline can sustain +# multiple NEON instructions per cycle, but dual NEON issue could +# not be observed, and for NEON-only sequences IPC(*) was found to +# be limited by 1:-( 0.33 and 0.66 were measured for sequences with +# ILPs(*) of 1 and 2 respectively. This in turn means that you can +# even find yourself striving, as I did here, for achieving IPC +# adequate to one delivered by Cortex A8 [for reference, it's +# 0.5 for ILP of 1, and 1 for higher ILPs]. +# +# (*) ILP, instruction-level parallelism, how many instructions +# *can* execute at the same time. IPC, instructions per cycle, +# indicates how many instructions actually execute. # Byte order [in]dependence. ========================================= # @@ -220,16 +237,20 @@ WORD64(0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c) WORD64(0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a) WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817) .size K512,.-K512 +#if __ARM_MAX_ARCH__>=7 .LOPENSSL_armcap: .word OPENSSL_armcap_P-sha512_block_data_order .skip 32-4 +#else +.skip 32 +#endif .global sha512_block_data_order .type sha512_block_data_order,%function sha512_block_data_order: sub r3,pc,#8 @ sha512_block_data_order add $len,$inp,$len,lsl#7 @ len to point at the end of inp -#if __ARM_ARCH__>=7 +#if __ARM_MAX_ARCH__>=7 ldr r12,.LOPENSSL_armcap ldr r12,[r3,r12] @ OPENSSL_armcap_P tst r12,#1 @@ -457,40 +478,40 @@ $code.=<<___ if ($i<16 || $i&1); vld1.64 {@X[$i%16]},[$inp]! @ handles unaligned #endif vshr.u64 $t1,$e,#@Sigma1[1] +#if $i>0 + vadd.i64 $a,$Maj @ h+=Maj from the past +#endif vshr.u64 $t2,$e,#@Sigma1[2] ___ $code.=<<___; vld1.64 {$K},[$Ktbl,:64]! @ K[i++] vsli.64 $t0,$e,#`64-@Sigma1[0]` vsli.64 $t1,$e,#`64-@Sigma1[1]` + vmov $Ch,$e vsli.64 $t2,$e,#`64-@Sigma1[2]` #if $i<16 && defined(__ARMEL__) vrev64.8 @X[$i],@X[$i] #endif - vadd.i64 $T1,$K,$h - veor $Ch,$f,$g - veor $t0,$t1 - vand $Ch,$e - veor $t0,$t2 @ Sigma1(e) - veor $Ch,$g @ Ch(e,f,g) - vadd.i64 $T1,$t0 + veor $t1,$t0 + vbsl $Ch,$f,$g @ Ch(e,f,g) vshr.u64 $t0,$a,#@Sigma0[0] - vadd.i64 $T1,$Ch + veor $t2,$t1 @ Sigma1(e) + vadd.i64 $T1,$Ch,$h vshr.u64 $t1,$a,#@Sigma0[1] - vshr.u64 $t2,$a,#@Sigma0[2] vsli.64 $t0,$a,#`64-@Sigma0[0]` + vadd.i64 $T1,$t2 + vshr.u64 $t2,$a,#@Sigma0[2] + vadd.i64 $K,@X[$i%16] vsli.64 $t1,$a,#`64-@Sigma0[1]` + veor $Maj,$a,$b vsli.64 $t2,$a,#`64-@Sigma0[2]` - vadd.i64 $T1,@X[$i%16] - vorr $Maj,$a,$c - vand $Ch,$a,$c veor $h,$t0,$t1 - vand $Maj,$b + vadd.i64 $T1,$K + vbsl $Maj,$c,$b @ Maj(a,b,c) veor $h,$t2 @ Sigma0(a) - vorr $Maj,$Ch @ Maj(a,b,c) - vadd.i64 $h,$T1 vadd.i64 $d,$T1 - vadd.i64 $h,$Maj + vadd.i64 $Maj,$T1 + @ vadd.i64 $h,$Maj ___ } @@ -508,6 +529,7 @@ $i /= 2; $code.=<<___; vshr.u64 $t0,@X[($i+7)%8],#@sigma1[0] vshr.u64 $t1,@X[($i+7)%8],#@sigma1[1] + vadd.i64 @_[0],d30 @ h+=Maj from the past vshr.u64 $s1,@X[($i+7)%8],#@sigma1[2] vsli.64 $t0,@X[($i+7)%8],#`64-@sigma1[0]` vext.8 $s0,@X[$i%8],@X[($i+1)%8],#8 @ X[i+1] @@ -533,7 +555,8 @@ ___ } $code.=<<___; -#if __ARM_ARCH__>=7 +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a .fpu neon .align 4 @@ -554,6 +577,7 @@ for(;$i<32;$i++) { &NEON_16_79($i,@V); unshift(@V,pop(@V)); } $code.=<<___; bne .L16_79_neon + vadd.i64 $A,d30 @ h+=Maj from the past vldmia $ctx,{d24-d31} @ load context to temp vadd.i64 q8,q12 @ vectorized accumulate vadd.i64 q9,q13 @@ -565,7 +589,7 @@ $code.=<<___; bne .Loop_neon vldmia sp!,{d8-d15} @ epilogue - bx lr + ret @ bx lr #endif ___ } @@ -573,10 +597,13 @@ $code.=<<___; .size sha512_block_data_order,.-sha512_block_data_order .asciz "SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by " .align 2 +#if __ARM_MAX_ARCH__>=7 .comm OPENSSL_armcap_P,4,4 +#endif ___ $code =~ s/\`([^\`]*)\`/eval $1/gem; $code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 +$code =~ s/\bret\b/bx lr/gm; print $code; close STDOUT; # enforce flush diff --git a/crypto/sha/asm/sha512-armv8.pl b/crypto/sha/asm/sha512-armv8.pl new file mode 100755 index 000000000000..f7b36b986a61 --- /dev/null +++ b/crypto/sha/asm/sha512-armv8.pl @@ -0,0 +1,422 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# SHA256/512 for ARMv8. +# +# Performance in cycles per processed byte and improvement coefficient +# over code generated with "default" compiler: +# +# SHA256-hw SHA256(*) SHA512 +# Apple A7 1.97 10.5 (+33%) 6.73 (-1%(**)) +# Cortex-A53 2.38 15.5 (+115%) 10.0 (+150%(***)) +# Cortex-A57 2.31 11.6 (+86%) 7.51 (+260%(***)) +# Denver 2.01 10.5 (+26%) 6.70 (+8%) +# X-Gene 20.0 (+100%) 12.8 (+300%(***)) +# +# (*) Software SHA256 results are of lesser relevance, presented +# mostly for informational purposes. +# (**) The result is a trade-off: it's possible to improve it by +# 10% (or by 1 cycle per round), but at the cost of 20% loss +# on Cortex-A53 (or by 4 cycles per round). +# (***) Super-impressive coefficients over gcc-generated code are +# indication of some compiler "pathology", most notably code +# generated with -mgeneral-regs-only is significanty faster +# and the gap is only 40-90%. + +$flavour=shift; +$output=shift; +open STDOUT,">$output"; + +if ($output =~ /512/) { + $BITS=512; + $SZ=8; + @Sigma0=(28,34,39); + @Sigma1=(14,18,41); + @sigma0=(1, 8, 7); + @sigma1=(19,61, 6); + $rounds=80; + $reg_t="x"; +} else { + $BITS=256; + $SZ=4; + @Sigma0=( 2,13,22); + @Sigma1=( 6,11,25); + @sigma0=( 7,18, 3); + @sigma1=(17,19,10); + $rounds=64; + $reg_t="w"; +} + +$func="sha${BITS}_block_data_order"; + +($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30)); + +@X=map("$reg_t$_",(3..15,0..2)); +@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27)); +($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28)); + +sub BODY_00_xx { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +my $j=($i+1)&15; +my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]); + $T0=@X[$i+3] if ($i<11); + +$code.=<<___ if ($i<16); +#ifndef __ARMEB__ + rev @X[$i],@X[$i] // $i +#endif +___ +$code.=<<___ if ($i<13 && ($i&1)); + ldp @X[$i+1],@X[$i+2],[$inp],#2*$SZ +___ +$code.=<<___ if ($i==13); + ldp @X[14],@X[15],[$inp] +___ +$code.=<<___ if ($i>=14); + ldr @X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`] +___ +$code.=<<___ if ($i>0 && $i<16); + add $a,$a,$t1 // h+=Sigma0(a) +___ +$code.=<<___ if ($i>=11); + str @X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`] +___ +# While ARMv8 specifies merged rotate-n-logical operation such as +# 'eor x,y,z,ror#n', it was found to negatively affect performance +# on Apple A7. The reason seems to be that it requires even 'y' to +# be available earlier. This means that such merged instruction is +# not necessarily best choice on critical path... On the other hand +# Cortex-A5x handles merged instructions much better than disjoint +# rotate and logical... See (**) footnote above. +$code.=<<___ if ($i<15); + ror $t0,$e,#$Sigma1[0] + add $h,$h,$t2 // h+=K[i] + eor $T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]` + and $t1,$f,$e + bic $t2,$g,$e + add $h,$h,@X[$i&15] // h+=X[i] + orr $t1,$t1,$t2 // Ch(e,f,g) + eor $t2,$a,$b // a^b, b^c in next round + eor $t0,$t0,$T0,ror#$Sigma1[1] // Sigma1(e) + ror $T0,$a,#$Sigma0[0] + add $h,$h,$t1 // h+=Ch(e,f,g) + eor $t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]` + add $h,$h,$t0 // h+=Sigma1(e) + and $t3,$t3,$t2 // (b^c)&=(a^b) + add $d,$d,$h // d+=h + eor $t3,$t3,$b // Maj(a,b,c) + eor $t1,$T0,$t1,ror#$Sigma0[1] // Sigma0(a) + add $h,$h,$t3 // h+=Maj(a,b,c) + ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round + //add $h,$h,$t1 // h+=Sigma0(a) +___ +$code.=<<___ if ($i>=15); + ror $t0,$e,#$Sigma1[0] + add $h,$h,$t2 // h+=K[i] + ror $T1,@X[($j+1)&15],#$sigma0[0] + and $t1,$f,$e + ror $T2,@X[($j+14)&15],#$sigma1[0] + bic $t2,$g,$e + ror $T0,$a,#$Sigma0[0] + add $h,$h,@X[$i&15] // h+=X[i] + eor $t0,$t0,$e,ror#$Sigma1[1] + eor $T1,$T1,@X[($j+1)&15],ror#$sigma0[1] + orr $t1,$t1,$t2 // Ch(e,f,g) + eor $t2,$a,$b // a^b, b^c in next round + eor $t0,$t0,$e,ror#$Sigma1[2] // Sigma1(e) + eor $T0,$T0,$a,ror#$Sigma0[1] + add $h,$h,$t1 // h+=Ch(e,f,g) + and $t3,$t3,$t2 // (b^c)&=(a^b) + eor $T2,$T2,@X[($j+14)&15],ror#$sigma1[1] + eor $T1,$T1,@X[($j+1)&15],lsr#$sigma0[2] // sigma0(X[i+1]) + add $h,$h,$t0 // h+=Sigma1(e) + eor $t3,$t3,$b // Maj(a,b,c) + eor $t1,$T0,$a,ror#$Sigma0[2] // Sigma0(a) + eor $T2,$T2,@X[($j+14)&15],lsr#$sigma1[2] // sigma1(X[i+14]) + add @X[$j],@X[$j],@X[($j+9)&15] + add $d,$d,$h // d+=h + add $h,$h,$t3 // h+=Maj(a,b,c) + ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round + add @X[$j],@X[$j],$T1 + add $h,$h,$t1 // h+=Sigma0(a) + add @X[$j],@X[$j],$T2 +___ + ($t2,$t3)=($t3,$t2); +} + +$code.=<<___; +#include "arm_arch.h" + +.text + +.globl $func +.type $func,%function +.align 6 +$func: +___ +$code.=<<___ if ($SZ==4); + ldr x16,.LOPENSSL_armcap_P + adr x17,.LOPENSSL_armcap_P + add x16,x16,x17 + ldr w16,[x16] + tst w16,#ARMV8_SHA256 + b.ne .Lv8_entry +___ +$code.=<<___; + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#4*$SZ + + ldp $A,$B,[$ctx] // load context + ldp $C,$D,[$ctx,#2*$SZ] + ldp $E,$F,[$ctx,#4*$SZ] + add $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input + ldp $G,$H,[$ctx,#6*$SZ] + adr $Ktbl,K$BITS + stp $ctx,$num,[x29,#96] + +.Loop: + ldp @X[0],@X[1],[$inp],#2*$SZ + ldr $t2,[$Ktbl],#$SZ // *K++ + eor $t3,$B,$C // magic seed + str $inp,[x29,#112] +___ +for ($i=0;$i<16;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); } +$code.=".Loop_16_xx:\n"; +for (;$i<32;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + cbnz $t2,.Loop_16_xx + + ldp $ctx,$num,[x29,#96] + ldr $inp,[x29,#112] + sub $Ktbl,$Ktbl,#`$SZ*($rounds+1)` // rewind + + ldp @X[0],@X[1],[$ctx] + ldp @X[2],@X[3],[$ctx,#2*$SZ] + add $inp,$inp,#14*$SZ // advance input pointer + ldp @X[4],@X[5],[$ctx,#4*$SZ] + add $A,$A,@X[0] + ldp @X[6],@X[7],[$ctx,#6*$SZ] + add $B,$B,@X[1] + add $C,$C,@X[2] + add $D,$D,@X[3] + stp $A,$B,[$ctx] + add $E,$E,@X[4] + add $F,$F,@X[5] + stp $C,$D,[$ctx,#2*$SZ] + add $G,$G,@X[6] + add $H,$H,@X[7] + cmp $inp,$num + stp $E,$F,[$ctx,#4*$SZ] + stp $G,$H,[$ctx,#6*$SZ] + b.ne .Loop + + ldp x19,x20,[x29,#16] + add sp,sp,#4*$SZ + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#128 + ret +.size $func,.-$func + +.align 6 +.type K$BITS,%object +K$BITS: +___ +$code.=<<___ if ($SZ==8); + .quad 0x428a2f98d728ae22,0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538,0x59f111f1b605d019 + .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242,0x12835b0145706fbe + .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x983e5152ee66dfab,0xa831c66d2db43210 + .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 + .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xd192e819d6ef5218,0xd69906245565a910 + .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x90befffa23631e28,0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xca273eceea26619c,0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x28db77f523047d84,0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 + .quad 0 // terminator +___ +$code.=<<___ if ($SZ==4); + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + .long 0 //terminator +___ +$code.=<<___; +.size K$BITS,.-K$BITS +.align 3 +.LOPENSSL_armcap_P: + .quad OPENSSL_armcap_P-. +.asciz "SHA$BITS block transform for ARMv8, CRYPTOGAMS by " +.align 2 +___ + +if ($SZ==4) { +my $Ktbl="x3"; + +my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2)); +my @MSG=map("v$_.16b",(4..7)); +my ($W0,$W1)=("v16.4s","v17.4s"); +my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b"); + +$code.=<<___; +.type sha256_block_armv8,%function +.align 6 +sha256_block_armv8: +.Lv8_entry: + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ld1.32 {$ABCD,$EFGH},[$ctx] + adr $Ktbl,K256 + +.Loop_hw: + ld1 {@MSG[0]-@MSG[3]},[$inp],#64 + sub $num,$num,#1 + ld1.32 {$W0},[$Ktbl],#16 + rev32 @MSG[0],@MSG[0] + rev32 @MSG[1],@MSG[1] + rev32 @MSG[2],@MSG[2] + rev32 @MSG[3],@MSG[3] + orr $ABCD_SAVE,$ABCD,$ABCD // offload + orr $EFGH_SAVE,$EFGH,$EFGH +___ +for($i=0;$i<12;$i++) { +$code.=<<___; + ld1.32 {$W1},[$Ktbl],#16 + add.i32 $W0,$W0,@MSG[0] + sha256su0 @MSG[0],@MSG[1] + orr $abcd,$ABCD,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + sha256su1 @MSG[0],@MSG[2],@MSG[3] +___ + ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG)); +} +$code.=<<___; + ld1.32 {$W1},[$Ktbl],#16 + add.i32 $W0,$W0,@MSG[0] + orr $abcd,$ABCD,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + + ld1.32 {$W0},[$Ktbl],#16 + add.i32 $W1,$W1,@MSG[1] + orr $abcd,$ABCD,$ABCD + sha256h $ABCD,$EFGH,$W1 + sha256h2 $EFGH,$abcd,$W1 + + ld1.32 {$W1},[$Ktbl] + add.i32 $W0,$W0,@MSG[2] + sub $Ktbl,$Ktbl,#$rounds*$SZ-16 // rewind + orr $abcd,$ABCD,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + + add.i32 $W1,$W1,@MSG[3] + orr $abcd,$ABCD,$ABCD + sha256h $ABCD,$EFGH,$W1 + sha256h2 $EFGH,$abcd,$W1 + + add.i32 $ABCD,$ABCD,$ABCD_SAVE + add.i32 $EFGH,$EFGH,$EFGH_SAVE + + cbnz $num,.Loop_hw + + st1.32 {$ABCD,$EFGH},[$ctx] + + ldr x29,[sp],#16 + ret +.size sha256_block_armv8,.-sha256_block_armv8 +___ +} + +$code.=<<___; +.comm OPENSSL_armcap_P,4,4 +___ + +{ my %opcode = ( + "sha256h" => 0x5e004000, "sha256h2" => 0x5e005000, + "sha256su0" => 0x5e282800, "sha256su1" => 0x5e006000 ); + + sub unsha256 { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o + && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5)|($3<<16), + $mnemonic,$arg; + } +} + +foreach(split("\n",$code)) { + + s/\`([^\`]*)\`/eval($1)/geo; + + s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/geo; + + s/\.\w?32\b//o and s/\.16b/\.4s/go; + m/(ld|st)1[^\[]+\[0\]/o and s/\.4s/\.s/go; + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/sha/asm/sha512-ia64.pl b/crypto/sha/asm/sha512-ia64.pl index 1c6ce56522ed..59f889a09594 100755 --- a/crypto/sha/asm/sha512-ia64.pl +++ b/crypto/sha/asm/sha512-ia64.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -48,16 +48,22 @@ # because on Itanium 1 stall on MM result is accompanied by # pipeline flush, which takes 6 cycles:-( # -# Resulting performance numbers for 900MHz Itanium 2 system: +# June 2012 # -# The 'numbers' are in 1000s of bytes per second processed. -# type 16 bytes 64 bytes 256 bytes 1024 bytes 8192 bytes -# sha1(*) 6210.14k 20376.30k 52447.83k 85870.05k 105478.12k -# sha256 7476.45k 20572.05k 41538.34k 56062.29k 62093.18k -# sha512 4996.56k 20026.28k 47597.20k 85278.79k 111501.31k +# Improve performance by 15-20%. Note about "rules of engagement" +# above. Contemporary cores are equipped with additional shifter, +# so that they should perform even better than below, presumably +# by ~10%. # -# (*) SHA1 numbers are for HP-UX compiler and are presented purely -# for reference purposes. I bet it can improved too... +###################################################################### +# Current performance in cycles per processed byte for Itanium 2 +# pre-9000 series [little-endian] system: +# +# SHA1(*) 5.7 +# SHA256 12.6 +# SHA512 6.7 +# +# (*) SHA1 result is presented purely for reference purposes. # # To generate code, pass the file name with either 256 or 512 in its # name and compiler flags. @@ -106,8 +112,8 @@ if (!defined($big_endian)) { $big_endian=(unpack('L',pack('N',1))==1); } $code=<<___; -.ident \"$output, version 1.1\" -.ident \"IA-64 ISA artwork by Andy Polyakov \" +.ident \"$output, version 2.0\" +.ident \"IA-64 ISA artwork by Andy Polyakov \" .explicit .text @@ -115,26 +121,25 @@ pfssave=r2; lcsave=r3; prsave=r14; K=r15; -A=r16; B=r17; C=r18; D=r19; -E=r20; F=r21; G=r22; H=r23; +A_=r16; B_=r17; C_=r18; D_=r19; +E_=r20; F_=r21; G_=r22; H_=r23; T1=r24; T2=r25; s0=r26; s1=r27; t0=r28; t1=r29; Ktbl=r30; ctx=r31; // 1st arg -input=r48; // 2nd arg -num=r49; // 3rd arg -sgm0=r50; sgm1=r51; // small constants -A_=r54; B_=r55; C_=r56; D_=r57; -E_=r58; F_=r59; G_=r60; H_=r61; +input=r56; // 2nd arg +num=r57; // 3rd arg +sgm0=r58; sgm1=r59; // small constants // void $func (SHA_CTX *ctx, const void *in,size_t num[,int host]) .global $func# .proc $func# .align 32 +.skip 16 $func: .prologue .save ar.pfs,pfssave -{ .mmi; alloc pfssave=ar.pfs,3,27,0,16 +{ .mmi; alloc pfssave=ar.pfs,3,25,0,24 $ADDP ctx=0,r32 // 1st arg .save ar.lc,lcsave mov lcsave=ar.lc } @@ -145,11 +150,9 @@ $func: .body { .mib; add r8=0*$SZ,ctx - add r9=1*$SZ,ctx - brp.loop.imp .L_first16,.L_first16_end-16 } + add r9=1*$SZ,ctx } { .mib; add r10=2*$SZ,ctx - add r11=3*$SZ,ctx - brp.loop.imp .L_rest,.L_rest_end-16 };; + add r11=3*$SZ,ctx };; // load A-H .Lpic_point: @@ -164,7 +167,7 @@ $func: add Ktbl=($TABLE#-.Lpic_point),Ktbl } { .mmi; $LDW G_=[r10] $LDW H_=[r11] - cmp.ne p0,p16=0,r0 };; // used in sha256_block + cmp.ne p0,p16=0,r0 };; ___ $code.=<<___ if ($BITS==64); { .mii; and r8=7,input @@ -179,50 +182,26 @@ $code.=<<___ if ($BITS==64); ___ $code.=<<___; .L_outer: -.rotr X[16] -{ .mmi; mov A=A_ - mov B=B_ +.rotr R[8],X[16] +A=R[0]; B=R[1]; C=R[2]; D=R[3]; E=R[4]; F=R[5]; G=R[6]; H=R[7] +{ .mmi; ld1 X[15]=[input],$SZ // eliminated in sha512 + mov A=A_ mov ar.lc=14 } -{ .mmi; mov C=C_ - mov D=D_ - mov E=E_ } -{ .mmi; mov F=F_ - mov G=G_ - mov ar.ec=2 } -{ .mmi; ld1 X[15]=[input],$SZ // eliminated in 64-bit +{ .mmi; mov B=B_ + mov C=C_ + mov D=D_ } +{ .mmi; mov E=E_ + mov F=F_ + mov ar.ec=2 };; +{ .mmi; mov G=G_ mov H=H_ - mov sgm1=$sigma1[2] };; - -___ -$t0="t0", $t1="t1", $code.=<<___ if ($BITS==32); -.align 32 -.L_first16: -{ .mmi; add r9=1-$SZ,input - add r10=2-$SZ,input - add r11=3-$SZ,input };; -{ .mmi; ld1 r9=[r9] - ld1 r10=[r10] - dep.z $t1=E,32,32 } -{ .mmi; $LDW K=[Ktbl],$SZ - ld1 r11=[r11] - zxt4 E=E };; -{ .mii; or $t1=$t1,E - dep X[15]=X[15],r9,8,8 - dep r11=r10,r11,8,8 };; -{ .mmi; and T1=F,E - and T2=A,B - dep X[15]=X[15],r11,16,16 } -{ .mmi; andcm r8=G,E - and r9=A,C - mux2 $t0=A,0x44 };; // copy lower half to upper -{ .mmi; (p16) ld1 X[15-1]=[input],$SZ // prefetch - xor T1=T1,r8 // T1=((e & f) ^ (~e & g)) - _rotr r11=$t1,$Sigma1[0] } // ROTR(e,14) -{ .mib; and r10=B,C - xor T2=T2,r9 };; + mov sgm1=$sigma1[2] } +{ .mib; mov r8=0 + add r9=1-$SZ,input + brp.loop.imp .L_first16,.L_first16_end-16 };; ___ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); -// in 64-bit mode I load whole X[16] at once and take care of alignment... +// in sha512 case I load whole X[16] at once and take care of alignment... { .mmi; add r8=1*$SZ,input add r9=2*$SZ,input add r10=3*$SZ,input };; @@ -248,7 +227,9 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); $LDW X[ 2]=[r8],4*$SZ (p15) br.cond.dpnt.many .L7byte };; { .mmb; $LDW X[ 1]=[r9],4*$SZ - $LDW X[ 0]=[r10],4*$SZ + $LDW X[ 0]=[r10],4*$SZ } +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev // eliminated on big-endian br.many .L_first16 };; .L1byte: { .mmi; $LDW X[13]=[r9],4*$SZ @@ -281,7 +262,9 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); shrp X[ 3]=X[ 3],X[ 2],56 } { .mii; shrp X[ 2]=X[ 2],X[ 1],56 shrp X[ 1]=X[ 1],X[ 0],56 } -{ .mib; shrp X[ 0]=X[ 0],T1,56 +{ .mib; shrp X[ 0]=X[ 0],T1,56 } +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev // eliminated on big-endian br.many .L_first16 };; .L2byte: { .mmi; $LDW X[11]=[input],4*$SZ @@ -313,7 +296,9 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); shrp X[ 2]=X[ 2],X[ 1],48 } { .mii; shrp X[ 1]=X[ 1],X[ 0],48 shrp X[ 0]=X[ 0],T1,48 } -{ .mfb; br.many .L_first16 };; +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev // eliminated on big-endian + br.many .L_first16 };; .L3byte: { .mmi; $LDW X[ 9]=[r9],4*$SZ $LDW X[ 8]=[r10],4*$SZ @@ -341,7 +326,9 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); shrp X[ 3]=X[ 3],X[ 2],40 } { .mii; shrp X[ 2]=X[ 2],X[ 1],40 shrp X[ 1]=X[ 1],X[ 0],40 } -{ .mib; shrp X[ 0]=X[ 0],T1,40 +{ .mib; shrp X[ 0]=X[ 0],T1,40 } +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev // eliminated on big-endian br.many .L_first16 };; .L4byte: { .mmi; $LDW X[ 7]=[input],4*$SZ @@ -369,7 +356,9 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); shrp X[ 2]=X[ 2],X[ 1],32 } { .mii; shrp X[ 1]=X[ 1],X[ 0],32 shrp X[ 0]=X[ 0],T1,32 } -{ .mfb; br.many .L_first16 };; +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev // eliminated on big-endian + br.many .L_first16 };; .L5byte: { .mmi; $LDW X[ 5]=[r9],4*$SZ $LDW X[ 4]=[r10],4*$SZ @@ -393,7 +382,9 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); shrp X[ 3]=X[ 3],X[ 2],24 } { .mii; shrp X[ 2]=X[ 2],X[ 1],24 shrp X[ 1]=X[ 1],X[ 0],24 } -{ .mib; shrp X[ 0]=X[ 0],T1,24 +{ .mib; shrp X[ 0]=X[ 0],T1,24 } +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev // eliminated on big-endian br.many .L_first16 };; .L6byte: { .mmi; $LDW X[ 3]=[input],4*$SZ @@ -417,7 +408,9 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); shrp X[ 2]=X[ 2],X[ 1],16 } { .mii; shrp X[ 1]=X[ 1],X[ 0],16 shrp X[ 0]=X[ 0],T1,16 } -{ .mfb; br.many .L_first16 };; +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev // eliminated on big-endian + br.many .L_first16 };; .L7byte: { .mmi; $LDW X[ 1]=[r9],4*$SZ $LDW X[ 0]=[r10],4*$SZ @@ -437,128 +430,146 @@ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); shrp X[ 3]=X[ 3],X[ 2],8 } { .mii; shrp X[ 2]=X[ 2],X[ 1],8 shrp X[ 1]=X[ 1],X[ 0],8 } -{ .mib; shrp X[ 0]=X[ 0],T1,8 - br.many .L_first16 };; +{ .mib; shrp X[ 0]=X[ 0],T1,8 } +{ .mib; mov r8=0 + mux1 X[15]=X[15],\@rev };; // eliminated on big-endian .align 32 .L_first16: { .mmi; $LDW K=[Ktbl],$SZ - and T1=F,E - and T2=A,B } -{ .mmi; //$LDW X[15]=[input],$SZ // X[i]=*input++ + add A=A,r8 // H+=Sigma(0) from the past + _rotr r10=$t1,$Sigma1[0] } // ROTR(e,14) +{ .mmi; and T1=F,E andcm r8=G,E - and r9=A,C };; -{ .mmi; xor T1=T1,r8 //T1=((e & f) ^ (~e & g)) - and r10=B,C - _rotr r11=$t1,$Sigma1[0] } // ROTR(e,14) -{ .mmi; xor T2=T2,r9 - mux1 X[15]=X[15],\@rev };; // eliminated in big-endian + (p16) mux1 X[14]=X[14],\@rev };; // eliminated on big-endian +{ .mmi; and T2=A,B + and r9=A,C + _rotr r11=$t1,$Sigma1[1] } // ROTR(e,41) +{ .mmi; xor T1=T1,r8 // T1=((e & f) ^ (~e & g)) + and r8=B,C };; +___ +$t0="t0", $t1="t1", $code.=<<___ if ($BITS==32); +.align 32 +.L_first16: +{ .mmi; add A=A,r8 // H+=Sigma(0) from the past + add r10=2-$SZ,input + add r11=3-$SZ,input };; +{ .mmi; ld1 r9=[r9] + ld1 r10=[r10] + dep.z $t1=E,32,32 } +{ .mmi; ld1 r11=[r11] + $LDW K=[Ktbl],$SZ + zxt4 E=E };; +{ .mii; or $t1=$t1,E + dep X[15]=X[15],r9,8,8 + mux2 $t0=A,0x44 };; // copy lower half to upper +{ .mmi; and T1=F,E + andcm r8=G,E + dep r11=r10,r11,8,8 };; +{ .mmi; and T2=A,B + and r9=A,C + dep X[15]=X[15],r11,16,16 };; +{ .mmi; (p16) ld1 X[15-1]=[input],$SZ // prefetch + xor T1=T1,r8 // T1=((e & f) ^ (~e & g)) + _rotr r10=$t1,$Sigma1[0] } // ROTR(e,14) +{ .mmi; and r8=B,C + _rotr r11=$t1,$Sigma1[1] };; // ROTR(e,18) ___ $code.=<<___; -{ .mib; add T1=T1,H // T1=Ch(e,f,g)+h - _rotr r8=$t1,$Sigma1[1] } // ROTR(e,18) -{ .mib; xor T2=T2,r10 // T2=((a & b) ^ (a & c) ^ (b & c)) - mov H=G };; -{ .mib; xor r11=r8,r11 - _rotr r9=$t1,$Sigma1[2] } // ROTR(e,41) -{ .mib; mov G=F - mov F=E };; -{ .mib; xor r9=r9,r11 // r9=Sigma1(e) - _rotr r10=$t0,$Sigma0[0] } // ROTR(a,28) -{ .mib; add T1=T1,K // T1=Ch(e,f,g)+h+K512[i] - mov E=D };; -{ .mib; add T1=T1,r9 // T1+=Sigma1(e) - _rotr r11=$t0,$Sigma0[1] } // ROTR(a,34) -{ .mib; mov D=C - mov C=B };; -{ .mib; add T1=T1,X[15] // T1+=X[i] - _rotr r8=$t0,$Sigma0[2] } // ROTR(a,39) -{ .mib; xor r10=r10,r11 - mux2 X[15]=X[15],0x44 };; // eliminated in 64-bit -{ .mmi; xor r10=r8,r10 // r10=Sigma0(a) - mov B=A - add A=T1,T2 };; -{ .mib; add E=E,T1 - add A=A,r10 // T2=Maj(a,b,c)+Sigma0(a) - br.ctop.sptk .L_first16 };; +{ .mmi; add T1=T1,H // T1=Ch(e,f,g)+h + xor r10=r10,r11 + _rotr r11=$t1,$Sigma1[2] } // ROTR(e,41) +{ .mmi; xor T2=T2,r9 + add K=K,X[15] };; +{ .mmi; add T1=T1,K // T1+=K[i]+X[i] + xor T2=T2,r8 // T2=((a & b) ^ (a & c) ^ (b & c)) + _rotr r8=$t0,$Sigma0[0] } // ROTR(a,28) +{ .mmi; xor r11=r11,r10 // Sigma1(e) + _rotr r9=$t0,$Sigma0[1] };; // ROTR(a,34) +{ .mmi; add T1=T1,r11 // T+=Sigma1(e) + xor r8=r8,r9 + _rotr r9=$t0,$Sigma0[2] };; // ROTR(a,39) +{ .mmi; xor r8=r8,r9 // Sigma0(a) + add D=D,T1 + mux2 H=X[15],0x44 } // mov H=X[15] in sha512 +{ .mib; (p16) add r9=1-$SZ,input // not used in sha512 + add X[15]=T1,T2 // H=T1+Maj(a,b,c) + br.ctop.sptk .L_first16 };; .L_first16_end: -{ .mii; mov ar.lc=$rounds-17 - mov ar.ec=1 };; +{ .mib; mov ar.lc=$rounds-17 + brp.loop.imp .L_rest,.L_rest_end-16 } +{ .mib; mov ar.ec=1 + br.many .L_rest };; .align 32 .L_rest: -.rotr X[16] -{ .mib; $LDW K=[Ktbl],$SZ +{ .mmi; $LDW K=[Ktbl],$SZ + add A=A,r8 // H+=Sigma0(a) from the past _rotr r8=X[15-1],$sigma0[0] } // ROTR(s0,1) -{ .mib; $ADD X[15]=X[15],X[15-9] // X[i&0xF]+=X[(i+9)&0xF] - $SHRU s0=X[15-1],sgm0 };; // s0=X[(i+1)&0xF]>>7 +{ .mmi; add X[15]=X[15],X[15-9] // X[i&0xF]+=X[(i+9)&0xF] + $SHRU s0=X[15-1],sgm0 };; // s0=X[(i+1)&0xF]>>7 { .mib; and T1=F,E _rotr r9=X[15-1],$sigma0[1] } // ROTR(s0,8) { .mib; andcm r10=G,E - $SHRU s1=X[15-14],sgm1 };; // s1=X[(i+14)&0xF]>>6 + $SHRU s1=X[15-14],sgm1 };; // s1=X[(i+14)&0xF]>>6 +// Pair of mmi; splits on Itanium 1 and prevents pipeline flush +// upon $SHRU output usage { .mmi; xor T1=T1,r10 // T1=((e & f) ^ (~e & g)) xor r9=r8,r9 - _rotr r10=X[15-14],$sigma1[0] };;// ROTR(s1,19) -{ .mib; and T2=A,B - _rotr r11=X[15-14],$sigma1[1] }// ROTR(s1,61) -{ .mib; and r8=A,C };; + _rotr r10=X[15-14],$sigma1[0] }// ROTR(s1,19) +{ .mmi; and T2=A,B + and r8=A,C + _rotr r11=X[15-14],$sigma1[1] };;// ROTR(s1,61) ___ $t0="t0", $t1="t1", $code.=<<___ if ($BITS==32); -// I adhere to mmi; in order to hold Itanium 1 back and avoid 6 cycle -// pipeline flush in last bundle. Note that even on Itanium2 the -// latter stalls for one clock cycle... -{ .mmi; xor s0=s0,r9 // s0=sigma0(X[(i+1)&0xF]) - dep.z $t1=E,32,32 } -{ .mmi; xor r10=r11,r10 - zxt4 E=E };; -{ .mmi; or $t1=$t1,E - xor s1=s1,r10 // s1=sigma1(X[(i+14)&0xF]) - mux2 $t0=A,0x44 };; // copy lower half to upper +{ .mib; xor s0=s0,r9 // s0=sigma0(X[(i+1)&0xF]) + dep.z $t1=E,32,32 } +{ .mib; xor r10=r11,r10 + zxt4 E=E };; +{ .mii; xor s1=s1,r10 // s1=sigma1(X[(i+14)&0xF]) + shrp r9=E,$t1,32+$Sigma1[0] // ROTR(e,14) + mux2 $t0=A,0x44 };; // copy lower half to upper +// Pair of mmi; splits on Itanium 1 and prevents pipeline flush +// upon mux2 output usage { .mmi; xor T2=T2,r8 - _rotr r9=$t1,$Sigma1[0] } // ROTR(e,14) + shrp r8=E,$t1,32+$Sigma1[1]} // ROTR(e,18) { .mmi; and r10=B,C add T1=T1,H // T1=Ch(e,f,g)+h - $ADD X[15]=X[15],s0 };; // X[i&0xF]+=sigma0(X[(i+1)&0xF]) + or $t1=$t1,E };; ___ $t0="A", $t1="E", $code.=<<___ if ($BITS==64); { .mib; xor s0=s0,r9 // s0=sigma0(X[(i+1)&0xF]) - _rotr r9=$t1,$Sigma1[0] } // ROTR(e,14) + _rotr r9=$t1,$Sigma1[0] } // ROTR(e,14) { .mib; xor r10=r11,r10 - xor T2=T2,r8 };; + xor T2=T2,r8 };; { .mib; xor s1=s1,r10 // s1=sigma1(X[(i+14)&0xF]) - add T1=T1,H } + _rotr r8=$t1,$Sigma1[1] } // ROTR(e,18) { .mib; and r10=B,C - $ADD X[15]=X[15],s0 };; // X[i&0xF]+=sigma0(X[(i+1)&0xF]) + add T1=T1,H };; // T1+=H ___ $code.=<<___; -{ .mmi; xor T2=T2,r10 // T2=((a & b) ^ (a & c) ^ (b & c)) - mov H=G - _rotr r8=$t1,$Sigma1[1] };; // ROTR(e,18) -{ .mmi; xor r11=r8,r9 - $ADD X[15]=X[15],s1 // X[i&0xF]+=sigma1(X[(i+14)&0xF]) - _rotr r9=$t1,$Sigma1[2] } // ROTR(e,41) -{ .mmi; mov G=F - mov F=E };; -{ .mib; xor r9=r9,r11 // r9=Sigma1(e) - _rotr r10=$t0,$Sigma0[0] } // ROTR(a,28) -{ .mib; add T1=T1,K // T1=Ch(e,f,g)+h+K512[i] - mov E=D };; -{ .mib; add T1=T1,r9 // T1+=Sigma1(e) - _rotr r11=$t0,$Sigma0[1] } // ROTR(a,34) -{ .mib; mov D=C - mov C=B };; -{ .mmi; add T1=T1,X[15] // T1+=X[i] - xor r10=r10,r11 - _rotr r8=$t0,$Sigma0[2] };; // ROTR(a,39) -{ .mmi; xor r10=r8,r10 // r10=Sigma0(a) - mov B=A - add A=T1,T2 };; -{ .mib; add E=E,T1 - add A=A,r10 // T2=Maj(a,b,c)+Sigma0(a) - br.ctop.sptk .L_rest };; +{ .mib; xor r9=r9,r8 + _rotr r8=$t1,$Sigma1[2] } // ROTR(e,41) +{ .mib; xor T2=T2,r10 // T2=((a & b) ^ (a & c) ^ (b & c)) + add X[15]=X[15],s0 };; // X[i]+=sigma0(X[i+1]) +{ .mmi; xor r9=r9,r8 // Sigma1(e) + add X[15]=X[15],s1 // X[i]+=sigma0(X[i+14]) + _rotr r8=$t0,$Sigma0[0] };; // ROTR(a,28) +{ .mmi; add K=K,X[15] + add T1=T1,r9 // T1+=Sigma1(e) + _rotr r9=$t0,$Sigma0[1] };; // ROTR(a,34) +{ .mmi; add T1=T1,K // T1+=K[i]+X[i] + xor r8=r8,r9 + _rotr r9=$t0,$Sigma0[2] };; // ROTR(a,39) +{ .mib; add D=D,T1 + mux2 H=X[15],0x44 } // mov H=X[15] in sha512 +{ .mib; xor r8=r8,r9 // Sigma0(a) + add X[15]=T1,T2 // H=T1+Maj(a,b,c) + br.ctop.sptk .L_rest };; .L_rest_end: +{ .mmi; add A=A,r8 };; // H+=Sigma0(a) from the past { .mmi; add A_=A_,A add B_=B_,B add C_=C_,C } @@ -590,17 +601,19 @@ $code.=<<___; .endp $func# ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -$code =~ s/_rotr(\s+)([^=]+)=([^,]+),([0-9]+)/shrp$1$2=$3,$3,$4/gm; -if ($BITS==64) { - $code =~ s/mux2(\s+)\S+/nop.i$1 0x0/gm; - $code =~ s/mux1(\s+)\S+/nop.i$1 0x0/gm if ($big_endian); - $code =~ s/(shrp\s+X\[[^=]+)=([^,]+),([^,]+),([1-9]+)/$1=$3,$2,64-$4/gm +foreach(split($/,$code)) { + s/\`([^\`]*)\`/eval $1/gem; + s/_rotr(\s+)([^=]+)=([^,]+),([0-9]+)/shrp$1$2=$3,$3,$4/gm; + if ($BITS==64) { + s/mux2(\s+)([^=]+)=([^,]+),\S+/mov$1 $2=$3/gm; + s/mux1(\s+)\S+/nop.i$1 0x0/gm if ($big_endian); + s/(shrp\s+X\[[^=]+)=([^,]+),([^,]+),([1-9]+)/$1=$3,$2,64-$4/gm if (!$big_endian); - $code =~ s/ld1(\s+)X\[\S+/nop.m$1 0x0/gm; -} + s/ld1(\s+)X\[\S+/nop.m$1 0x0/gm; + } -print $code; + print $_,"\n"; +} print<<___ if ($BITS==32); .align 64 diff --git a/crypto/sha/asm/sha512-mips.pl b/crypto/sha/asm/sha512-mips.pl index 6807a2c7220b..b468cfb4569e 100755 --- a/crypto/sha/asm/sha512-mips.pl +++ b/crypto/sha/asm/sha512-mips.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -17,6 +17,10 @@ # ~17%, but it comes for free, because it's same instruction sequence. # Improvement coefficients are for aligned input. +# September 2012. +# +# Add MIPS[32|64]R2 code (>25% less instructions). + ###################################################################### # There is a number of MIPS ABI in use, O32 and N32/64 are most # widely used. Then there is a new contender: NUBI. It appears that if @@ -45,7 +49,7 @@ # ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); # ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); # -$flavour = shift; # supported flavours are o32,n32,64,nubi32,nubi64 +$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64 if ($flavour =~ /64|n32/i) { $PTR_ADD="dadd"; # incidentally works even on n32 @@ -83,6 +87,7 @@ if ($output =~ /512/) { $SLL="dsll"; # shift left logical $SRL="dsrl"; # shift right logical $ADDU="daddu"; + $ROTR="drotr"; @Sigma0=(28,34,39); @Sigma1=(14,18,41); @sigma0=( 7, 1, 8); # right shift first @@ -97,6 +102,7 @@ if ($output =~ /512/) { $SLL="sll"; # shift left logical $SRL="srl"; # shift right logical $ADDU="addu"; + $ROTR="rotr"; @Sigma0=( 2,13,22); @Sigma1=( 6,11,25); @sigma0=( 3, 7,18); # right shift first @@ -124,6 +130,10 @@ $code.=<<___ if ($i<15); ${LD}r @X[1],`($i+1)*$SZ+$LSB`($inp) ___ $code.=<<___ if (!$big_endian && $i<16 && $SZ==4); +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + wsbh @X[0],@X[0] # byte swap($i) + rotr @X[0],@X[0],16 +#else srl $tmp0,@X[0],24 # byte swap($i) srl $tmp1,@X[0],8 andi $tmp2,@X[0],0xFF00 @@ -133,8 +143,13 @@ $code.=<<___ if (!$big_endian && $i<16 && $SZ==4); or @X[0],$tmp0 or $tmp1,$tmp2 or @X[0],$tmp1 +#endif ___ $code.=<<___ if (!$big_endian && $i<16 && $SZ==8); +#if defined(_MIPS_ARCH_MIPS64R2) + dsbh @X[0],@X[0] # byte swap($i) + dshd @X[0],@X[0] +#else ori $tmp0,$zero,0xFF dsll $tmp2,$tmp0,32 or $tmp0,$tmp2 # 0x000000FF000000FF @@ -153,8 +168,31 @@ $code.=<<___ if (!$big_endian && $i<16 && $SZ==8); dsrl $tmp1,@X[0],32 dsll @X[0],32 or @X[0],$tmp1 +#endif ___ $code.=<<___; +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + xor $tmp2,$f,$g # $i + $ROTR $tmp0,$e,@Sigma1[0] + $ADDU $T1,$X[0],$h + $ROTR $tmp1,$e,@Sigma1[1] + and $tmp2,$e + $ROTR $h,$e,@Sigma1[2] + xor $tmp0,$tmp1 + $ROTR $tmp1,$a,@Sigma0[0] + xor $tmp2,$g # Ch(e,f,g) + xor $tmp0,$h # Sigma1(e) + + $ROTR $h,$a,@Sigma0[1] + $ADDU $T1,$tmp2 + $LD $tmp2,`$i*$SZ`($Ktbl) # K[$i] + xor $h,$tmp1 + $ROTR $tmp1,$a,@Sigma0[2] + $ADDU $T1,$tmp0 + and $tmp0,$b,$c + xor $h,$tmp1 # Sigma0(a) + xor $tmp1,$b,$c +#else $ADDU $T1,$X[0],$h # $i $SRL $h,$e,@Sigma1[0] xor $tmp2,$f,$g @@ -184,16 +222,15 @@ $code.=<<___; xor $h,$tmp1 $SLL $tmp1,$a,`$SZ*8-@Sigma0[0]` xor $h,$tmp0 - $ST @X[0],`($i%16)*$SZ`($sp) # offload to ring buffer + and $tmp0,$b,$c xor $h,$tmp1 # Sigma0(a) - - or $tmp0,$a,$b - and $tmp1,$a,$b - and $tmp0,$c - or $tmp1,$tmp0 # Maj(a,b,c) + xor $tmp1,$b,$c +#endif + $ST @X[0],`($i%16)*$SZ`($sp) # offload to ring buffer + $ADDU $h,$tmp0 + and $tmp1,$a $ADDU $T1,$tmp2 # +=K[$i] - $ADDU $h,$tmp1 - + $ADDU $h,$tmp1 # +=Maj(a,b,c) $ADDU $d,$T1 $ADDU $h,$T1 ___ @@ -207,6 +244,20 @@ my $i=@_[0]; my ($tmp0,$tmp1,$tmp2,$tmp3)=(@X[4],@X[5],@X[6],@X[7]); $code.=<<___; +#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2) + $SRL $tmp2,@X[1],@sigma0[0] # Xupdate($i) + $ROTR $tmp0,@X[1],@sigma0[1] + $ADDU @X[0],@X[9] # +=X[i+9] + xor $tmp2,$tmp0 + $ROTR $tmp0,@X[1],@sigma0[2] + + $SRL $tmp3,@X[14],@sigma1[0] + $ROTR $tmp1,@X[14],@sigma1[1] + xor $tmp2,$tmp0 # sigma0(X[i+1]) + $ROTR $tmp0,@X[14],@sigma1[2] + xor $tmp3,$tmp1 + $ADDU @X[0],$tmp2 +#else $SRL $tmp2,@X[1],@sigma0[0] # Xupdate($i) $ADDU @X[0],@X[9] # +=X[i+9] $SLL $tmp1,@X[1],`$SZ*8-@sigma0[2]` @@ -227,7 +278,7 @@ $code.=<<___; xor $tmp3,$tmp0 $SRL $tmp0,@X[14],@sigma1[2] xor $tmp3,$tmp1 - +#endif xor $tmp3,$tmp0 # sigma1(X[i+14]) $ADDU @X[0],$tmp3 ___ @@ -242,9 +293,13 @@ $code.=<<___; # include #endif +#if defined(__mips_smartmips) && !defined(_MIPS_ARCH_MIPS32R2) +#define _MIPS_ARCH_MIPS32R2 +#endif + .text .set noat -#if !defined(__vxworks) || defined(__pic__) +#if !defined(__mips_eabi) && (!defined(__vxworks) || defined(__pic__)) .option pic2 #endif diff --git a/crypto/sha/asm/sha512-ppc.pl b/crypto/sha/asm/sha512-ppc.pl index 6b44a68e599e..734f3c1ca0f0 100755 --- a/crypto/sha/asm/sha512-ppc.pl +++ b/crypto/sha/asm/sha512-ppc.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. @@ -9,8 +9,7 @@ # I let hardware handle unaligned input, except on page boundaries # (see below for details). Otherwise straightforward implementation -# with X vector in register bank. The module is big-endian [which is -# not big deal as there're no little-endian targets left around]. +# with X vector in register bank. # sha256 | sha512 # -m64 -m32 | -m64 -m32 @@ -56,6 +55,8 @@ if ($flavour =~ /64/) { $PUSH="stw"; } else { die "nonsense $flavour"; } +$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0; + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or @@ -64,7 +65,7 @@ die "can't locate ppc-xlate.pl"; open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; if ($output =~ /512/) { - $func="sha512_block_data_order"; + $func="sha512_block_ppc"; $SZ=8; @Sigma0=(28,34,39); @Sigma1=(14,18,41); @@ -76,7 +77,7 @@ if ($output =~ /512/) { $ROR="rotrdi"; $SHR="srdi"; } else { - $func="sha256_block_data_order"; + $func="sha256_block_ppc"; $SZ=4; @Sigma0=( 2,13,22); @Sigma1=( 6,11,25); @@ -110,7 +111,7 @@ $B ="r9"; $C ="r10"; $D ="r11"; $E ="r12"; -$F ="r13"; $F="r2" if ($SIZE_T==8);# reassigned to exempt TLS pointer +$F =$t1; $t1 = "r0"; # stay away from "r13"; $G ="r14"; $H ="r15"; @@ -118,24 +119,23 @@ $H ="r15"; @X=("r16","r17","r18","r19","r20","r21","r22","r23", "r24","r25","r26","r27","r28","r29","r30","r31"); -$inp="r31"; # reassigned $inp! aliases with @X[15] +$inp="r31" if($SZ==4 || $SIZE_T==8); # reassigned $inp! aliases with @X[15] sub ROUND_00_15 { my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; $code.=<<___; - $LD $T,`$i*$SZ`($Tbl) $ROR $a0,$e,$Sigma1[0] $ROR $a1,$e,$Sigma1[1] and $t0,$f,$e - andc $t1,$g,$e - add $T,$T,$h xor $a0,$a0,$a1 + add $h,$h,$t1 + andc $t1,$g,$e $ROR $a1,$a1,`$Sigma1[2]-$Sigma1[1]` or $t0,$t0,$t1 ; Ch(e,f,g) - add $T,$T,@X[$i] + add $h,$h,@X[$i%16] xor $a0,$a0,$a1 ; Sigma1(e) - add $T,$T,$t0 - add $T,$T,$a0 + add $h,$h,$t0 + add $h,$h,$a0 $ROR $a0,$a,$Sigma0[0] $ROR $a1,$a,$Sigma0[1] @@ -146,9 +146,14 @@ $code.=<<___; xor $t0,$t0,$t1 and $t1,$b,$c xor $a0,$a0,$a1 ; Sigma0(a) - add $d,$d,$T + add $d,$d,$h xor $t0,$t0,$t1 ; Maj(a,b,c) - add $h,$T,$a0 +___ +$code.=<<___ if ($i<15); + $LD $t1,`($i+1)*$SZ`($Tbl) +___ +$code.=<<___; + add $h,$h,$a0 add $h,$h,$t0 ___ @@ -169,10 +174,11 @@ $code.=<<___; add @X[$i],@X[$i],@X[($i+9)%16] xor $a0,$a0,$a1 ; sigma0(X[(i+1)&0x0f]) xor $t0,$t0,$t1 ; sigma1(X[(i+14)&0x0f]) + $LD $t1,`$i*$SZ`($Tbl) add @X[$i],@X[$i],$a0 add @X[$i],@X[$i],$t0 ___ -&ROUND_00_15($i,$a,$b,$c,$d,$e,$f,$g,$h); +&ROUND_00_15($i+16,$a,$b,$c,$d,$e,$f,$g,$h); } $code=<<___; @@ -188,8 +194,6 @@ $func: $PUSH $ctx,`$FRAME-$SIZE_T*22`($sp) - $PUSH $toc,`$FRAME-$SIZE_T*20`($sp) - $PUSH r13,`$FRAME-$SIZE_T*19`($sp) $PUSH r14,`$FRAME-$SIZE_T*18`($sp) $PUSH r15,`$FRAME-$SIZE_T*17`($sp) $PUSH r16,`$FRAME-$SIZE_T*16`($sp) @@ -209,7 +213,10 @@ $func: $PUSH r30,`$FRAME-$SIZE_T*2`($sp) $PUSH r31,`$FRAME-$SIZE_T*1`($sp) $PUSH r0,`$FRAME+$LRSAVE`($sp) +___ +if ($SZ==4 || $SIZE_T==8) { +$code.=<<___; $LD $A,`0*$SZ`($ctx) mr $inp,r4 ; incarnate $inp $LD $B,`1*$SZ`($ctx) @@ -219,7 +226,16 @@ $func: $LD $F,`5*$SZ`($ctx) $LD $G,`6*$SZ`($ctx) $LD $H,`7*$SZ`($ctx) +___ +} else { + for ($i=16;$i<32;$i++) { + $code.=<<___; + lwz r$i,`$LITTLE_ENDIAN^(4*($i-16))`($ctx) +___ + } +} +$code.=<<___; bl LPICmeup LPICedup: andi. r0,$inp,3 @@ -255,6 +271,9 @@ Lunaligned: Lcross_page: li $t1,`16*$SZ/4` mtctr $t1 +___ +if ($SZ==4 || $SIZE_T==8) { +$code.=<<___; addi r20,$sp,$LOCALS ; aligned spot below the frame Lmemcpy: lbz r16,0($inp) @@ -268,7 +287,26 @@ Lmemcpy: stb r19,3(r20) addi r20,r20,4 bdnz Lmemcpy +___ +} else { +$code.=<<___; + addi r12,$sp,$LOCALS ; aligned spot below the frame +Lmemcpy: + lbz r8,0($inp) + lbz r9,1($inp) + lbz r10,2($inp) + lbz r11,3($inp) + addi $inp,$inp,4 + stb r8,0(r12) + stb r9,1(r12) + stb r10,2(r12) + stb r11,3(r12) + addi r12,r12,4 + bdnz Lmemcpy +___ +} +$code.=<<___; $PUSH $inp,`$FRAME-$SIZE_T*26`($sp) ; save real inp addi $t1,$sp,`$LOCALS+16*$SZ` ; fictitious end pointer addi $inp,$sp,$LOCALS ; fictitious inp pointer @@ -283,8 +321,6 @@ Lmemcpy: Ldone: $POP r0,`$FRAME+$LRSAVE`($sp) - $POP $toc,`$FRAME-$SIZE_T*20`($sp) - $POP r13,`$FRAME-$SIZE_T*19`($sp) $POP r14,`$FRAME-$SIZE_T*18`($sp) $POP r15,`$FRAME-$SIZE_T*17`($sp) $POP r16,`$FRAME-$SIZE_T*16`($sp) @@ -309,27 +345,48 @@ Ldone: .long 0 .byte 0,12,4,1,0x80,18,3,0 .long 0 +___ +if ($SZ==4 || $SIZE_T==8) { +$code.=<<___; .align 4 Lsha2_block_private: + $LD $t1,0($Tbl) ___ for($i=0;$i<16;$i++) { -$code.=<<___ if ($SZ==4); +$code.=<<___ if ($SZ==4 && !$LITTLE_ENDIAN); lwz @X[$i],`$i*$SZ`($inp) ___ +$code.=<<___ if ($SZ==4 && $LITTLE_ENDIAN); + lwz $a0,`$i*$SZ`($inp) + rotlwi @X[$i],$a0,8 + rlwimi @X[$i],$a0,24,0,7 + rlwimi @X[$i],$a0,24,16,23 +___ # 64-bit loads are split to 2x32-bit ones, as CPU can't handle # unaligned 64-bit loads, only 32-bit ones... -$code.=<<___ if ($SZ==8); +$code.=<<___ if ($SZ==8 && !$LITTLE_ENDIAN); lwz $t0,`$i*$SZ`($inp) lwz @X[$i],`$i*$SZ+4`($inp) insrdi @X[$i],$t0,32,0 +___ +$code.=<<___ if ($SZ==8 && $LITTLE_ENDIAN); + lwz $a0,`$i*$SZ`($inp) + lwz $a1,`$i*$SZ+4`($inp) + rotlwi $t0,$a0,8 + rotlwi @X[$i],$a1,8 + rlwimi $t0,$a0,24,0,7 + rlwimi @X[$i],$a1,24,0,7 + rlwimi $t0,$a0,24,16,23 + rlwimi @X[$i],$a1,24,16,23 + insrdi @X[$i],$t0,32,0 ___ &ROUND_00_15($i,@V); unshift(@V,pop(@V)); } $code.=<<___; - li $T,`$rounds/16-1` - mtctr $T + li $t0,`$rounds/16-1` + mtctr $t0 .align 4 Lrounds: addi $Tbl,$Tbl,`16*$SZ` @@ -377,7 +434,282 @@ $code.=<<___; blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +.size $func,.-$func +___ +} else { +######################################################################## +# SHA512 for PPC32, X vector is off-loaded to stack... +# +# | sha512 +# | -m32 +# ----------------------+----------------------- +# PPC74x0,gcc-4.0.1 | +48% +# POWER6,gcc-4.4.6 | +124%(*) +# POWER7,gcc-4.4.6 | +79%(*) +# e300,gcc-4.1.0 | +167% +# +# (*) ~1/3 of -m64 result [and ~20% better than -m32 code generated +# by xlc-12.1] + +my $XOFF=$LOCALS; + +my @V=map("r$_",(16..31)); # A..H + +my ($s0,$s1,$t0,$t1,$t2,$t3,$a0,$a1,$a2,$a3)=map("r$_",(0,5,6,8..12,14,15)); +my ($x0,$x1)=("r3","r4"); # zaps $ctx and $inp + +sub ROUND_00_15_ppc32 { +my ($i, $ahi,$alo,$bhi,$blo,$chi,$clo,$dhi,$dlo, + $ehi,$elo,$fhi,$flo,$ghi,$glo,$hhi,$hlo)=@_; + +$code.=<<___; + lwz $t2,`$SZ*($i%16)+($LITTLE_ENDIAN^4)`($Tbl) + xor $a0,$flo,$glo + lwz $t3,`$SZ*($i%16)+($LITTLE_ENDIAN^0)`($Tbl) + xor $a1,$fhi,$ghi + addc $hlo,$hlo,$t0 ; h+=x[i] + stw $t0,`$XOFF+0+$SZ*($i%16)`($sp) ; save x[i] + + srwi $s0,$elo,$Sigma1[0] + srwi $s1,$ehi,$Sigma1[0] + and $a0,$a0,$elo + adde $hhi,$hhi,$t1 + and $a1,$a1,$ehi + stw $t1,`$XOFF+4+$SZ*($i%16)`($sp) + srwi $t0,$elo,$Sigma1[1] + srwi $t1,$ehi,$Sigma1[1] + addc $hlo,$hlo,$t2 ; h+=K512[i] + insrwi $s0,$ehi,$Sigma1[0],0 + insrwi $s1,$elo,$Sigma1[0],0 + xor $a0,$a0,$glo ; Ch(e,f,g) + adde $hhi,$hhi,$t3 + xor $a1,$a1,$ghi + insrwi $t0,$ehi,$Sigma1[1],0 + insrwi $t1,$elo,$Sigma1[1],0 + addc $hlo,$hlo,$a0 ; h+=Ch(e,f,g) + srwi $t2,$ehi,$Sigma1[2]-32 + srwi $t3,$elo,$Sigma1[2]-32 + xor $s0,$s0,$t0 + xor $s1,$s1,$t1 + insrwi $t2,$elo,$Sigma1[2]-32,0 + insrwi $t3,$ehi,$Sigma1[2]-32,0 + xor $a0,$alo,$blo ; a^b, b^c in next round + adde $hhi,$hhi,$a1 + xor $a1,$ahi,$bhi + xor $s0,$s0,$t2 ; Sigma1(e) + xor $s1,$s1,$t3 + + srwi $t0,$alo,$Sigma0[0] + and $a2,$a2,$a0 + addc $hlo,$hlo,$s0 ; h+=Sigma1(e) + and $a3,$a3,$a1 + srwi $t1,$ahi,$Sigma0[0] + srwi $s0,$ahi,$Sigma0[1]-32 + adde $hhi,$hhi,$s1 + srwi $s1,$alo,$Sigma0[1]-32 + insrwi $t0,$ahi,$Sigma0[0],0 + insrwi $t1,$alo,$Sigma0[0],0 + xor $a2,$a2,$blo ; Maj(a,b,c) + addc $dlo,$dlo,$hlo ; d+=h + xor $a3,$a3,$bhi + insrwi $s0,$alo,$Sigma0[1]-32,0 + insrwi $s1,$ahi,$Sigma0[1]-32,0 + adde $dhi,$dhi,$hhi + srwi $t2,$ahi,$Sigma0[2]-32 + srwi $t3,$alo,$Sigma0[2]-32 + xor $s0,$s0,$t0 + addc $hlo,$hlo,$a2 ; h+=Maj(a,b,c) + xor $s1,$s1,$t1 + insrwi $t2,$alo,$Sigma0[2]-32,0 + insrwi $t3,$ahi,$Sigma0[2]-32,0 + adde $hhi,$hhi,$a3 +___ +$code.=<<___ if ($i>=15); + lwz $t0,`$XOFF+0+$SZ*(($i+2)%16)`($sp) + lwz $t1,`$XOFF+4+$SZ*(($i+2)%16)`($sp) +___ +$code.=<<___ if ($i<15 && !$LITTLE_ENDIAN); + lwz $t1,`$SZ*($i+1)+0`($inp) + lwz $t0,`$SZ*($i+1)+4`($inp) ___ +$code.=<<___ if ($i<15 && $LITTLE_ENDIAN); + lwz $a2,`$SZ*($i+1)+0`($inp) + lwz $a3,`$SZ*($i+1)+4`($inp) + rotlwi $t1,$a2,8 + rotlwi $t0,$a3,8 + rlwimi $t1,$a2,24,0,7 + rlwimi $t0,$a3,24,0,7 + rlwimi $t1,$a2,24,16,23 + rlwimi $t0,$a3,24,16,23 +___ +$code.=<<___; + xor $s0,$s0,$t2 ; Sigma0(a) + xor $s1,$s1,$t3 + addc $hlo,$hlo,$s0 ; h+=Sigma0(a) + adde $hhi,$hhi,$s1 +___ +$code.=<<___ if ($i==15); + lwz $x0,`$XOFF+0+$SZ*(($i+1)%16)`($sp) + lwz $x1,`$XOFF+4+$SZ*(($i+1)%16)`($sp) +___ +} +sub ROUND_16_xx_ppc32 { +my ($i, $ahi,$alo,$bhi,$blo,$chi,$clo,$dhi,$dlo, + $ehi,$elo,$fhi,$flo,$ghi,$glo,$hhi,$hlo)=@_; + +$code.=<<___; + srwi $s0,$t0,$sigma0[0] + srwi $s1,$t1,$sigma0[0] + srwi $t2,$t0,$sigma0[1] + srwi $t3,$t1,$sigma0[1] + insrwi $s0,$t1,$sigma0[0],0 + insrwi $s1,$t0,$sigma0[0],0 + srwi $a0,$t0,$sigma0[2] + insrwi $t2,$t1,$sigma0[1],0 + insrwi $t3,$t0,$sigma0[1],0 + insrwi $a0,$t1,$sigma0[2],0 + xor $s0,$s0,$t2 + lwz $t2,`$XOFF+0+$SZ*(($i+14)%16)`($sp) + srwi $a1,$t1,$sigma0[2] + xor $s1,$s1,$t3 + lwz $t3,`$XOFF+4+$SZ*(($i+14)%16)`($sp) + xor $a0,$a0,$s0 + srwi $s0,$t2,$sigma1[0] + xor $a1,$a1,$s1 + srwi $s1,$t3,$sigma1[0] + addc $x0,$x0,$a0 ; x[i]+=sigma0(x[i+1]) + srwi $a0,$t3,$sigma1[1]-32 + insrwi $s0,$t3,$sigma1[0],0 + insrwi $s1,$t2,$sigma1[0],0 + adde $x1,$x1,$a1 + srwi $a1,$t2,$sigma1[1]-32 + + insrwi $a0,$t2,$sigma1[1]-32,0 + srwi $t2,$t2,$sigma1[2] + insrwi $a1,$t3,$sigma1[1]-32,0 + insrwi $t2,$t3,$sigma1[2],0 + xor $s0,$s0,$a0 + lwz $a0,`$XOFF+0+$SZ*(($i+9)%16)`($sp) + srwi $t3,$t3,$sigma1[2] + xor $s1,$s1,$a1 + lwz $a1,`$XOFF+4+$SZ*(($i+9)%16)`($sp) + xor $s0,$s0,$t2 + addc $x0,$x0,$a0 ; x[i]+=x[i+9] + xor $s1,$s1,$t3 + adde $x1,$x1,$a1 + addc $x0,$x0,$s0 ; x[i]+=sigma1(x[i+14]) + adde $x1,$x1,$s1 +___ + ($t0,$t1,$x0,$x1) = ($x0,$x1,$t0,$t1); + &ROUND_00_15_ppc32(@_); +} + +$code.=<<___; +.align 4 +Lsha2_block_private: +___ +$code.=<<___ if (!$LITTLE_ENDIAN); + lwz $t1,0($inp) + xor $a2,@V[3],@V[5] ; B^C, magic seed + lwz $t0,4($inp) + xor $a3,@V[2],@V[4] +___ +$code.=<<___ if ($LITTLE_ENDIAN); + lwz $a1,0($inp) + xor $a2,@V[3],@V[5] ; B^C, magic seed + lwz $a0,4($inp) + xor $a3,@V[2],@V[4] + rotlwi $t1,$a1,8 + rotlwi $t0,$a0,8 + rlwimi $t1,$a1,24,0,7 + rlwimi $t0,$a0,24,0,7 + rlwimi $t1,$a1,24,16,23 + rlwimi $t0,$a0,24,16,23 +___ +for($i=0;$i<16;$i++) { + &ROUND_00_15_ppc32($i,@V); + unshift(@V,pop(@V)); unshift(@V,pop(@V)); + ($a0,$a1,$a2,$a3) = ($a2,$a3,$a0,$a1); +} +$code.=<<___; + li $a0,`$rounds/16-1` + mtctr $a0 +.align 4 +Lrounds: + addi $Tbl,$Tbl,`16*$SZ` +___ +for(;$i<32;$i++) { + &ROUND_16_xx_ppc32($i,@V); + unshift(@V,pop(@V)); unshift(@V,pop(@V)); + ($a0,$a1,$a2,$a3) = ($a2,$a3,$a0,$a1); +} +$code.=<<___; + bdnz- Lrounds + + $POP $ctx,`$FRAME-$SIZE_T*22`($sp) + $POP $inp,`$FRAME-$SIZE_T*23`($sp) ; inp pointer + $POP $num,`$FRAME-$SIZE_T*24`($sp) ; end pointer + subi $Tbl,$Tbl,`($rounds-16)*$SZ` ; rewind Tbl + + lwz $t0,`$LITTLE_ENDIAN^0`($ctx) + lwz $t1,`$LITTLE_ENDIAN^4`($ctx) + lwz $t2,`$LITTLE_ENDIAN^8`($ctx) + lwz $t3,`$LITTLE_ENDIAN^12`($ctx) + lwz $a0,`$LITTLE_ENDIAN^16`($ctx) + lwz $a1,`$LITTLE_ENDIAN^20`($ctx) + lwz $a2,`$LITTLE_ENDIAN^24`($ctx) + addc @V[1],@V[1],$t1 + lwz $a3,`$LITTLE_ENDIAN^28`($ctx) + adde @V[0],@V[0],$t0 + lwz $t0,`$LITTLE_ENDIAN^32`($ctx) + addc @V[3],@V[3],$t3 + lwz $t1,`$LITTLE_ENDIAN^36`($ctx) + adde @V[2],@V[2],$t2 + lwz $t2,`$LITTLE_ENDIAN^40`($ctx) + addc @V[5],@V[5],$a1 + lwz $t3,`$LITTLE_ENDIAN^44`($ctx) + adde @V[4],@V[4],$a0 + lwz $a0,`$LITTLE_ENDIAN^48`($ctx) + addc @V[7],@V[7],$a3 + lwz $a1,`$LITTLE_ENDIAN^52`($ctx) + adde @V[6],@V[6],$a2 + lwz $a2,`$LITTLE_ENDIAN^56`($ctx) + addc @V[9],@V[9],$t1 + lwz $a3,`$LITTLE_ENDIAN^60`($ctx) + adde @V[8],@V[8],$t0 + stw @V[0],`$LITTLE_ENDIAN^0`($ctx) + stw @V[1],`$LITTLE_ENDIAN^4`($ctx) + addc @V[11],@V[11],$t3 + stw @V[2],`$LITTLE_ENDIAN^8`($ctx) + stw @V[3],`$LITTLE_ENDIAN^12`($ctx) + adde @V[10],@V[10],$t2 + stw @V[4],`$LITTLE_ENDIAN^16`($ctx) + stw @V[5],`$LITTLE_ENDIAN^20`($ctx) + addc @V[13],@V[13],$a1 + stw @V[6],`$LITTLE_ENDIAN^24`($ctx) + stw @V[7],`$LITTLE_ENDIAN^28`($ctx) + adde @V[12],@V[12],$a0 + stw @V[8],`$LITTLE_ENDIAN^32`($ctx) + stw @V[9],`$LITTLE_ENDIAN^36`($ctx) + addc @V[15],@V[15],$a3 + stw @V[10],`$LITTLE_ENDIAN^40`($ctx) + stw @V[11],`$LITTLE_ENDIAN^44`($ctx) + adde @V[14],@V[14],$a2 + stw @V[12],`$LITTLE_ENDIAN^48`($ctx) + stw @V[13],`$LITTLE_ENDIAN^52`($ctx) + stw @V[14],`$LITTLE_ENDIAN^56`($ctx) + stw @V[15],`$LITTLE_ENDIAN^60`($ctx) + + addi $inp,$inp,`16*$SZ` ; advance inp + $PUSH $inp,`$FRAME-$SIZE_T*23`($sp) + $UCMP $inp,$num + bne Lsha2_block_private + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +.size $func,.-$func +___ +} # Ugly hack here, because PPC assembler syntax seem to vary too # much from platforms to platform... @@ -395,46 +727,46 @@ LPICmeup: .space `64-9*4` ___ $code.=<<___ if ($SZ==8); - .long 0x428a2f98,0xd728ae22,0x71374491,0x23ef65cd - .long 0xb5c0fbcf,0xec4d3b2f,0xe9b5dba5,0x8189dbbc - .long 0x3956c25b,0xf348b538,0x59f111f1,0xb605d019 - .long 0x923f82a4,0xaf194f9b,0xab1c5ed5,0xda6d8118 - .long 0xd807aa98,0xa3030242,0x12835b01,0x45706fbe - .long 0x243185be,0x4ee4b28c,0x550c7dc3,0xd5ffb4e2 - .long 0x72be5d74,0xf27b896f,0x80deb1fe,0x3b1696b1 - .long 0x9bdc06a7,0x25c71235,0xc19bf174,0xcf692694 - .long 0xe49b69c1,0x9ef14ad2,0xefbe4786,0x384f25e3 - .long 0x0fc19dc6,0x8b8cd5b5,0x240ca1cc,0x77ac9c65 - .long 0x2de92c6f,0x592b0275,0x4a7484aa,0x6ea6e483 - .long 0x5cb0a9dc,0xbd41fbd4,0x76f988da,0x831153b5 - .long 0x983e5152,0xee66dfab,0xa831c66d,0x2db43210 - .long 0xb00327c8,0x98fb213f,0xbf597fc7,0xbeef0ee4 - .long 0xc6e00bf3,0x3da88fc2,0xd5a79147,0x930aa725 - .long 0x06ca6351,0xe003826f,0x14292967,0x0a0e6e70 - .long 0x27b70a85,0x46d22ffc,0x2e1b2138,0x5c26c926 - .long 0x4d2c6dfc,0x5ac42aed,0x53380d13,0x9d95b3df - .long 0x650a7354,0x8baf63de,0x766a0abb,0x3c77b2a8 - .long 0x81c2c92e,0x47edaee6,0x92722c85,0x1482353b - .long 0xa2bfe8a1,0x4cf10364,0xa81a664b,0xbc423001 - .long 0xc24b8b70,0xd0f89791,0xc76c51a3,0x0654be30 - .long 0xd192e819,0xd6ef5218,0xd6990624,0x5565a910 - .long 0xf40e3585,0x5771202a,0x106aa070,0x32bbd1b8 - .long 0x19a4c116,0xb8d2d0c8,0x1e376c08,0x5141ab53 - .long 0x2748774c,0xdf8eeb99,0x34b0bcb5,0xe19b48a8 - .long 0x391c0cb3,0xc5c95a63,0x4ed8aa4a,0xe3418acb - .long 0x5b9cca4f,0x7763e373,0x682e6ff3,0xd6b2b8a3 - .long 0x748f82ee,0x5defb2fc,0x78a5636f,0x43172f60 - .long 0x84c87814,0xa1f0ab72,0x8cc70208,0x1a6439ec - .long 0x90befffa,0x23631e28,0xa4506ceb,0xde82bde9 - .long 0xbef9a3f7,0xb2c67915,0xc67178f2,0xe372532b - .long 0xca273ece,0xea26619c,0xd186b8c7,0x21c0c207 - .long 0xeada7dd6,0xcde0eb1e,0xf57d4f7f,0xee6ed178 - .long 0x06f067aa,0x72176fba,0x0a637dc5,0xa2c898a6 - .long 0x113f9804,0xbef90dae,0x1b710b35,0x131c471b - .long 0x28db77f5,0x23047d84,0x32caab7b,0x40c72493 - .long 0x3c9ebe0a,0x15c9bebc,0x431d67c4,0x9c100d4c - .long 0x4cc5d4be,0xcb3e42b6,0x597f299c,0xfc657e2a - .long 0x5fcb6fab,0x3ad6faec,0x6c44198c,0x4a475817 + .quad 0x428a2f98d728ae22,0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538,0x59f111f1b605d019 + .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242,0x12835b0145706fbe + .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x983e5152ee66dfab,0xa831c66d2db43210 + .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 + .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xd192e819d6ef5218,0xd69906245565a910 + .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x90befffa23631e28,0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xca273eceea26619c,0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x28db77f523047d84,0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 ___ $code.=<<___ if ($SZ==4); .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 diff --git a/crypto/sha/asm/sha512-sparcv9.pl b/crypto/sha/asm/sha512-sparcv9.pl index 585740789e63..5a9c15d1d34a 100755 --- a/crypto/sha/asm/sha512-sparcv9.pl +++ b/crypto/sha/asm/sha512-sparcv9.pl @@ -5,6 +5,8 @@ # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. +# +# Hardware SPARC T4 support by David S. Miller . # ==================================================================== # SHA256 performance improvement over compiler generated code varies @@ -41,11 +43,11 @@ # loads are always slower than one 64-bit load. Once again this # is unlike pre-T1 UltraSPARC, where, if scheduled appropriately, # 2x32-bit loads can be as fast as 1x64-bit ones. - -$bits=32; -for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } -if ($bits==64) { $bias=2047; $frame=192; } -else { $bias=0; $frame=112; } +# +# SPARC T4 SHA256/512 hardware achieves 3.17/2.01 cycles per byte, +# which is 9.3x/11.1x faster than software. Multi-process benchmark +# saturates at 11.5x single-process result on 8-core processor, or +# ~11/16GBps per 2.85GHz socket. $output=shift; open STDOUT,">$output"; @@ -170,6 +172,7 @@ $code.=<<___ if ($i==0); ld [$inp+16],%l4 ld [$inp+20],%l5 ld [$inp+24],%l6 + cmp $tmp31,0 ld [$inp+28],%l7 ___ $code.=<<___ if ($i<15); @@ -182,29 +185,29 @@ $code.=<<___ if ($i<15); or @pair[1],$tmp2,$tmp2 `"ld [$inp+".eval(32+4+$i*8)."],@pair[1]" if ($i<12)` add $h,$tmp2,$T1 - $ST $tmp2,[%sp+`$bias+$frame+$i*$SZ`] + $ST $tmp2,[%sp+STACK_BIAS+STACK_FRAME+`$i*$SZ`] ___ $code.=<<___ if ($i==12); - brnz,a $tmp31,.+8 + bnz,a,pn %icc,.+8 ld [$inp+128],%l0 ___ $code.=<<___ if ($i==15); - ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+0`],%l2 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+1)%16)*$SZ+0`],%l2 sllx @pair[1],$tmp31,$tmp2 ! Xload($i) add $tmp31,32,$tmp0 - ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+4`],%l3 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+1)%16)*$SZ+4`],%l3 sllx @pair[0],$tmp0,$tmp1 - ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+0`],%l4 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+9)%16)*$SZ+0`],%l4 srlx @pair[2],$tmp32,@pair[1] or $tmp1,$tmp2,$tmp2 - ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+4`],%l5 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+9)%16)*$SZ+4`],%l5 or @pair[1],$tmp2,$tmp2 - ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+0`],%l6 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+14)%16)*$SZ+0`],%l6 add $h,$tmp2,$T1 - $ST $tmp2,[%sp+`$bias+$frame+$i*$SZ`] - ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+4`],%l7 - ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+0`],%l0 - ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+4`],%l1 + $ST $tmp2,[%sp+STACK_BIAS+STACK_FRAME+`$i*$SZ`] + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+14)%16)*$SZ+4`],%l7 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+0)%16)*$SZ+0`],%l0 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+0)%16)*$SZ+4`],%l1 ___ } if ($SZ==8); @@ -340,9 +343,9 @@ $code.=<<___; or %l3,$tmp0,$tmp0 srlx $tmp0,@sigma0[0],$T1 - ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+0`],%l2 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+1)%16)*$SZ+0`],%l2 sllx $tmp0,`64-@sigma0[2]`,$tmp1 - ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+4`],%l3 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+1)%16)*$SZ+4`],%l3 srlx $tmp0,@sigma0[1],$tmp0 xor $tmp1,$T1,$T1 sllx $tmp1,`@sigma0[2]-@sigma0[1]`,$tmp1 @@ -354,9 +357,9 @@ $code.=<<___; or %l7,$tmp2,$tmp2 srlx $tmp2,@sigma1[0],$tmp1 - ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+0`],%l6 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+14)%16)*$SZ+0`],%l6 sllx $tmp2,`64-@sigma1[2]`,$tmp0 - ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+4`],%l7 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+14)%16)*$SZ+4`],%l7 srlx $tmp2,@sigma1[1],$tmp2 xor $tmp0,$tmp1,$tmp1 sllx $tmp0,`@sigma1[2]-@sigma1[1]`,$tmp0 @@ -365,27 +368,30 @@ $code.=<<___; xor $tmp0,$tmp1,$tmp1 sllx %l4,32,$tmp0 xor $tmp2,$tmp1,$tmp1 ! sigma1(X[$i+14]) - ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+0`],%l4 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+9)%16)*$SZ+0`],%l4 or %l5,$tmp0,$tmp0 - ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+4`],%l5 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+9)%16)*$SZ+4`],%l5 sllx %l0,32,$tmp2 add $tmp1,$T1,$T1 - ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+0`],%l0 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+0)%16)*$SZ+0`],%l0 or %l1,$tmp2,$tmp2 add $tmp0,$T1,$T1 ! +=X[$i+9] - ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+4`],%l1 + ld [%sp+STACK_BIAS+STACK_FRAME+`(($i+1+0)%16)*$SZ+4`],%l1 add $tmp2,$T1,$T1 ! +=X[$i] - $ST $T1,[%sp+`$bias+$frame+($i%16)*$SZ`] + $ST $T1,[%sp+STACK_BIAS+STACK_FRAME+`($i%16)*$SZ`] ___ &BODY_00_15(@_); } if ($SZ==8); -$code.=<<___ if ($bits==64); +$code.=<<___; +#include "sparc_arch.h" + +#ifdef __arch64__ .register %g2,#scratch .register %g3,#scratch -___ -$code.=<<___; +#endif + .section ".text",#alloc,#execinstr .align 64 @@ -457,9 +463,203 @@ ___ } $code.=<<___; .size K${label},.-K${label} + +#ifdef __PIC__ +SPARC_PIC_THUNK(%g1) +#endif + .globl sha${label}_block_data_order +.align 32 sha${label}_block_data_order: - save %sp,`-$frame-$locals`,%sp + SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5) + ld [%g1+4],%g1 ! OPENSSL_sparcv9cap_P[1] + + andcc %g1, CFR_SHA${label}, %g0 + be .Lsoftware + nop +___ +$code.=<<___ if ($SZ==8); # SHA512 + ldd [%o0 + 0x00], %f0 ! load context + ldd [%o0 + 0x08], %f2 + ldd [%o0 + 0x10], %f4 + ldd [%o0 + 0x18], %f6 + ldd [%o0 + 0x20], %f8 + ldd [%o0 + 0x28], %f10 + andcc %o1, 0x7, %g0 + ldd [%o0 + 0x30], %f12 + bne,pn %icc, .Lhwunaligned + ldd [%o0 + 0x38], %f14 + +.Lhwaligned_loop: + ldd [%o1 + 0x00], %f16 + ldd [%o1 + 0x08], %f18 + ldd [%o1 + 0x10], %f20 + ldd [%o1 + 0x18], %f22 + ldd [%o1 + 0x20], %f24 + ldd [%o1 + 0x28], %f26 + ldd [%o1 + 0x30], %f28 + ldd [%o1 + 0x38], %f30 + ldd [%o1 + 0x40], %f32 + ldd [%o1 + 0x48], %f34 + ldd [%o1 + 0x50], %f36 + ldd [%o1 + 0x58], %f38 + ldd [%o1 + 0x60], %f40 + ldd [%o1 + 0x68], %f42 + ldd [%o1 + 0x70], %f44 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x78], %f46 + add %o1, 0x80, %o1 + prefetch [%o1 + 63], 20 + prefetch [%o1 + 64+63], 20 + + .word 0x81b02860 ! SHA512 + + bne,pt SIZE_T_CC, .Lhwaligned_loop + nop + +.Lhwfinish: + std %f0, [%o0 + 0x00] ! store context + std %f2, [%o0 + 0x08] + std %f4, [%o0 + 0x10] + std %f6, [%o0 + 0x18] + std %f8, [%o0 + 0x20] + std %f10, [%o0 + 0x28] + std %f12, [%o0 + 0x30] + retl + std %f14, [%o0 + 0x38] + +.align 16 +.Lhwunaligned: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f18 +.Lhwunaligned_loop: + ldd [%o1 + 0x08], %f20 + ldd [%o1 + 0x10], %f22 + ldd [%o1 + 0x18], %f24 + ldd [%o1 + 0x20], %f26 + ldd [%o1 + 0x28], %f28 + ldd [%o1 + 0x30], %f30 + ldd [%o1 + 0x38], %f32 + ldd [%o1 + 0x40], %f34 + ldd [%o1 + 0x48], %f36 + ldd [%o1 + 0x50], %f38 + ldd [%o1 + 0x58], %f40 + ldd [%o1 + 0x60], %f42 + ldd [%o1 + 0x68], %f44 + ldd [%o1 + 0x70], %f46 + ldd [%o1 + 0x78], %f48 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x80], %f50 + add %o1, 0x80, %o1 + prefetch [%o1 + 63], 20 + prefetch [%o1 + 64+63], 20 + + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + faligndata %f26, %f28, %f24 + faligndata %f28, %f30, %f26 + faligndata %f30, %f32, %f28 + faligndata %f32, %f34, %f30 + faligndata %f34, %f36, %f32 + faligndata %f36, %f38, %f34 + faligndata %f38, %f40, %f36 + faligndata %f40, %f42, %f38 + faligndata %f42, %f44, %f40 + faligndata %f44, %f46, %f42 + faligndata %f46, %f48, %f44 + faligndata %f48, %f50, %f46 + + .word 0x81b02860 ! SHA512 + + bne,pt SIZE_T_CC, .Lhwunaligned_loop + for %f50, %f50, %f18 ! %f18=%f50 + + ba .Lhwfinish + nop +___ +$code.=<<___ if ($SZ==4); # SHA256 + ld [%o0 + 0x00], %f0 + ld [%o0 + 0x04], %f1 + ld [%o0 + 0x08], %f2 + ld [%o0 + 0x0c], %f3 + ld [%o0 + 0x10], %f4 + ld [%o0 + 0x14], %f5 + andcc %o1, 0x7, %g0 + ld [%o0 + 0x18], %f6 + bne,pn %icc, .Lhwunaligned + ld [%o0 + 0x1c], %f7 + +.Lhwloop: + ldd [%o1 + 0x00], %f8 + ldd [%o1 + 0x08], %f10 + ldd [%o1 + 0x10], %f12 + ldd [%o1 + 0x18], %f14 + ldd [%o1 + 0x20], %f16 + ldd [%o1 + 0x28], %f18 + ldd [%o1 + 0x30], %f20 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x38], %f22 + add %o1, 0x40, %o1 + prefetch [%o1 + 63], 20 + + .word 0x81b02840 ! SHA256 + + bne,pt SIZE_T_CC, .Lhwloop + nop + +.Lhwfinish: + st %f0, [%o0 + 0x00] ! store context + st %f1, [%o0 + 0x04] + st %f2, [%o0 + 0x08] + st %f3, [%o0 + 0x0c] + st %f4, [%o0 + 0x10] + st %f5, [%o0 + 0x14] + st %f6, [%o0 + 0x18] + retl + st %f7, [%o0 + 0x1c] + +.align 8 +.Lhwunaligned: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f10 +.Lhwunaligned_loop: + ldd [%o1 + 0x08], %f12 + ldd [%o1 + 0x10], %f14 + ldd [%o1 + 0x18], %f16 + ldd [%o1 + 0x20], %f18 + ldd [%o1 + 0x28], %f20 + ldd [%o1 + 0x30], %f22 + ldd [%o1 + 0x38], %f24 + subcc %o2, 1, %o2 ! done yet? + ldd [%o1 + 0x40], %f26 + add %o1, 0x40, %o1 + prefetch [%o1 + 63], 20 + + faligndata %f10, %f12, %f8 + faligndata %f12, %f14, %f10 + faligndata %f14, %f16, %f12 + faligndata %f16, %f18, %f14 + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + + .word 0x81b02840 ! SHA256 + + bne,pt SIZE_T_CC, .Lhwunaligned_loop + for %f26, %f26, %f10 ! %f10=%f26 + + ba .Lhwfinish + nop +___ +$code.=<<___; +.align 16 +.Lsoftware: + save %sp,-STACK_FRAME-$locals,%sp and $inp,`$align-1`,$tmp31 sllx $len,`log(16*$SZ)/log(2)`,$len andn $inp,`$align-1`,$inp @@ -578,7 +778,7 @@ ___ $code.=<<___; add $inp,`16*$SZ`,$inp ! advance inp cmp $inp,$len - bne `$bits==64?"%xcc":"%icc"`,.Lloop + bne SIZE_T_CC,.Lloop sub $Ktbl,`($rounds-16)*$SZ`,$Ktbl ! rewind Ktbl ret @@ -589,6 +789,62 @@ $code.=<<___; .align 4 ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -print $code; +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my $ref,$opf; +my %visopf = ( "faligndata" => 0x048, + "for" => 0x07c ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} +sub unalignaddr { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my $ref="$mnemonic\t$rs1,$rs2,$rd"; + + foreach ($rs1,$rs2,$rd) { + if (/%([goli])([0-7])/) { $_=$bias{$1}+$2; } + else { return $ref; } + } + return sprintf ".word\t0x%08x !%s", + 0x81b00300|$rd<<25|$rs1<<14|$rs2, + $ref; +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/\b(f[^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ + &unvis($1,$2,$3,$4) + /ge; + s/\b(alignaddr)\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ + &unalignaddr($1,$2,$3,$4) + /ge; + + print $_,"\n"; +} + close STDOUT; diff --git a/crypto/sha/asm/sha512-x86_64.pl b/crypto/sha/asm/sha512-x86_64.pl index 8d5167855781..b7b44b441136 100755 --- a/crypto/sha/asm/sha512-x86_64.pl +++ b/crypto/sha/asm/sha512-x86_64.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl # # ==================================================================== -# Written by Andy Polyakov for the OpenSSL +# Written by Andy Polyakov for the OpenSSL # project. Rights for redistribution and usage in source and binary # forms are granted according to the OpenSSL license. # ==================================================================== @@ -39,6 +39,64 @@ # contrary, 64-bit version, sha512_block, is ~30% *slower* than 32-bit # sha256_block:-( This is presumably because 64-bit shifts/rotates # apparently are not atomic instructions, but implemented in microcode. +# +# May 2012. +# +# Optimization including one of Pavel Semjanov's ideas, alternative +# Maj, resulted in >=5% improvement on most CPUs, +20% SHA256 and +# unfortunately -2% SHA512 on P4 [which nobody should care about +# that much]. +# +# June 2012. +# +# Add SIMD code paths, see below for improvement coefficients. SSSE3 +# code path was not attempted for SHA512, because improvement is not +# estimated to be high enough, noticeably less than 9%, to justify +# the effort, not on pre-AVX processors. [Obviously with exclusion +# for VIA Nano, but it has SHA512 instruction that is faster and +# should be used instead.] For reference, corresponding estimated +# upper limit for improvement for SSSE3 SHA256 is 28%. The fact that +# higher coefficients are observed on VIA Nano and Bulldozer has more +# to do with specifics of their architecture [which is topic for +# separate discussion]. +# +# November 2012. +# +# Add AVX2 code path. Two consecutive input blocks are loaded to +# 256-bit %ymm registers, with data from first block to least +# significant 128-bit halves and data from second to most significant. +# The data is then processed with same SIMD instruction sequence as +# for AVX, but with %ymm as operands. Side effect is increased stack +# frame, 448 additional bytes in SHA256 and 1152 in SHA512, and 1.2KB +# code size increase. +# +# March 2014. +# +# Add support for Intel SHA Extensions. + +###################################################################### +# Current performance in cycles per processed byte (less is better): +# +# SHA256 SSSE3 AVX/XOP(*) SHA512 AVX/XOP(*) +# +# AMD K8 14.9 - - 9.57 - +# P4 17.3 - - 30.8 - +# Core 2 15.6 13.8(+13%) - 9.97 - +# Westmere 14.8 12.3(+19%) - 9.58 - +# Sandy Bridge 17.4 14.2(+23%) 11.6(+50%(**)) 11.2 8.10(+38%(**)) +# Ivy Bridge 12.6 10.5(+20%) 10.3(+22%) 8.17 7.22(+13%) +# Haswell 12.2 9.28(+31%) 7.80(+56%) 7.66 5.40(+42%) +# Bulldozer 21.1 13.6(+54%) 13.6(+54%(***)) 13.5 8.58(+57%) +# VIA Nano 23.0 16.5(+39%) - 14.7 - +# Atom 23.0 18.9(+22%) - 14.7 - +# Silvermont 27.4 20.6(+33%) - 17.5 - +# +# (*) whichever best applicable; +# (**) switch from ror to shrd stands for fair share of improvement; +# (***) execution time is fully determined by remaining integer-only +# part, body_00_15; reducing the amount of SIMD instructions +# below certain limit makes no difference/sense; to conserve +# space SHA256 XOP code path is therefore omitted; $flavour = shift; $output = shift; @@ -51,6 +109,28 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or die "can't locate x86_64-xlate.pl"; +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22); +} + +if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.09) + ($1>=2.10); +} + +if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); +} + +if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); +} + +$shaext=1; ### set to zero if compiling for 1.0.1 +$avx=1 if (!$shaext && $avx); + open OUT,"| \"$^X\" $xlate $flavour $output"; *STDOUT=*OUT; @@ -60,7 +140,7 @@ if ($output =~ /512/) { $SZ=8; @ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%rax","%rbx","%rcx","%rdx", "%r8", "%r9", "%r10","%r11"); - ($T1,$a0,$a1,$a2)=("%r12","%r13","%r14","%r15"); + ($T1,$a0,$a1,$a2,$a3)=("%r12","%r13","%r14","%r15","%rdi"); @Sigma0=(28,34,39); @Sigma1=(14,18,41); @sigma0=(1, 8, 7); @@ -72,7 +152,7 @@ if ($output =~ /512/) { $SZ=4; @ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%eax","%ebx","%ecx","%edx", "%r8d","%r9d","%r10d","%r11d"); - ($T1,$a0,$a1,$a2)=("%r12d","%r13d","%r14d","%r15d"); + ($T1,$a0,$a1,$a2,$a3)=("%r12d","%r13d","%r14d","%r15d","%edi"); @Sigma0=( 2,13,22); @Sigma1=( 6,11,25); @sigma0=( 7,18, 3); @@ -80,8 +160,7 @@ if ($output =~ /512/) { $rounds=64; } -$ctx="%rdi"; # 1st arg -$round="%rdi"; # zaps $ctx +$ctx="%rdi"; # 1st arg, zapped by $a3 $inp="%rsi"; # 2nd arg $Tbl="%rbp"; @@ -94,47 +173,51 @@ $framesz="16*$SZ+4*8"; sub ROUND_00_15() { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; + my $STRIDE=$SZ; + $STRIDE += 16 if ($i%(16/$SZ)==(16/$SZ-1)); $code.=<<___; ror \$`$Sigma1[2]-$Sigma1[1]`,$a0 mov $f,$a2 - mov $T1,`$SZ*($i&0xf)`(%rsp) - ror \$`$Sigma0[2]-$Sigma0[1]`,$a1 xor $e,$a0 + ror \$`$Sigma0[2]-$Sigma0[1]`,$a1 xor $g,$a2 # f^g - ror \$`$Sigma1[1]-$Sigma1[0]`,$a0 - add $h,$T1 # T1+=h + mov $T1,`$SZ*($i&0xf)`(%rsp) xor $a,$a1 - - add ($Tbl,$round,$SZ),$T1 # T1+=K[round] and $e,$a2 # (f^g)&e - mov $b,$h + + ror \$`$Sigma1[1]-$Sigma1[0]`,$a0 + add $h,$T1 # T1+=h + xor $g,$a2 # Ch(e,f,g)=((f^g)&e)^g ror \$`$Sigma0[1]-$Sigma0[0]`,$a1 xor $e,$a0 - xor $g,$a2 # Ch(e,f,g)=((f^g)&e)^g + add $a2,$T1 # T1+=Ch(e,f,g) - xor $c,$h # b^c + mov $a,$a2 + add ($Tbl),$T1 # T1+=K[round] xor $a,$a1 - add $a2,$T1 # T1+=Ch(e,f,g) - mov $b,$a2 + xor $b,$a2 # a^b, b^c in next round ror \$$Sigma1[0],$a0 # Sigma1(e) - and $a,$h # h=(b^c)&a - and $c,$a2 # b&c + mov $b,$h + and $a2,$a3 ror \$$Sigma0[0],$a1 # Sigma0(a) add $a0,$T1 # T1+=Sigma1(e) - add $a2,$h # h+=b&c (completes +=Maj(a,b,c) + xor $a3,$h # h=Maj(a,b,c)=Ch(a^b,c,b) add $T1,$d # d+=T1 add $T1,$h # h+=T1 - lea 1($round),$round # round++ - add $a1,$h # h+=Sigma0(a) + lea $STRIDE($Tbl),$Tbl # round++ +___ +$code.=<<___ if ($i<15); + add $a1,$h # h+=Sigma0(a) ___ + ($a2,$a3) = ($a3,$a2); } sub ROUND_16_XX() @@ -142,29 +225,28 @@ sub ROUND_16_XX() $code.=<<___; mov `$SZ*(($i+1)&0xf)`(%rsp),$a0 - mov `$SZ*(($i+14)&0xf)`(%rsp),$a1 - mov $a0,$T1 - mov $a1,$a2 - - ror \$`$sigma0[1]-$sigma0[0]`,$T1 - xor $a0,$T1 - shr \$$sigma0[2],$a0 - - ror \$$sigma0[0],$T1 - xor $T1,$a0 # sigma0(X[(i+1)&0xf]) - mov `$SZ*(($i+9)&0xf)`(%rsp),$T1 + mov `$SZ*(($i+14)&0xf)`(%rsp),$a2 + mov $a0,$T1 + ror \$`$sigma0[1]-$sigma0[0]`,$a0 + add $a1,$a # modulo-scheduled h+=Sigma0(a) + mov $a2,$a1 ror \$`$sigma1[1]-$sigma1[0]`,$a2 + + xor $T1,$a0 + shr \$$sigma0[2],$T1 + ror \$$sigma0[0],$a0 xor $a1,$a2 shr \$$sigma1[2],$a1 ror \$$sigma1[0],$a2 - add $a0,$T1 - xor $a2,$a1 # sigma1(X[(i+14)&0xf]) + xor $a0,$T1 # sigma0(X[(i+1)&0xf]) + xor $a1,$a2 # sigma1(X[(i+14)&0xf]) + add `$SZ*(($i+9)&0xf)`(%rsp),$T1 add `$SZ*($i&0xf)`(%rsp),$T1 mov $e,$a0 - add $a1,$T1 + add $a2,$T1 mov $a,$a1 ___ &ROUND_00_15(@_); @@ -173,10 +255,43 @@ ___ $code=<<___; .text +.extern OPENSSL_ia32cap_P .globl $func -.type $func,\@function,4 +.type $func,\@function,3 .align 16 $func: +___ +$code.=<<___ if ($SZ==4 || $avx); + lea OPENSSL_ia32cap_P(%rip),%r11 + mov 0(%r11),%r9d + mov 4(%r11),%r10d + mov 8(%r11),%r11d +___ +$code.=<<___ if ($SZ==4 && $shaext); + test \$`1<<29`,%r11d # check for SHA + jnz _shaext_shortcut +___ +$code.=<<___ if ($avx && $SZ==8); + test \$`1<<11`,%r10d # check for XOP + jnz .Lxop_shortcut +___ +$code.=<<___ if ($avx>1); + and \$`1<<8|1<<5|1<<3`,%r11d # check for BMI2+AVX2+BMI1 + cmp \$`1<<8|1<<5|1<<3`,%r11d + je .Lavx2_shortcut +___ +$code.=<<___ if ($avx); + and \$`1<<30`,%r9d # mask "Intel CPU" bit + and \$`1<<28|1<<9`,%r10d # mask AVX and SSSE3 bits + or %r9d,%r10d + cmp \$`1<<28|1<<9|1<<30`,%r10d + je .Lavx_shortcut +___ +$code.=<<___ if ($SZ==4); + test \$`1<<9`,%r10d + jnz .Lssse3_shortcut +___ +$code.=<<___; push %rbx push %rbp push %r12 @@ -194,8 +309,6 @@ $func: mov %r11,$_rsp # save copy of %rsp .Lprologue: - lea $TABLE(%rip),$Tbl - mov $SZ*0($ctx),$A mov $SZ*1($ctx),$B mov $SZ*2($ctx),$C @@ -208,7 +321,9 @@ $func: .align 16 .Lloop: - xor $round,$round + mov $B,$a3 + lea $TABLE(%rip),$Tbl + xor $C,$a3 # magic ___ for($i=0;$i<16;$i++) { $code.=" mov $SZ*$i($inp),$T1\n"; @@ -229,10 +344,11 @@ ___ } $code.=<<___; - cmp \$$rounds,$round - jb .Lrounds_16_xx + cmpb \$0,`$SZ-1`($Tbl) + jnz .Lrounds_16_xx mov $_ctx,$ctx + add $a1,$A # modulo-scheduled h+=Sigma0(a) lea 16*$SZ($inp),$inp add $SZ*0($ctx),$A @@ -275,145 +391,1861 @@ $code.=<<___; .type $TABLE,\@object $TABLE: .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f + .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f + .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff + .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff + .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 + .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 + .asciz "SHA256 block transform for x86_64, CRYPTOGAMS by " ___ } else { $code.=<<___; .align 64 .type $TABLE,\@object $TABLE: + .quad 0x428a2f98d728ae22,0x7137449123ef65cd .quad 0x428a2f98d728ae22,0x7137449123ef65cd .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538,0x59f111f1b605d019 .quad 0x3956c25bf348b538,0x59f111f1b605d019 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242,0x12835b0145706fbe .quad 0xd807aa98a3030242,0x12835b0145706fbe .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x983e5152ee66dfab,0xa831c66d2db43210 .quad 0x983e5152ee66dfab,0xa831c66d2db43210 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xd192e819d6ef5218,0xd69906245565a910 .quad 0xd192e819d6ef5218,0xd69906245565a910 .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x90befffa23631e28,0xa4506cebde82bde9 .quad 0x90befffa23631e28,0xa4506cebde82bde9 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xca273eceea26619c,0xd186b8c721c0c207 .quad 0xca273eceea26619c,0xd186b8c721c0c207 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x28db77f523047d84,0x32caab7b40c72493 .quad 0x28db77f523047d84,0x32caab7b40c72493 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 + .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 + + .quad 0x0001020304050607,0x08090a0b0c0d0e0f + .quad 0x0001020304050607,0x08090a0b0c0d0e0f + .asciz "SHA512 block transform for x86_64, CRYPTOGAMS by " ___ } -# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, -# CONTEXT *context,DISPATCHER_CONTEXT *disp) -if ($win64) { -$rec="%rcx"; -$frame="%rdx"; -$context="%r8"; -$disp="%r9"; +###################################################################### +# SIMD code paths +# +if ($SZ==4 && $shaext) {{{ +###################################################################### +# Intel SHA Extensions implementation of SHA256 update function. +# +my ($ctx,$inp,$num,$Tbl)=("%rdi","%rsi","%rdx","%rcx"); + +my ($Wi,$ABEF,$CDGH,$TMP,$BSWAP,$ABEF_SAVE,$CDGH_SAVE)=map("%xmm$_",(0..2,7..10)); +my @MSG=map("%xmm$_",(3..6)); $code.=<<___; -.extern __imp_RtlVirtualUnwind -.type se_handler,\@abi-omnipotent +.type sha256_block_data_order_shaext,\@function,3 +.align 64 +sha256_block_data_order_shaext: +_shaext_shortcut: +___ +$code.=<<___ if ($win64); + lea `-8-5*16`(%rsp),%rsp + movaps %xmm6,-8-5*16(%rax) + movaps %xmm7,-8-4*16(%rax) + movaps %xmm8,-8-3*16(%rax) + movaps %xmm9,-8-2*16(%rax) + movaps %xmm10,-8-1*16(%rax) +.Lprologue_shaext: +___ +$code.=<<___; + lea K256+0x80(%rip),$Tbl + movdqu ($ctx),$ABEF # DCBA + movdqu 16($ctx),$CDGH # HGFE + movdqa 0x200-0x80($Tbl),$TMP # byte swap mask + + pshufd \$0x1b,$ABEF,$Wi # ABCD + pshufd \$0xb1,$ABEF,$ABEF # CDAB + pshufd \$0x1b,$CDGH,$CDGH # EFGH + movdqa $TMP,$BSWAP # offload + palignr \$8,$CDGH,$ABEF # ABEF + punpcklqdq $Wi,$CDGH # CDGH + jmp .Loop_shaext + .align 16 -se_handler: - push %rsi - push %rdi +.Loop_shaext: + movdqu ($inp),@MSG[0] + movdqu 0x10($inp),@MSG[1] + movdqu 0x20($inp),@MSG[2] + pshufb $TMP,@MSG[0] + movdqu 0x30($inp),@MSG[3] + + movdqa 0*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + pshufb $TMP,@MSG[1] + movdqa $CDGH,$CDGH_SAVE # offload + sha256rnds2 $ABEF,$CDGH # 0-3 + pshufd \$0x0e,$Wi,$Wi + nop + movdqa $ABEF,$ABEF_SAVE # offload + sha256rnds2 $CDGH,$ABEF + + movdqa 1*32-0x80($Tbl),$Wi + paddd @MSG[1],$Wi + pshufb $TMP,@MSG[2] + sha256rnds2 $ABEF,$CDGH # 4-7 + pshufd \$0x0e,$Wi,$Wi + lea 0x40($inp),$inp + sha256msg1 @MSG[1],@MSG[0] + sha256rnds2 $CDGH,$ABEF + + movdqa 2*32-0x80($Tbl),$Wi + paddd @MSG[2],$Wi + pshufb $TMP,@MSG[3] + sha256rnds2 $ABEF,$CDGH # 8-11 + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[3],$TMP + palignr \$4,@MSG[2],$TMP + nop + paddd $TMP,@MSG[0] + sha256msg1 @MSG[2],@MSG[1] + sha256rnds2 $CDGH,$ABEF + + movdqa 3*32-0x80($Tbl),$Wi + paddd @MSG[3],$Wi + sha256msg2 @MSG[3],@MSG[0] + sha256rnds2 $ABEF,$CDGH # 12-15 + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[0],$TMP + palignr \$4,@MSG[3],$TMP + nop + paddd $TMP,@MSG[1] + sha256msg1 @MSG[3],@MSG[2] + sha256rnds2 $CDGH,$ABEF +___ +for($i=4;$i<16-3;$i++) { +$code.=<<___; + movdqa $i*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + sha256msg2 @MSG[0],@MSG[1] + sha256rnds2 $ABEF,$CDGH # 16-19... + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[1],$TMP + palignr \$4,@MSG[0],$TMP + nop + paddd $TMP,@MSG[2] + sha256msg1 @MSG[0],@MSG[3] + sha256rnds2 $CDGH,$ABEF +___ + push(@MSG,shift(@MSG)); +} +$code.=<<___; + movdqa 13*32-0x80($Tbl),$Wi + paddd @MSG[0],$Wi + sha256msg2 @MSG[0],@MSG[1] + sha256rnds2 $ABEF,$CDGH # 52-55 + pshufd \$0x0e,$Wi,$Wi + movdqa @MSG[1],$TMP + palignr \$4,@MSG[0],$TMP + sha256rnds2 $CDGH,$ABEF + paddd $TMP,@MSG[2] + + movdqa 14*32-0x80($Tbl),$Wi + paddd @MSG[1],$Wi + sha256rnds2 $ABEF,$CDGH # 56-59 + pshufd \$0x0e,$Wi,$Wi + sha256msg2 @MSG[1],@MSG[2] + movdqa $BSWAP,$TMP + sha256rnds2 $CDGH,$ABEF + + movdqa 15*32-0x80($Tbl),$Wi + paddd @MSG[2],$Wi + nop + sha256rnds2 $ABEF,$CDGH # 60-63 + pshufd \$0x0e,$Wi,$Wi + dec $num + nop + sha256rnds2 $CDGH,$ABEF + + paddd $CDGH_SAVE,$CDGH + paddd $ABEF_SAVE,$ABEF + jnz .Loop_shaext + + pshufd \$0xb1,$CDGH,$CDGH # DCHG + pshufd \$0x1b,$ABEF,$TMP # FEBA + pshufd \$0xb1,$ABEF,$ABEF # BAFE + punpckhqdq $CDGH,$ABEF # DCBA + palignr \$8,$TMP,$CDGH # HGFE + + movdqu $ABEF,($ctx) + movdqu $CDGH,16($ctx) +___ +$code.=<<___ if ($win64); + movaps -8-5*16(%rax),%xmm6 + movaps -8-4*16(%rax),%xmm7 + movaps -8-3*16(%rax),%xmm8 + movaps -8-2*16(%rax),%xmm9 + movaps -8-1*16(%rax),%xmm10 + mov %rax,%rsp +.Lepilogue_shaext: +___ +$code.=<<___; + ret +.size sha256_block_data_order_shaext,.-sha256_block_data_order_shaext +___ +}}} +{{{ + +my $a4=$T1; +my ($a,$b,$c,$d,$e,$f,$g,$h); + +sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; + my $arg = pop; + $arg = "\$$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; +} + +sub body_00_15 () { + ( + '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'. + + '&ror ($a0,$Sigma1[2]-$Sigma1[1])', + '&mov ($a,$a1)', + '&mov ($a4,$f)', + + '&ror ($a1,$Sigma0[2]-$Sigma0[1])', + '&xor ($a0,$e)', + '&xor ($a4,$g)', # f^g + + '&ror ($a0,$Sigma1[1]-$Sigma1[0])', + '&xor ($a1,$a)', + '&and ($a4,$e)', # (f^g)&e + + '&xor ($a0,$e)', + '&add ($h,$SZ*($i&15)."(%rsp)")', # h+=X[i]+K[i] + '&mov ($a2,$a)', + + '&xor ($a4,$g)', # Ch(e,f,g)=((f^g)&e)^g + '&ror ($a1,$Sigma0[1]-$Sigma0[0])', + '&xor ($a2,$b)', # a^b, b^c in next round + + '&add ($h,$a4)', # h+=Ch(e,f,g) + '&ror ($a0,$Sigma1[0])', # Sigma1(e) + '&and ($a3,$a2)', # (b^c)&(a^b) + + '&xor ($a1,$a)', + '&add ($h,$a0)', # h+=Sigma1(e) + '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b) + + '&ror ($a1,$Sigma0[0])', # Sigma0(a) + '&add ($d,$h)', # d+=h + '&add ($h,$a3)', # h+=Maj(a,b,c) + + '&mov ($a0,$d)', + '&add ($a1,$h);'. # h+=Sigma0(a) + '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;' + ); +} + +###################################################################### +# SSSE3 code path +# +if ($SZ==4) { # SHA256 only +my @X = map("%xmm$_",(0..3)); +my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9)); + +$code.=<<___; +.type ${func}_ssse3,\@function,3 +.align 64 +${func}_ssse3: +.Lssse3_shortcut: push %rbx push %rbp push %r12 push %r13 push %r14 push %r15 - pushfq - sub \$64,%rsp + mov %rsp,%r11 # copy %rsp + shl \$4,%rdx # num*16 + sub \$`$framesz+$win64*16*4`,%rsp + lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ + and \$-64,%rsp # align stack frame + mov $ctx,$_ctx # save ctx, 1st arg + mov $inp,$_inp # save inp, 2nd arh + mov %rdx,$_end # save end pointer, "3rd" arg + mov %r11,$_rsp # save copy of %rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,16*$SZ+32(%rsp) + movaps %xmm7,16*$SZ+48(%rsp) + movaps %xmm8,16*$SZ+64(%rsp) + movaps %xmm9,16*$SZ+80(%rsp) +___ +$code.=<<___; +.Lprologue_ssse3: - mov 120($context),%rax # pull context->Rax - mov 248($context),%rbx # pull context->Rip + mov $SZ*0($ctx),$A + mov $SZ*1($ctx),$B + mov $SZ*2($ctx),$C + mov $SZ*3($ctx),$D + mov $SZ*4($ctx),$E + mov $SZ*5($ctx),$F + mov $SZ*6($ctx),$G + mov $SZ*7($ctx),$H +___ - lea .Lprologue(%rip),%r10 - cmp %r10,%rbx # context->Rip<.Lprologue - jb .Lin_prologue +$code.=<<___; + #movdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4 + #movdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5 + jmp .Lloop_ssse3 +.align 16 +.Lloop_ssse3: + movdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + movdqu 0x00($inp),@X[0] + movdqu 0x10($inp),@X[1] + movdqu 0x20($inp),@X[2] + pshufb $t3,@X[0] + movdqu 0x30($inp),@X[3] + lea $TABLE(%rip),$Tbl + pshufb $t3,@X[1] + movdqa 0x00($Tbl),$t0 + movdqa 0x20($Tbl),$t1 + pshufb $t3,@X[2] + paddd @X[0],$t0 + movdqa 0x40($Tbl),$t2 + pshufb $t3,@X[3] + movdqa 0x60($Tbl),$t3 + paddd @X[1],$t1 + paddd @X[2],$t2 + paddd @X[3],$t3 + movdqa $t0,0x00(%rsp) + mov $A,$a1 + movdqa $t1,0x10(%rsp) + mov $B,$a3 + movdqa $t2,0x20(%rsp) + xor $C,$a3 # magic + movdqa $t3,0x30(%rsp) + mov $E,$a0 + jmp .Lssse3_00_47 - mov 152($context),%rax # pull context->Rsp +.align 16 +.Lssse3_00_47: + sub \$`-16*2*$SZ`,$Tbl # size optimization +___ +sub Xupdate_256_SSSE3 () { + ( + '&movdqa ($t0,@X[1]);', + '&movdqa ($t3,@X[3])', + '&palignr ($t0,@X[0],$SZ)', # X[1..4] + '&palignr ($t3,@X[2],$SZ);', # X[9..12] + '&movdqa ($t1,$t0)', + '&movdqa ($t2,$t0);', + '&psrld ($t0,$sigma0[2])', + '&paddd (@X[0],$t3);', # X[0..3] += X[9..12] + '&psrld ($t2,$sigma0[0])', + '&pshufd ($t3,@X[3],0b11111010)',# X[14..15] + '&pslld ($t1,8*$SZ-$sigma0[1]);'. + '&pxor ($t0,$t2)', + '&psrld ($t2,$sigma0[1]-$sigma0[0]);'. + '&pxor ($t0,$t1)', + '&pslld ($t1,$sigma0[1]-$sigma0[0]);'. + '&pxor ($t0,$t2);', + '&movdqa ($t2,$t3)', + '&pxor ($t0,$t1);', # sigma0(X[1..4]) + '&psrld ($t3,$sigma1[2])', + '&paddd (@X[0],$t0);', # X[0..3] += sigma0(X[1..4]) + '&psrlq ($t2,$sigma1[0])', + '&pxor ($t3,$t2);', + '&psrlq ($t2,$sigma1[1]-$sigma1[0])', + '&pxor ($t3,$t2)', + '&pshufb ($t3,$t4)', # sigma1(X[14..15]) + '&paddd (@X[0],$t3)', # X[0..1] += sigma1(X[14..15]) + '&pshufd ($t3,@X[0],0b01010000)',# X[16..17] + '&movdqa ($t2,$t3);', + '&psrld ($t3,$sigma1[2])', + '&psrlq ($t2,$sigma1[0])', + '&pxor ($t3,$t2);', + '&psrlq ($t2,$sigma1[1]-$sigma1[0])', + '&pxor ($t3,$t2);', + '&movdqa ($t2,16*2*$j."($Tbl)")', + '&pshufb ($t3,$t5)', + '&paddd (@X[0],$t3)' # X[2..3] += sigma1(X[16..17]) + ); +} - lea .Lepilogue(%rip),%r10 - cmp %r10,%rbx # context->Rip>=.Lepilogue - jae .Lin_prologue +sub SSSE3_256_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 104 instructions - mov 16*$SZ+3*8(%rax),%rax # pull $_rsp - lea 48(%rax),%rax + if (0) { + foreach (Xupdate_256_SSSE3()) { # 36 instructions + eval; + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + } + } else { # squeeze extra 4% on Westmere and 19% on Atom + eval(shift(@insns)); #@ + &movdqa ($t0,@X[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa ($t3,@X[3]); + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); #@ + eval(shift(@insns)); + &palignr ($t0,@X[0],$SZ); # X[1..4] + eval(shift(@insns)); + eval(shift(@insns)); + &palignr ($t3,@X[2],$SZ); # X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); #@ + &movdqa ($t1,$t0); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa ($t2,$t0); + eval(shift(@insns)); #@ + eval(shift(@insns)); + &psrld ($t0,$sigma0[2]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@X[0],$t3); # X[0..3] += X[9..12] + eval(shift(@insns)); #@ + eval(shift(@insns)); + &psrld ($t2,$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &pshufd ($t3,@X[3],0b11111010); # X[4..15] + eval(shift(@insns)); + eval(shift(@insns)); #@ + &pslld ($t1,8*$SZ-$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &pxor ($t0,$t2); + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); #@ + &psrld ($t2,$sigma0[1]-$sigma0[0]); + eval(shift(@insns)); + &pxor ($t0,$t1); + eval(shift(@insns)); + eval(shift(@insns)); + &pslld ($t1,$sigma0[1]-$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &pxor ($t0,$t2); + eval(shift(@insns)); + eval(shift(@insns)); #@ + &movdqa ($t2,$t3); + eval(shift(@insns)); + eval(shift(@insns)); + &pxor ($t0,$t1); # sigma0(X[1..4]) + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + &psrld ($t3,$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@X[0],$t0); # X[0..3] += sigma0(X[1..4]) + eval(shift(@insns)); #@ + eval(shift(@insns)); + &psrlq ($t2,$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &pxor ($t3,$t2); + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); #@ + &psrlq ($t2,$sigma1[1]-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &pxor ($t3,$t2); + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + #&pshufb ($t3,$t4); # sigma1(X[14..15]) + &pshufd ($t3,$t3,0b10000000); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &psrldq ($t3,8); + eval(shift(@insns)); + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); #@ + &paddd (@X[0],$t3); # X[0..1] += sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &pshufd ($t3,@X[0],0b01010000); # X[16..17] + eval(shift(@insns)); + eval(shift(@insns)); #@ + eval(shift(@insns)); + &movdqa ($t2,$t3); + eval(shift(@insns)); + eval(shift(@insns)); + &psrld ($t3,$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); #@ + &psrlq ($t2,$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &pxor ($t3,$t2); + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); #@ + eval(shift(@insns)); + &psrlq ($t2,$sigma1[1]-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &pxor ($t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); #@ + #&pshufb ($t3,$t5); + &pshufd ($t3,$t3,0b00001000); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa ($t2,16*2*$j."($Tbl)"); + eval(shift(@insns)); #@ + eval(shift(@insns)); + &pslldq ($t3,8); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@X[0],$t3); # X[2..3] += sigma1(X[16..17]) + eval(shift(@insns)); #@ + eval(shift(@insns)); + eval(shift(@insns)); + } + &paddd ($t2,@X[0]); + foreach (@insns) { eval; } # remaining instructions + &movdqa (16*$j."(%rsp)",$t2); +} - mov -8(%rax),%rbx - mov -16(%rax),%rbp - mov -24(%rax),%r12 - mov -32(%rax),%r13 - mov -40(%rax),%r14 - mov -48(%rax),%r15 - mov %rbx,144($context) # restore context->Rbx - mov %rbp,160($context) # restore context->Rbp - mov %r12,216($context) # restore context->R12 - mov %r13,224($context) # restore context->R13 - mov %r14,232($context) # restore context->R14 - mov %r15,240($context) # restore context->R15 + for ($i=0,$j=0; $j<4; $j++) { + &SSSE3_256_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0); + &jne (".Lssse3_00_47"); -.Lin_prologue: - mov 8(%rax),%rdi - mov 16(%rax),%rsi - mov %rax,152($context) # restore context->Rsp - mov %rsi,168($context) # restore context->Rsi - mov %rdi,176($context) # restore context->Rdi + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } +$code.=<<___; + mov $_ctx,$ctx + mov $a1,$A - mov 40($disp),%rdi # disp->ContextRecord - mov $context,%rsi # context - mov \$154,%ecx # sizeof(CONTEXT) - .long 0xa548f3fc # cld; rep movsq + add $SZ*0($ctx),$A + lea 16*$SZ($inp),$inp + add $SZ*1($ctx),$B + add $SZ*2($ctx),$C + add $SZ*3($ctx),$D + add $SZ*4($ctx),$E + add $SZ*5($ctx),$F + add $SZ*6($ctx),$G + add $SZ*7($ctx),$H - mov $disp,%rsi - xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER - mov 8(%rsi),%rdx # arg2, disp->ImageBase - mov 0(%rsi),%r8 # arg3, disp->ControlPc - mov 16(%rsi),%r9 # arg4, disp->FunctionEntry - mov 40(%rsi),%r10 # disp->ContextRecord - lea 56(%rsi),%r11 # &disp->HandlerData - lea 24(%rsi),%r12 # &disp->EstablisherFrame - mov %r10,32(%rsp) # arg5 - mov %r11,40(%rsp) # arg6 + cmp $_end,$inp + + mov $A,$SZ*0($ctx) + mov $B,$SZ*1($ctx) + mov $C,$SZ*2($ctx) + mov $D,$SZ*3($ctx) + mov $E,$SZ*4($ctx) + mov $F,$SZ*5($ctx) + mov $G,$SZ*6($ctx) + mov $H,$SZ*7($ctx) + jb .Lloop_ssse3 + + mov $_rsp,%rsi +___ +$code.=<<___ if ($win64); + movaps 16*$SZ+32(%rsp),%xmm6 + movaps 16*$SZ+48(%rsp),%xmm7 + movaps 16*$SZ+64(%rsp),%xmm8 + movaps 16*$SZ+80(%rsp),%xmm9 +___ +$code.=<<___; + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_ssse3: + ret +.size ${func}_ssse3,.-${func}_ssse3 +___ +} + +if ($avx) {{ +###################################################################### +# XOP code path +# +if ($SZ==8) { # SHA512 only +$code.=<<___; +.type ${func}_xop,\@function,3 +.align 64 +${func}_xop: +.Lxop_shortcut: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + mov %rsp,%r11 # copy %rsp + shl \$4,%rdx # num*16 + sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp + lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ + and \$-64,%rsp # align stack frame + mov $ctx,$_ctx # save ctx, 1st arg + mov $inp,$_inp # save inp, 2nd arh + mov %rdx,$_end # save end pointer, "3rd" arg + mov %r11,$_rsp # save copy of %rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,16*$SZ+32(%rsp) + movaps %xmm7,16*$SZ+48(%rsp) + movaps %xmm8,16*$SZ+64(%rsp) + movaps %xmm9,16*$SZ+80(%rsp) +___ +$code.=<<___ if ($win64 && $SZ>4); + movaps %xmm10,16*$SZ+96(%rsp) + movaps %xmm11,16*$SZ+112(%rsp) +___ +$code.=<<___; +.Lprologue_xop: + + vzeroupper + mov $SZ*0($ctx),$A + mov $SZ*1($ctx),$B + mov $SZ*2($ctx),$C + mov $SZ*3($ctx),$D + mov $SZ*4($ctx),$E + mov $SZ*5($ctx),$F + mov $SZ*6($ctx),$G + mov $SZ*7($ctx),$H + jmp .Lloop_xop +___ + if ($SZ==4) { # SHA256 + my @X = map("%xmm$_",(0..3)); + my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7)); + +$code.=<<___; +.align 16 +.Lloop_xop: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu 0x00($inp),@X[0] + vmovdqu 0x10($inp),@X[1] + vmovdqu 0x20($inp),@X[2] + vmovdqu 0x30($inp),@X[3] + vpshufb $t3,@X[0],@X[0] + lea $TABLE(%rip),$Tbl + vpshufb $t3,@X[1],@X[1] + vpshufb $t3,@X[2],@X[2] + vpaddd 0x00($Tbl),@X[0],$t0 + vpshufb $t3,@X[3],@X[3] + vpaddd 0x20($Tbl),@X[1],$t1 + vpaddd 0x40($Tbl),@X[2],$t2 + vpaddd 0x60($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + mov $A,$a1 + vmovdqa $t1,0x10(%rsp) + mov $B,$a3 + vmovdqa $t2,0x20(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x30(%rsp) + mov $E,$a0 + jmp .Lxop_00_47 + +.align 16 +.Lxop_00_47: + sub \$`-16*2*$SZ`,$Tbl # size optimization +___ +sub XOP_256_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 104 instructions + + &vpalignr ($t0,@X[1],@X[0],$SZ); # X[1..4] + eval(shift(@insns)); + eval(shift(@insns)); + &vpalignr ($t3,@X[3],@X[2],$SZ); # X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t1,$t0,8*$SZ-$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrld ($t0,$t0,$sigma0[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t3); # X[0..3] += X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t2,$t1,$sigma0[1]-$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t0,$t0,$t1); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t3,@X[3],8*$SZ-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t0,$t0,$t2); # sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrld ($t2,@X[3],$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t0); # X[0..3] += sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t1); # sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrldq ($t3,$t3,8); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t3,@X[0],8*$SZ-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrld ($t2,@X[0],$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t1); # sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpslldq ($t3,$t3,8); # 22 instructions + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[0],@X[0],$t3); # X[2..3] += sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd ($t2,@X[0],16*2*$j."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa (16*$j."(%rsp)",$t2); +} + + for ($i=0,$j=0; $j<4; $j++) { + &XOP_256_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0); + &jne (".Lxop_00_47"); + + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } + + } else { # SHA512 + my @X = map("%xmm$_",(0..7)); + my ($t0,$t1,$t2,$t3) = map("%xmm$_",(8..11)); + +$code.=<<___; +.align 16 +.Lloop_xop: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu 0x00($inp),@X[0] + lea $TABLE+0x80(%rip),$Tbl # size optimization + vmovdqu 0x10($inp),@X[1] + vmovdqu 0x20($inp),@X[2] + vpshufb $t3,@X[0],@X[0] + vmovdqu 0x30($inp),@X[3] + vpshufb $t3,@X[1],@X[1] + vmovdqu 0x40($inp),@X[4] + vpshufb $t3,@X[2],@X[2] + vmovdqu 0x50($inp),@X[5] + vpshufb $t3,@X[3],@X[3] + vmovdqu 0x60($inp),@X[6] + vpshufb $t3,@X[4],@X[4] + vmovdqu 0x70($inp),@X[7] + vpshufb $t3,@X[5],@X[5] + vpaddq -0x80($Tbl),@X[0],$t0 + vpshufb $t3,@X[6],@X[6] + vpaddq -0x60($Tbl),@X[1],$t1 + vpshufb $t3,@X[7],@X[7] + vpaddq -0x40($Tbl),@X[2],$t2 + vpaddq -0x20($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + vpaddq 0x00($Tbl),@X[4],$t0 + vmovdqa $t1,0x10(%rsp) + vpaddq 0x20($Tbl),@X[5],$t1 + vmovdqa $t2,0x20(%rsp) + vpaddq 0x40($Tbl),@X[6],$t2 + vmovdqa $t3,0x30(%rsp) + vpaddq 0x60($Tbl),@X[7],$t3 + vmovdqa $t0,0x40(%rsp) + mov $A,$a1 + vmovdqa $t1,0x50(%rsp) + mov $B,$a3 + vmovdqa $t2,0x60(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x70(%rsp) + mov $E,$a0 + jmp .Lxop_00_47 + +.align 16 +.Lxop_00_47: + add \$`16*2*$SZ`,$Tbl +___ +sub XOP_512_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body); # 52 instructions + + &vpalignr ($t0,@X[1],@X[0],$SZ); # X[1..2] + eval(shift(@insns)); + eval(shift(@insns)); + &vpalignr ($t3,@X[5],@X[4],$SZ); # X[9..10] + eval(shift(@insns)); + eval(shift(@insns)); + &vprotq ($t1,$t0,8*$SZ-$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrlq ($t0,$t0,$sigma0[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddq (@X[0],@X[0],$t3); # X[0..1] += X[9..10] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotq ($t2,$t1,$sigma0[1]-$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t0,$t0,$t1); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vprotq ($t3,@X[7],8*$SZ-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t0,$t0,$t2); # sigma0(X[1..2]) + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrlq ($t2,@X[7],$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddq (@X[0],@X[0],$t0); # X[0..1] += sigma0(X[1..2]) + eval(shift(@insns)); + eval(shift(@insns)); + &vprotq ($t1,$t3,$sigma1[1]-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t2); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor ($t3,$t3,$t1); # sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddq (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa (16*$j."(%rsp)",$t2); +} + + for ($i=0,$j=0; $j<8; $j++) { + &XOP_512_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmpb ($SZ-1+16*2*$SZ-0x80."($Tbl)",0); + &jne (".Lxop_00_47"); + + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } +} +$code.=<<___; + mov $_ctx,$ctx + mov $a1,$A + + add $SZ*0($ctx),$A + lea 16*$SZ($inp),$inp + add $SZ*1($ctx),$B + add $SZ*2($ctx),$C + add $SZ*3($ctx),$D + add $SZ*4($ctx),$E + add $SZ*5($ctx),$F + add $SZ*6($ctx),$G + add $SZ*7($ctx),$H + + cmp $_end,$inp + + mov $A,$SZ*0($ctx) + mov $B,$SZ*1($ctx) + mov $C,$SZ*2($ctx) + mov $D,$SZ*3($ctx) + mov $E,$SZ*4($ctx) + mov $F,$SZ*5($ctx) + mov $G,$SZ*6($ctx) + mov $H,$SZ*7($ctx) + jb .Lloop_xop + + mov $_rsp,%rsi + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*$SZ+32(%rsp),%xmm6 + movaps 16*$SZ+48(%rsp),%xmm7 + movaps 16*$SZ+64(%rsp),%xmm8 + movaps 16*$SZ+80(%rsp),%xmm9 +___ +$code.=<<___ if ($win64 && $SZ>4); + movaps 16*$SZ+96(%rsp),%xmm10 + movaps 16*$SZ+112(%rsp),%xmm11 +___ +$code.=<<___; + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_xop: + ret +.size ${func}_xop,.-${func}_xop +___ +} +###################################################################### +# AVX+shrd code path +# +local *ror = sub { &shrd(@_[0],@_) }; + +$code.=<<___; +.type ${func}_avx,\@function,3 +.align 64 +${func}_avx: +.Lavx_shortcut: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + mov %rsp,%r11 # copy %rsp + shl \$4,%rdx # num*16 + sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp + lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ + and \$-64,%rsp # align stack frame + mov $ctx,$_ctx # save ctx, 1st arg + mov $inp,$_inp # save inp, 2nd arh + mov %rdx,$_end # save end pointer, "3rd" arg + mov %r11,$_rsp # save copy of %rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,16*$SZ+32(%rsp) + movaps %xmm7,16*$SZ+48(%rsp) + movaps %xmm8,16*$SZ+64(%rsp) + movaps %xmm9,16*$SZ+80(%rsp) +___ +$code.=<<___ if ($win64 && $SZ>4); + movaps %xmm10,16*$SZ+96(%rsp) + movaps %xmm11,16*$SZ+112(%rsp) +___ +$code.=<<___; +.Lprologue_avx: + + vzeroupper + mov $SZ*0($ctx),$A + mov $SZ*1($ctx),$B + mov $SZ*2($ctx),$C + mov $SZ*3($ctx),$D + mov $SZ*4($ctx),$E + mov $SZ*5($ctx),$F + mov $SZ*6($ctx),$G + mov $SZ*7($ctx),$H +___ + if ($SZ==4) { # SHA256 + my @X = map("%xmm$_",(0..3)); + my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9)); + +$code.=<<___; + vmovdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4 + vmovdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5 + jmp .Lloop_avx +.align 16 +.Lloop_avx: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu 0x00($inp),@X[0] + vmovdqu 0x10($inp),@X[1] + vmovdqu 0x20($inp),@X[2] + vmovdqu 0x30($inp),@X[3] + vpshufb $t3,@X[0],@X[0] + lea $TABLE(%rip),$Tbl + vpshufb $t3,@X[1],@X[1] + vpshufb $t3,@X[2],@X[2] + vpaddd 0x00($Tbl),@X[0],$t0 + vpshufb $t3,@X[3],@X[3] + vpaddd 0x20($Tbl),@X[1],$t1 + vpaddd 0x40($Tbl),@X[2],$t2 + vpaddd 0x60($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + mov $A,$a1 + vmovdqa $t1,0x10(%rsp) + mov $B,$a3 + vmovdqa $t2,0x20(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x30(%rsp) + mov $E,$a0 + jmp .Lavx_00_47 + +.align 16 +.Lavx_00_47: + sub \$`-16*2*$SZ`,$Tbl # size optimization +___ +sub Xupdate_256_AVX () { + ( + '&vpalignr ($t0,@X[1],@X[0],$SZ)', # X[1..4] + '&vpalignr ($t3,@X[3],@X[2],$SZ)', # X[9..12] + '&vpsrld ($t2,$t0,$sigma0[0]);', + '&vpaddd (@X[0],@X[0],$t3)', # X[0..3] += X[9..12] + '&vpsrld ($t3,$t0,$sigma0[2])', + '&vpslld ($t1,$t0,8*$SZ-$sigma0[1]);', + '&vpxor ($t0,$t3,$t2)', + '&vpshufd ($t3,@X[3],0b11111010)',# X[14..15] + '&vpsrld ($t2,$t2,$sigma0[1]-$sigma0[0]);', + '&vpxor ($t0,$t0,$t1)', + '&vpslld ($t1,$t1,$sigma0[1]-$sigma0[0]);', + '&vpxor ($t0,$t0,$t2)', + '&vpsrld ($t2,$t3,$sigma1[2]);', + '&vpxor ($t0,$t0,$t1)', # sigma0(X[1..4]) + '&vpsrlq ($t3,$t3,$sigma1[0]);', + '&vpaddd (@X[0],@X[0],$t0)', # X[0..3] += sigma0(X[1..4]) + '&vpxor ($t2,$t2,$t3);', + '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])', + '&vpxor ($t2,$t2,$t3)', + '&vpshufb ($t2,$t2,$t4)', # sigma1(X[14..15]) + '&vpaddd (@X[0],@X[0],$t2)', # X[0..1] += sigma1(X[14..15]) + '&vpshufd ($t3,@X[0],0b01010000)',# X[16..17] + '&vpsrld ($t2,$t3,$sigma1[2])', + '&vpsrlq ($t3,$t3,$sigma1[0])', + '&vpxor ($t2,$t2,$t3);', + '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])', + '&vpxor ($t2,$t2,$t3)', + '&vpshufb ($t2,$t2,$t5)', + '&vpaddd (@X[0],@X[0],$t2)' # X[2..3] += sigma1(X[16..17]) + ); +} + +sub AVX_256_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 104 instructions + + foreach (Xupdate_256_AVX()) { # 29 instructions + eval; + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + } + &vpaddd ($t2,@X[0],16*2*$j."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa (16*$j."(%rsp)",$t2); +} + + for ($i=0,$j=0; $j<4; $j++) { + &AVX_256_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0); + &jne (".Lavx_00_47"); + + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } + + } else { # SHA512 + my @X = map("%xmm$_",(0..7)); + my ($t0,$t1,$t2,$t3) = map("%xmm$_",(8..11)); + +$code.=<<___; + jmp .Lloop_avx +.align 16 +.Lloop_avx: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu 0x00($inp),@X[0] + lea $TABLE+0x80(%rip),$Tbl # size optimization + vmovdqu 0x10($inp),@X[1] + vmovdqu 0x20($inp),@X[2] + vpshufb $t3,@X[0],@X[0] + vmovdqu 0x30($inp),@X[3] + vpshufb $t3,@X[1],@X[1] + vmovdqu 0x40($inp),@X[4] + vpshufb $t3,@X[2],@X[2] + vmovdqu 0x50($inp),@X[5] + vpshufb $t3,@X[3],@X[3] + vmovdqu 0x60($inp),@X[6] + vpshufb $t3,@X[4],@X[4] + vmovdqu 0x70($inp),@X[7] + vpshufb $t3,@X[5],@X[5] + vpaddq -0x80($Tbl),@X[0],$t0 + vpshufb $t3,@X[6],@X[6] + vpaddq -0x60($Tbl),@X[1],$t1 + vpshufb $t3,@X[7],@X[7] + vpaddq -0x40($Tbl),@X[2],$t2 + vpaddq -0x20($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + vpaddq 0x00($Tbl),@X[4],$t0 + vmovdqa $t1,0x10(%rsp) + vpaddq 0x20($Tbl),@X[5],$t1 + vmovdqa $t2,0x20(%rsp) + vpaddq 0x40($Tbl),@X[6],$t2 + vmovdqa $t3,0x30(%rsp) + vpaddq 0x60($Tbl),@X[7],$t3 + vmovdqa $t0,0x40(%rsp) + mov $A,$a1 + vmovdqa $t1,0x50(%rsp) + mov $B,$a3 + vmovdqa $t2,0x60(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x70(%rsp) + mov $E,$a0 + jmp .Lavx_00_47 + +.align 16 +.Lavx_00_47: + add \$`16*2*$SZ`,$Tbl +___ +sub Xupdate_512_AVX () { + ( + '&vpalignr ($t0,@X[1],@X[0],$SZ)', # X[1..2] + '&vpalignr ($t3,@X[5],@X[4],$SZ)', # X[9..10] + '&vpsrlq ($t2,$t0,$sigma0[0])', + '&vpaddq (@X[0],@X[0],$t3);', # X[0..1] += X[9..10] + '&vpsrlq ($t3,$t0,$sigma0[2])', + '&vpsllq ($t1,$t0,8*$SZ-$sigma0[1]);', + '&vpxor ($t0,$t3,$t2)', + '&vpsrlq ($t2,$t2,$sigma0[1]-$sigma0[0]);', + '&vpxor ($t0,$t0,$t1)', + '&vpsllq ($t1,$t1,$sigma0[1]-$sigma0[0]);', + '&vpxor ($t0,$t0,$t2)', + '&vpsrlq ($t3,@X[7],$sigma1[2]);', + '&vpxor ($t0,$t0,$t1)', # sigma0(X[1..2]) + '&vpsllq ($t2,@X[7],8*$SZ-$sigma1[1]);', + '&vpaddq (@X[0],@X[0],$t0)', # X[0..1] += sigma0(X[1..2]) + '&vpsrlq ($t1,@X[7],$sigma1[0]);', + '&vpxor ($t3,$t3,$t2)', + '&vpsllq ($t2,$t2,$sigma1[1]-$sigma1[0]);', + '&vpxor ($t3,$t3,$t1)', + '&vpsrlq ($t1,$t1,$sigma1[1]-$sigma1[0]);', + '&vpxor ($t3,$t3,$t2)', + '&vpxor ($t3,$t3,$t1)', # sigma1(X[14..15]) + '&vpaddq (@X[0],@X[0],$t3)', # X[0..1] += sigma1(X[14..15]) + ); +} + +sub AVX_512_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body); # 52 instructions + + foreach (Xupdate_512_AVX()) { # 23 instructions + eval; + eval(shift(@insns)); + eval(shift(@insns)); + } + &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa (16*$j."(%rsp)",$t2); +} + + for ($i=0,$j=0; $j<8; $j++) { + &AVX_512_00_47($j,\&body_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &cmpb ($SZ-1+16*2*$SZ-0x80."($Tbl)",0); + &jne (".Lavx_00_47"); + + for ($i=0; $i<16; ) { + foreach(body_00_15()) { eval; } + } +} +$code.=<<___; + mov $_ctx,$ctx + mov $a1,$A + + add $SZ*0($ctx),$A + lea 16*$SZ($inp),$inp + add $SZ*1($ctx),$B + add $SZ*2($ctx),$C + add $SZ*3($ctx),$D + add $SZ*4($ctx),$E + add $SZ*5($ctx),$F + add $SZ*6($ctx),$G + add $SZ*7($ctx),$H + + cmp $_end,$inp + + mov $A,$SZ*0($ctx) + mov $B,$SZ*1($ctx) + mov $C,$SZ*2($ctx) + mov $D,$SZ*3($ctx) + mov $E,$SZ*4($ctx) + mov $F,$SZ*5($ctx) + mov $G,$SZ*6($ctx) + mov $H,$SZ*7($ctx) + jb .Lloop_avx + + mov $_rsp,%rsi + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*$SZ+32(%rsp),%xmm6 + movaps 16*$SZ+48(%rsp),%xmm7 + movaps 16*$SZ+64(%rsp),%xmm8 + movaps 16*$SZ+80(%rsp),%xmm9 +___ +$code.=<<___ if ($win64 && $SZ>4); + movaps 16*$SZ+96(%rsp),%xmm10 + movaps 16*$SZ+112(%rsp),%xmm11 +___ +$code.=<<___; + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_avx: + ret +.size ${func}_avx,.-${func}_avx +___ + +if ($avx>1) {{ +###################################################################### +# AVX2+BMI code path +# +my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp +my $PUSH8=8*2*$SZ; +use integer; + +sub bodyx_00_15 () { + # at start $a1 should be zero, $a3 - $b^$c and $a4 copy of $f + ( + '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'. + + '&add ($h,(32*($i/(16/$SZ))+$SZ*($i%(16/$SZ)))%$PUSH8.$base)', # h+=X[i]+K[i] + '&and ($a4,$e)', # f&e + '&rorx ($a0,$e,$Sigma1[2])', + '&rorx ($a2,$e,$Sigma1[1])', + + '&lea ($a,"($a,$a1)")', # h+=Sigma0(a) from the past + '&lea ($h,"($h,$a4)")', + '&andn ($a4,$e,$g)', # ~e&g + '&xor ($a0,$a2)', + + '&rorx ($a1,$e,$Sigma1[0])', + '&lea ($h,"($h,$a4)")', # h+=Ch(e,f,g)=(e&f)+(~e&g) + '&xor ($a0,$a1)', # Sigma1(e) + '&mov ($a2,$a)', + + '&rorx ($a4,$a,$Sigma0[2])', + '&lea ($h,"($h,$a0)")', # h+=Sigma1(e) + '&xor ($a2,$b)', # a^b, b^c in next round + '&rorx ($a1,$a,$Sigma0[1])', + + '&rorx ($a0,$a,$Sigma0[0])', + '&lea ($d,"($d,$h)")', # d+=h + '&and ($a3,$a2)', # (b^c)&(a^b) + '&xor ($a1,$a4)', + + '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b) + '&xor ($a1,$a0)', # Sigma0(a) + '&lea ($h,"($h,$a3)");'. # h+=Maj(a,b,c) + '&mov ($a4,$e)', # copy of f in future + + '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;' + ); + # and at the finish one has to $a+=$a1 +} + +$code.=<<___; +.type ${func}_avx2,\@function,3 +.align 64 +${func}_avx2: +.Lavx2_shortcut: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + mov %rsp,%r11 # copy %rsp + sub \$`2*$SZ*$rounds+4*8+$win64*16*($SZ==4?4:6)`,%rsp + shl \$4,%rdx # num*16 + and \$-256*$SZ,%rsp # align stack frame + lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ + add \$`2*$SZ*($rounds-8)`,%rsp + mov $ctx,$_ctx # save ctx, 1st arg + mov $inp,$_inp # save inp, 2nd arh + mov %rdx,$_end # save end pointer, "3rd" arg + mov %r11,$_rsp # save copy of %rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,16*$SZ+32(%rsp) + movaps %xmm7,16*$SZ+48(%rsp) + movaps %xmm8,16*$SZ+64(%rsp) + movaps %xmm9,16*$SZ+80(%rsp) +___ +$code.=<<___ if ($win64 && $SZ>4); + movaps %xmm10,16*$SZ+96(%rsp) + movaps %xmm11,16*$SZ+112(%rsp) +___ +$code.=<<___; +.Lprologue_avx2: + + vzeroupper + sub \$-16*$SZ,$inp # inp++, size optimization + mov $SZ*0($ctx),$A + mov $inp,%r12 # borrow $T1 + mov $SZ*1($ctx),$B + cmp %rdx,$inp # $_end + mov $SZ*2($ctx),$C + cmove %rsp,%r12 # next block or random data + mov $SZ*3($ctx),$D + mov $SZ*4($ctx),$E + mov $SZ*5($ctx),$F + mov $SZ*6($ctx),$G + mov $SZ*7($ctx),$H +___ + if ($SZ==4) { # SHA256 + my @X = map("%ymm$_",(0..3)); + my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%ymm$_",(4..9)); + +$code.=<<___; + vmovdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4 + vmovdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5 + jmp .Loop_avx2 +.align 16 +.Loop_avx2: + vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3 + vmovdqu -16*$SZ+0($inp),%xmm0 + vmovdqu -16*$SZ+16($inp),%xmm1 + vmovdqu -16*$SZ+32($inp),%xmm2 + vmovdqu -16*$SZ+48($inp),%xmm3 + #mov $inp,$_inp # offload $inp + vinserti128 \$1,(%r12),@X[0],@X[0] + vinserti128 \$1,16(%r12),@X[1],@X[1] + vpshufb $t3,@X[0],@X[0] + vinserti128 \$1,32(%r12),@X[2],@X[2] + vpshufb $t3,@X[1],@X[1] + vinserti128 \$1,48(%r12),@X[3],@X[3] + + lea $TABLE(%rip),$Tbl + vpshufb $t3,@X[2],@X[2] + vpaddd 0x00($Tbl),@X[0],$t0 + vpshufb $t3,@X[3],@X[3] + vpaddd 0x20($Tbl),@X[1],$t1 + vpaddd 0x40($Tbl),@X[2],$t2 + vpaddd 0x60($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + xor $a1,$a1 + vmovdqa $t1,0x20(%rsp) + lea -$PUSH8(%rsp),%rsp + mov $B,$a3 + vmovdqa $t2,0x00(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x20(%rsp) + mov $F,$a4 + sub \$-16*2*$SZ,$Tbl # size optimization + jmp .Lavx2_00_47 + +.align 16 +.Lavx2_00_47: +___ + +sub AVX2_256_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body,&$body,&$body); # 96 instructions +my $base = "+2*$PUSH8(%rsp)"; + + &lea ("%rsp","-$PUSH8(%rsp)") if (($j%2)==0); + foreach (Xupdate_256_AVX()) { # 29 instructions + eval; + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + } + &vpaddd ($t2,@X[0],16*2*$j."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa ((32*$j)%$PUSH8."(%rsp)",$t2); +} + + for ($i=0,$j=0; $j<4; $j++) { + &AVX2_256_00_47($j,\&bodyx_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &lea ($Tbl,16*2*$SZ."($Tbl)"); + &cmpb (($SZ-1)."($Tbl)",0); + &jne (".Lavx2_00_47"); + + for ($i=0; $i<16; ) { + my $base=$i<8?"+$PUSH8(%rsp)":"(%rsp)"; + foreach(bodyx_00_15()) { eval; } + } + } else { # SHA512 + my @X = map("%ymm$_",(0..7)); + my ($t0,$t1,$t2,$t3) = map("%ymm$_",(8..11)); + +$code.=<<___; + jmp .Loop_avx2 +.align 16 +.Loop_avx2: + vmovdqu -16*$SZ($inp),%xmm0 + vmovdqu -16*$SZ+16($inp),%xmm1 + vmovdqu -16*$SZ+32($inp),%xmm2 + lea $TABLE+0x80(%rip),$Tbl # size optimization + vmovdqu -16*$SZ+48($inp),%xmm3 + vmovdqu -16*$SZ+64($inp),%xmm4 + vmovdqu -16*$SZ+80($inp),%xmm5 + vmovdqu -16*$SZ+96($inp),%xmm6 + vmovdqu -16*$SZ+112($inp),%xmm7 + #mov $inp,$_inp # offload $inp + vmovdqa `$SZ*2*$rounds-0x80`($Tbl),$t2 + vinserti128 \$1,(%r12),@X[0],@X[0] + vinserti128 \$1,16(%r12),@X[1],@X[1] + vpshufb $t2,@X[0],@X[0] + vinserti128 \$1,32(%r12),@X[2],@X[2] + vpshufb $t2,@X[1],@X[1] + vinserti128 \$1,48(%r12),@X[3],@X[3] + vpshufb $t2,@X[2],@X[2] + vinserti128 \$1,64(%r12),@X[4],@X[4] + vpshufb $t2,@X[3],@X[3] + vinserti128 \$1,80(%r12),@X[5],@X[5] + vpshufb $t2,@X[4],@X[4] + vinserti128 \$1,96(%r12),@X[6],@X[6] + vpshufb $t2,@X[5],@X[5] + vinserti128 \$1,112(%r12),@X[7],@X[7] + + vpaddq -0x80($Tbl),@X[0],$t0 + vpshufb $t2,@X[6],@X[6] + vpaddq -0x60($Tbl),@X[1],$t1 + vpshufb $t2,@X[7],@X[7] + vpaddq -0x40($Tbl),@X[2],$t2 + vpaddq -0x20($Tbl),@X[3],$t3 + vmovdqa $t0,0x00(%rsp) + vpaddq 0x00($Tbl),@X[4],$t0 + vmovdqa $t1,0x20(%rsp) + vpaddq 0x20($Tbl),@X[5],$t1 + vmovdqa $t2,0x40(%rsp) + vpaddq 0x40($Tbl),@X[6],$t2 + vmovdqa $t3,0x60(%rsp) + lea -$PUSH8(%rsp),%rsp + vpaddq 0x60($Tbl),@X[7],$t3 + vmovdqa $t0,0x00(%rsp) + xor $a1,$a1 + vmovdqa $t1,0x20(%rsp) + mov $B,$a3 + vmovdqa $t2,0x40(%rsp) + xor $C,$a3 # magic + vmovdqa $t3,0x60(%rsp) + mov $F,$a4 + add \$16*2*$SZ,$Tbl + jmp .Lavx2_00_47 + +.align 16 +.Lavx2_00_47: +___ + +sub AVX2_512_00_47 () { +my $j = shift; +my $body = shift; +my @X = @_; +my @insns = (&$body,&$body); # 48 instructions +my $base = "+2*$PUSH8(%rsp)"; + + &lea ("%rsp","-$PUSH8(%rsp)") if (($j%4)==0); + foreach (Xupdate_512_AVX()) { # 23 instructions + eval; + if ($_ !~ /\;$/) { + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + } + } + &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)"); + foreach (@insns) { eval; } # remaining instructions + &vmovdqa ((32*$j)%$PUSH8."(%rsp)",$t2); +} + + for ($i=0,$j=0; $j<8; $j++) { + &AVX2_512_00_47($j,\&bodyx_00_15,@X); + push(@X,shift(@X)); # rotate(@X) + } + &lea ($Tbl,16*2*$SZ."($Tbl)"); + &cmpb (($SZ-1-0x80)."($Tbl)",0); + &jne (".Lavx2_00_47"); + + for ($i=0; $i<16; ) { + my $base=$i<8?"+$PUSH8(%rsp)":"(%rsp)"; + foreach(bodyx_00_15()) { eval; } + } +} +$code.=<<___; + mov `2*$SZ*$rounds`(%rsp),$ctx # $_ctx + add $a1,$A + #mov `2*$SZ*$rounds+8`(%rsp),$inp # $_inp + lea `2*$SZ*($rounds-8)`(%rsp),$Tbl + + add $SZ*0($ctx),$A + add $SZ*1($ctx),$B + add $SZ*2($ctx),$C + add $SZ*3($ctx),$D + add $SZ*4($ctx),$E + add $SZ*5($ctx),$F + add $SZ*6($ctx),$G + add $SZ*7($ctx),$H + + mov $A,$SZ*0($ctx) + mov $B,$SZ*1($ctx) + mov $C,$SZ*2($ctx) + mov $D,$SZ*3($ctx) + mov $E,$SZ*4($ctx) + mov $F,$SZ*5($ctx) + mov $G,$SZ*6($ctx) + mov $H,$SZ*7($ctx) + + cmp `$PUSH8+2*8`($Tbl),$inp # $_end + je .Ldone_avx2 + + xor $a1,$a1 + mov $B,$a3 + xor $C,$a3 # magic + mov $F,$a4 + jmp .Lower_avx2 +.align 16 +.Lower_avx2: +___ + for ($i=0; $i<8; ) { + my $base="+16($Tbl)"; + foreach(bodyx_00_15()) { eval; } + } +$code.=<<___; + lea -$PUSH8($Tbl),$Tbl + cmp %rsp,$Tbl + jae .Lower_avx2 + + mov `2*$SZ*$rounds`(%rsp),$ctx # $_ctx + add $a1,$A + #mov `2*$SZ*$rounds+8`(%rsp),$inp # $_inp + lea `2*$SZ*($rounds-8)`(%rsp),%rsp + + add $SZ*0($ctx),$A + add $SZ*1($ctx),$B + add $SZ*2($ctx),$C + add $SZ*3($ctx),$D + add $SZ*4($ctx),$E + add $SZ*5($ctx),$F + lea `2*16*$SZ`($inp),$inp # inp+=2 + add $SZ*6($ctx),$G + mov $inp,%r12 + add $SZ*7($ctx),$H + cmp $_end,$inp + + mov $A,$SZ*0($ctx) + cmove %rsp,%r12 # next block or stale data + mov $B,$SZ*1($ctx) + mov $C,$SZ*2($ctx) + mov $D,$SZ*3($ctx) + mov $E,$SZ*4($ctx) + mov $F,$SZ*5($ctx) + mov $G,$SZ*6($ctx) + mov $H,$SZ*7($ctx) + + jbe .Loop_avx2 + lea (%rsp),$Tbl + +.Ldone_avx2: + lea ($Tbl),%rsp + mov $_rsp,%rsi + vzeroupper +___ +$code.=<<___ if ($win64); + movaps 16*$SZ+32(%rsp),%xmm6 + movaps 16*$SZ+48(%rsp),%xmm7 + movaps 16*$SZ+64(%rsp),%xmm8 + movaps 16*$SZ+80(%rsp),%xmm9 +___ +$code.=<<___ if ($win64 && $SZ>4); + movaps 16*$SZ+96(%rsp),%xmm10 + movaps 16*$SZ+112(%rsp),%xmm11 +___ +$code.=<<___; + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue_avx2: + ret +.size ${func}_avx2,.-${func}_avx2 +___ +}} +}}}}} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HanderlData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lin_prologue +___ +$code.=<<___ if ($avx>1); + lea .Lavx2_shortcut(%rip),%r10 + cmp %r10,%rbx # context->RipRbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R15 + + lea .Lepilogue(%rip),%r10 + cmp %r10,%rbx + jb .Lin_prologue # non-AVX code + + lea 16*$SZ+4*8(%rsi),%rsi # Xmm6- save area + lea 512($context),%rdi # &context.Xmm6 + mov \$`$SZ==4?8:12`,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lin_prologue: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 mov %r12,48(%rsp) # arg7 mov %rcx,56(%rsp) # arg8, (NULL) call *__imp_RtlVirtualUnwind(%rip) @@ -431,21 +2263,136 @@ se_handler: pop %rsi ret .size se_handler,.-se_handler +___ + +$code.=<<___ if ($SZ==4 && $shaext); +.type shaext_handler,\@abi-omnipotent +.align 16 +shaext_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + lea .Lprologue_shaext(%rip),%r10 + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lin_prologue + + lea .Lepilogue_shaext(%rip),%r10 + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lin_prologue + + lea -8-5*16(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$10,%ecx + .long 0xa548f3fc # cld; rep movsq + + jmp .Lin_prologue +.size shaext_handler,.-shaext_handler +___ +$code.=<<___; .section .pdata .align 4 .rva .LSEH_begin_$func .rva .LSEH_end_$func .rva .LSEH_info_$func - +___ +$code.=<<___ if ($SZ==4 && $shaext); + .rva .LSEH_begin_${func}_shaext + .rva .LSEH_end_${func}_shaext + .rva .LSEH_info_${func}_shaext +___ +$code.=<<___ if ($SZ==4); + .rva .LSEH_begin_${func}_ssse3 + .rva .LSEH_end_${func}_ssse3 + .rva .LSEH_info_${func}_ssse3 +___ +$code.=<<___ if ($avx && $SZ==8); + .rva .LSEH_begin_${func}_xop + .rva .LSEH_end_${func}_xop + .rva .LSEH_info_${func}_xop +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_${func}_avx + .rva .LSEH_end_${func}_avx + .rva .LSEH_info_${func}_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_${func}_avx2 + .rva .LSEH_end_${func}_avx2 + .rva .LSEH_info_${func}_avx2 +___ +$code.=<<___; .section .xdata .align 8 .LSEH_info_$func: .byte 9,0,0,0 .rva se_handler + .rva .Lprologue,.Lepilogue # HandlerData[] +___ +$code.=<<___ if ($SZ==4 && $shaext); +.LSEH_info_${func}_shaext: + .byte 9,0,0,0 + .rva shaext_handler +___ +$code.=<<___ if ($SZ==4); +.LSEH_info_${func}_ssse3: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_ssse3,.Lepilogue_ssse3 # HandlerData[] ___ +$code.=<<___ if ($avx && $SZ==8); +.LSEH_info_${func}_xop: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_xop,.Lepilogue_xop # HandlerData[] +___ +$code.=<<___ if ($avx); +.LSEH_info_${func}_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[] +___ +$code.=<<___ if ($avx>1); +.LSEH_info_${func}_avx2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lprologue_avx2,.Lepilogue_avx2 # HandlerData[] +___ +} + +sub sha256op38 { + my $instr = shift; + my %opcodelet = ( + "sha256rnds2" => 0xcb, + "sha256msg1" => 0xcc, + "sha256msg2" => 0xcd ); + + if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-7]),\s*%xmm([0-7])/) { + my @opcode=(0x0f,0x38); + push @opcode,$opcodelet{$instr}; + push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M + return ".byte\t".join(',',@opcode); + } else { + return $instr."\t".@_[0]; + } } -$code =~ s/\`([^\`]*)\`/eval $1/gem; -print $code; +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo; + + print $_,"\n"; +} close STDOUT; diff --git a/crypto/sha/asm/sha512p8-ppc.pl b/crypto/sha/asm/sha512p8-ppc.pl new file mode 100755 index 000000000000..47189502c6cc --- /dev/null +++ b/crypto/sha/asm/sha512p8-ppc.pl @@ -0,0 +1,424 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA256/512 for PowerISA v2.07. +# +# Accurate performance measurements are problematic, because it's +# always virtualized setup with possibly throttled processor. +# Relative comparison is therefore more informative. This module is +# ~60% faster than integer-only sha512-ppc.pl. To anchor to something +# else, SHA256 is 24% slower than sha1-ppc.pl and 2.5x slower than +# hardware-assisted aes-128-cbc encrypt. SHA512 is 20% faster than +# sha1-ppc.pl and 1.6x slower than aes-128-cbc. Another interesting +# result is degree of computational resources' utilization. POWER8 is +# "massively multi-threaded chip" and difference between single- and +# maximum multi-process benchmark results tells that utlization is +# whooping 94%. For sha512-ppc.pl we get [not unimpressive] 84% and +# for sha1-ppc.pl - 73%. 100% means that multi-process result equals +# to single-process one, given that all threads end up on the same +# physical core. + +$flavour=shift; +$output =shift; + +if ($flavour =~ /64/) { + $SIZE_T=8; + $LRSAVE=2*$SIZE_T; + $STU="stdu"; + $POP="ld"; + $PUSH="std"; +} elsif ($flavour =~ /32/) { + $SIZE_T=4; + $LRSAVE=$SIZE_T; + $STU="stwu"; + $POP="lwz"; + $PUSH="stw"; +} else { die "nonsense $flavour"; } + +$LENDIAN=($flavour=~/le/); + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; + +if ($output =~ /512/) { + $bits=512; + $SZ=8; + $sz="d"; + $rounds=80; +} else { + $bits=256; + $SZ=4; + $sz="w"; + $rounds=64; +} + +$func="sha${bits}_block_p8"; +$FRAME=8*$SIZE_T; + +$sp ="r1"; +$toc="r2"; +$ctx="r3"; +$inp="r4"; +$num="r5"; +$Tbl="r6"; +$idx="r7"; +$lrsave="r8"; +$offload="r11"; +$vrsave="r12"; +($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,10,26..31)); + $x00=0 if ($flavour =~ /osx/); + +@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("v$_",(0..7)); +@X=map("v$_",(8..23)); +($Ki,$Func,$S0,$S1,$s0,$s1,$lemask)=map("v$_",(24..31)); + +sub ROUND { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +my $j=($i+1)%16; + +$code.=<<___ if ($i<15 && ($i%(16/$SZ))==(16/$SZ-1)); + lvx_u @X[$i+1],0,$inp ; load X[i] in advance + addi $inp,$inp,16 +___ +$code.=<<___ if ($i<16 && ($i%(16/$SZ))); + vsldoi @X[$i],@X[$i-1],@X[$i-1],$SZ +___ +$code.=<<___ if ($LENDIAN && $i<16 && ($i%(16/$SZ))==0); + vperm @X[$i],@X[$i],@X[$i],$lemask +___ +$code.=<<___; + `"vshasigma${sz} $s0,@X[($j+1)%16],0,0" if ($i>=15)` + vsel $Func,$g,$f,$e ; Ch(e,f,g) + vshasigma${sz} $S1,$e,1,15 ; Sigma1(e) + vaddu${sz}m $h,$h,@X[$i%16] ; h+=X[i] + vshasigma${sz} $S0,$a,1,0 ; Sigma0(a) + `"vshasigma${sz} $s1,@X[($j+14)%16],0,15" if ($i>=15)` + vaddu${sz}m $h,$h,$Func ; h+=Ch(e,f,g) + vxor $Func,$a,$b + `"vaddu${sz}m @X[$j],@X[$j],@X[($j+9)%16]" if ($i>=15)` + vaddu${sz}m $h,$h,$S1 ; h+=Sigma1(e) + vsel $Func,$b,$c,$Func ; Maj(a,b,c) + vaddu${sz}m $g,$g,$Ki ; future h+=K[i] + vaddu${sz}m $d,$d,$h ; d+=h + vaddu${sz}m $S0,$S0,$Func ; Sigma0(a)+Maj(a,b,c) + `"vaddu${sz}m @X[$j],@X[$j],$s0" if ($i>=15)` + lvx $Ki,$idx,$Tbl ; load next K[i] + addi $idx,$idx,16 + vaddu${sz}m $h,$h,$S0 ; h+=Sigma0(a)+Maj(a,b,c) + `"vaddu${sz}m @X[$j],@X[$j],$s1" if ($i>=15)` +___ +} + +$code=<<___; +.machine "any" +.text + +.globl $func +.align 6 +$func: + $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) + mflr $lrsave + li r10,`$FRAME+8*16+15` + li r11,`$FRAME+8*16+31` + stvx v20,r10,$sp # ABI says so + addi r10,r10,32 + mfspr $vrsave,256 + stvx v21,r11,$sp + addi r11,r11,32 + stvx v22,r10,$sp + addi r10,r10,32 + stvx v23,r11,$sp + addi r11,r11,32 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + li r11,-1 + stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave + li $x10,0x10 + $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) + li $x20,0x20 + $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) + li $x30,0x30 + $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) + li $x40,0x40 + $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) + li $x50,0x50 + $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) + li $x60,0x60 + $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) + li $x70,0x70 + $PUSH $lrsave,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp) + mtspr 256,r11 + + bl LPICmeup + addi $offload,$sp,$FRAME+15 +___ +$code.=<<___ if ($LENDIAN); + li $idx,8 + lvsl $lemask,0,$idx + vspltisb $Ki,0x0f + vxor $lemask,$lemask,$Ki +___ +$code.=<<___ if ($SZ==4); + lvx_4w $A,$x00,$ctx + lvx_4w $E,$x10,$ctx + vsldoi $B,$A,$A,4 # unpack + vsldoi $C,$A,$A,8 + vsldoi $D,$A,$A,12 + vsldoi $F,$E,$E,4 + vsldoi $G,$E,$E,8 + vsldoi $H,$E,$E,12 +___ +$code.=<<___ if ($SZ==8); + lvx_u $A,$x00,$ctx + lvx_u $C,$x10,$ctx + lvx_u $E,$x20,$ctx + vsldoi $B,$A,$A,8 # unpack + lvx_u $G,$x30,$ctx + vsldoi $D,$C,$C,8 + vsldoi $F,$E,$E,8 + vsldoi $H,$G,$G,8 +___ +$code.=<<___; + li r0,`($rounds-16)/16` # inner loop counter + b Loop +.align 5 +Loop: + lvx $Ki,$x00,$Tbl + li $idx,16 + lvx_u @X[0],0,$inp + addi $inp,$inp,16 + stvx $A,$x00,$offload # offload $A-$H + stvx $B,$x10,$offload + stvx $C,$x20,$offload + stvx $D,$x30,$offload + stvx $E,$x40,$offload + stvx $F,$x50,$offload + stvx $G,$x60,$offload + stvx $H,$x70,$offload + vaddu${sz}m $H,$H,$Ki # h+K[i] + lvx $Ki,$idx,$Tbl + addi $idx,$idx,16 +___ +for ($i=0;$i<16;$i++) { &ROUND($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + mtctr r0 + b L16_xx +.align 5 +L16_xx: +___ +for (;$i<32;$i++) { &ROUND($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + bdnz L16_xx + + lvx @X[2],$x00,$offload + subic. $num,$num,1 + lvx @X[3],$x10,$offload + vaddu${sz}m $A,$A,@X[2] + lvx @X[4],$x20,$offload + vaddu${sz}m $B,$B,@X[3] + lvx @X[5],$x30,$offload + vaddu${sz}m $C,$C,@X[4] + lvx @X[6],$x40,$offload + vaddu${sz}m $D,$D,@X[5] + lvx @X[7],$x50,$offload + vaddu${sz}m $E,$E,@X[6] + lvx @X[8],$x60,$offload + vaddu${sz}m $F,$F,@X[7] + lvx @X[9],$x70,$offload + vaddu${sz}m $G,$G,@X[8] + vaddu${sz}m $H,$H,@X[9] + bne Loop +___ +$code.=<<___ if ($SZ==4); + lvx @X[0],$idx,$Tbl + addi $idx,$idx,16 + vperm $A,$A,$B,$Ki # pack the answer + lvx @X[1],$idx,$Tbl + vperm $E,$E,$F,$Ki + vperm $A,$A,$C,@X[0] + vperm $E,$E,$G,@X[0] + vperm $A,$A,$D,@X[1] + vperm $E,$E,$H,@X[1] + stvx_4w $A,$x00,$ctx + stvx_4w $E,$x10,$ctx +___ +$code.=<<___ if ($SZ==8); + vperm $A,$A,$B,$Ki # pack the answer + vperm $C,$C,$D,$Ki + vperm $E,$E,$F,$Ki + vperm $G,$G,$H,$Ki + stvx_u $A,$x00,$ctx + stvx_u $C,$x10,$ctx + stvx_u $E,$x20,$ctx + stvx_u $G,$x30,$ctx +___ +$code.=<<___; + li r10,`$FRAME+8*16+15` + mtlr $lrsave + li r11,`$FRAME+8*16+31` + mtspr 256,$vrsave + lvx v20,r10,$sp # ABI says so + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) + $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) + $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) + $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) + $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) + $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) + addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` + blr + .long 0 + .byte 0,12,4,1,0x80,6,3,0 + .long 0 +.size $func,.-$func +___ + +# Ugly hack here, because PPC assembler syntax seem to vary too +# much from platforms to platform... +$code.=<<___; +.align 6 +LPICmeup: + mflr r0 + bcl 20,31,\$+4 + mflr $Tbl ; vvvvvv "distance" between . and 1st data entry + addi $Tbl,$Tbl,`64-8` + mtlr r0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + .space `64-9*4` +___ + +if ($SZ==8) { + local *table = sub { + foreach(@_) { $code.=".quad $_,$_\n"; } + }; + table( + "0x428a2f98d728ae22","0x7137449123ef65cd", + "0xb5c0fbcfec4d3b2f","0xe9b5dba58189dbbc", + "0x3956c25bf348b538","0x59f111f1b605d019", + "0x923f82a4af194f9b","0xab1c5ed5da6d8118", + "0xd807aa98a3030242","0x12835b0145706fbe", + "0x243185be4ee4b28c","0x550c7dc3d5ffb4e2", + "0x72be5d74f27b896f","0x80deb1fe3b1696b1", + "0x9bdc06a725c71235","0xc19bf174cf692694", + "0xe49b69c19ef14ad2","0xefbe4786384f25e3", + "0x0fc19dc68b8cd5b5","0x240ca1cc77ac9c65", + "0x2de92c6f592b0275","0x4a7484aa6ea6e483", + "0x5cb0a9dcbd41fbd4","0x76f988da831153b5", + "0x983e5152ee66dfab","0xa831c66d2db43210", + "0xb00327c898fb213f","0xbf597fc7beef0ee4", + "0xc6e00bf33da88fc2","0xd5a79147930aa725", + "0x06ca6351e003826f","0x142929670a0e6e70", + "0x27b70a8546d22ffc","0x2e1b21385c26c926", + "0x4d2c6dfc5ac42aed","0x53380d139d95b3df", + "0x650a73548baf63de","0x766a0abb3c77b2a8", + "0x81c2c92e47edaee6","0x92722c851482353b", + "0xa2bfe8a14cf10364","0xa81a664bbc423001", + "0xc24b8b70d0f89791","0xc76c51a30654be30", + "0xd192e819d6ef5218","0xd69906245565a910", + "0xf40e35855771202a","0x106aa07032bbd1b8", + "0x19a4c116b8d2d0c8","0x1e376c085141ab53", + "0x2748774cdf8eeb99","0x34b0bcb5e19b48a8", + "0x391c0cb3c5c95a63","0x4ed8aa4ae3418acb", + "0x5b9cca4f7763e373","0x682e6ff3d6b2b8a3", + "0x748f82ee5defb2fc","0x78a5636f43172f60", + "0x84c87814a1f0ab72","0x8cc702081a6439ec", + "0x90befffa23631e28","0xa4506cebde82bde9", + "0xbef9a3f7b2c67915","0xc67178f2e372532b", + "0xca273eceea26619c","0xd186b8c721c0c207", + "0xeada7dd6cde0eb1e","0xf57d4f7fee6ed178", + "0x06f067aa72176fba","0x0a637dc5a2c898a6", + "0x113f9804bef90dae","0x1b710b35131c471b", + "0x28db77f523047d84","0x32caab7b40c72493", + "0x3c9ebe0a15c9bebc","0x431d67c49c100d4c", + "0x4cc5d4becb3e42b6","0x597f299cfc657e2a", + "0x5fcb6fab3ad6faec","0x6c44198c4a475817","0"); +$code.=<<___ if (!$LENDIAN); +.quad 0x0001020304050607,0x1011121314151617 +___ +$code.=<<___ if ($LENDIAN); # quad-swapped +.quad 0x1011121314151617,0x0001020304050607 +___ +} else { + local *table = sub { + foreach(@_) { $code.=".long $_,$_,$_,$_\n"; } + }; + table( + "0x428a2f98","0x71374491","0xb5c0fbcf","0xe9b5dba5", + "0x3956c25b","0x59f111f1","0x923f82a4","0xab1c5ed5", + "0xd807aa98","0x12835b01","0x243185be","0x550c7dc3", + "0x72be5d74","0x80deb1fe","0x9bdc06a7","0xc19bf174", + "0xe49b69c1","0xefbe4786","0x0fc19dc6","0x240ca1cc", + "0x2de92c6f","0x4a7484aa","0x5cb0a9dc","0x76f988da", + "0x983e5152","0xa831c66d","0xb00327c8","0xbf597fc7", + "0xc6e00bf3","0xd5a79147","0x06ca6351","0x14292967", + "0x27b70a85","0x2e1b2138","0x4d2c6dfc","0x53380d13", + "0x650a7354","0x766a0abb","0x81c2c92e","0x92722c85", + "0xa2bfe8a1","0xa81a664b","0xc24b8b70","0xc76c51a3", + "0xd192e819","0xd6990624","0xf40e3585","0x106aa070", + "0x19a4c116","0x1e376c08","0x2748774c","0x34b0bcb5", + "0x391c0cb3","0x4ed8aa4a","0x5b9cca4f","0x682e6ff3", + "0x748f82ee","0x78a5636f","0x84c87814","0x8cc70208", + "0x90befffa","0xa4506ceb","0xbef9a3f7","0xc67178f2","0"); +$code.=<<___ if (!$LENDIAN); +.long 0x00010203,0x10111213,0x10111213,0x10111213 +.long 0x00010203,0x04050607,0x10111213,0x10111213 +.long 0x00010203,0x04050607,0x08090a0b,0x10111213 +___ +$code.=<<___ if ($LENDIAN); # word-swapped +.long 0x10111213,0x10111213,0x10111213,0x00010203 +.long 0x10111213,0x10111213,0x04050607,0x00010203 +.long 0x10111213,0x08090a0b,0x04050607,0x00010203 +___ +} +$code.=<<___; +.asciz "SHA${bits} for PowerISA 2.07, CRYPTOGAMS by " +.align 2 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/sha/sha512.c b/crypto/sha/sha512.c index de0aad8c9dfa..3bf66ae1987e 100644 --- a/crypto/sha/sha512.c +++ b/crypto/sha/sha512.c @@ -55,6 +55,7 @@ const char SHA512_version[] = "SHA-512" OPENSSL_VERSION_PTEXT; # if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(_M_AMD64) || defined(_M_X64) || \ defined(__s390__) || defined(__s390x__) || \ + defined(__aarch64__) || \ defined(SHA512_ASM) # define SHA512_BLOCK_CAN_MANAGE_UNALIGNED_DATA # endif @@ -353,6 +354,18 @@ static const SHA_LONG64 K512[80] = { asm ("rotrdi %0,%1,%2" \ : "=r"(ret) \ : "r"(a),"K"(n)); ret; }) +# elif defined(__aarch64__) +# define ROTR(a,n) ({ SHA_LONG64 ret; \ + asm ("ror %0,%1,%2" \ + : "=r"(ret) \ + : "r"(a),"I"(n)); ret; }) +# if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ + __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define PULL64(x) ({ SHA_LONG64 ret; \ + asm ("rev %0,%1" \ + : "=r"(ret) \ + : "r"(*((const SHA_LONG64 *)(&(x))))); ret; }) +# endif # endif # elif defined(_MSC_VER) # if defined(_WIN64) /* applies to both IA-64 and AMD64 */ diff --git a/crypto/sparc_arch.h b/crypto/sparc_arch.h new file mode 100644 index 000000000000..e30d322a4ae2 --- /dev/null +++ b/crypto/sparc_arch.h @@ -0,0 +1,101 @@ +#ifndef __SPARC_ARCH_H__ +# define __SPARC_ARCH_H__ + +# define SPARCV9_TICK_PRIVILEGED (1<<0) +# define SPARCV9_PREFER_FPU (1<<1) +# define SPARCV9_VIS1 (1<<2) +# define SPARCV9_VIS2 (1<<3)/* reserved */ +# define SPARCV9_FMADD (1<<4)/* reserved for SPARC64 V */ +# define SPARCV9_BLK (1<<5)/* VIS1 block copy */ +# define SPARCV9_VIS3 (1<<6) +# define SPARCV9_RANDOM (1<<7) +# define SPARCV9_64BIT_STACK (1<<8) + +/* + * OPENSSL_sparcv9cap_P[1] is copy of Compatibility Feature Register, + * %asr26, SPARC-T4 and later. There is no SPARCV9_CFR bit in + * OPENSSL_sparcv9cap_P[0], as %cfr copy is sufficient... + */ +# define CFR_AES 0x00000001/* Supports AES opcodes */ +# define CFR_DES 0x00000002/* Supports DES opcodes */ +# define CFR_KASUMI 0x00000004/* Supports KASUMI opcodes */ +# define CFR_CAMELLIA 0x00000008/* Supports CAMELLIA opcodes */ +# define CFR_MD5 0x00000010/* Supports MD5 opcodes */ +# define CFR_SHA1 0x00000020/* Supports SHA1 opcodes */ +# define CFR_SHA256 0x00000040/* Supports SHA256 opcodes */ +# define CFR_SHA512 0x00000080/* Supports SHA512 opcodes */ +# define CFR_MPMUL 0x00000100/* Supports MPMUL opcodes */ +# define CFR_MONTMUL 0x00000200/* Supports MONTMUL opcodes */ +# define CFR_MONTSQR 0x00000400/* Supports MONTSQR opcodes */ +# define CFR_CRC32C 0x00000800/* Supports CRC32C opcodes */ + +# if defined(OPENSSL_PIC) && !defined(__PIC__) +# define __PIC__ +# endif + +# if defined(__SUNPRO_C) && defined(__sparcv9) && !defined(__arch64__) +# define __arch64__ +# endif + +# define SPARC_PIC_THUNK(reg) \ + .align 32; \ +.Lpic_thunk: \ + jmp %o7 + 8; \ + add %o7, reg, reg; + +# define SPARC_PIC_THUNK_CALL(reg) \ + sethi %hi(_GLOBAL_OFFSET_TABLE_-4), reg; \ + call .Lpic_thunk; \ + or reg, %lo(_GLOBAL_OFFSET_TABLE_+4), reg; + +# if 1 +# define SPARC_SETUP_GOT_REG(reg) SPARC_PIC_THUNK_CALL(reg) +# else +# define SPARC_SETUP_GOT_REG(reg) \ + sethi %hi(_GLOBAL_OFFSET_TABLE_-4), reg; \ + call .+8; \ + or reg,%lo(_GLOBAL_OFFSET_TABLE_+4), reg; \ + add %o7, reg, reg +# endif + +# if defined(__arch64__) + +# define SPARC_LOAD_ADDRESS(SYM, reg) \ + setx SYM, %o7, reg; +# define LDPTR ldx +# define SIZE_T_CC %xcc +# define STACK_FRAME 192 +# define STACK_BIAS 2047 +# define STACK_7thARG (STACK_BIAS+176) + +# else + +# define SPARC_LOAD_ADDRESS(SYM, reg) \ + set SYM, reg; +# define LDPTR ld +# define SIZE_T_CC %icc +# define STACK_FRAME 112 +# define STACK_BIAS 0 +# define STACK_7thARG 92 +# define SPARC_LOAD_ADDRESS_LEAF(SYM,reg,tmp) SPARC_LOAD_ADDRESS(SYM,reg) + +# endif + +# ifdef __PIC__ +# undef SPARC_LOAD_ADDRESS +# undef SPARC_LOAD_ADDRESS_LEAF +# define SPARC_LOAD_ADDRESS(SYM, reg) \ + SPARC_SETUP_GOT_REG(reg); \ + sethi %hi(SYM), %o7; \ + or %o7, %lo(SYM), %o7; \ + LDPTR [reg + %o7], reg; +# endif + +# ifndef SPARC_LOAD_ADDRESS_LEAF +# define SPARC_LOAD_ADDRESS_LEAF(SYM, reg, tmp) \ + mov %o7, tmp; \ + SPARC_LOAD_ADDRESS(SYM, reg) \ + mov tmp, %o7; +# endif + +#endif /* __SPARC_ARCH_H__ */ diff --git a/crypto/sparccpuid.S b/crypto/sparccpuid.S index 0cc247e48971..eea2006fba18 100644 --- a/crypto/sparccpuid.S +++ b/crypto/sparccpuid.S @@ -251,6 +251,11 @@ _sparcv9_vis1_probe: ! UltraSPARC IIe 7 ! UltraSPARC III 7 ! UltraSPARC T1 24 +! SPARC T4 65(*) +! +! (*) result has lesser to do with VIS instruction latencies, rdtick +! appears that slow, but it does the trick in sense that FP and +! VIS code paths are still slower than integer-only ones. ! ! Numbers for T2 and SPARC64 V-VII are more than welcomed. ! @@ -260,6 +265,8 @@ _sparcv9_vis1_probe: .global _sparcv9_vis1_instrument .align 8 _sparcv9_vis1_instrument: + .word 0x81b00d80 !fxor %f0,%f0,%f0 + .word 0x85b08d82 !fxor %f2,%f2,%f2 .word 0x91410000 !rd %tick,%o0 .word 0x81b00d80 !fxor %f0,%f0,%f0 .word 0x85b08d82 !fxor %f2,%f2,%f2 @@ -314,6 +321,30 @@ _sparcv9_fmadd_probe: .type _sparcv9_fmadd_probe,#function .size _sparcv9_fmadd_probe,.-_sparcv9_fmadd_probe +.global _sparcv9_rdcfr +.align 8 +_sparcv9_rdcfr: + retl + .word 0x91468000 !rd %asr26,%o0 +.type _sparcv9_rdcfr,#function +.size _sparcv9_rdcfr,.-_sparcv9_rdcfr + +.global _sparcv9_vis3_probe +.align 8 +_sparcv9_vis3_probe: + retl + .word 0x81b022a0 !xmulx %g0,%g0,%g0 +.type _sparcv9_vis3_probe,#function +.size _sparcv9_vis3_probe,.-_sparcv9_vis3_probe + +.global _sparcv9_random +.align 8 +_sparcv9_random: + retl + .word 0x91b002a0 !random %o0 +.type _sparcv9_random,#function +.size _sparcv9_random,.-_sparcv9_vis3_probe + .global OPENSSL_cleanse .align 32 OPENSSL_cleanse: @@ -397,6 +428,102 @@ OPENSSL_cleanse: .type OPENSSL_cleanse,#function .size OPENSSL_cleanse,.-OPENSSL_cleanse +.global _sparcv9_vis1_instrument_bus +.align 8 +_sparcv9_vis1_instrument_bus: + mov %o1,%o3 ! save cnt + .word 0x99410000 !rd %tick,%o4 ! tick + mov %o4,%o5 ! lasttick = tick + set 0,%g4 ! diff + + andn %o0,63,%g1 + .word 0xc1985e00 !ldda [%g1]0xf0,%f0 ! block load + .word 0x8143e040 !membar #Sync + .word 0xc1b85c00 !stda %f0,[%g1]0xe0 ! block store and commit + .word 0x8143e040 !membar #Sync + ld [%o0],%o4 + add %o4,%g4,%g4 + .word 0xc9e2100c !cas [%o0],%o4,%g4 + +.Loop: .word 0x99410000 !rd %tick,%o4 + sub %o4,%o5,%g4 ! diff=tick-lasttick + mov %o4,%o5 ! lasttick=tick + + andn %o0,63,%g1 + .word 0xc1985e00 !ldda [%g1]0xf0,%f0 ! block load + .word 0x8143e040 !membar #Sync + .word 0xc1b85c00 !stda %f0,[%g1]0xe0 ! block store and commit + .word 0x8143e040 !membar #Sync + ld [%o0],%o4 + add %o4,%g4,%g4 + .word 0xc9e2100c !cas [%o0],%o4,%g4 + subcc %o1,1,%o1 ! --$cnt + bnz .Loop + add %o0,4,%o0 ! ++$out + + retl + mov %o3,%o0 +.type _sparcv9_vis1_instrument_bus,#function +.size _sparcv9_vis1_instrument_bus,.-_sparcv9_vis1_instrument_bus + +.global _sparcv9_vis1_instrument_bus2 +.align 8 +_sparcv9_vis1_instrument_bus2: + mov %o1,%o3 ! save cnt + sll %o1,2,%o1 ! cnt*=4 + + .word 0x99410000 !rd %tick,%o4 ! tick + mov %o4,%o5 ! lasttick = tick + set 0,%g4 ! diff + + andn %o0,63,%g1 + .word 0xc1985e00 !ldda [%g1]0xf0,%f0 ! block load + .word 0x8143e040 !membar #Sync + .word 0xc1b85c00 !stda %f0,[%g1]0xe0 ! block store and commit + .word 0x8143e040 !membar #Sync + ld [%o0],%o4 + add %o4,%g4,%g4 + .word 0xc9e2100c !cas [%o0],%o4,%g4 + + .word 0x99410000 !rd %tick,%o4 ! tick + sub %o4,%o5,%g4 ! diff=tick-lasttick + mov %o4,%o5 ! lasttick=tick + mov %g4,%g5 ! lastdiff=diff +.Loop2: + andn %o0,63,%g1 + .word 0xc1985e00 !ldda [%g1]0xf0,%f0 ! block load + .word 0x8143e040 !membar #Sync + .word 0xc1b85c00 !stda %f0,[%g1]0xe0 ! block store and commit + .word 0x8143e040 !membar #Sync + ld [%o0],%o4 + add %o4,%g4,%g4 + .word 0xc9e2100c !cas [%o0],%o4,%g4 + + subcc %o2,1,%o2 ! --max + bz .Ldone2 + nop + + .word 0x99410000 !rd %tick,%o4 ! tick + sub %o4,%o5,%g4 ! diff=tick-lasttick + mov %o4,%o5 ! lasttick=tick + cmp %g4,%g5 + mov %g4,%g5 ! lastdiff=diff + + .word 0x83408000 !rd %ccr,%g1 + and %g1,4,%g1 ! isolate zero flag + xor %g1,4,%g1 ! flip zero flag + + subcc %o1,%g1,%o1 ! conditional --$cnt + bnz .Loop2 + add %o0,%g1,%o0 ! conditional ++$out + +.Ldone2: + srl %o1,2,%o1 + retl + sub %o3,%o1,%o0 +.type _sparcv9_vis1_instrument_bus2,#function +.size _sparcv9_vis1_instrument_bus2,.-_sparcv9_vis1_instrument_bus2 + .section ".init",#alloc,#execinstr call OPENSSL_cpuid_setup nop diff --git a/crypto/sparcv9cap.c b/crypto/sparcv9cap.c index d9f986f6b968..8bf2846929b1 100644 --- a/crypto/sparcv9cap.c +++ b/crypto/sparcv9cap.c @@ -4,30 +4,68 @@ #include #include #include +#include #include -#define SPARCV9_TICK_PRIVILEGED (1<<0) -#define SPARCV9_PREFER_FPU (1<<1) -#define SPARCV9_VIS1 (1<<2) -#define SPARCV9_VIS2 (1<<3) /* reserved */ -#define SPARCV9_FMADD (1<<4) /* reserved for SPARC64 V */ +#include "sparc_arch.h" -static int OPENSSL_sparcv9cap_P = SPARCV9_TICK_PRIVILEGED; +#if defined(__GNUC__) && defined(__linux) +__attribute__ ((visibility("hidden"))) +#endif +unsigned int OPENSSL_sparcv9cap_P[2] = { SPARCV9_TICK_PRIVILEGED, 0 }; int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, int num) { + int bn_mul_mont_vis3(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, + const BN_ULONG *np, const BN_ULONG *n0, int num); int bn_mul_mont_fpu(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, int num); int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, int num); - if (num >= 8 && !(num & 1) && - (OPENSSL_sparcv9cap_P & (SPARCV9_PREFER_FPU | SPARCV9_VIS1)) == - (SPARCV9_PREFER_FPU | SPARCV9_VIS1)) - return bn_mul_mont_fpu(rp, ap, bp, np, n0, num); - else - return bn_mul_mont_int(rp, ap, bp, np, n0, num); + if (!(num & 1) && num >= 6) { + if ((num & 15) == 0 && num <= 64 && + (OPENSSL_sparcv9cap_P[1] & (CFR_MONTMUL | CFR_MONTSQR)) == + (CFR_MONTMUL | CFR_MONTSQR)) { + typedef int (*bn_mul_mont_f) (BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *bp, + const BN_ULONG *np, + const BN_ULONG *n0); + int bn_mul_mont_t4_8(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *bp, const BN_ULONG *np, + const BN_ULONG *n0); + int bn_mul_mont_t4_16(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *bp, const BN_ULONG *np, + const BN_ULONG *n0); + int bn_mul_mont_t4_24(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *bp, const BN_ULONG *np, + const BN_ULONG *n0); + int bn_mul_mont_t4_32(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *bp, const BN_ULONG *np, + const BN_ULONG *n0); + static const bn_mul_mont_f funcs[4] = { + bn_mul_mont_t4_8, bn_mul_mont_t4_16, + bn_mul_mont_t4_24, bn_mul_mont_t4_32 + }; + bn_mul_mont_f worker = funcs[num / 16 - 1]; + + if ((*worker) (rp, ap, bp, np, n0)) + return 1; + /* retry once and fall back */ + if ((*worker) (rp, ap, bp, np, n0)) + return 1; + return bn_mul_mont_vis3(rp, ap, bp, np, n0, num); + } + if ((OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3)) + return bn_mul_mont_vis3(rp, ap, bp, np, n0, num); + else if (num >= 8 && + (OPENSSL_sparcv9cap_P[0] & + (SPARCV9_PREFER_FPU | SPARCV9_VIS1)) == + (SPARCV9_PREFER_FPU | SPARCV9_VIS1)) + return bn_mul_mont_fpu(rp, ap, bp, np, n0, num); + } + return bn_mul_mont_int(rp, ap, bp, np, n0, num); } unsigned long _sparcv9_rdtick(void); @@ -35,10 +73,15 @@ void _sparcv9_vis1_probe(void); unsigned long _sparcv9_vis1_instrument(void); void _sparcv9_vis2_probe(void); void _sparcv9_fmadd_probe(void); +unsigned long _sparcv9_rdcfr(void); +void _sparcv9_vis3_probe(void); +unsigned long _sparcv9_random(void); +size_t _sparcv9_vis1_instrument_bus(unsigned int *, size_t); +size_t _sparcv9_vis1_instrument_bus2(unsigned int *, size_t, size_t); unsigned long OPENSSL_rdtsc(void) { - if (OPENSSL_sparcv9cap_P & SPARCV9_TICK_PRIVILEGED) + if (OPENSSL_sparcv9cap_P[0] & SPARCV9_TICK_PRIVILEGED) #if defined(__sun) && defined(__SVR4) return gethrtime(); #else @@ -48,6 +91,24 @@ unsigned long OPENSSL_rdtsc(void) return _sparcv9_rdtick(); } +size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt) +{ + if ((OPENSSL_sparcv9cap_P[0] & (SPARCV9_TICK_PRIVILEGED | SPARCV9_BLK)) == + SPARCV9_BLK) + return _sparcv9_vis1_instrument_bus(out, cnt); + else + return 0; +} + +size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max) +{ + if ((OPENSSL_sparcv9cap_P[0] & (SPARCV9_TICK_PRIVILEGED | SPARCV9_BLK)) == + SPARCV9_BLK) + return _sparcv9_vis1_instrument_bus2(out, cnt, max); + else + return 0; +} + #if 0 && defined(__sun) && defined(__SVR4) /* * This code path is disabled, because of incompatibility of libdevinfo.so.1 @@ -74,17 +135,17 @@ static int walk_nodename(di_node_t node, di_node_name_t di_node_name) if (!strcmp(name, "SUNW,UltraSPARC") || /* covers II,III,IV */ !strncmp(name, "SUNW,UltraSPARC-I", 17)) { - OPENSSL_sparcv9cap_P |= SPARCV9_PREFER_FPU | SPARCV9_VIS1; + OPENSSL_sparcv9cap_P[0] |= SPARCV9_PREFER_FPU | SPARCV9_VIS1; /* %tick is privileged only on UltraSPARC-I/II, but not IIe */ if (name[14] != '\0' && name[17] != '\0' && name[18] != '\0') - OPENSSL_sparcv9cap_P &= ~SPARCV9_TICK_PRIVILEGED; + OPENSSL_sparcv9cap_P[0] &= ~SPARCV9_TICK_PRIVILEGED; return DI_WALK_TERMINATE; } /* This is expected to catch remaining UltraSPARCs, such as T1 */ else if (!strncmp(name, "SUNW,UltraSPARC", 15)) { - OPENSSL_sparcv9cap_P &= ~SPARCV9_TICK_PRIVILEGED; + OPENSSL_sparcv9cap_P[0] &= ~SPARCV9_TICK_PRIVILEGED; return DI_WALK_TERMINATE; } @@ -103,22 +164,22 @@ void OPENSSL_cpuid_setup(void) trigger = 1; if ((e = getenv("OPENSSL_sparcv9cap"))) { - OPENSSL_sparcv9cap_P = strtoul(e, NULL, 0); + OPENSSL_sparcv9cap_P[0] = strtoul(e, NULL, 0); return; } if (sysinfo(SI_MACHINE, si, sizeof(si)) > 0) { if (strcmp(si, "sun4v")) /* FPU is preferred for all CPUs, but US-T1/2 */ - OPENSSL_sparcv9cap_P |= SPARCV9_PREFER_FPU; + OPENSSL_sparcv9cap_P[0] |= SPARCV9_PREFER_FPU; } if (sysinfo(SI_ISALIST, si, sizeof(si)) > 0) { if (strstr(si, "+vis")) - OPENSSL_sparcv9cap_P |= SPARCV9_VIS1; + OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS1 | SPARCV9_BLK; if (strstr(si, "+vis2")) { - OPENSSL_sparcv9cap_P |= SPARCV9_VIS2; - OPENSSL_sparcv9cap_P &= ~SPARCV9_TICK_PRIVILEGED; + OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS2; + OPENSSL_sparcv9cap_P[0] &= ~SPARCV9_TICK_PRIVILEGED; return; } } @@ -188,12 +249,14 @@ void OPENSSL_cpuid_setup(void) trigger = 1; if ((e = getenv("OPENSSL_sparcv9cap"))) { - OPENSSL_sparcv9cap_P = strtoul(e, NULL, 0); + OPENSSL_sparcv9cap_P[0] = strtoul(e, NULL, 0); + if ((e = strchr(e, ':'))) + OPENSSL_sparcv9cap_P[1] = strtoul(e + 1, NULL, 0); return; } /* Initial value, fits UltraSPARC-I&II... */ - OPENSSL_sparcv9cap_P = SPARCV9_PREFER_FPU | SPARCV9_TICK_PRIVILEGED; + OPENSSL_sparcv9cap_P[0] = SPARCV9_PREFER_FPU | SPARCV9_TICK_PRIVILEGED; sigfillset(&all_masked); sigdelset(&all_masked, SIGILL); @@ -216,30 +279,68 @@ void OPENSSL_cpuid_setup(void) if (sigsetjmp(common_jmp, 1) == 0) { _sparcv9_rdtick(); - OPENSSL_sparcv9cap_P &= ~SPARCV9_TICK_PRIVILEGED; + OPENSSL_sparcv9cap_P[0] &= ~SPARCV9_TICK_PRIVILEGED; } if (sigsetjmp(common_jmp, 1) == 0) { _sparcv9_vis1_probe(); - OPENSSL_sparcv9cap_P |= SPARCV9_VIS1; + OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS1 | SPARCV9_BLK; /* detect UltraSPARC-Tx, see sparccpud.S for details... */ if (_sparcv9_vis1_instrument() >= 12) - OPENSSL_sparcv9cap_P &= ~(SPARCV9_VIS1 | SPARCV9_PREFER_FPU); + OPENSSL_sparcv9cap_P[0] &= ~(SPARCV9_VIS1 | SPARCV9_PREFER_FPU); else { _sparcv9_vis2_probe(); - OPENSSL_sparcv9cap_P |= SPARCV9_VIS2; + OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS2; } } if (sigsetjmp(common_jmp, 1) == 0) { _sparcv9_fmadd_probe(); - OPENSSL_sparcv9cap_P |= SPARCV9_FMADD; + OPENSSL_sparcv9cap_P[0] |= SPARCV9_FMADD; + } + + /* + * VIS3 flag is tested independently from VIS1, unlike VIS2 that is, + * because VIS3 defines even integer instructions. + */ + if (sigsetjmp(common_jmp, 1) == 0) { + _sparcv9_vis3_probe(); + OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS3; + } +# if 0 /* was planned at some point but never + * implemented in hardware */ + if (sigsetjmp(common_jmp, 1) == 0) { + (void)_sparcv9_random(); + OPENSSL_sparcv9cap_P[0] |= SPARCV9_RANDOM; + } +# endif + + /* + * In wait for better solution _sparcv9_rdcfr is masked by + * VIS3 flag, because it goes to uninterruptable endless + * loop on UltraSPARC II running Solaris. Things might be + * different on Linux... + */ + if ((OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) && + sigsetjmp(common_jmp, 1) == 0) { + OPENSSL_sparcv9cap_P[1] = (unsigned int)_sparcv9_rdcfr(); } sigaction(SIGBUS, &bus_oact, NULL); sigaction(SIGILL, &ill_oact, NULL); sigprocmask(SIG_SETMASK, &oset, NULL); + + if (sizeof(size_t) == 8) + OPENSSL_sparcv9cap_P[0] |= SPARCV9_64BIT_STACK; +# ifdef __linux + else { + int ret = syscall(340); + + if (ret >= 0 && ret & 1) + OPENSSL_sparcv9cap_P[0] |= SPARCV9_64BIT_STACK; + } +# endif } #endif diff --git a/crypto/srp/Makefile b/crypto/srp/Makefile index 763953384b19..414af7bc66b1 100644 --- a/crypto/srp/Makefile +++ b/crypto/srp/Makefile @@ -37,6 +37,9 @@ lib: $(LIBOBJ) $(RANLIB) $(LIB) || echo Never mind. @touch lib +files: + $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO + links: @$(PERL) $(TOP)/util/mklink.pl ../../include/openssl $(EXHEADER) @$(PERL) $(TOP)/util/mklink.pl ../../test $(TEST) diff --git a/crypto/srp/srptest.c b/crypto/srp/srptest.c index 451c70e40f56..00e001bd6b83 100644 --- a/crypto/srp/srptest.c +++ b/crypto/srp/srptest.c @@ -148,6 +148,7 @@ int main(int argc, char **argv) ERR_remove_thread_state(NULL); ERR_free_strings(); CRYPTO_mem_leaks(bio_err); + BIO_free(bio_err); return 0; } diff --git a/crypto/stack/safestack.h b/crypto/stack/safestack.h index 519649b6ed0e..1d4f87eab34d 100644 --- a/crypto/stack/safestack.h +++ b/crypto/stack/safestack.h @@ -75,12 +75,12 @@ extern "C" { # define CHECKED_STACK_OF(type, p) \ ((_STACK*) (1 ? p : (STACK_OF(type)*)0)) +# define CHECKED_SK_COPY_FUNC(type, p) \ + ((void *(*)(void *)) ((1 ? p : (type *(*)(const type *))0))) + # define CHECKED_SK_FREE_FUNC(type, p) \ ((void (*)(void *)) ((1 ? p : (void (*)(type *))0))) -# define CHECKED_SK_FREE_FUNC2(type, p) \ - ((void (*)(void *)) ((1 ? p : (void (*)(type))0))) - # define CHECKED_SK_CMP_FUNC(type, p) \ ((int (*)(const void *, const void *)) \ ((1 ? p : (int (*)(const type * const *, const type * const *))0))) @@ -177,6 +177,8 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) (STACK_OF(type) *)sk_dup(CHECKED_STACK_OF(type, st)) # define SKM_sk_pop_free(type, st, free_func) \ sk_pop_free(CHECKED_STACK_OF(type, st), CHECKED_SK_FREE_FUNC(type, free_func)) +# define SKM_sk_deep_copy(type, st, copy_func, free_func) \ + (STACK_OF(type) *)sk_deep_copy(CHECKED_STACK_OF(type, st), CHECKED_SK_COPY_FUNC(type, copy_func), CHECKED_SK_FREE_FUNC(type, free_func)) # define SKM_sk_shift(type, st) \ (type *)sk_shift(CHECKED_STACK_OF(type, st)) # define SKM_sk_pop(type, st) \ @@ -226,6 +228,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ACCESS_DESCRIPTION_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ACCESS_DESCRIPTION, (st), (cmp)) # define sk_ACCESS_DESCRIPTION_dup(st) SKM_sk_dup(ACCESS_DESCRIPTION, st) # define sk_ACCESS_DESCRIPTION_pop_free(st, free_func) SKM_sk_pop_free(ACCESS_DESCRIPTION, (st), (free_func)) +# define sk_ACCESS_DESCRIPTION_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ACCESS_DESCRIPTION, (st), (copy_func), (free_func)) # define sk_ACCESS_DESCRIPTION_shift(st) SKM_sk_shift(ACCESS_DESCRIPTION, (st)) # define sk_ACCESS_DESCRIPTION_pop(st) SKM_sk_pop(ACCESS_DESCRIPTION, (st)) # define sk_ACCESS_DESCRIPTION_sort(st) SKM_sk_sort(ACCESS_DESCRIPTION, (st)) @@ -247,6 +250,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASIdOrRange_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASIdOrRange, (st), (cmp)) # define sk_ASIdOrRange_dup(st) SKM_sk_dup(ASIdOrRange, st) # define sk_ASIdOrRange_pop_free(st, free_func) SKM_sk_pop_free(ASIdOrRange, (st), (free_func)) +# define sk_ASIdOrRange_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASIdOrRange, (st), (copy_func), (free_func)) # define sk_ASIdOrRange_shift(st) SKM_sk_shift(ASIdOrRange, (st)) # define sk_ASIdOrRange_pop(st) SKM_sk_pop(ASIdOrRange, (st)) # define sk_ASIdOrRange_sort(st) SKM_sk_sort(ASIdOrRange, (st)) @@ -268,6 +272,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASN1_GENERALSTRING_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASN1_GENERALSTRING, (st), (cmp)) # define sk_ASN1_GENERALSTRING_dup(st) SKM_sk_dup(ASN1_GENERALSTRING, st) # define sk_ASN1_GENERALSTRING_pop_free(st, free_func) SKM_sk_pop_free(ASN1_GENERALSTRING, (st), (free_func)) +# define sk_ASN1_GENERALSTRING_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASN1_GENERALSTRING, (st), (copy_func), (free_func)) # define sk_ASN1_GENERALSTRING_shift(st) SKM_sk_shift(ASN1_GENERALSTRING, (st)) # define sk_ASN1_GENERALSTRING_pop(st) SKM_sk_pop(ASN1_GENERALSTRING, (st)) # define sk_ASN1_GENERALSTRING_sort(st) SKM_sk_sort(ASN1_GENERALSTRING, (st)) @@ -289,6 +294,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASN1_INTEGER_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASN1_INTEGER, (st), (cmp)) # define sk_ASN1_INTEGER_dup(st) SKM_sk_dup(ASN1_INTEGER, st) # define sk_ASN1_INTEGER_pop_free(st, free_func) SKM_sk_pop_free(ASN1_INTEGER, (st), (free_func)) +# define sk_ASN1_INTEGER_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASN1_INTEGER, (st), (copy_func), (free_func)) # define sk_ASN1_INTEGER_shift(st) SKM_sk_shift(ASN1_INTEGER, (st)) # define sk_ASN1_INTEGER_pop(st) SKM_sk_pop(ASN1_INTEGER, (st)) # define sk_ASN1_INTEGER_sort(st) SKM_sk_sort(ASN1_INTEGER, (st)) @@ -310,6 +316,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASN1_OBJECT_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASN1_OBJECT, (st), (cmp)) # define sk_ASN1_OBJECT_dup(st) SKM_sk_dup(ASN1_OBJECT, st) # define sk_ASN1_OBJECT_pop_free(st, free_func) SKM_sk_pop_free(ASN1_OBJECT, (st), (free_func)) +# define sk_ASN1_OBJECT_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASN1_OBJECT, (st), (copy_func), (free_func)) # define sk_ASN1_OBJECT_shift(st) SKM_sk_shift(ASN1_OBJECT, (st)) # define sk_ASN1_OBJECT_pop(st) SKM_sk_pop(ASN1_OBJECT, (st)) # define sk_ASN1_OBJECT_sort(st) SKM_sk_sort(ASN1_OBJECT, (st)) @@ -331,6 +338,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASN1_STRING_TABLE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASN1_STRING_TABLE, (st), (cmp)) # define sk_ASN1_STRING_TABLE_dup(st) SKM_sk_dup(ASN1_STRING_TABLE, st) # define sk_ASN1_STRING_TABLE_pop_free(st, free_func) SKM_sk_pop_free(ASN1_STRING_TABLE, (st), (free_func)) +# define sk_ASN1_STRING_TABLE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASN1_STRING_TABLE, (st), (copy_func), (free_func)) # define sk_ASN1_STRING_TABLE_shift(st) SKM_sk_shift(ASN1_STRING_TABLE, (st)) # define sk_ASN1_STRING_TABLE_pop(st) SKM_sk_pop(ASN1_STRING_TABLE, (st)) # define sk_ASN1_STRING_TABLE_sort(st) SKM_sk_sort(ASN1_STRING_TABLE, (st)) @@ -352,6 +360,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASN1_TYPE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASN1_TYPE, (st), (cmp)) # define sk_ASN1_TYPE_dup(st) SKM_sk_dup(ASN1_TYPE, st) # define sk_ASN1_TYPE_pop_free(st, free_func) SKM_sk_pop_free(ASN1_TYPE, (st), (free_func)) +# define sk_ASN1_TYPE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASN1_TYPE, (st), (copy_func), (free_func)) # define sk_ASN1_TYPE_shift(st) SKM_sk_shift(ASN1_TYPE, (st)) # define sk_ASN1_TYPE_pop(st) SKM_sk_pop(ASN1_TYPE, (st)) # define sk_ASN1_TYPE_sort(st) SKM_sk_sort(ASN1_TYPE, (st)) @@ -373,6 +382,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASN1_UTF8STRING_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASN1_UTF8STRING, (st), (cmp)) # define sk_ASN1_UTF8STRING_dup(st) SKM_sk_dup(ASN1_UTF8STRING, st) # define sk_ASN1_UTF8STRING_pop_free(st, free_func) SKM_sk_pop_free(ASN1_UTF8STRING, (st), (free_func)) +# define sk_ASN1_UTF8STRING_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASN1_UTF8STRING, (st), (copy_func), (free_func)) # define sk_ASN1_UTF8STRING_shift(st) SKM_sk_shift(ASN1_UTF8STRING, (st)) # define sk_ASN1_UTF8STRING_pop(st) SKM_sk_pop(ASN1_UTF8STRING, (st)) # define sk_ASN1_UTF8STRING_sort(st) SKM_sk_sort(ASN1_UTF8STRING, (st)) @@ -394,6 +404,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ASN1_VALUE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ASN1_VALUE, (st), (cmp)) # define sk_ASN1_VALUE_dup(st) SKM_sk_dup(ASN1_VALUE, st) # define sk_ASN1_VALUE_pop_free(st, free_func) SKM_sk_pop_free(ASN1_VALUE, (st), (free_func)) +# define sk_ASN1_VALUE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ASN1_VALUE, (st), (copy_func), (free_func)) # define sk_ASN1_VALUE_shift(st) SKM_sk_shift(ASN1_VALUE, (st)) # define sk_ASN1_VALUE_pop(st) SKM_sk_pop(ASN1_VALUE, (st)) # define sk_ASN1_VALUE_sort(st) SKM_sk_sort(ASN1_VALUE, (st)) @@ -415,6 +426,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_BIO_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(BIO, (st), (cmp)) # define sk_BIO_dup(st) SKM_sk_dup(BIO, st) # define sk_BIO_pop_free(st, free_func) SKM_sk_pop_free(BIO, (st), (free_func)) +# define sk_BIO_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(BIO, (st), (copy_func), (free_func)) # define sk_BIO_shift(st) SKM_sk_shift(BIO, (st)) # define sk_BIO_pop(st) SKM_sk_pop(BIO, (st)) # define sk_BIO_sort(st) SKM_sk_sort(BIO, (st)) @@ -436,6 +448,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_BY_DIR_ENTRY_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(BY_DIR_ENTRY, (st), (cmp)) # define sk_BY_DIR_ENTRY_dup(st) SKM_sk_dup(BY_DIR_ENTRY, st) # define sk_BY_DIR_ENTRY_pop_free(st, free_func) SKM_sk_pop_free(BY_DIR_ENTRY, (st), (free_func)) +# define sk_BY_DIR_ENTRY_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(BY_DIR_ENTRY, (st), (copy_func), (free_func)) # define sk_BY_DIR_ENTRY_shift(st) SKM_sk_shift(BY_DIR_ENTRY, (st)) # define sk_BY_DIR_ENTRY_pop(st) SKM_sk_pop(BY_DIR_ENTRY, (st)) # define sk_BY_DIR_ENTRY_sort(st) SKM_sk_sort(BY_DIR_ENTRY, (st)) @@ -457,6 +470,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_BY_DIR_HASH_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(BY_DIR_HASH, (st), (cmp)) # define sk_BY_DIR_HASH_dup(st) SKM_sk_dup(BY_DIR_HASH, st) # define sk_BY_DIR_HASH_pop_free(st, free_func) SKM_sk_pop_free(BY_DIR_HASH, (st), (free_func)) +# define sk_BY_DIR_HASH_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(BY_DIR_HASH, (st), (copy_func), (free_func)) # define sk_BY_DIR_HASH_shift(st) SKM_sk_shift(BY_DIR_HASH, (st)) # define sk_BY_DIR_HASH_pop(st) SKM_sk_pop(BY_DIR_HASH, (st)) # define sk_BY_DIR_HASH_sort(st) SKM_sk_sort(BY_DIR_HASH, (st)) @@ -478,10 +492,33 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CMS_CertificateChoices_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CMS_CertificateChoices, (st), (cmp)) # define sk_CMS_CertificateChoices_dup(st) SKM_sk_dup(CMS_CertificateChoices, st) # define sk_CMS_CertificateChoices_pop_free(st, free_func) SKM_sk_pop_free(CMS_CertificateChoices, (st), (free_func)) +# define sk_CMS_CertificateChoices_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CMS_CertificateChoices, (st), (copy_func), (free_func)) # define sk_CMS_CertificateChoices_shift(st) SKM_sk_shift(CMS_CertificateChoices, (st)) # define sk_CMS_CertificateChoices_pop(st) SKM_sk_pop(CMS_CertificateChoices, (st)) # define sk_CMS_CertificateChoices_sort(st) SKM_sk_sort(CMS_CertificateChoices, (st)) # define sk_CMS_CertificateChoices_is_sorted(st) SKM_sk_is_sorted(CMS_CertificateChoices, (st)) +# define sk_CMS_RecipientEncryptedKey_new(cmp) SKM_sk_new(CMS_RecipientEncryptedKey, (cmp)) +# define sk_CMS_RecipientEncryptedKey_new_null() SKM_sk_new_null(CMS_RecipientEncryptedKey) +# define sk_CMS_RecipientEncryptedKey_free(st) SKM_sk_free(CMS_RecipientEncryptedKey, (st)) +# define sk_CMS_RecipientEncryptedKey_num(st) SKM_sk_num(CMS_RecipientEncryptedKey, (st)) +# define sk_CMS_RecipientEncryptedKey_value(st, i) SKM_sk_value(CMS_RecipientEncryptedKey, (st), (i)) +# define sk_CMS_RecipientEncryptedKey_set(st, i, val) SKM_sk_set(CMS_RecipientEncryptedKey, (st), (i), (val)) +# define sk_CMS_RecipientEncryptedKey_zero(st) SKM_sk_zero(CMS_RecipientEncryptedKey, (st)) +# define sk_CMS_RecipientEncryptedKey_push(st, val) SKM_sk_push(CMS_RecipientEncryptedKey, (st), (val)) +# define sk_CMS_RecipientEncryptedKey_unshift(st, val) SKM_sk_unshift(CMS_RecipientEncryptedKey, (st), (val)) +# define sk_CMS_RecipientEncryptedKey_find(st, val) SKM_sk_find(CMS_RecipientEncryptedKey, (st), (val)) +# define sk_CMS_RecipientEncryptedKey_find_ex(st, val) SKM_sk_find_ex(CMS_RecipientEncryptedKey, (st), (val)) +# define sk_CMS_RecipientEncryptedKey_delete(st, i) SKM_sk_delete(CMS_RecipientEncryptedKey, (st), (i)) +# define sk_CMS_RecipientEncryptedKey_delete_ptr(st, ptr) SKM_sk_delete_ptr(CMS_RecipientEncryptedKey, (st), (ptr)) +# define sk_CMS_RecipientEncryptedKey_insert(st, val, i) SKM_sk_insert(CMS_RecipientEncryptedKey, (st), (val), (i)) +# define sk_CMS_RecipientEncryptedKey_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CMS_RecipientEncryptedKey, (st), (cmp)) +# define sk_CMS_RecipientEncryptedKey_dup(st) SKM_sk_dup(CMS_RecipientEncryptedKey, st) +# define sk_CMS_RecipientEncryptedKey_pop_free(st, free_func) SKM_sk_pop_free(CMS_RecipientEncryptedKey, (st), (free_func)) +# define sk_CMS_RecipientEncryptedKey_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CMS_RecipientEncryptedKey, (st), (copy_func), (free_func)) +# define sk_CMS_RecipientEncryptedKey_shift(st) SKM_sk_shift(CMS_RecipientEncryptedKey, (st)) +# define sk_CMS_RecipientEncryptedKey_pop(st) SKM_sk_pop(CMS_RecipientEncryptedKey, (st)) +# define sk_CMS_RecipientEncryptedKey_sort(st) SKM_sk_sort(CMS_RecipientEncryptedKey, (st)) +# define sk_CMS_RecipientEncryptedKey_is_sorted(st) SKM_sk_is_sorted(CMS_RecipientEncryptedKey, (st)) # define sk_CMS_RecipientInfo_new(cmp) SKM_sk_new(CMS_RecipientInfo, (cmp)) # define sk_CMS_RecipientInfo_new_null() SKM_sk_new_null(CMS_RecipientInfo) # define sk_CMS_RecipientInfo_free(st) SKM_sk_free(CMS_RecipientInfo, (st)) @@ -499,6 +536,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CMS_RecipientInfo_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CMS_RecipientInfo, (st), (cmp)) # define sk_CMS_RecipientInfo_dup(st) SKM_sk_dup(CMS_RecipientInfo, st) # define sk_CMS_RecipientInfo_pop_free(st, free_func) SKM_sk_pop_free(CMS_RecipientInfo, (st), (free_func)) +# define sk_CMS_RecipientInfo_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CMS_RecipientInfo, (st), (copy_func), (free_func)) # define sk_CMS_RecipientInfo_shift(st) SKM_sk_shift(CMS_RecipientInfo, (st)) # define sk_CMS_RecipientInfo_pop(st) SKM_sk_pop(CMS_RecipientInfo, (st)) # define sk_CMS_RecipientInfo_sort(st) SKM_sk_sort(CMS_RecipientInfo, (st)) @@ -520,6 +558,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CMS_RevocationInfoChoice_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CMS_RevocationInfoChoice, (st), (cmp)) # define sk_CMS_RevocationInfoChoice_dup(st) SKM_sk_dup(CMS_RevocationInfoChoice, st) # define sk_CMS_RevocationInfoChoice_pop_free(st, free_func) SKM_sk_pop_free(CMS_RevocationInfoChoice, (st), (free_func)) +# define sk_CMS_RevocationInfoChoice_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CMS_RevocationInfoChoice, (st), (copy_func), (free_func)) # define sk_CMS_RevocationInfoChoice_shift(st) SKM_sk_shift(CMS_RevocationInfoChoice, (st)) # define sk_CMS_RevocationInfoChoice_pop(st) SKM_sk_pop(CMS_RevocationInfoChoice, (st)) # define sk_CMS_RevocationInfoChoice_sort(st) SKM_sk_sort(CMS_RevocationInfoChoice, (st)) @@ -541,6 +580,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CMS_SignerInfo_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CMS_SignerInfo, (st), (cmp)) # define sk_CMS_SignerInfo_dup(st) SKM_sk_dup(CMS_SignerInfo, st) # define sk_CMS_SignerInfo_pop_free(st, free_func) SKM_sk_pop_free(CMS_SignerInfo, (st), (free_func)) +# define sk_CMS_SignerInfo_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CMS_SignerInfo, (st), (copy_func), (free_func)) # define sk_CMS_SignerInfo_shift(st) SKM_sk_shift(CMS_SignerInfo, (st)) # define sk_CMS_SignerInfo_pop(st) SKM_sk_pop(CMS_SignerInfo, (st)) # define sk_CMS_SignerInfo_sort(st) SKM_sk_sort(CMS_SignerInfo, (st)) @@ -562,6 +602,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CONF_IMODULE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CONF_IMODULE, (st), (cmp)) # define sk_CONF_IMODULE_dup(st) SKM_sk_dup(CONF_IMODULE, st) # define sk_CONF_IMODULE_pop_free(st, free_func) SKM_sk_pop_free(CONF_IMODULE, (st), (free_func)) +# define sk_CONF_IMODULE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CONF_IMODULE, (st), (copy_func), (free_func)) # define sk_CONF_IMODULE_shift(st) SKM_sk_shift(CONF_IMODULE, (st)) # define sk_CONF_IMODULE_pop(st) SKM_sk_pop(CONF_IMODULE, (st)) # define sk_CONF_IMODULE_sort(st) SKM_sk_sort(CONF_IMODULE, (st)) @@ -583,6 +624,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CONF_MODULE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CONF_MODULE, (st), (cmp)) # define sk_CONF_MODULE_dup(st) SKM_sk_dup(CONF_MODULE, st) # define sk_CONF_MODULE_pop_free(st, free_func) SKM_sk_pop_free(CONF_MODULE, (st), (free_func)) +# define sk_CONF_MODULE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CONF_MODULE, (st), (copy_func), (free_func)) # define sk_CONF_MODULE_shift(st) SKM_sk_shift(CONF_MODULE, (st)) # define sk_CONF_MODULE_pop(st) SKM_sk_pop(CONF_MODULE, (st)) # define sk_CONF_MODULE_sort(st) SKM_sk_sort(CONF_MODULE, (st)) @@ -604,6 +646,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CONF_VALUE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CONF_VALUE, (st), (cmp)) # define sk_CONF_VALUE_dup(st) SKM_sk_dup(CONF_VALUE, st) # define sk_CONF_VALUE_pop_free(st, free_func) SKM_sk_pop_free(CONF_VALUE, (st), (free_func)) +# define sk_CONF_VALUE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CONF_VALUE, (st), (copy_func), (free_func)) # define sk_CONF_VALUE_shift(st) SKM_sk_shift(CONF_VALUE, (st)) # define sk_CONF_VALUE_pop(st) SKM_sk_pop(CONF_VALUE, (st)) # define sk_CONF_VALUE_sort(st) SKM_sk_sort(CONF_VALUE, (st)) @@ -625,6 +668,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CRYPTO_EX_DATA_FUNCS_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CRYPTO_EX_DATA_FUNCS, (st), (cmp)) # define sk_CRYPTO_EX_DATA_FUNCS_dup(st) SKM_sk_dup(CRYPTO_EX_DATA_FUNCS, st) # define sk_CRYPTO_EX_DATA_FUNCS_pop_free(st, free_func) SKM_sk_pop_free(CRYPTO_EX_DATA_FUNCS, (st), (free_func)) +# define sk_CRYPTO_EX_DATA_FUNCS_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CRYPTO_EX_DATA_FUNCS, (st), (copy_func), (free_func)) # define sk_CRYPTO_EX_DATA_FUNCS_shift(st) SKM_sk_shift(CRYPTO_EX_DATA_FUNCS, (st)) # define sk_CRYPTO_EX_DATA_FUNCS_pop(st) SKM_sk_pop(CRYPTO_EX_DATA_FUNCS, (st)) # define sk_CRYPTO_EX_DATA_FUNCS_sort(st) SKM_sk_sort(CRYPTO_EX_DATA_FUNCS, (st)) @@ -646,6 +690,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_CRYPTO_dynlock_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(CRYPTO_dynlock, (st), (cmp)) # define sk_CRYPTO_dynlock_dup(st) SKM_sk_dup(CRYPTO_dynlock, st) # define sk_CRYPTO_dynlock_pop_free(st, free_func) SKM_sk_pop_free(CRYPTO_dynlock, (st), (free_func)) +# define sk_CRYPTO_dynlock_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(CRYPTO_dynlock, (st), (copy_func), (free_func)) # define sk_CRYPTO_dynlock_shift(st) SKM_sk_shift(CRYPTO_dynlock, (st)) # define sk_CRYPTO_dynlock_pop(st) SKM_sk_pop(CRYPTO_dynlock, (st)) # define sk_CRYPTO_dynlock_sort(st) SKM_sk_sort(CRYPTO_dynlock, (st)) @@ -667,6 +712,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_DIST_POINT_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(DIST_POINT, (st), (cmp)) # define sk_DIST_POINT_dup(st) SKM_sk_dup(DIST_POINT, st) # define sk_DIST_POINT_pop_free(st, free_func) SKM_sk_pop_free(DIST_POINT, (st), (free_func)) +# define sk_DIST_POINT_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(DIST_POINT, (st), (copy_func), (free_func)) # define sk_DIST_POINT_shift(st) SKM_sk_shift(DIST_POINT, (st)) # define sk_DIST_POINT_pop(st) SKM_sk_pop(DIST_POINT, (st)) # define sk_DIST_POINT_sort(st) SKM_sk_sort(DIST_POINT, (st)) @@ -688,6 +734,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ENGINE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ENGINE, (st), (cmp)) # define sk_ENGINE_dup(st) SKM_sk_dup(ENGINE, st) # define sk_ENGINE_pop_free(st, free_func) SKM_sk_pop_free(ENGINE, (st), (free_func)) +# define sk_ENGINE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ENGINE, (st), (copy_func), (free_func)) # define sk_ENGINE_shift(st) SKM_sk_shift(ENGINE, (st)) # define sk_ENGINE_pop(st) SKM_sk_pop(ENGINE, (st)) # define sk_ENGINE_sort(st) SKM_sk_sort(ENGINE, (st)) @@ -709,6 +756,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ENGINE_CLEANUP_ITEM_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ENGINE_CLEANUP_ITEM, (st), (cmp)) # define sk_ENGINE_CLEANUP_ITEM_dup(st) SKM_sk_dup(ENGINE_CLEANUP_ITEM, st) # define sk_ENGINE_CLEANUP_ITEM_pop_free(st, free_func) SKM_sk_pop_free(ENGINE_CLEANUP_ITEM, (st), (free_func)) +# define sk_ENGINE_CLEANUP_ITEM_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ENGINE_CLEANUP_ITEM, (st), (copy_func), (free_func)) # define sk_ENGINE_CLEANUP_ITEM_shift(st) SKM_sk_shift(ENGINE_CLEANUP_ITEM, (st)) # define sk_ENGINE_CLEANUP_ITEM_pop(st) SKM_sk_pop(ENGINE_CLEANUP_ITEM, (st)) # define sk_ENGINE_CLEANUP_ITEM_sort(st) SKM_sk_sort(ENGINE_CLEANUP_ITEM, (st)) @@ -730,6 +778,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_ESS_CERT_ID_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(ESS_CERT_ID, (st), (cmp)) # define sk_ESS_CERT_ID_dup(st) SKM_sk_dup(ESS_CERT_ID, st) # define sk_ESS_CERT_ID_pop_free(st, free_func) SKM_sk_pop_free(ESS_CERT_ID, (st), (free_func)) +# define sk_ESS_CERT_ID_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(ESS_CERT_ID, (st), (copy_func), (free_func)) # define sk_ESS_CERT_ID_shift(st) SKM_sk_shift(ESS_CERT_ID, (st)) # define sk_ESS_CERT_ID_pop(st) SKM_sk_pop(ESS_CERT_ID, (st)) # define sk_ESS_CERT_ID_sort(st) SKM_sk_sort(ESS_CERT_ID, (st)) @@ -751,6 +800,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_EVP_MD_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(EVP_MD, (st), (cmp)) # define sk_EVP_MD_dup(st) SKM_sk_dup(EVP_MD, st) # define sk_EVP_MD_pop_free(st, free_func) SKM_sk_pop_free(EVP_MD, (st), (free_func)) +# define sk_EVP_MD_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(EVP_MD, (st), (copy_func), (free_func)) # define sk_EVP_MD_shift(st) SKM_sk_shift(EVP_MD, (st)) # define sk_EVP_MD_pop(st) SKM_sk_pop(EVP_MD, (st)) # define sk_EVP_MD_sort(st) SKM_sk_sort(EVP_MD, (st)) @@ -772,6 +822,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_EVP_PBE_CTL_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(EVP_PBE_CTL, (st), (cmp)) # define sk_EVP_PBE_CTL_dup(st) SKM_sk_dup(EVP_PBE_CTL, st) # define sk_EVP_PBE_CTL_pop_free(st, free_func) SKM_sk_pop_free(EVP_PBE_CTL, (st), (free_func)) +# define sk_EVP_PBE_CTL_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(EVP_PBE_CTL, (st), (copy_func), (free_func)) # define sk_EVP_PBE_CTL_shift(st) SKM_sk_shift(EVP_PBE_CTL, (st)) # define sk_EVP_PBE_CTL_pop(st) SKM_sk_pop(EVP_PBE_CTL, (st)) # define sk_EVP_PBE_CTL_sort(st) SKM_sk_sort(EVP_PBE_CTL, (st)) @@ -793,6 +844,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_EVP_PKEY_ASN1_METHOD_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(EVP_PKEY_ASN1_METHOD, (st), (cmp)) # define sk_EVP_PKEY_ASN1_METHOD_dup(st) SKM_sk_dup(EVP_PKEY_ASN1_METHOD, st) # define sk_EVP_PKEY_ASN1_METHOD_pop_free(st, free_func) SKM_sk_pop_free(EVP_PKEY_ASN1_METHOD, (st), (free_func)) +# define sk_EVP_PKEY_ASN1_METHOD_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(EVP_PKEY_ASN1_METHOD, (st), (copy_func), (free_func)) # define sk_EVP_PKEY_ASN1_METHOD_shift(st) SKM_sk_shift(EVP_PKEY_ASN1_METHOD, (st)) # define sk_EVP_PKEY_ASN1_METHOD_pop(st) SKM_sk_pop(EVP_PKEY_ASN1_METHOD, (st)) # define sk_EVP_PKEY_ASN1_METHOD_sort(st) SKM_sk_sort(EVP_PKEY_ASN1_METHOD, (st)) @@ -814,6 +866,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_EVP_PKEY_METHOD_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(EVP_PKEY_METHOD, (st), (cmp)) # define sk_EVP_PKEY_METHOD_dup(st) SKM_sk_dup(EVP_PKEY_METHOD, st) # define sk_EVP_PKEY_METHOD_pop_free(st, free_func) SKM_sk_pop_free(EVP_PKEY_METHOD, (st), (free_func)) +# define sk_EVP_PKEY_METHOD_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(EVP_PKEY_METHOD, (st), (copy_func), (free_func)) # define sk_EVP_PKEY_METHOD_shift(st) SKM_sk_shift(EVP_PKEY_METHOD, (st)) # define sk_EVP_PKEY_METHOD_pop(st) SKM_sk_pop(EVP_PKEY_METHOD, (st)) # define sk_EVP_PKEY_METHOD_sort(st) SKM_sk_sort(EVP_PKEY_METHOD, (st)) @@ -835,6 +888,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_GENERAL_NAME_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(GENERAL_NAME, (st), (cmp)) # define sk_GENERAL_NAME_dup(st) SKM_sk_dup(GENERAL_NAME, st) # define sk_GENERAL_NAME_pop_free(st, free_func) SKM_sk_pop_free(GENERAL_NAME, (st), (free_func)) +# define sk_GENERAL_NAME_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(GENERAL_NAME, (st), (copy_func), (free_func)) # define sk_GENERAL_NAME_shift(st) SKM_sk_shift(GENERAL_NAME, (st)) # define sk_GENERAL_NAME_pop(st) SKM_sk_pop(GENERAL_NAME, (st)) # define sk_GENERAL_NAME_sort(st) SKM_sk_sort(GENERAL_NAME, (st)) @@ -856,6 +910,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_GENERAL_NAMES_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(GENERAL_NAMES, (st), (cmp)) # define sk_GENERAL_NAMES_dup(st) SKM_sk_dup(GENERAL_NAMES, st) # define sk_GENERAL_NAMES_pop_free(st, free_func) SKM_sk_pop_free(GENERAL_NAMES, (st), (free_func)) +# define sk_GENERAL_NAMES_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(GENERAL_NAMES, (st), (copy_func), (free_func)) # define sk_GENERAL_NAMES_shift(st) SKM_sk_shift(GENERAL_NAMES, (st)) # define sk_GENERAL_NAMES_pop(st) SKM_sk_pop(GENERAL_NAMES, (st)) # define sk_GENERAL_NAMES_sort(st) SKM_sk_sort(GENERAL_NAMES, (st)) @@ -877,6 +932,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_GENERAL_SUBTREE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(GENERAL_SUBTREE, (st), (cmp)) # define sk_GENERAL_SUBTREE_dup(st) SKM_sk_dup(GENERAL_SUBTREE, st) # define sk_GENERAL_SUBTREE_pop_free(st, free_func) SKM_sk_pop_free(GENERAL_SUBTREE, (st), (free_func)) +# define sk_GENERAL_SUBTREE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(GENERAL_SUBTREE, (st), (copy_func), (free_func)) # define sk_GENERAL_SUBTREE_shift(st) SKM_sk_shift(GENERAL_SUBTREE, (st)) # define sk_GENERAL_SUBTREE_pop(st) SKM_sk_pop(GENERAL_SUBTREE, (st)) # define sk_GENERAL_SUBTREE_sort(st) SKM_sk_sort(GENERAL_SUBTREE, (st)) @@ -898,6 +954,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_IPAddressFamily_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(IPAddressFamily, (st), (cmp)) # define sk_IPAddressFamily_dup(st) SKM_sk_dup(IPAddressFamily, st) # define sk_IPAddressFamily_pop_free(st, free_func) SKM_sk_pop_free(IPAddressFamily, (st), (free_func)) +# define sk_IPAddressFamily_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(IPAddressFamily, (st), (copy_func), (free_func)) # define sk_IPAddressFamily_shift(st) SKM_sk_shift(IPAddressFamily, (st)) # define sk_IPAddressFamily_pop(st) SKM_sk_pop(IPAddressFamily, (st)) # define sk_IPAddressFamily_sort(st) SKM_sk_sort(IPAddressFamily, (st)) @@ -919,6 +976,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_IPAddressOrRange_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(IPAddressOrRange, (st), (cmp)) # define sk_IPAddressOrRange_dup(st) SKM_sk_dup(IPAddressOrRange, st) # define sk_IPAddressOrRange_pop_free(st, free_func) SKM_sk_pop_free(IPAddressOrRange, (st), (free_func)) +# define sk_IPAddressOrRange_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(IPAddressOrRange, (st), (copy_func), (free_func)) # define sk_IPAddressOrRange_shift(st) SKM_sk_shift(IPAddressOrRange, (st)) # define sk_IPAddressOrRange_pop(st) SKM_sk_pop(IPAddressOrRange, (st)) # define sk_IPAddressOrRange_sort(st) SKM_sk_sort(IPAddressOrRange, (st)) @@ -940,6 +998,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_APREQBODY_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_APREQBODY, (st), (cmp)) # define sk_KRB5_APREQBODY_dup(st) SKM_sk_dup(KRB5_APREQBODY, st) # define sk_KRB5_APREQBODY_pop_free(st, free_func) SKM_sk_pop_free(KRB5_APREQBODY, (st), (free_func)) +# define sk_KRB5_APREQBODY_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_APREQBODY, (st), (copy_func), (free_func)) # define sk_KRB5_APREQBODY_shift(st) SKM_sk_shift(KRB5_APREQBODY, (st)) # define sk_KRB5_APREQBODY_pop(st) SKM_sk_pop(KRB5_APREQBODY, (st)) # define sk_KRB5_APREQBODY_sort(st) SKM_sk_sort(KRB5_APREQBODY, (st)) @@ -961,6 +1020,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_AUTHDATA_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_AUTHDATA, (st), (cmp)) # define sk_KRB5_AUTHDATA_dup(st) SKM_sk_dup(KRB5_AUTHDATA, st) # define sk_KRB5_AUTHDATA_pop_free(st, free_func) SKM_sk_pop_free(KRB5_AUTHDATA, (st), (free_func)) +# define sk_KRB5_AUTHDATA_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_AUTHDATA, (st), (copy_func), (free_func)) # define sk_KRB5_AUTHDATA_shift(st) SKM_sk_shift(KRB5_AUTHDATA, (st)) # define sk_KRB5_AUTHDATA_pop(st) SKM_sk_pop(KRB5_AUTHDATA, (st)) # define sk_KRB5_AUTHDATA_sort(st) SKM_sk_sort(KRB5_AUTHDATA, (st)) @@ -982,6 +1042,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_AUTHENTBODY_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_AUTHENTBODY, (st), (cmp)) # define sk_KRB5_AUTHENTBODY_dup(st) SKM_sk_dup(KRB5_AUTHENTBODY, st) # define sk_KRB5_AUTHENTBODY_pop_free(st, free_func) SKM_sk_pop_free(KRB5_AUTHENTBODY, (st), (free_func)) +# define sk_KRB5_AUTHENTBODY_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_AUTHENTBODY, (st), (copy_func), (free_func)) # define sk_KRB5_AUTHENTBODY_shift(st) SKM_sk_shift(KRB5_AUTHENTBODY, (st)) # define sk_KRB5_AUTHENTBODY_pop(st) SKM_sk_pop(KRB5_AUTHENTBODY, (st)) # define sk_KRB5_AUTHENTBODY_sort(st) SKM_sk_sort(KRB5_AUTHENTBODY, (st)) @@ -1003,6 +1064,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_CHECKSUM_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_CHECKSUM, (st), (cmp)) # define sk_KRB5_CHECKSUM_dup(st) SKM_sk_dup(KRB5_CHECKSUM, st) # define sk_KRB5_CHECKSUM_pop_free(st, free_func) SKM_sk_pop_free(KRB5_CHECKSUM, (st), (free_func)) +# define sk_KRB5_CHECKSUM_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_CHECKSUM, (st), (copy_func), (free_func)) # define sk_KRB5_CHECKSUM_shift(st) SKM_sk_shift(KRB5_CHECKSUM, (st)) # define sk_KRB5_CHECKSUM_pop(st) SKM_sk_pop(KRB5_CHECKSUM, (st)) # define sk_KRB5_CHECKSUM_sort(st) SKM_sk_sort(KRB5_CHECKSUM, (st)) @@ -1024,6 +1086,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_ENCDATA_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_ENCDATA, (st), (cmp)) # define sk_KRB5_ENCDATA_dup(st) SKM_sk_dup(KRB5_ENCDATA, st) # define sk_KRB5_ENCDATA_pop_free(st, free_func) SKM_sk_pop_free(KRB5_ENCDATA, (st), (free_func)) +# define sk_KRB5_ENCDATA_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_ENCDATA, (st), (copy_func), (free_func)) # define sk_KRB5_ENCDATA_shift(st) SKM_sk_shift(KRB5_ENCDATA, (st)) # define sk_KRB5_ENCDATA_pop(st) SKM_sk_pop(KRB5_ENCDATA, (st)) # define sk_KRB5_ENCDATA_sort(st) SKM_sk_sort(KRB5_ENCDATA, (st)) @@ -1045,6 +1108,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_ENCKEY_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_ENCKEY, (st), (cmp)) # define sk_KRB5_ENCKEY_dup(st) SKM_sk_dup(KRB5_ENCKEY, st) # define sk_KRB5_ENCKEY_pop_free(st, free_func) SKM_sk_pop_free(KRB5_ENCKEY, (st), (free_func)) +# define sk_KRB5_ENCKEY_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_ENCKEY, (st), (copy_func), (free_func)) # define sk_KRB5_ENCKEY_shift(st) SKM_sk_shift(KRB5_ENCKEY, (st)) # define sk_KRB5_ENCKEY_pop(st) SKM_sk_pop(KRB5_ENCKEY, (st)) # define sk_KRB5_ENCKEY_sort(st) SKM_sk_sort(KRB5_ENCKEY, (st)) @@ -1066,6 +1130,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_PRINCNAME_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_PRINCNAME, (st), (cmp)) # define sk_KRB5_PRINCNAME_dup(st) SKM_sk_dup(KRB5_PRINCNAME, st) # define sk_KRB5_PRINCNAME_pop_free(st, free_func) SKM_sk_pop_free(KRB5_PRINCNAME, (st), (free_func)) +# define sk_KRB5_PRINCNAME_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_PRINCNAME, (st), (copy_func), (free_func)) # define sk_KRB5_PRINCNAME_shift(st) SKM_sk_shift(KRB5_PRINCNAME, (st)) # define sk_KRB5_PRINCNAME_pop(st) SKM_sk_pop(KRB5_PRINCNAME, (st)) # define sk_KRB5_PRINCNAME_sort(st) SKM_sk_sort(KRB5_PRINCNAME, (st)) @@ -1087,6 +1152,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_KRB5_TKTBODY_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(KRB5_TKTBODY, (st), (cmp)) # define sk_KRB5_TKTBODY_dup(st) SKM_sk_dup(KRB5_TKTBODY, st) # define sk_KRB5_TKTBODY_pop_free(st, free_func) SKM_sk_pop_free(KRB5_TKTBODY, (st), (free_func)) +# define sk_KRB5_TKTBODY_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(KRB5_TKTBODY, (st), (copy_func), (free_func)) # define sk_KRB5_TKTBODY_shift(st) SKM_sk_shift(KRB5_TKTBODY, (st)) # define sk_KRB5_TKTBODY_pop(st) SKM_sk_pop(KRB5_TKTBODY, (st)) # define sk_KRB5_TKTBODY_sort(st) SKM_sk_sort(KRB5_TKTBODY, (st)) @@ -1108,6 +1174,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_MEM_OBJECT_DATA_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(MEM_OBJECT_DATA, (st), (cmp)) # define sk_MEM_OBJECT_DATA_dup(st) SKM_sk_dup(MEM_OBJECT_DATA, st) # define sk_MEM_OBJECT_DATA_pop_free(st, free_func) SKM_sk_pop_free(MEM_OBJECT_DATA, (st), (free_func)) +# define sk_MEM_OBJECT_DATA_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(MEM_OBJECT_DATA, (st), (copy_func), (free_func)) # define sk_MEM_OBJECT_DATA_shift(st) SKM_sk_shift(MEM_OBJECT_DATA, (st)) # define sk_MEM_OBJECT_DATA_pop(st) SKM_sk_pop(MEM_OBJECT_DATA, (st)) # define sk_MEM_OBJECT_DATA_sort(st) SKM_sk_sort(MEM_OBJECT_DATA, (st)) @@ -1129,6 +1196,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_MIME_HEADER_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(MIME_HEADER, (st), (cmp)) # define sk_MIME_HEADER_dup(st) SKM_sk_dup(MIME_HEADER, st) # define sk_MIME_HEADER_pop_free(st, free_func) SKM_sk_pop_free(MIME_HEADER, (st), (free_func)) +# define sk_MIME_HEADER_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(MIME_HEADER, (st), (copy_func), (free_func)) # define sk_MIME_HEADER_shift(st) SKM_sk_shift(MIME_HEADER, (st)) # define sk_MIME_HEADER_pop(st) SKM_sk_pop(MIME_HEADER, (st)) # define sk_MIME_HEADER_sort(st) SKM_sk_sort(MIME_HEADER, (st)) @@ -1150,6 +1218,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_MIME_PARAM_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(MIME_PARAM, (st), (cmp)) # define sk_MIME_PARAM_dup(st) SKM_sk_dup(MIME_PARAM, st) # define sk_MIME_PARAM_pop_free(st, free_func) SKM_sk_pop_free(MIME_PARAM, (st), (free_func)) +# define sk_MIME_PARAM_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(MIME_PARAM, (st), (copy_func), (free_func)) # define sk_MIME_PARAM_shift(st) SKM_sk_shift(MIME_PARAM, (st)) # define sk_MIME_PARAM_pop(st) SKM_sk_pop(MIME_PARAM, (st)) # define sk_MIME_PARAM_sort(st) SKM_sk_sort(MIME_PARAM, (st)) @@ -1171,6 +1240,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_NAME_FUNCS_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(NAME_FUNCS, (st), (cmp)) # define sk_NAME_FUNCS_dup(st) SKM_sk_dup(NAME_FUNCS, st) # define sk_NAME_FUNCS_pop_free(st, free_func) SKM_sk_pop_free(NAME_FUNCS, (st), (free_func)) +# define sk_NAME_FUNCS_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(NAME_FUNCS, (st), (copy_func), (free_func)) # define sk_NAME_FUNCS_shift(st) SKM_sk_shift(NAME_FUNCS, (st)) # define sk_NAME_FUNCS_pop(st) SKM_sk_pop(NAME_FUNCS, (st)) # define sk_NAME_FUNCS_sort(st) SKM_sk_sort(NAME_FUNCS, (st)) @@ -1192,6 +1262,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_OCSP_CERTID_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(OCSP_CERTID, (st), (cmp)) # define sk_OCSP_CERTID_dup(st) SKM_sk_dup(OCSP_CERTID, st) # define sk_OCSP_CERTID_pop_free(st, free_func) SKM_sk_pop_free(OCSP_CERTID, (st), (free_func)) +# define sk_OCSP_CERTID_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(OCSP_CERTID, (st), (copy_func), (free_func)) # define sk_OCSP_CERTID_shift(st) SKM_sk_shift(OCSP_CERTID, (st)) # define sk_OCSP_CERTID_pop(st) SKM_sk_pop(OCSP_CERTID, (st)) # define sk_OCSP_CERTID_sort(st) SKM_sk_sort(OCSP_CERTID, (st)) @@ -1213,6 +1284,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_OCSP_ONEREQ_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(OCSP_ONEREQ, (st), (cmp)) # define sk_OCSP_ONEREQ_dup(st) SKM_sk_dup(OCSP_ONEREQ, st) # define sk_OCSP_ONEREQ_pop_free(st, free_func) SKM_sk_pop_free(OCSP_ONEREQ, (st), (free_func)) +# define sk_OCSP_ONEREQ_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(OCSP_ONEREQ, (st), (copy_func), (free_func)) # define sk_OCSP_ONEREQ_shift(st) SKM_sk_shift(OCSP_ONEREQ, (st)) # define sk_OCSP_ONEREQ_pop(st) SKM_sk_pop(OCSP_ONEREQ, (st)) # define sk_OCSP_ONEREQ_sort(st) SKM_sk_sort(OCSP_ONEREQ, (st)) @@ -1234,6 +1306,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_OCSP_RESPID_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(OCSP_RESPID, (st), (cmp)) # define sk_OCSP_RESPID_dup(st) SKM_sk_dup(OCSP_RESPID, st) # define sk_OCSP_RESPID_pop_free(st, free_func) SKM_sk_pop_free(OCSP_RESPID, (st), (free_func)) +# define sk_OCSP_RESPID_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(OCSP_RESPID, (st), (copy_func), (free_func)) # define sk_OCSP_RESPID_shift(st) SKM_sk_shift(OCSP_RESPID, (st)) # define sk_OCSP_RESPID_pop(st) SKM_sk_pop(OCSP_RESPID, (st)) # define sk_OCSP_RESPID_sort(st) SKM_sk_sort(OCSP_RESPID, (st)) @@ -1255,6 +1328,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_OCSP_SINGLERESP_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(OCSP_SINGLERESP, (st), (cmp)) # define sk_OCSP_SINGLERESP_dup(st) SKM_sk_dup(OCSP_SINGLERESP, st) # define sk_OCSP_SINGLERESP_pop_free(st, free_func) SKM_sk_pop_free(OCSP_SINGLERESP, (st), (free_func)) +# define sk_OCSP_SINGLERESP_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(OCSP_SINGLERESP, (st), (copy_func), (free_func)) # define sk_OCSP_SINGLERESP_shift(st) SKM_sk_shift(OCSP_SINGLERESP, (st)) # define sk_OCSP_SINGLERESP_pop(st) SKM_sk_pop(OCSP_SINGLERESP, (st)) # define sk_OCSP_SINGLERESP_sort(st) SKM_sk_sort(OCSP_SINGLERESP, (st)) @@ -1276,6 +1350,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_PKCS12_SAFEBAG_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(PKCS12_SAFEBAG, (st), (cmp)) # define sk_PKCS12_SAFEBAG_dup(st) SKM_sk_dup(PKCS12_SAFEBAG, st) # define sk_PKCS12_SAFEBAG_pop_free(st, free_func) SKM_sk_pop_free(PKCS12_SAFEBAG, (st), (free_func)) +# define sk_PKCS12_SAFEBAG_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(PKCS12_SAFEBAG, (st), (copy_func), (free_func)) # define sk_PKCS12_SAFEBAG_shift(st) SKM_sk_shift(PKCS12_SAFEBAG, (st)) # define sk_PKCS12_SAFEBAG_pop(st) SKM_sk_pop(PKCS12_SAFEBAG, (st)) # define sk_PKCS12_SAFEBAG_sort(st) SKM_sk_sort(PKCS12_SAFEBAG, (st)) @@ -1297,6 +1372,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_PKCS7_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(PKCS7, (st), (cmp)) # define sk_PKCS7_dup(st) SKM_sk_dup(PKCS7, st) # define sk_PKCS7_pop_free(st, free_func) SKM_sk_pop_free(PKCS7, (st), (free_func)) +# define sk_PKCS7_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(PKCS7, (st), (copy_func), (free_func)) # define sk_PKCS7_shift(st) SKM_sk_shift(PKCS7, (st)) # define sk_PKCS7_pop(st) SKM_sk_pop(PKCS7, (st)) # define sk_PKCS7_sort(st) SKM_sk_sort(PKCS7, (st)) @@ -1318,6 +1394,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_PKCS7_RECIP_INFO_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(PKCS7_RECIP_INFO, (st), (cmp)) # define sk_PKCS7_RECIP_INFO_dup(st) SKM_sk_dup(PKCS7_RECIP_INFO, st) # define sk_PKCS7_RECIP_INFO_pop_free(st, free_func) SKM_sk_pop_free(PKCS7_RECIP_INFO, (st), (free_func)) +# define sk_PKCS7_RECIP_INFO_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(PKCS7_RECIP_INFO, (st), (copy_func), (free_func)) # define sk_PKCS7_RECIP_INFO_shift(st) SKM_sk_shift(PKCS7_RECIP_INFO, (st)) # define sk_PKCS7_RECIP_INFO_pop(st) SKM_sk_pop(PKCS7_RECIP_INFO, (st)) # define sk_PKCS7_RECIP_INFO_sort(st) SKM_sk_sort(PKCS7_RECIP_INFO, (st)) @@ -1339,6 +1416,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_PKCS7_SIGNER_INFO_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(PKCS7_SIGNER_INFO, (st), (cmp)) # define sk_PKCS7_SIGNER_INFO_dup(st) SKM_sk_dup(PKCS7_SIGNER_INFO, st) # define sk_PKCS7_SIGNER_INFO_pop_free(st, free_func) SKM_sk_pop_free(PKCS7_SIGNER_INFO, (st), (free_func)) +# define sk_PKCS7_SIGNER_INFO_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(PKCS7_SIGNER_INFO, (st), (copy_func), (free_func)) # define sk_PKCS7_SIGNER_INFO_shift(st) SKM_sk_shift(PKCS7_SIGNER_INFO, (st)) # define sk_PKCS7_SIGNER_INFO_pop(st) SKM_sk_pop(PKCS7_SIGNER_INFO, (st)) # define sk_PKCS7_SIGNER_INFO_sort(st) SKM_sk_sort(PKCS7_SIGNER_INFO, (st)) @@ -1360,6 +1438,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_POLICYINFO_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(POLICYINFO, (st), (cmp)) # define sk_POLICYINFO_dup(st) SKM_sk_dup(POLICYINFO, st) # define sk_POLICYINFO_pop_free(st, free_func) SKM_sk_pop_free(POLICYINFO, (st), (free_func)) +# define sk_POLICYINFO_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(POLICYINFO, (st), (copy_func), (free_func)) # define sk_POLICYINFO_shift(st) SKM_sk_shift(POLICYINFO, (st)) # define sk_POLICYINFO_pop(st) SKM_sk_pop(POLICYINFO, (st)) # define sk_POLICYINFO_sort(st) SKM_sk_sort(POLICYINFO, (st)) @@ -1381,6 +1460,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_POLICYQUALINFO_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(POLICYQUALINFO, (st), (cmp)) # define sk_POLICYQUALINFO_dup(st) SKM_sk_dup(POLICYQUALINFO, st) # define sk_POLICYQUALINFO_pop_free(st, free_func) SKM_sk_pop_free(POLICYQUALINFO, (st), (free_func)) +# define sk_POLICYQUALINFO_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(POLICYQUALINFO, (st), (copy_func), (free_func)) # define sk_POLICYQUALINFO_shift(st) SKM_sk_shift(POLICYQUALINFO, (st)) # define sk_POLICYQUALINFO_pop(st) SKM_sk_pop(POLICYQUALINFO, (st)) # define sk_POLICYQUALINFO_sort(st) SKM_sk_sort(POLICYQUALINFO, (st)) @@ -1402,10 +1482,33 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_POLICY_MAPPING_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(POLICY_MAPPING, (st), (cmp)) # define sk_POLICY_MAPPING_dup(st) SKM_sk_dup(POLICY_MAPPING, st) # define sk_POLICY_MAPPING_pop_free(st, free_func) SKM_sk_pop_free(POLICY_MAPPING, (st), (free_func)) +# define sk_POLICY_MAPPING_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(POLICY_MAPPING, (st), (copy_func), (free_func)) # define sk_POLICY_MAPPING_shift(st) SKM_sk_shift(POLICY_MAPPING, (st)) # define sk_POLICY_MAPPING_pop(st) SKM_sk_pop(POLICY_MAPPING, (st)) # define sk_POLICY_MAPPING_sort(st) SKM_sk_sort(POLICY_MAPPING, (st)) # define sk_POLICY_MAPPING_is_sorted(st) SKM_sk_is_sorted(POLICY_MAPPING, (st)) +# define sk_SCT_new(cmp) SKM_sk_new(SCT, (cmp)) +# define sk_SCT_new_null() SKM_sk_new_null(SCT) +# define sk_SCT_free(st) SKM_sk_free(SCT, (st)) +# define sk_SCT_num(st) SKM_sk_num(SCT, (st)) +# define sk_SCT_value(st, i) SKM_sk_value(SCT, (st), (i)) +# define sk_SCT_set(st, i, val) SKM_sk_set(SCT, (st), (i), (val)) +# define sk_SCT_zero(st) SKM_sk_zero(SCT, (st)) +# define sk_SCT_push(st, val) SKM_sk_push(SCT, (st), (val)) +# define sk_SCT_unshift(st, val) SKM_sk_unshift(SCT, (st), (val)) +# define sk_SCT_find(st, val) SKM_sk_find(SCT, (st), (val)) +# define sk_SCT_find_ex(st, val) SKM_sk_find_ex(SCT, (st), (val)) +# define sk_SCT_delete(st, i) SKM_sk_delete(SCT, (st), (i)) +# define sk_SCT_delete_ptr(st, ptr) SKM_sk_delete_ptr(SCT, (st), (ptr)) +# define sk_SCT_insert(st, val, i) SKM_sk_insert(SCT, (st), (val), (i)) +# define sk_SCT_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SCT, (st), (cmp)) +# define sk_SCT_dup(st) SKM_sk_dup(SCT, st) +# define sk_SCT_pop_free(st, free_func) SKM_sk_pop_free(SCT, (st), (free_func)) +# define sk_SCT_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SCT, (st), (copy_func), (free_func)) +# define sk_SCT_shift(st) SKM_sk_shift(SCT, (st)) +# define sk_SCT_pop(st) SKM_sk_pop(SCT, (st)) +# define sk_SCT_sort(st) SKM_sk_sort(SCT, (st)) +# define sk_SCT_is_sorted(st) SKM_sk_is_sorted(SCT, (st)) # define sk_SRP_gN_new(cmp) SKM_sk_new(SRP_gN, (cmp)) # define sk_SRP_gN_new_null() SKM_sk_new_null(SRP_gN) # define sk_SRP_gN_free(st) SKM_sk_free(SRP_gN, (st)) @@ -1423,6 +1526,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_SRP_gN_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SRP_gN, (st), (cmp)) # define sk_SRP_gN_dup(st) SKM_sk_dup(SRP_gN, st) # define sk_SRP_gN_pop_free(st, free_func) SKM_sk_pop_free(SRP_gN, (st), (free_func)) +# define sk_SRP_gN_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SRP_gN, (st), (copy_func), (free_func)) # define sk_SRP_gN_shift(st) SKM_sk_shift(SRP_gN, (st)) # define sk_SRP_gN_pop(st) SKM_sk_pop(SRP_gN, (st)) # define sk_SRP_gN_sort(st) SKM_sk_sort(SRP_gN, (st)) @@ -1444,6 +1548,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_SRP_gN_cache_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SRP_gN_cache, (st), (cmp)) # define sk_SRP_gN_cache_dup(st) SKM_sk_dup(SRP_gN_cache, st) # define sk_SRP_gN_cache_pop_free(st, free_func) SKM_sk_pop_free(SRP_gN_cache, (st), (free_func)) +# define sk_SRP_gN_cache_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SRP_gN_cache, (st), (copy_func), (free_func)) # define sk_SRP_gN_cache_shift(st) SKM_sk_shift(SRP_gN_cache, (st)) # define sk_SRP_gN_cache_pop(st) SKM_sk_pop(SRP_gN_cache, (st)) # define sk_SRP_gN_cache_sort(st) SKM_sk_sort(SRP_gN_cache, (st)) @@ -1465,6 +1570,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_SRP_user_pwd_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SRP_user_pwd, (st), (cmp)) # define sk_SRP_user_pwd_dup(st) SKM_sk_dup(SRP_user_pwd, st) # define sk_SRP_user_pwd_pop_free(st, free_func) SKM_sk_pop_free(SRP_user_pwd, (st), (free_func)) +# define sk_SRP_user_pwd_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SRP_user_pwd, (st), (copy_func), (free_func)) # define sk_SRP_user_pwd_shift(st) SKM_sk_shift(SRP_user_pwd, (st)) # define sk_SRP_user_pwd_pop(st) SKM_sk_pop(SRP_user_pwd, (st)) # define sk_SRP_user_pwd_sort(st) SKM_sk_sort(SRP_user_pwd, (st)) @@ -1486,6 +1592,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_SRTP_PROTECTION_PROFILE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SRTP_PROTECTION_PROFILE, (st), (cmp)) # define sk_SRTP_PROTECTION_PROFILE_dup(st) SKM_sk_dup(SRTP_PROTECTION_PROFILE, st) # define sk_SRTP_PROTECTION_PROFILE_pop_free(st, free_func) SKM_sk_pop_free(SRTP_PROTECTION_PROFILE, (st), (free_func)) +# define sk_SRTP_PROTECTION_PROFILE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SRTP_PROTECTION_PROFILE, (st), (copy_func), (free_func)) # define sk_SRTP_PROTECTION_PROFILE_shift(st) SKM_sk_shift(SRTP_PROTECTION_PROFILE, (st)) # define sk_SRTP_PROTECTION_PROFILE_pop(st) SKM_sk_pop(SRTP_PROTECTION_PROFILE, (st)) # define sk_SRTP_PROTECTION_PROFILE_sort(st) SKM_sk_sort(SRTP_PROTECTION_PROFILE, (st)) @@ -1507,6 +1614,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_SSL_CIPHER_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SSL_CIPHER, (st), (cmp)) # define sk_SSL_CIPHER_dup(st) SKM_sk_dup(SSL_CIPHER, st) # define sk_SSL_CIPHER_pop_free(st, free_func) SKM_sk_pop_free(SSL_CIPHER, (st), (free_func)) +# define sk_SSL_CIPHER_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SSL_CIPHER, (st), (copy_func), (free_func)) # define sk_SSL_CIPHER_shift(st) SKM_sk_shift(SSL_CIPHER, (st)) # define sk_SSL_CIPHER_pop(st) SKM_sk_pop(SSL_CIPHER, (st)) # define sk_SSL_CIPHER_sort(st) SKM_sk_sort(SSL_CIPHER, (st)) @@ -1528,6 +1636,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_SSL_COMP_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SSL_COMP, (st), (cmp)) # define sk_SSL_COMP_dup(st) SKM_sk_dup(SSL_COMP, st) # define sk_SSL_COMP_pop_free(st, free_func) SKM_sk_pop_free(SSL_COMP, (st), (free_func)) +# define sk_SSL_COMP_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SSL_COMP, (st), (copy_func), (free_func)) # define sk_SSL_COMP_shift(st) SKM_sk_shift(SSL_COMP, (st)) # define sk_SSL_COMP_pop(st) SKM_sk_pop(SSL_COMP, (st)) # define sk_SSL_COMP_sort(st) SKM_sk_sort(SSL_COMP, (st)) @@ -1549,6 +1658,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_STACK_OF_X509_NAME_ENTRY_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(STACK_OF_X509_NAME_ENTRY, (st), (cmp)) # define sk_STACK_OF_X509_NAME_ENTRY_dup(st) SKM_sk_dup(STACK_OF_X509_NAME_ENTRY, st) # define sk_STACK_OF_X509_NAME_ENTRY_pop_free(st, free_func) SKM_sk_pop_free(STACK_OF_X509_NAME_ENTRY, (st), (free_func)) +# define sk_STACK_OF_X509_NAME_ENTRY_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(STACK_OF_X509_NAME_ENTRY, (st), (copy_func), (free_func)) # define sk_STACK_OF_X509_NAME_ENTRY_shift(st) SKM_sk_shift(STACK_OF_X509_NAME_ENTRY, (st)) # define sk_STACK_OF_X509_NAME_ENTRY_pop(st) SKM_sk_pop(STACK_OF_X509_NAME_ENTRY, (st)) # define sk_STACK_OF_X509_NAME_ENTRY_sort(st) SKM_sk_sort(STACK_OF_X509_NAME_ENTRY, (st)) @@ -1570,6 +1680,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_STORE_ATTR_INFO_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(STORE_ATTR_INFO, (st), (cmp)) # define sk_STORE_ATTR_INFO_dup(st) SKM_sk_dup(STORE_ATTR_INFO, st) # define sk_STORE_ATTR_INFO_pop_free(st, free_func) SKM_sk_pop_free(STORE_ATTR_INFO, (st), (free_func)) +# define sk_STORE_ATTR_INFO_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(STORE_ATTR_INFO, (st), (copy_func), (free_func)) # define sk_STORE_ATTR_INFO_shift(st) SKM_sk_shift(STORE_ATTR_INFO, (st)) # define sk_STORE_ATTR_INFO_pop(st) SKM_sk_pop(STORE_ATTR_INFO, (st)) # define sk_STORE_ATTR_INFO_sort(st) SKM_sk_sort(STORE_ATTR_INFO, (st)) @@ -1591,6 +1702,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_STORE_OBJECT_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(STORE_OBJECT, (st), (cmp)) # define sk_STORE_OBJECT_dup(st) SKM_sk_dup(STORE_OBJECT, st) # define sk_STORE_OBJECT_pop_free(st, free_func) SKM_sk_pop_free(STORE_OBJECT, (st), (free_func)) +# define sk_STORE_OBJECT_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(STORE_OBJECT, (st), (copy_func), (free_func)) # define sk_STORE_OBJECT_shift(st) SKM_sk_shift(STORE_OBJECT, (st)) # define sk_STORE_OBJECT_pop(st) SKM_sk_pop(STORE_OBJECT, (st)) # define sk_STORE_OBJECT_sort(st) SKM_sk_sort(STORE_OBJECT, (st)) @@ -1612,6 +1724,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_SXNETID_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(SXNETID, (st), (cmp)) # define sk_SXNETID_dup(st) SKM_sk_dup(SXNETID, st) # define sk_SXNETID_pop_free(st, free_func) SKM_sk_pop_free(SXNETID, (st), (free_func)) +# define sk_SXNETID_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(SXNETID, (st), (copy_func), (free_func)) # define sk_SXNETID_shift(st) SKM_sk_shift(SXNETID, (st)) # define sk_SXNETID_pop(st) SKM_sk_pop(SXNETID, (st)) # define sk_SXNETID_sort(st) SKM_sk_sort(SXNETID, (st)) @@ -1633,6 +1746,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_UI_STRING_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(UI_STRING, (st), (cmp)) # define sk_UI_STRING_dup(st) SKM_sk_dup(UI_STRING, st) # define sk_UI_STRING_pop_free(st, free_func) SKM_sk_pop_free(UI_STRING, (st), (free_func)) +# define sk_UI_STRING_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(UI_STRING, (st), (copy_func), (free_func)) # define sk_UI_STRING_shift(st) SKM_sk_shift(UI_STRING, (st)) # define sk_UI_STRING_pop(st) SKM_sk_pop(UI_STRING, (st)) # define sk_UI_STRING_sort(st) SKM_sk_sort(UI_STRING, (st)) @@ -1654,6 +1768,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509, (st), (cmp)) # define sk_X509_dup(st) SKM_sk_dup(X509, st) # define sk_X509_pop_free(st, free_func) SKM_sk_pop_free(X509, (st), (free_func)) +# define sk_X509_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509, (st), (copy_func), (free_func)) # define sk_X509_shift(st) SKM_sk_shift(X509, (st)) # define sk_X509_pop(st) SKM_sk_pop(X509, (st)) # define sk_X509_sort(st) SKM_sk_sort(X509, (st)) @@ -1675,6 +1790,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509V3_EXT_METHOD_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509V3_EXT_METHOD, (st), (cmp)) # define sk_X509V3_EXT_METHOD_dup(st) SKM_sk_dup(X509V3_EXT_METHOD, st) # define sk_X509V3_EXT_METHOD_pop_free(st, free_func) SKM_sk_pop_free(X509V3_EXT_METHOD, (st), (free_func)) +# define sk_X509V3_EXT_METHOD_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509V3_EXT_METHOD, (st), (copy_func), (free_func)) # define sk_X509V3_EXT_METHOD_shift(st) SKM_sk_shift(X509V3_EXT_METHOD, (st)) # define sk_X509V3_EXT_METHOD_pop(st) SKM_sk_pop(X509V3_EXT_METHOD, (st)) # define sk_X509V3_EXT_METHOD_sort(st) SKM_sk_sort(X509V3_EXT_METHOD, (st)) @@ -1696,6 +1812,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_ALGOR_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_ALGOR, (st), (cmp)) # define sk_X509_ALGOR_dup(st) SKM_sk_dup(X509_ALGOR, st) # define sk_X509_ALGOR_pop_free(st, free_func) SKM_sk_pop_free(X509_ALGOR, (st), (free_func)) +# define sk_X509_ALGOR_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_ALGOR, (st), (copy_func), (free_func)) # define sk_X509_ALGOR_shift(st) SKM_sk_shift(X509_ALGOR, (st)) # define sk_X509_ALGOR_pop(st) SKM_sk_pop(X509_ALGOR, (st)) # define sk_X509_ALGOR_sort(st) SKM_sk_sort(X509_ALGOR, (st)) @@ -1717,6 +1834,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_ATTRIBUTE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_ATTRIBUTE, (st), (cmp)) # define sk_X509_ATTRIBUTE_dup(st) SKM_sk_dup(X509_ATTRIBUTE, st) # define sk_X509_ATTRIBUTE_pop_free(st, free_func) SKM_sk_pop_free(X509_ATTRIBUTE, (st), (free_func)) +# define sk_X509_ATTRIBUTE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_ATTRIBUTE, (st), (copy_func), (free_func)) # define sk_X509_ATTRIBUTE_shift(st) SKM_sk_shift(X509_ATTRIBUTE, (st)) # define sk_X509_ATTRIBUTE_pop(st) SKM_sk_pop(X509_ATTRIBUTE, (st)) # define sk_X509_ATTRIBUTE_sort(st) SKM_sk_sort(X509_ATTRIBUTE, (st)) @@ -1738,6 +1856,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_CRL_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_CRL, (st), (cmp)) # define sk_X509_CRL_dup(st) SKM_sk_dup(X509_CRL, st) # define sk_X509_CRL_pop_free(st, free_func) SKM_sk_pop_free(X509_CRL, (st), (free_func)) +# define sk_X509_CRL_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_CRL, (st), (copy_func), (free_func)) # define sk_X509_CRL_shift(st) SKM_sk_shift(X509_CRL, (st)) # define sk_X509_CRL_pop(st) SKM_sk_pop(X509_CRL, (st)) # define sk_X509_CRL_sort(st) SKM_sk_sort(X509_CRL, (st)) @@ -1759,6 +1878,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_EXTENSION_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_EXTENSION, (st), (cmp)) # define sk_X509_EXTENSION_dup(st) SKM_sk_dup(X509_EXTENSION, st) # define sk_X509_EXTENSION_pop_free(st, free_func) SKM_sk_pop_free(X509_EXTENSION, (st), (free_func)) +# define sk_X509_EXTENSION_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_EXTENSION, (st), (copy_func), (free_func)) # define sk_X509_EXTENSION_shift(st) SKM_sk_shift(X509_EXTENSION, (st)) # define sk_X509_EXTENSION_pop(st) SKM_sk_pop(X509_EXTENSION, (st)) # define sk_X509_EXTENSION_sort(st) SKM_sk_sort(X509_EXTENSION, (st)) @@ -1780,6 +1900,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_INFO_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_INFO, (st), (cmp)) # define sk_X509_INFO_dup(st) SKM_sk_dup(X509_INFO, st) # define sk_X509_INFO_pop_free(st, free_func) SKM_sk_pop_free(X509_INFO, (st), (free_func)) +# define sk_X509_INFO_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_INFO, (st), (copy_func), (free_func)) # define sk_X509_INFO_shift(st) SKM_sk_shift(X509_INFO, (st)) # define sk_X509_INFO_pop(st) SKM_sk_pop(X509_INFO, (st)) # define sk_X509_INFO_sort(st) SKM_sk_sort(X509_INFO, (st)) @@ -1801,6 +1922,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_LOOKUP_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_LOOKUP, (st), (cmp)) # define sk_X509_LOOKUP_dup(st) SKM_sk_dup(X509_LOOKUP, st) # define sk_X509_LOOKUP_pop_free(st, free_func) SKM_sk_pop_free(X509_LOOKUP, (st), (free_func)) +# define sk_X509_LOOKUP_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_LOOKUP, (st), (copy_func), (free_func)) # define sk_X509_LOOKUP_shift(st) SKM_sk_shift(X509_LOOKUP, (st)) # define sk_X509_LOOKUP_pop(st) SKM_sk_pop(X509_LOOKUP, (st)) # define sk_X509_LOOKUP_sort(st) SKM_sk_sort(X509_LOOKUP, (st)) @@ -1822,6 +1944,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_NAME_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_NAME, (st), (cmp)) # define sk_X509_NAME_dup(st) SKM_sk_dup(X509_NAME, st) # define sk_X509_NAME_pop_free(st, free_func) SKM_sk_pop_free(X509_NAME, (st), (free_func)) +# define sk_X509_NAME_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_NAME, (st), (copy_func), (free_func)) # define sk_X509_NAME_shift(st) SKM_sk_shift(X509_NAME, (st)) # define sk_X509_NAME_pop(st) SKM_sk_pop(X509_NAME, (st)) # define sk_X509_NAME_sort(st) SKM_sk_sort(X509_NAME, (st)) @@ -1843,6 +1966,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_NAME_ENTRY_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_NAME_ENTRY, (st), (cmp)) # define sk_X509_NAME_ENTRY_dup(st) SKM_sk_dup(X509_NAME_ENTRY, st) # define sk_X509_NAME_ENTRY_pop_free(st, free_func) SKM_sk_pop_free(X509_NAME_ENTRY, (st), (free_func)) +# define sk_X509_NAME_ENTRY_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_NAME_ENTRY, (st), (copy_func), (free_func)) # define sk_X509_NAME_ENTRY_shift(st) SKM_sk_shift(X509_NAME_ENTRY, (st)) # define sk_X509_NAME_ENTRY_pop(st) SKM_sk_pop(X509_NAME_ENTRY, (st)) # define sk_X509_NAME_ENTRY_sort(st) SKM_sk_sort(X509_NAME_ENTRY, (st)) @@ -1864,6 +1988,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_OBJECT_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_OBJECT, (st), (cmp)) # define sk_X509_OBJECT_dup(st) SKM_sk_dup(X509_OBJECT, st) # define sk_X509_OBJECT_pop_free(st, free_func) SKM_sk_pop_free(X509_OBJECT, (st), (free_func)) +# define sk_X509_OBJECT_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_OBJECT, (st), (copy_func), (free_func)) # define sk_X509_OBJECT_shift(st) SKM_sk_shift(X509_OBJECT, (st)) # define sk_X509_OBJECT_pop(st) SKM_sk_pop(X509_OBJECT, (st)) # define sk_X509_OBJECT_sort(st) SKM_sk_sort(X509_OBJECT, (st)) @@ -1885,6 +2010,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_POLICY_DATA_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_POLICY_DATA, (st), (cmp)) # define sk_X509_POLICY_DATA_dup(st) SKM_sk_dup(X509_POLICY_DATA, st) # define sk_X509_POLICY_DATA_pop_free(st, free_func) SKM_sk_pop_free(X509_POLICY_DATA, (st), (free_func)) +# define sk_X509_POLICY_DATA_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_POLICY_DATA, (st), (copy_func), (free_func)) # define sk_X509_POLICY_DATA_shift(st) SKM_sk_shift(X509_POLICY_DATA, (st)) # define sk_X509_POLICY_DATA_pop(st) SKM_sk_pop(X509_POLICY_DATA, (st)) # define sk_X509_POLICY_DATA_sort(st) SKM_sk_sort(X509_POLICY_DATA, (st)) @@ -1906,6 +2032,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_POLICY_NODE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_POLICY_NODE, (st), (cmp)) # define sk_X509_POLICY_NODE_dup(st) SKM_sk_dup(X509_POLICY_NODE, st) # define sk_X509_POLICY_NODE_pop_free(st, free_func) SKM_sk_pop_free(X509_POLICY_NODE, (st), (free_func)) +# define sk_X509_POLICY_NODE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_POLICY_NODE, (st), (copy_func), (free_func)) # define sk_X509_POLICY_NODE_shift(st) SKM_sk_shift(X509_POLICY_NODE, (st)) # define sk_X509_POLICY_NODE_pop(st) SKM_sk_pop(X509_POLICY_NODE, (st)) # define sk_X509_POLICY_NODE_sort(st) SKM_sk_sort(X509_POLICY_NODE, (st)) @@ -1927,6 +2054,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_PURPOSE_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_PURPOSE, (st), (cmp)) # define sk_X509_PURPOSE_dup(st) SKM_sk_dup(X509_PURPOSE, st) # define sk_X509_PURPOSE_pop_free(st, free_func) SKM_sk_pop_free(X509_PURPOSE, (st), (free_func)) +# define sk_X509_PURPOSE_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_PURPOSE, (st), (copy_func), (free_func)) # define sk_X509_PURPOSE_shift(st) SKM_sk_shift(X509_PURPOSE, (st)) # define sk_X509_PURPOSE_pop(st) SKM_sk_pop(X509_PURPOSE, (st)) # define sk_X509_PURPOSE_sort(st) SKM_sk_sort(X509_PURPOSE, (st)) @@ -1948,6 +2076,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_REVOKED_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_REVOKED, (st), (cmp)) # define sk_X509_REVOKED_dup(st) SKM_sk_dup(X509_REVOKED, st) # define sk_X509_REVOKED_pop_free(st, free_func) SKM_sk_pop_free(X509_REVOKED, (st), (free_func)) +# define sk_X509_REVOKED_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_REVOKED, (st), (copy_func), (free_func)) # define sk_X509_REVOKED_shift(st) SKM_sk_shift(X509_REVOKED, (st)) # define sk_X509_REVOKED_pop(st) SKM_sk_pop(X509_REVOKED, (st)) # define sk_X509_REVOKED_sort(st) SKM_sk_sort(X509_REVOKED, (st)) @@ -1969,6 +2098,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_TRUST_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_TRUST, (st), (cmp)) # define sk_X509_TRUST_dup(st) SKM_sk_dup(X509_TRUST, st) # define sk_X509_TRUST_pop_free(st, free_func) SKM_sk_pop_free(X509_TRUST, (st), (free_func)) +# define sk_X509_TRUST_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_TRUST, (st), (copy_func), (free_func)) # define sk_X509_TRUST_shift(st) SKM_sk_shift(X509_TRUST, (st)) # define sk_X509_TRUST_pop(st) SKM_sk_pop(X509_TRUST, (st)) # define sk_X509_TRUST_sort(st) SKM_sk_sort(X509_TRUST, (st)) @@ -1990,6 +2120,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_X509_VERIFY_PARAM_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(X509_VERIFY_PARAM, (st), (cmp)) # define sk_X509_VERIFY_PARAM_dup(st) SKM_sk_dup(X509_VERIFY_PARAM, st) # define sk_X509_VERIFY_PARAM_pop_free(st, free_func) SKM_sk_pop_free(X509_VERIFY_PARAM, (st), (free_func)) +# define sk_X509_VERIFY_PARAM_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(X509_VERIFY_PARAM, (st), (copy_func), (free_func)) # define sk_X509_VERIFY_PARAM_shift(st) SKM_sk_shift(X509_VERIFY_PARAM, (st)) # define sk_X509_VERIFY_PARAM_pop(st) SKM_sk_pop(X509_VERIFY_PARAM, (st)) # define sk_X509_VERIFY_PARAM_sort(st) SKM_sk_sort(X509_VERIFY_PARAM, (st)) @@ -2011,6 +2142,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_nid_triple_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(nid_triple, (st), (cmp)) # define sk_nid_triple_dup(st) SKM_sk_dup(nid_triple, st) # define sk_nid_triple_pop_free(st, free_func) SKM_sk_pop_free(nid_triple, (st), (free_func)) +# define sk_nid_triple_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(nid_triple, (st), (copy_func), (free_func)) # define sk_nid_triple_shift(st) SKM_sk_shift(nid_triple, (st)) # define sk_nid_triple_pop(st) SKM_sk_pop(nid_triple, (st)) # define sk_nid_triple_sort(st) SKM_sk_sort(nid_triple, (st)) @@ -2032,6 +2164,7 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_void_set_cmp_func(st, cmp) SKM_sk_set_cmp_func(void, (st), (cmp)) # define sk_void_dup(st) SKM_sk_dup(void, st) # define sk_void_pop_free(st, free_func) SKM_sk_pop_free(void, (st), (free_func)) +# define sk_void_deep_copy(st, copy_func, free_func) SKM_sk_deep_copy(void, (st), (copy_func), (free_func)) # define sk_void_shift(st) SKM_sk_shift(void, (st)) # define sk_void_pop(st) SKM_sk_pop(void, (st)) # define sk_void_sort(st) SKM_sk_sort(void, (st)) @@ -2042,7 +2175,8 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_OPENSSL_STRING_find(st, val) sk_find(CHECKED_STACK_OF(OPENSSL_STRING, st), CHECKED_PTR_OF(char, val)) # define sk_OPENSSL_STRING_value(st, i) ((OPENSSL_STRING)sk_value(CHECKED_STACK_OF(OPENSSL_STRING, st), i)) # define sk_OPENSSL_STRING_num(st) SKM_sk_num(OPENSSL_STRING, st) -# define sk_OPENSSL_STRING_pop_free(st, free_func) sk_pop_free(CHECKED_STACK_OF(OPENSSL_STRING, st), CHECKED_SK_FREE_FUNC2(OPENSSL_STRING, free_func)) +# define sk_OPENSSL_STRING_pop_free(st, free_func) sk_pop_free(CHECKED_STACK_OF(OPENSSL_STRING, st), CHECKED_SK_FREE_FUNC(char, free_func)) +# define sk_OPENSSL_STRING_deep_copy(st, copy_func, free_func) ((STACK_OF(OPENSSL_STRING) *)sk_deep_copy(CHECKED_STACK_OF(OPENSSL_STRING, st), CHECKED_SK_COPY_FUNC(char, copy_func), CHECKED_SK_FREE_FUNC(char, free_func))) # define sk_OPENSSL_STRING_insert(st, val, i) sk_insert(CHECKED_STACK_OF(OPENSSL_STRING, st), CHECKED_PTR_OF(char, val), i) # define sk_OPENSSL_STRING_free(st) SKM_sk_free(OPENSSL_STRING, st) # define sk_OPENSSL_STRING_set(st, i, val) sk_set(CHECKED_STACK_OF(OPENSSL_STRING, st), i, CHECKED_PTR_OF(char, val)) @@ -2065,7 +2199,8 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_OPENSSL_BLOCK_find(st, val) sk_find(CHECKED_STACK_OF(OPENSSL_BLOCK, st), CHECKED_PTR_OF(void, val)) # define sk_OPENSSL_BLOCK_value(st, i) ((OPENSSL_BLOCK)sk_value(CHECKED_STACK_OF(OPENSSL_BLOCK, st), i)) # define sk_OPENSSL_BLOCK_num(st) SKM_sk_num(OPENSSL_BLOCK, st) -# define sk_OPENSSL_BLOCK_pop_free(st, free_func) sk_pop_free(CHECKED_STACK_OF(OPENSSL_BLOCK, st), CHECKED_SK_FREE_FUNC2(OPENSSL_BLOCK, free_func)) +# define sk_OPENSSL_BLOCK_pop_free(st, free_func) sk_pop_free(CHECKED_STACK_OF(OPENSSL_BLOCK, st), CHECKED_SK_FREE_FUNC(void, free_func)) +# define sk_OPENSSL_BLOCK_deep_copy(st, copy_func, free_func) ((STACK_OF(OPENSSL_BLOCK) *)sk_deep_copy(CHECKED_STACK_OF(OPENSSL_BLOCK, st), CHECKED_SK_COPY_FUNC(void, copy_func), CHECKED_SK_FREE_FUNC(void, free_func))) # define sk_OPENSSL_BLOCK_insert(st, val, i) sk_insert(CHECKED_STACK_OF(OPENSSL_BLOCK, st), CHECKED_PTR_OF(void, val), i) # define sk_OPENSSL_BLOCK_free(st) SKM_sk_free(OPENSSL_BLOCK, st) # define sk_OPENSSL_BLOCK_set(st, i, val) sk_set(CHECKED_STACK_OF(OPENSSL_BLOCK, st), i, CHECKED_PTR_OF(void, val)) @@ -2088,7 +2223,8 @@ DECLARE_SPECIAL_STACK_OF(OPENSSL_BLOCK, void) # define sk_OPENSSL_PSTRING_find(st, val) sk_find(CHECKED_STACK_OF(OPENSSL_PSTRING, st), CHECKED_PTR_OF(OPENSSL_STRING, val)) # define sk_OPENSSL_PSTRING_value(st, i) ((OPENSSL_PSTRING)sk_value(CHECKED_STACK_OF(OPENSSL_PSTRING, st), i)) # define sk_OPENSSL_PSTRING_num(st) SKM_sk_num(OPENSSL_PSTRING, st) -# define sk_OPENSSL_PSTRING_pop_free(st, free_func) sk_pop_free(CHECKED_STACK_OF(OPENSSL_PSTRING, st), CHECKED_SK_FREE_FUNC2(OPENSSL_PSTRING, free_func)) +# define sk_OPENSSL_PSTRING_pop_free(st, free_func) sk_pop_free(CHECKED_STACK_OF(OPENSSL_PSTRING, st), CHECKED_SK_FREE_FUNC(OPENSSL_STRING, free_func)) +# define sk_OPENSSL_PSTRING_deep_copy(st, copy_func, free_func) ((STACK_OF(OPENSSL_PSTRING) *)sk_deep_copy(CHECKED_STACK_OF(OPENSSL_PSTRING, st), CHECKED_SK_COPY_FUNC(OPENSSL_STRING, copy_func), CHECKED_SK_FREE_FUNC(OPENSSL_STRING, free_func))) # define sk_OPENSSL_PSTRING_insert(st, val, i) sk_insert(CHECKED_STACK_OF(OPENSSL_PSTRING, st), CHECKED_PTR_OF(OPENSSL_STRING, val), i) # define sk_OPENSSL_PSTRING_free(st) SKM_sk_free(OPENSSL_PSTRING, st) # define sk_OPENSSL_PSTRING_set(st, i, val) sk_set(CHECKED_STACK_OF(OPENSSL_PSTRING, st), i, CHECKED_PTR_OF(OPENSSL_STRING, val)) diff --git a/crypto/stack/stack.c b/crypto/stack/stack.c index 331f907190f7..de437acf6a5c 100644 --- a/crypto/stack/stack.c +++ b/crypto/stack/stack.c @@ -115,6 +115,40 @@ _STACK *sk_dup(_STACK *sk) return (NULL); } +_STACK *sk_deep_copy(_STACK *sk, void *(*copy_func) (void *), + void (*free_func) (void *)) +{ + _STACK *ret; + int i; + + if ((ret = OPENSSL_malloc(sizeof(_STACK))) == NULL) + return ret; + ret->comp = sk->comp; + ret->sorted = sk->sorted; + ret->num = sk->num; + ret->num_alloc = sk->num > MIN_NODES ? sk->num : MIN_NODES; + ret->data = OPENSSL_malloc(sizeof(char *) * ret->num_alloc); + if (ret->data == NULL) { + OPENSSL_free(ret); + return NULL; + } + for (i = 0; i < ret->num_alloc; i++) + ret->data[i] = NULL; + + for (i = 0; i < ret->num; ++i) { + if (sk->data[i] == NULL) + continue; + if ((ret->data[i] = copy_func(sk->data[i])) == NULL) { + while (--i >= 0) + if (ret->data[i] != NULL) + free_func(ret->data[i]); + sk_free(ret); + return NULL; + } + } + return ret; +} + _STACK *sk_new_null(void) { return sk_new((int (*)(const void *, const void *))0); diff --git a/crypto/stack/stack.h b/crypto/stack/stack.h index 8d6e939a8031..eb07216659d0 100644 --- a/crypto/stack/stack.h +++ b/crypto/stack/stack.h @@ -83,6 +83,7 @@ _STACK *sk_new(int (*cmp) (const void *, const void *)); _STACK *sk_new_null(void); void sk_free(_STACK *); void sk_pop_free(_STACK *st, void (*func) (void *)); +_STACK *sk_deep_copy(_STACK *, void *(*)(void *), void (*)(void *)); int sk_insert(_STACK *sk, void *data, int where); void *sk_delete(_STACK *st, int loc); void *sk_delete_ptr(_STACK *st, void *p); diff --git a/crypto/symhacks.h b/crypto/symhacks.h index 2eadf7f3df2b..239fa4fb1b77 100644 --- a/crypto/symhacks.h +++ b/crypto/symhacks.h @@ -166,7 +166,7 @@ # undef CRYPTO_get_locked_mem_ex_functions # define CRYPTO_get_locked_mem_ex_functions CRYPTO_get_locked_mem_ex_funcs -/* Hack some long SSL names */ +/* Hack some long SSL/TLS names */ # undef SSL_CTX_set_default_verify_paths # define SSL_CTX_set_default_verify_paths SSL_CTX_set_def_verify_paths # undef SSL_get_ex_data_X509_STORE_CTX_idx @@ -183,6 +183,10 @@ # define SSL_CTX_set_default_passwd_cb_userdata SSL_CTX_set_def_passwd_cb_ud # undef SSL_COMP_get_compression_methods # define SSL_COMP_get_compression_methods SSL_COMP_get_compress_methods +# undef SSL_COMP_set0_compression_methods +# define SSL_COMP_set0_compression_methods SSL_COMP_set0_compress_methods +# undef SSL_COMP_free_compression_methods +# define SSL_COMP_free_compression_methods SSL_COMP_free_compress_methods # undef ssl_add_clienthello_renegotiate_ext # define ssl_add_clienthello_renegotiate_ext ssl_add_clienthello_reneg_ext # undef ssl_add_serverhello_renegotiate_ext @@ -211,6 +215,16 @@ # define SSL_CTX_set_next_protos_advertised_cb SSL_CTX_set_next_protos_adv_cb # undef SSL_CTX_set_next_proto_select_cb # define SSL_CTX_set_next_proto_select_cb SSL_CTX_set_next_proto_sel_cb + +# undef tls1_send_server_supplemental_data +# define tls1_send_server_supplemental_data tls1_send_server_suppl_data +# undef tls1_send_client_supplemental_data +# define tls1_send_client_supplemental_data tls1_send_client_suppl_data +# undef tls1_get_server_supplemental_data +# define tls1_get_server_supplemental_data tls1_get_server_suppl_data +# undef tls1_get_client_supplemental_data +# define tls1_get_client_supplemental_data tls1_get_client_suppl_data + # undef ssl3_cbc_record_digest_supported # define ssl3_cbc_record_digest_supported ssl3_cbc_record_digest_support # undef ssl_check_clienthello_tlsext_late @@ -218,7 +232,11 @@ # undef ssl_check_clienthello_tlsext_early # define ssl_check_clienthello_tlsext_early ssl_check_clihello_tlsext_early -/* Hack some long ENGINE names */ +/* Hack some RSA long names */ +# undef RSA_padding_check_PKCS1_OAEP_mgf1 +# define RSA_padding_check_PKCS1_OAEP_mgf1 RSA_pad_check_PKCS1_OAEP_mgf1 + +/* Hack some ENGINE long names */ # undef ENGINE_get_default_BN_mod_exp_crt # define ENGINE_get_default_BN_mod_exp_crt ENGINE_get_def_BN_mod_exp_crt # undef ENGINE_set_default_BN_mod_exp_crt @@ -427,6 +445,18 @@ # define CMS_OriginatorIdentifierOrKey_it CMS_OriginatorIdOrKey_it # undef cms_SignerIdentifier_get0_signer_id # define cms_SignerIdentifier_get0_signer_id cms_SignerId_get0_signer_id +# undef CMS_RecipientInfo_kari_get0_orig_id +# define CMS_RecipientInfo_kari_get0_orig_id CMS_RecipInfo_kari_get0_orig_id +# undef CMS_RecipientInfo_kari_get0_reks +# define CMS_RecipientInfo_kari_get0_reks CMS_RecipInfo_kari_get0_reks +# undef CMS_RecipientEncryptedKey_cert_cmp +# define CMS_RecipientEncryptedKey_cert_cmp CMS_RecipEncryptedKey_cert_cmp +# undef CMS_RecipientInfo_kari_set0_pkey +# define CMS_RecipientInfo_kari_set0_pkey CMS_RecipInfo_kari_set0_pkey +# undef CMS_RecipientEncryptedKey_get0_id +# define CMS_RecipientEncryptedKey_get0_id CMS_RecipEncryptedKey_get0_id +# undef CMS_RecipientInfo_kari_orig_id_cmp +# define CMS_RecipientInfo_kari_orig_id_cmp CMS_RecipInfo_kari_orig_id_cmp /* Hack some long DTLS1 names */ # undef dtls1_retransmit_buffered_messages diff --git a/crypto/ts/ts_rsp_sign.c b/crypto/ts/ts_rsp_sign.c index 031d872e2cf6..db6ce3241f73 100644 --- a/crypto/ts/ts_rsp_sign.c +++ b/crypto/ts/ts_rsp_sign.c @@ -238,7 +238,6 @@ int TS_RESP_CTX_set_def_policy(TS_RESP_CTX *ctx, ASN1_OBJECT *def_policy) int TS_RESP_CTX_set_certs(TS_RESP_CTX *ctx, STACK_OF(X509) *certs) { - int i; if (ctx->certs) { sk_X509_pop_free(ctx->certs, X509_free); @@ -246,14 +245,10 @@ int TS_RESP_CTX_set_certs(TS_RESP_CTX *ctx, STACK_OF(X509) *certs) } if (!certs) return 1; - if (!(ctx->certs = sk_X509_dup(certs))) { + if (!(ctx->certs = X509_chain_up_ref(certs))) { TSerr(TS_F_TS_RESP_CTX_SET_CERTS, ERR_R_MALLOC_FAILURE); return 0; } - for (i = 0; i < sk_X509_num(ctx->certs); ++i) { - X509 *cert = sk_X509_value(ctx->certs, i); - CRYPTO_add(&cert->references, +1, CRYPTO_LOCK_X509); - } return 1; } diff --git a/crypto/ts/ts_rsp_verify.c b/crypto/ts/ts_rsp_verify.c index 32b4d9923f46..3ce765dfa1b6 100644 --- a/crypto/ts/ts_rsp_verify.c +++ b/crypto/ts/ts_rsp_verify.c @@ -637,7 +637,7 @@ static int TS_compute_imprint(BIO *data, TS_TST_INFO *tst_info, X509_ALGOR_free(*md_alg); OPENSSL_free(*imprint); *imprint_len = 0; - *imprint = NULL; + *imprint = 0; return 0; } diff --git a/crypto/ui/ui_openssl.c b/crypto/ui/ui_openssl.c index 829ea8691eb5..5d66276418fc 100644 --- a/crypto/ui/ui_openssl.c +++ b/crypto/ui/ui_openssl.c @@ -208,7 +208,7 @@ # elif !defined(OPENSSL_SYS_VMS) \ && !defined(OPENSSL_SYS_MSDOS) \ && !defined(OPENSSL_SYS_MACINTOSH_CLASSIC) \ - && !defined(MAC_OS_GUSI_SOURCE) \ + && !defined(MAC_OS_GUSI_SOURCE) \ && !defined(OPENSSL_SYS_VXWORKS) \ && !defined(OPENSSL_SYS_NETWARE) # define TERMIOS diff --git a/crypto/whrlpool/asm/wp-mmx.pl b/crypto/whrlpool/asm/wp-mmx.pl index cb2381c22ba1..c584e5b92b25 100755 --- a/crypto/whrlpool/asm/wp-mmx.pl +++ b/crypto/whrlpool/asm/wp-mmx.pl @@ -118,34 +118,36 @@ $tbl="ebp"; &movq (@mm[0],&QWP(2048*$SCALE,$tbl,"esi",8)); # rc[r] &mov ("eax",&DWP(0,"esp")); &mov ("ebx",&DWP(4,"esp")); + &movz ("ecx",&LB("eax")); + &movz ("edx",&HB("eax")); for($i=0;$i<8;$i++) { my $func = ($i==0)? \&movq : \&pxor; - &movb (&LB("ecx"),&LB("eax")); - &movb (&LB("edx"),&HB("eax")); + &shr ("eax",16); &scale ("esi","ecx"); + &movz ("ecx",&LB("eax")); &scale ("edi","edx"); - &shr ("eax",16); + &movz ("edx",&HB("eax")); &pxor (@mm[0],&QWP(&row(0),$tbl,"esi",8)); &$func (@mm[1],&QWP(&row(1),$tbl,"edi",8)); - &movb (&LB("ecx"),&LB("eax")); - &movb (&LB("edx"),&HB("eax")); &mov ("eax",&DWP(($i+1)*8,"esp")); &scale ("esi","ecx"); + &movz ("ecx",&LB("ebx")); &scale ("edi","edx"); + &movz ("edx",&HB("ebx")); &$func (@mm[2],&QWP(&row(2),$tbl,"esi",8)); &$func (@mm[3],&QWP(&row(3),$tbl,"edi",8)); - &movb (&LB("ecx"),&LB("ebx")); - &movb (&LB("edx"),&HB("ebx")); + &shr ("ebx",16); &scale ("esi","ecx"); + &movz ("ecx",&LB("ebx")); &scale ("edi","edx"); - &shr ("ebx",16); + &movz ("edx",&HB("ebx")); &$func (@mm[4],&QWP(&row(4),$tbl,"esi",8)); &$func (@mm[5],&QWP(&row(5),$tbl,"edi",8)); - &movb (&LB("ecx"),&LB("ebx")); - &movb (&LB("edx"),&HB("ebx")); &mov ("ebx",&DWP(($i+1)*8+4,"esp")); &scale ("esi","ecx"); + &movz ("ecx",&LB("eax")); &scale ("edi","edx"); + &movz ("edx",&HB("eax")); &$func (@mm[6],&QWP(&row(6),$tbl,"esi",8)); &$func (@mm[7],&QWP(&row(7),$tbl,"edi",8)); push(@mm,shift(@mm)); @@ -154,32 +156,32 @@ for($i=0;$i<8;$i++) { for($i=0;$i<8;$i++) { &movq(&QWP($i*8,"esp"),@mm[$i]); } # K=L for($i=0;$i<8;$i++) { - &movb (&LB("ecx"),&LB("eax")); - &movb (&LB("edx"),&HB("eax")); + &shr ("eax",16); &scale ("esi","ecx"); + &movz ("ecx",&LB("eax")); &scale ("edi","edx"); - &shr ("eax",16); + &movz ("edx",&HB("eax")); &pxor (@mm[0],&QWP(&row(0),$tbl,"esi",8)); &pxor (@mm[1],&QWP(&row(1),$tbl,"edi",8)); - &movb (&LB("ecx"),&LB("eax")); - &movb (&LB("edx"),&HB("eax")); &mov ("eax",&DWP(64+($i+1)*8,"esp")) if ($i<7); &scale ("esi","ecx"); + &movz ("ecx",&LB("ebx")); &scale ("edi","edx"); + &movz ("edx",&HB("ebx")); &pxor (@mm[2],&QWP(&row(2),$tbl,"esi",8)); &pxor (@mm[3],&QWP(&row(3),$tbl,"edi",8)); - &movb (&LB("ecx"),&LB("ebx")); - &movb (&LB("edx"),&HB("ebx")); + &shr ("ebx",16); &scale ("esi","ecx"); + &movz ("ecx",&LB("ebx")); &scale ("edi","edx"); - &shr ("ebx",16); + &movz ("edx",&HB("ebx")); &pxor (@mm[4],&QWP(&row(4),$tbl,"esi",8)); &pxor (@mm[5],&QWP(&row(5),$tbl,"edi",8)); - &movb (&LB("ecx"),&LB("ebx")); - &movb (&LB("edx"),&HB("ebx")); &mov ("ebx",&DWP(64+($i+1)*8+4,"esp")) if ($i<7); &scale ("esi","ecx"); + &movz ("ecx",&LB("eax")); &scale ("edi","edx"); + &movz ("edx",&HB("eax")); &pxor (@mm[6],&QWP(&row(6),$tbl,"esi",8)); &pxor (@mm[7],&QWP(&row(7),$tbl,"edi",8)); push(@mm,shift(@mm)); diff --git a/crypto/whrlpool/asm/wp-x86_64.pl b/crypto/whrlpool/asm/wp-x86_64.pl index 24b2ff60c38b..5a3bdbcf20d1 100755 --- a/crypto/whrlpool/asm/wp-x86_64.pl +++ b/crypto/whrlpool/asm/wp-x86_64.pl @@ -91,41 +91,44 @@ for($i=0;$i<8;$i++) { $code.="mov @mm[$i],64+$i*8(%rsp)\n"; } # S=L $code.=<<___; xor %rsi,%rsi mov %rsi,24(%rbx) # zero round counter + jmp .Lround .align 16 .Lround: mov 4096(%rbp,%rsi,8),@mm[0] # rc[r] mov 0(%rsp),%eax mov 4(%rsp),%ebx + movz %al,%ecx + movz %ah,%edx ___ for($i=0;$i<8;$i++) { my $func = ($i==0)? "mov" : "xor"; $code.=<<___; - mov %al,%cl - mov %ah,%dl + shr \$16,%eax lea (%rcx,%rcx),%rsi + movz %al,%ecx lea (%rdx,%rdx),%rdi - shr \$16,%eax + movz %ah,%edx xor 0(%rbp,%rsi,8),@mm[0] $func 7(%rbp,%rdi,8),@mm[1] - mov %al,%cl - mov %ah,%dl mov $i*8+8(%rsp),%eax # ($i+1)*8 lea (%rcx,%rcx),%rsi + movz %bl,%ecx lea (%rdx,%rdx),%rdi + movz %bh,%edx $func 6(%rbp,%rsi,8),@mm[2] $func 5(%rbp,%rdi,8),@mm[3] - mov %bl,%cl - mov %bh,%dl + shr \$16,%ebx lea (%rcx,%rcx),%rsi + movz %bl,%ecx lea (%rdx,%rdx),%rdi - shr \$16,%ebx + movz %bh,%edx $func 4(%rbp,%rsi,8),@mm[4] $func 3(%rbp,%rdi,8),@mm[5] - mov %bl,%cl - mov %bh,%dl mov $i*8+8+4(%rsp),%ebx # ($i+1)*8+4 lea (%rcx,%rcx),%rsi + movz %al,%ecx lea (%rdx,%rdx),%rdi + movz %ah,%edx $func 2(%rbp,%rsi,8),@mm[6] $func 1(%rbp,%rdi,8),@mm[7] ___ @@ -134,32 +137,32 @@ ___ for($i=0;$i<8;$i++) { $code.="mov @mm[$i],$i*8(%rsp)\n"; } # K=L for($i=0;$i<8;$i++) { $code.=<<___; - mov %al,%cl - mov %ah,%dl + shr \$16,%eax lea (%rcx,%rcx),%rsi + movz %al,%ecx lea (%rdx,%rdx),%rdi - shr \$16,%eax + movz %ah,%edx xor 0(%rbp,%rsi,8),@mm[0] xor 7(%rbp,%rdi,8),@mm[1] - mov %al,%cl - mov %ah,%dl `"mov 64+$i*8+8(%rsp),%eax" if($i<7);` # 64+($i+1)*8 lea (%rcx,%rcx),%rsi + movz %bl,%ecx lea (%rdx,%rdx),%rdi + movz %bh,%edx xor 6(%rbp,%rsi,8),@mm[2] xor 5(%rbp,%rdi,8),@mm[3] - mov %bl,%cl - mov %bh,%dl + shr \$16,%ebx lea (%rcx,%rcx),%rsi + movz %bl,%ecx lea (%rdx,%rdx),%rdi - shr \$16,%ebx + movz %bh,%edx xor 4(%rbp,%rsi,8),@mm[4] xor 3(%rbp,%rdi,8),@mm[5] - mov %bl,%cl - mov %bh,%dl `"mov 64+$i*8+8+4(%rsp),%ebx" if($i<7);` # 64+($i+1)*8+4 lea (%rcx,%rcx),%rsi + movz %al,%ecx lea (%rdx,%rdx),%rdi + movz %ah,%edx xor 2(%rbp,%rsi,8),@mm[6] xor 1(%rbp,%rdi,8),@mm[7] ___ diff --git a/crypto/x509/Makefile b/crypto/x509/Makefile index aac3ece18cb2..bf197a1d932a 100644 --- a/crypto/x509/Makefile +++ b/crypto/x509/Makefile @@ -33,7 +33,7 @@ LIBOBJ= x509_def.o x509_d2.o x509_r2x.o x509_cmp.o \ SRC= $(LIBSRC) EXHEADER= x509.h x509_vfy.h -HEADER= $(EXHEADER) +HEADER= $(EXHEADER) vpm_int.h ALL= $(GENERAL) $(SRC) $(HEADER) @@ -314,7 +314,7 @@ x509_vfy.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h x509_vfy.o: ../../include/openssl/sha.h ../../include/openssl/stack.h x509_vfy.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h x509_vfy.o: ../../include/openssl/x509_vfy.h ../../include/openssl/x509v3.h -x509_vfy.o: ../cryptlib.h x509_vfy.c +x509_vfy.o: ../cryptlib.h vpm_int.h x509_vfy.c x509_vpm.o: ../../e_os.h ../../include/openssl/asn1.h x509_vpm.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h x509_vpm.o: ../../include/openssl/conf.h ../../include/openssl/crypto.h @@ -328,7 +328,7 @@ x509_vpm.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h x509_vpm.o: ../../include/openssl/sha.h ../../include/openssl/stack.h x509_vpm.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h x509_vpm.o: ../../include/openssl/x509_vfy.h ../../include/openssl/x509v3.h -x509_vpm.o: ../cryptlib.h x509_vpm.c +x509_vpm.o: ../cryptlib.h vpm_int.h x509_vpm.c x509cset.o: ../../e_os.h ../../include/openssl/asn1.h x509cset.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h x509cset.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h @@ -395,15 +395,17 @@ x509type.o: ../../include/openssl/sha.h ../../include/openssl/stack.h x509type.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h x509type.o: ../../include/openssl/x509_vfy.h ../cryptlib.h x509type.c x_all.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h -x_all.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h -x_all.o: ../../include/openssl/dsa.h ../../include/openssl/e_os2.h -x_all.o: ../../include/openssl/ec.h ../../include/openssl/ecdh.h -x_all.o: ../../include/openssl/ecdsa.h ../../include/openssl/err.h -x_all.o: ../../include/openssl/evp.h ../../include/openssl/lhash.h -x_all.o: ../../include/openssl/obj_mac.h ../../include/openssl/objects.h +x_all.o: ../../include/openssl/buffer.h ../../include/openssl/conf.h +x_all.o: ../../include/openssl/crypto.h ../../include/openssl/dsa.h +x_all.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h +x_all.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h +x_all.o: ../../include/openssl/err.h ../../include/openssl/evp.h +x_all.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h +x_all.o: ../../include/openssl/objects.h ../../include/openssl/ocsp.h x_all.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h x_all.o: ../../include/openssl/ossl_typ.h ../../include/openssl/pkcs7.h x_all.o: ../../include/openssl/rsa.h ../../include/openssl/safestack.h x_all.o: ../../include/openssl/sha.h ../../include/openssl/stack.h x_all.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h -x_all.o: ../../include/openssl/x509_vfy.h ../cryptlib.h x_all.c +x_all.o: ../../include/openssl/x509_vfy.h ../../include/openssl/x509v3.h +x_all.o: ../cryptlib.h x_all.c diff --git a/crypto/x509/verify_extra_test.c b/crypto/x509/verify_extra_test.c index a1e41f2822bd..08509f013194 100644 --- a/crypto/x509/verify_extra_test.c +++ b/crypto/x509/verify_extra_test.c @@ -168,8 +168,7 @@ static int test_alt_chains_cert_forgery(void) i = X509_verify_cert(sctx); - if(i == 0 && X509_STORE_CTX_get_error(sctx) - == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT) { + if(i == 0 && X509_STORE_CTX_get_error(sctx) == X509_V_ERR_INVALID_CA) { /* This is the result we were expecting: Test passed */ ret = 1; } diff --git a/crypto/x509/vpm_int.h b/crypto/x509/vpm_int.h new file mode 100644 index 000000000000..9c55defc512a --- /dev/null +++ b/crypto/x509/vpm_int.h @@ -0,0 +1,70 @@ +/* vpm_int.h */ +/* + * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project + * 2013. + */ +/* ==================================================================== + * Copyright (c) 2013 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + +/* internal only structure to hold additional X509_VERIFY_PARAM data */ + +struct X509_VERIFY_PARAM_ID_st { + STACK_OF(OPENSSL_STRING) *hosts; /* Set of acceptable names */ + unsigned int hostflags; /* Flags to control matching features */ + char *peername; /* Matching hostname in peer certificate */ + char *email; /* If not NULL email address to match */ + size_t emaillen; + unsigned char *ip; /* If not NULL IP address to match */ + size_t iplen; /* Length of IP address */ +}; diff --git a/crypto/x509/x509.h b/crypto/x509/x509.h index a4911741005e..99337b849a52 100644 --- a/crypto/x509/x509.h +++ b/crypto/x509/x509.h @@ -361,6 +361,7 @@ typedef struct x509_cert_pair_st { # define X509_FLAG_NO_SIGDUMP (1L << 9) # define X509_FLAG_NO_AUX (1L << 10) # define X509_FLAG_NO_ATTRIBUTES (1L << 11) +# define X509_FLAG_NO_IDS (1L << 12) /* Flags specific to X509_NAME_print_ex() */ @@ -645,10 +646,12 @@ int X509_signature_print(BIO *bp, X509_ALGOR *alg, ASN1_STRING *sig); int X509_sign(X509 *x, EVP_PKEY *pkey, const EVP_MD *md); int X509_sign_ctx(X509 *x, EVP_MD_CTX *ctx); +int X509_http_nbio(OCSP_REQ_CTX *rctx, X509 **pcert); int X509_REQ_sign(X509_REQ *x, EVP_PKEY *pkey, const EVP_MD *md); int X509_REQ_sign_ctx(X509_REQ *x, EVP_MD_CTX *ctx); int X509_CRL_sign(X509_CRL *x, EVP_PKEY *pkey, const EVP_MD *md); int X509_CRL_sign_ctx(X509_CRL *x, EVP_MD_CTX *ctx); +int X509_CRL_http_nbio(OCSP_REQ_CTX *rctx, X509_CRL **pcrl); int NETSCAPE_SPKI_sign(NETSCAPE_SPKI *x, EVP_PKEY *pkey, const EVP_MD *md); int X509_pubkey_digest(const X509 *data, const EVP_MD *type, @@ -745,6 +748,7 @@ X509 *X509_dup(X509 *x509); X509_ATTRIBUTE *X509_ATTRIBUTE_dup(X509_ATTRIBUTE *xa); X509_EXTENSION *X509_EXTENSION_dup(X509_EXTENSION *ex); X509_CRL *X509_CRL_dup(X509_CRL *crl); +X509_REVOKED *X509_REVOKED_dup(X509_REVOKED *rev); X509_REQ *X509_REQ_dup(X509_REQ *req); X509_ALGOR *X509_ALGOR_dup(X509_ALGOR *xn); int X509_ALGOR_set0(X509_ALGOR *alg, ASN1_OBJECT *aobj, int ptype, @@ -828,6 +832,12 @@ void *X509_get_ex_data(X509 *r, int idx); int i2d_X509_AUX(X509 *a, unsigned char **pp); X509 *d2i_X509_AUX(X509 **a, const unsigned char **pp, long length); +int i2d_re_X509_tbs(X509 *x, unsigned char **pp); + +void X509_get0_signature(ASN1_BIT_STRING **psig, X509_ALGOR **palg, + const X509 *x); +int X509_get_signature_nid(const X509 *x); + int X509_alias_set1(X509 *x, unsigned char *name, int len); int X509_keyid_set1(X509 *x, unsigned char *id, int len); unsigned char *X509_alias_get0(X509 *x, int *len); @@ -939,9 +949,17 @@ int X509_CRL_sort(X509_CRL *crl); int X509_REVOKED_set_serialNumber(X509_REVOKED *x, ASN1_INTEGER *serial); int X509_REVOKED_set_revocationDate(X509_REVOKED *r, ASN1_TIME *tm); +X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, + EVP_PKEY *skey, const EVP_MD *md, unsigned int flags); + int X509_REQ_check_private_key(X509_REQ *x509, EVP_PKEY *pkey); int X509_check_private_key(X509 *x509, EVP_PKEY *pkey); +int X509_chain_check_suiteb(int *perror_depth, + X509 *x, STACK_OF(X509) *chain, + unsigned long flags); +int X509_CRL_check_suiteb(X509_CRL *crl, EVP_PKEY *pk, unsigned long flags); +STACK_OF(X509) *X509_chain_up_ref(STACK_OF(X509) *chain); int X509_issuer_and_serial_cmp(const X509 *a, const X509 *b); unsigned long X509_issuer_and_serial_hash(X509 *a); @@ -1236,6 +1254,7 @@ void ERR_load_X509_strings(void); # define X509_F_X509_ATTRIBUTE_GET0_DATA 139 # define X509_F_X509_ATTRIBUTE_SET1_DATA 138 # define X509_F_X509_CHECK_PRIVATE_KEY 128 +# define X509_F_X509_CRL_DIFF 105 # define X509_F_X509_CRL_PRINT_FP 147 # define X509_F_X509_EXTENSION_CREATE_BY_NID 108 # define X509_F_X509_EXTENSION_CREATE_BY_OBJ 109 @@ -1268,20 +1287,27 @@ void ERR_load_X509_strings(void); # define X509_F_X509_VERIFY_CERT 127 /* Reason codes. */ +# define X509_R_AKID_MISMATCH 110 # define X509_R_BAD_X509_FILETYPE 100 # define X509_R_BASE64_DECODE_ERROR 118 # define X509_R_CANT_CHECK_DH_KEY 114 # define X509_R_CERT_ALREADY_IN_HASH_TABLE 101 +# define X509_R_CRL_ALREADY_DELTA 127 +# define X509_R_CRL_VERIFY_FAILURE 131 # define X509_R_ERR_ASN1_LIB 102 +# define X509_R_IDP_MISMATCH 128 # define X509_R_INVALID_DIRECTORY 113 # define X509_R_INVALID_FIELD_NAME 119 # define X509_R_INVALID_TRUST 123 +# define X509_R_ISSUER_MISMATCH 129 # define X509_R_KEY_TYPE_MISMATCH 115 # define X509_R_KEY_VALUES_MISMATCH 116 # define X509_R_LOADING_CERT_DIR 103 # define X509_R_LOADING_DEFAULTS 104 # define X509_R_METHOD_NOT_SUPPORTED 124 +# define X509_R_NEWER_CRL_NOT_NEWER 132 # define X509_R_NO_CERT_SET_FOR_US_TO_VERIFY 105 +# define X509_R_NO_CRL_NUMBER 130 # define X509_R_PUBLIC_KEY_DECODE_ERROR 125 # define X509_R_PUBLIC_KEY_ENCODE_ERROR 126 # define X509_R_SHOULD_RETRY 106 diff --git a/crypto/x509/x509_cmp.c b/crypto/x509/x509_cmp.c index 3c5b717c15a3..49c71b91280e 100644 --- a/crypto/x509/x509_cmp.c +++ b/crypto/x509/x509_cmp.c @@ -179,11 +179,23 @@ unsigned long X509_subject_name_hash_old(X509 *x) */ int X509_cmp(const X509 *a, const X509 *b) { + int rv; /* ensure hash is valid */ X509_check_purpose((X509 *)a, -1, 0); X509_check_purpose((X509 *)b, -1, 0); - return memcmp(a->sha1_hash, b->sha1_hash, SHA_DIGEST_LENGTH); + rv = memcmp(a->sha1_hash, b->sha1_hash, SHA_DIGEST_LENGTH); + if (rv) + return rv; + /* Check for match against stored encoding too */ + if (!a->cert_info->enc.modified && !b->cert_info->enc.modified) { + rv = (int)(a->cert_info->enc.len - b->cert_info->enc.len); + if (rv) + return rv; + return memcmp(a->cert_info->enc.enc, b->cert_info->enc.enc, + a->cert_info->enc.len); + } + return rv; } #endif @@ -339,3 +351,148 @@ int X509_check_private_key(X509 *x, EVP_PKEY *k) return 1; return 0; } + +/* + * Check a suite B algorithm is permitted: pass in a public key and the NID + * of its signature (or 0 if no signature). The pflags is a pointer to a + * flags field which must contain the suite B verification flags. + */ + +#ifndef OPENSSL_NO_EC + +static int check_suite_b(EVP_PKEY *pkey, int sign_nid, unsigned long *pflags) +{ + const EC_GROUP *grp = NULL; + int curve_nid; + if (pkey && pkey->type == EVP_PKEY_EC) + grp = EC_KEY_get0_group(pkey->pkey.ec); + if (!grp) + return X509_V_ERR_SUITE_B_INVALID_ALGORITHM; + curve_nid = EC_GROUP_get_curve_name(grp); + /* Check curve is consistent with LOS */ + if (curve_nid == NID_secp384r1) { /* P-384 */ + /* + * Check signature algorithm is consistent with curve. + */ + if (sign_nid != -1 && sign_nid != NID_ecdsa_with_SHA384) + return X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM; + if (!(*pflags & X509_V_FLAG_SUITEB_192_LOS)) + return X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED; + /* If we encounter P-384 we cannot use P-256 later */ + *pflags &= ~X509_V_FLAG_SUITEB_128_LOS_ONLY; + } else if (curve_nid == NID_X9_62_prime256v1) { /* P-256 */ + if (sign_nid != -1 && sign_nid != NID_ecdsa_with_SHA256) + return X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM; + if (!(*pflags & X509_V_FLAG_SUITEB_128_LOS_ONLY)) + return X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED; + } else + return X509_V_ERR_SUITE_B_INVALID_CURVE; + + return X509_V_OK; +} + +int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain, + unsigned long flags) +{ + int rv, i, sign_nid; + EVP_PKEY *pk = NULL; + unsigned long tflags; + if (!(flags & X509_V_FLAG_SUITEB_128_LOS)) + return X509_V_OK; + tflags = flags; + /* If no EE certificate passed in must be first in chain */ + if (x == NULL) { + x = sk_X509_value(chain, 0); + i = 1; + } else + i = 0; + + if (X509_get_version(x) != 2) { + rv = X509_V_ERR_SUITE_B_INVALID_VERSION; + /* Correct error depth */ + i = 0; + goto end; + } + + pk = X509_get_pubkey(x); + /* Check EE key only */ + rv = check_suite_b(pk, -1, &tflags); + if (rv != X509_V_OK) { + /* Correct error depth */ + i = 0; + goto end; + } + for (; i < sk_X509_num(chain); i++) { + sign_nid = X509_get_signature_nid(x); + x = sk_X509_value(chain, i); + if (X509_get_version(x) != 2) { + rv = X509_V_ERR_SUITE_B_INVALID_VERSION; + goto end; + } + EVP_PKEY_free(pk); + pk = X509_get_pubkey(x); + rv = check_suite_b(pk, sign_nid, &tflags); + if (rv != X509_V_OK) + goto end; + } + + /* Final check: root CA signature */ + rv = check_suite_b(pk, X509_get_signature_nid(x), &tflags); + end: + if (pk) + EVP_PKEY_free(pk); + if (rv != X509_V_OK) { + /* Invalid signature or LOS errors are for previous cert */ + if ((rv == X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM + || rv == X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED) && i) + i--; + /* + * If we have LOS error and flags changed then we are signing P-384 + * with P-256. Use more meaninggul error. + */ + if (rv == X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED && flags != tflags) + rv = X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256; + if (perror_depth) + *perror_depth = i; + } + return rv; +} + +int X509_CRL_check_suiteb(X509_CRL *crl, EVP_PKEY *pk, unsigned long flags) +{ + int sign_nid; + if (!(flags & X509_V_FLAG_SUITEB_128_LOS)) + return X509_V_OK; + sign_nid = OBJ_obj2nid(crl->crl->sig_alg->algorithm); + return check_suite_b(pk, sign_nid, &flags); +} + +#else +int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain, + unsigned long flags) +{ + return 0; +} + +int X509_CRL_check_suiteb(X509_CRL *crl, EVP_PKEY *pk, unsigned long flags) +{ + return 0; +} + +#endif +/* + * Not strictly speaking an "up_ref" as a STACK doesn't have a reference + * count but it has the same effect by duping the STACK and upping the ref of + * each X509 structure. + */ +STACK_OF(X509) *X509_chain_up_ref(STACK_OF(X509) *chain) +{ + STACK_OF(X509) *ret; + int i; + ret = sk_X509_dup(chain); + for (i = 0; i < sk_X509_num(ret); i++) { + X509 *x = sk_X509_value(ret, i); + CRYPTO_add(&x->references, 1, CRYPTO_LOCK_X509); + } + return ret; +} diff --git a/crypto/x509/x509_err.c b/crypto/x509/x509_err.c index 61a19f74109b..43cde18e49a7 100644 --- a/crypto/x509/x509_err.c +++ b/crypto/x509/x509_err.c @@ -1,6 +1,6 @@ /* crypto/x509/x509_err.c */ /* ==================================================================== - * Copyright (c) 1999-2006 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2012 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -88,6 +88,7 @@ static ERR_STRING_DATA X509_str_functs[] = { {ERR_FUNC(X509_F_X509_ATTRIBUTE_GET0_DATA), "X509_ATTRIBUTE_get0_data"}, {ERR_FUNC(X509_F_X509_ATTRIBUTE_SET1_DATA), "X509_ATTRIBUTE_set1_data"}, {ERR_FUNC(X509_F_X509_CHECK_PRIVATE_KEY), "X509_check_private_key"}, + {ERR_FUNC(X509_F_X509_CRL_DIFF), "X509_CRL_diff"}, {ERR_FUNC(X509_F_X509_CRL_PRINT_FP), "X509_CRL_print_fp"}, {ERR_FUNC(X509_F_X509_EXTENSION_CREATE_BY_NID), "X509_EXTENSION_create_by_NID"}, @@ -131,22 +132,29 @@ static ERR_STRING_DATA X509_str_functs[] = { }; static ERR_STRING_DATA X509_str_reasons[] = { + {ERR_REASON(X509_R_AKID_MISMATCH), "akid mismatch"}, {ERR_REASON(X509_R_BAD_X509_FILETYPE), "bad x509 filetype"}, {ERR_REASON(X509_R_BASE64_DECODE_ERROR), "base64 decode error"}, {ERR_REASON(X509_R_CANT_CHECK_DH_KEY), "cant check dh key"}, {ERR_REASON(X509_R_CERT_ALREADY_IN_HASH_TABLE), "cert already in hash table"}, + {ERR_REASON(X509_R_CRL_ALREADY_DELTA), "crl already delta"}, + {ERR_REASON(X509_R_CRL_VERIFY_FAILURE), "crl verify failure"}, {ERR_REASON(X509_R_ERR_ASN1_LIB), "err asn1 lib"}, + {ERR_REASON(X509_R_IDP_MISMATCH), "idp mismatch"}, {ERR_REASON(X509_R_INVALID_DIRECTORY), "invalid directory"}, {ERR_REASON(X509_R_INVALID_FIELD_NAME), "invalid field name"}, {ERR_REASON(X509_R_INVALID_TRUST), "invalid trust"}, + {ERR_REASON(X509_R_ISSUER_MISMATCH), "issuer mismatch"}, {ERR_REASON(X509_R_KEY_TYPE_MISMATCH), "key type mismatch"}, {ERR_REASON(X509_R_KEY_VALUES_MISMATCH), "key values mismatch"}, {ERR_REASON(X509_R_LOADING_CERT_DIR), "loading cert dir"}, {ERR_REASON(X509_R_LOADING_DEFAULTS), "loading defaults"}, {ERR_REASON(X509_R_METHOD_NOT_SUPPORTED), "method not supported"}, + {ERR_REASON(X509_R_NEWER_CRL_NOT_NEWER), "newer crl not newer"}, {ERR_REASON(X509_R_NO_CERT_SET_FOR_US_TO_VERIFY), "no cert set for us to verify"}, + {ERR_REASON(X509_R_NO_CRL_NUMBER), "no crl number"}, {ERR_REASON(X509_R_PUBLIC_KEY_DECODE_ERROR), "public key decode error"}, {ERR_REASON(X509_R_PUBLIC_KEY_ENCODE_ERROR), "public key encode error"}, {ERR_REASON(X509_R_SHOULD_RETRY), "should retry"}, diff --git a/crypto/x509/x509_lu.c b/crypto/x509/x509_lu.c index 8415d1d8b353..b0d653903ff5 100644 --- a/crypto/x509/x509_lu.c +++ b/crypto/x509/x509_lu.c @@ -238,6 +238,19 @@ void X509_STORE_free(X509_STORE *vfy) if (vfy == NULL) return; + i = CRYPTO_add(&vfy->references, -1, CRYPTO_LOCK_X509_STORE); +#ifdef REF_PRINT + REF_PRINT("X509_STORE", vfy); +#endif + if (i > 0) + return; +#ifdef REF_CHECK + if (i < 0) { + fprintf(stderr, "X509_STORE_free, bad reference count\n"); + abort(); /* ok */ + } +#endif + sk = vfy->get_cert_methods; for (i = 0; i < sk_X509_LOOKUP_num(sk); i++) { lu = sk_X509_LOOKUP_value(sk, i); @@ -681,6 +694,19 @@ void X509_STORE_set_verify_cb(X509_STORE *ctx, ctx->verify_cb = verify_cb; } +void X509_STORE_set_lookup_crls_cb(X509_STORE *ctx, + STACK_OF(X509_CRL) *(*cb) (X509_STORE_CTX + *ctx, + X509_NAME *nm)) +{ + ctx->lookup_crls = cb; +} + +X509_STORE *X509_STORE_CTX_get0_store(X509_STORE_CTX *ctx) +{ + return ctx->ctx; +} + IMPLEMENT_STACK_OF(X509_LOOKUP) IMPLEMENT_STACK_OF(X509_OBJECT) diff --git a/crypto/x509/x509_set.c b/crypto/x509/x509_set.c index 4645777634b7..5b802bd6c754 100644 --- a/crypto/x509/x509_set.c +++ b/crypto/x509/x509_set.c @@ -67,6 +67,11 @@ int X509_set_version(X509 *x, long version) { if (x == NULL) return (0); + if (version == 0) { + M_ASN1_INTEGER_free(x->cert_info->version); + x->cert_info->version = NULL; + return (1); + } if (x->cert_info->version == NULL) { if ((x->cert_info->version = M_ASN1_INTEGER_new()) == NULL) return (0); diff --git a/crypto/x509/x509_trs.c b/crypto/x509/x509_trs.c index 7e444795a5e7..11e0763403ae 100644 --- a/crypto/x509/x509_trs.c +++ b/crypto/x509/x509_trs.c @@ -119,6 +119,14 @@ int X509_check_trust(X509 *x, int id, int flags) int idx; if (id == -1) return 1; + /* We get this as a default value */ + if (id == 0) { + int rv; + rv = obj_trust(NID_anyExtendedKeyUsage, x, 0); + if (rv != X509_TRUST_UNTRUSTED) + return rv; + return trust_compat(NULL, x, 0); + } idx = X509_TRUST_get_by_id(id); if (idx == -1) return default_trust(id, x, flags); diff --git a/crypto/x509/x509_txt.c b/crypto/x509/x509_txt.c index d834180ec443..3d46d3ff8366 100644 --- a/crypto/x509/x509_txt.c +++ b/crypto/x509/x509_txt.c @@ -184,6 +184,26 @@ const char *X509_verify_cert_error_string(long n) case X509_V_ERR_CRL_PATH_VALIDATION_ERROR: return ("CRL path validation error"); + case X509_V_ERR_SUITE_B_INVALID_VERSION: + return ("Suite B: certificate version invalid"); + case X509_V_ERR_SUITE_B_INVALID_ALGORITHM: + return ("Suite B: invalid public key algorithm"); + case X509_V_ERR_SUITE_B_INVALID_CURVE: + return ("Suite B: invalid ECC curve"); + case X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM: + return ("Suite B: invalid signature algorithm"); + case X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED: + return ("Suite B: curve not allowed for this LOS"); + case X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256: + return ("Suite B: cannot sign P-384 with P-256"); + + case X509_V_ERR_HOSTNAME_MISMATCH: + return ("Hostname mismatch"); + case X509_V_ERR_EMAIL_MISMATCH: + return ("Email address mismatch"); + case X509_V_ERR_IP_ADDRESS_MISMATCH: + return ("IP address mismatch"); + default: BIO_snprintf(buf, sizeof buf, "error number %ld", n); return (buf); diff --git a/crypto/x509/x509_vfy.c b/crypto/x509/x509_vfy.c index 7009ae63076e..a2f1dbefe352 100644 --- a/crypto/x509/x509_vfy.c +++ b/crypto/x509/x509_vfy.c @@ -69,6 +69,7 @@ #include #include #include +#include "vpm_int.h" /* CRL score values */ @@ -113,6 +114,7 @@ static int check_issued(X509_STORE_CTX *ctx, X509 *x, X509 *issuer); static X509 *find_issuer(X509_STORE_CTX *ctx, STACK_OF(X509) *sk, X509 *x); static int check_chain_extensions(X509_STORE_CTX *ctx); static int check_name_constraints(X509_STORE_CTX *ctx); +static int check_id(X509_STORE_CTX *ctx); static int check_trust(X509_STORE_CTX *ctx); static int check_revocation(X509_STORE_CTX *ctx); static int check_cert(X509_STORE_CTX *ctx); @@ -148,6 +150,40 @@ static int x509_subject_cmp(X509 **a, X509 **b) return X509_subject_name_cmp(*a, *b); } #endif +/* Return 1 is a certificate is self signed */ +static int cert_self_signed(X509 *x) +{ + X509_check_purpose(x, -1, 0); + if (x->ex_flags & EXFLAG_SS) + return 1; + else + return 0; +} + +/* Given a certificate try and find an exact match in the store */ + +static X509 *lookup_cert_match(X509_STORE_CTX *ctx, X509 *x) +{ + STACK_OF(X509) *certs; + X509 *xtmp = NULL; + int i; + /* Lookup all certs with matching subject name */ + certs = ctx->lookup_certs(ctx, X509_get_subject_name(x)); + if (certs == NULL) + return NULL; + /* Look for exact match */ + for (i = 0; i < sk_X509_num(certs); i++) { + xtmp = sk_X509_value(certs, i); + if (!X509_cmp(xtmp, x)) + break; + } + if (i < sk_X509_num(certs)) + CRYPTO_add(&xtmp->references, 1, CRYPTO_LOCK_X509); + else + xtmp = NULL; + sk_X509_pop_free(certs, X509_free); + return xtmp; +} int X509_verify_cert(X509_STORE_CTX *ctx) { @@ -205,8 +241,24 @@ int X509_verify_cert(X509_STORE_CTX *ctx) * later. */ /* If we are self signed, we break */ - if (ctx->check_issued(ctx, x, x)) + if (cert_self_signed(x)) break; + /* + * If asked see if we can find issuer in trusted store first + */ + if (ctx->param->flags & X509_V_FLAG_TRUSTED_FIRST) { + ok = ctx->get_issuer(&xtmp, ctx, x); + if (ok < 0) + return ok; + /* + * If successful for now free up cert so it will be picked up + * again later. + */ + if (ok > 0) { + X509_free(xtmp); + break; + } + } /* If we were passed a cert chain, use it first */ if (ctx->untrusted != NULL) { @@ -244,7 +296,7 @@ int X509_verify_cert(X509_STORE_CTX *ctx) */ i = sk_X509_num(ctx->chain); x = sk_X509_value(ctx->chain, i - 1); - if (ctx->check_issued(ctx, x, x)) { + if (cert_self_signed(x)) { /* we have a self signed certificate */ if (sk_X509_num(ctx->chain) == 1) { /* @@ -290,9 +342,10 @@ int X509_verify_cert(X509_STORE_CTX *ctx) if (depth < num) break; /* If we are self signed, we break */ - if (ctx->check_issued(ctx, x, x)) + if (cert_self_signed(x)) break; ok = ctx->get_issuer(&xtmp, ctx, x); + if (ok < 0) return ok; if (ok == 0) @@ -306,14 +359,22 @@ int X509_verify_cert(X509_STORE_CTX *ctx) num++; } + /* we now have our chain, lets check it... */ + i = check_trust(ctx); + + /* If explicitly rejected error */ + if (i == X509_TRUST_REJECTED) + goto end; /* - * If we haven't got a least one certificate from our store then check - * if there is an alternative chain that could be used. We only do this - * if the user hasn't switched off alternate chain checking + * If it's not explicitly trusted then check if there is an alternative + * chain that could be used. We only do this if we haven't already + * checked via TRUSTED_FIRST and the user hasn't switched off alternate + * chain checking */ retry = 0; - if (num == ctx->last_untrusted && - !(ctx->param->flags & X509_V_FLAG_NO_ALT_CHAINS)) { + if (i != X509_TRUST_TRUSTED + && !(ctx->param->flags & X509_V_FLAG_TRUSTED_FIRST) + && !(ctx->param->flags & X509_V_FLAG_NO_ALT_CHAINS)) { while (j-- > 1) { xtmp2 = sk_X509_value(ctx->chain, j - 1); ok = ctx->get_issuer(&xtmp, ctx, xtmp2); @@ -343,8 +404,12 @@ int X509_verify_cert(X509_STORE_CTX *ctx) } } while (retry); - /* Is last certificate looked up self signed? */ - if (!ctx->check_issued(ctx, x, x)) { + /* + * If not explicitly trusted then indicate error unless it's a single + * self signed certificate in which case we've indicated an error already + * and set bad_chain == 1 + */ + if (i != X509_TRUST_TRUSTED && !bad_chain) { if ((chain_ss == NULL) || !ctx->check_issued(ctx, x, chain_ss)) { if (ctx->last_untrusted >= num) ctx->error = X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY; @@ -381,10 +446,7 @@ int X509_verify_cert(X509_STORE_CTX *ctx) if (!ok) goto end; - /* The chain extensions are OK: check trust */ - - if (param->trust > 0) - ok = check_trust(ctx); + ok = check_id(ctx); if (!ok) goto end; @@ -401,6 +463,16 @@ int X509_verify_cert(X509_STORE_CTX *ctx) if (!ok) goto end; + i = X509_chain_check_suiteb(&ctx->error_depth, NULL, ctx->chain, + ctx->param->flags); + if (i != X509_V_OK) { + ctx->error = i; + ctx->current_cert = sk_X509_value(ctx->chain, ctx->error_depth); + ok = cb(0, ctx); + if (!ok) + goto end; + } + /* At this point, we have a chain and need to verify it */ if (ctx->verify != NULL) ok = ctx->verify(ctx); @@ -467,7 +539,6 @@ static int check_issued(X509_STORE_CTX *ctx, X509 *x, X509 *issuer) ctx->current_cert = x; ctx->current_issuer = issuer; return ctx->verify_cb(0, ctx); - return 0; } /* Alternative lookup method: look from a STACK stored in other_ctx */ @@ -667,30 +738,97 @@ static int check_name_constraints(X509_STORE_CTX *ctx) return 1; } -static int check_trust(X509_STORE_CTX *ctx) +static int check_id_error(X509_STORE_CTX *ctx, int errcode) { -#ifdef OPENSSL_NO_CHAIN_VERIFY + ctx->error = errcode; + ctx->current_cert = ctx->cert; + ctx->error_depth = 0; + return ctx->verify_cb(0, ctx); +} + +static int check_hosts(X509 *x, X509_VERIFY_PARAM_ID *id) +{ + int i; + int n = sk_OPENSSL_STRING_num(id->hosts); + char *name; + + for (i = 0; i < n; ++i) { + name = sk_OPENSSL_STRING_value(id->hosts, i); + if (X509_check_host(x, name, 0, id->hostflags, &id->peername) > 0) + return 1; + } + return n == 0; +} + +static int check_id(X509_STORE_CTX *ctx) +{ + X509_VERIFY_PARAM *vpm = ctx->param; + X509_VERIFY_PARAM_ID *id = vpm->id; + X509 *x = ctx->cert; + if (id->hosts && check_hosts(x, id) <= 0) { + if (!check_id_error(ctx, X509_V_ERR_HOSTNAME_MISMATCH)) + return 0; + } + if (id->email && X509_check_email(x, id->email, id->emaillen, 0) <= 0) { + if (!check_id_error(ctx, X509_V_ERR_EMAIL_MISMATCH)) + return 0; + } + if (id->ip && X509_check_ip(x, id->ip, id->iplen, 0) <= 0) { + if (!check_id_error(ctx, X509_V_ERR_IP_ADDRESS_MISMATCH)) + return 0; + } return 1; -#else +} + +static int check_trust(X509_STORE_CTX *ctx) +{ int i, ok; - X509 *x; + X509 *x = NULL; int (*cb) (int xok, X509_STORE_CTX *xctx); cb = ctx->verify_cb; -/* For now just check the last certificate in the chain */ - i = sk_X509_num(ctx->chain) - 1; - x = sk_X509_value(ctx->chain, i); - ok = X509_check_trust(x, ctx->param->trust, 0); - if (ok == X509_TRUST_TRUSTED) - return 1; - ctx->error_depth = i; - ctx->current_cert = x; - if (ok == X509_TRUST_REJECTED) - ctx->error = X509_V_ERR_CERT_REJECTED; - else - ctx->error = X509_V_ERR_CERT_UNTRUSTED; - ok = cb(0, ctx); - return ok; -#endif + /* Check all trusted certificates in chain */ + for (i = ctx->last_untrusted; i < sk_X509_num(ctx->chain); i++) { + x = sk_X509_value(ctx->chain, i); + ok = X509_check_trust(x, ctx->param->trust, 0); + /* If explicitly trusted return trusted */ + if (ok == X509_TRUST_TRUSTED) + return X509_TRUST_TRUSTED; + /* + * If explicitly rejected notify callback and reject if not + * overridden. + */ + if (ok == X509_TRUST_REJECTED) { + ctx->error_depth = i; + ctx->current_cert = x; + ctx->error = X509_V_ERR_CERT_REJECTED; + ok = cb(0, ctx); + if (!ok) + return X509_TRUST_REJECTED; + } + } + /* + * If we accept partial chains and have at least one trusted certificate + * return success. + */ + if (ctx->param->flags & X509_V_FLAG_PARTIAL_CHAIN) { + X509 *mx; + if (ctx->last_untrusted < sk_X509_num(ctx->chain)) + return X509_TRUST_TRUSTED; + x = sk_X509_value(ctx->chain, 0); + mx = lookup_cert_match(ctx, x); + if (mx) { + (void)sk_X509_set(ctx->chain, 0, mx); + X509_free(x); + ctx->last_untrusted = 0; + return X509_TRUST_TRUSTED; + } + } + + /* + * If no trusted certs in chain at all return untrusted and allow + * standard (no issuer cert) etc errors to be indicated. + */ + return X509_TRUST_UNTRUSTED; } static int check_revocation(X509_STORE_CTX *ctx) @@ -1409,6 +1547,14 @@ static int check_crl(X509_STORE_CTX *ctx, X509_CRL *crl) if (!ok) goto err; } else { + int rv; + rv = X509_CRL_check_suiteb(crl, ikey, ctx->param->flags); + if (rv != X509_V_OK) { + ctx->error = rv; + ok = ctx->verify_cb(0, ctx); + if (!ok) + goto err; + } /* Verify CRL signature */ if (X509_CRL_verify(crl, ikey) <= 0) { ctx->error = X509_V_ERR_CRL_SIGNATURE_FAILURE; @@ -1565,6 +1711,10 @@ static int internal_verify(X509_STORE_CTX *ctx) if (ctx->check_issued(ctx, xi, xi)) xs = xi; else { + if (ctx->param->flags & X509_V_FLAG_PARTIAL_CHAIN) { + xs = xi; + goto check_cert; + } if (n <= 0) { ctx->error = X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE; ctx->current_cert = xi; @@ -1610,6 +1760,7 @@ static int internal_verify(X509_STORE_CTX *ctx) xs->valid = 1; + check_cert: ok = check_cert_time(ctx, xs); if (!ok) goto end; @@ -1824,6 +1975,114 @@ int X509_get_pubkey_parameters(EVP_PKEY *pkey, STACK_OF(X509) *chain) return 1; } +/* Make a delta CRL as the diff between two full CRLs */ + +X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, + EVP_PKEY *skey, const EVP_MD *md, unsigned int flags) +{ + X509_CRL *crl = NULL; + int i; + STACK_OF(X509_REVOKED) *revs = NULL; + /* CRLs can't be delta already */ + if (base->base_crl_number || newer->base_crl_number) { + X509err(X509_F_X509_CRL_DIFF, X509_R_CRL_ALREADY_DELTA); + return NULL; + } + /* Base and new CRL must have a CRL number */ + if (!base->crl_number || !newer->crl_number) { + X509err(X509_F_X509_CRL_DIFF, X509_R_NO_CRL_NUMBER); + return NULL; + } + /* Issuer names must match */ + if (X509_NAME_cmp(X509_CRL_get_issuer(base), X509_CRL_get_issuer(newer))) { + X509err(X509_F_X509_CRL_DIFF, X509_R_ISSUER_MISMATCH); + return NULL; + } + /* AKID and IDP must match */ + if (!crl_extension_match(base, newer, NID_authority_key_identifier)) { + X509err(X509_F_X509_CRL_DIFF, X509_R_AKID_MISMATCH); + return NULL; + } + if (!crl_extension_match(base, newer, NID_issuing_distribution_point)) { + X509err(X509_F_X509_CRL_DIFF, X509_R_IDP_MISMATCH); + return NULL; + } + /* Newer CRL number must exceed full CRL number */ + if (ASN1_INTEGER_cmp(newer->crl_number, base->crl_number) <= 0) { + X509err(X509_F_X509_CRL_DIFF, X509_R_NEWER_CRL_NOT_NEWER); + return NULL; + } + /* CRLs must verify */ + if (skey && (X509_CRL_verify(base, skey) <= 0 || + X509_CRL_verify(newer, skey) <= 0)) { + X509err(X509_F_X509_CRL_DIFF, X509_R_CRL_VERIFY_FAILURE); + return NULL; + } + /* Create new CRL */ + crl = X509_CRL_new(); + if (!crl || !X509_CRL_set_version(crl, 1)) + goto memerr; + /* Set issuer name */ + if (!X509_CRL_set_issuer_name(crl, X509_CRL_get_issuer(newer))) + goto memerr; + + if (!X509_CRL_set_lastUpdate(crl, X509_CRL_get_lastUpdate(newer))) + goto memerr; + if (!X509_CRL_set_nextUpdate(crl, X509_CRL_get_nextUpdate(newer))) + goto memerr; + + /* Set base CRL number: must be critical */ + + if (!X509_CRL_add1_ext_i2d(crl, NID_delta_crl, base->crl_number, 1, 0)) + goto memerr; + + /* + * Copy extensions across from newest CRL to delta: this will set CRL + * number to correct value too. + */ + + for (i = 0; i < X509_CRL_get_ext_count(newer); i++) { + X509_EXTENSION *ext; + ext = X509_CRL_get_ext(newer, i); + if (!X509_CRL_add_ext(crl, ext, -1)) + goto memerr; + } + + /* Go through revoked entries, copying as needed */ + + revs = X509_CRL_get_REVOKED(newer); + + for (i = 0; i < sk_X509_REVOKED_num(revs); i++) { + X509_REVOKED *rvn, *rvtmp; + rvn = sk_X509_REVOKED_value(revs, i); + /* + * Add only if not also in base. TODO: need something cleverer here + * for some more complex CRLs covering multiple CAs. + */ + if (!X509_CRL_get0_by_serial(base, &rvtmp, rvn->serialNumber)) { + rvtmp = X509_REVOKED_dup(rvn); + if (!rvtmp) + goto memerr; + if (!X509_CRL_add0_revoked(crl, rvtmp)) { + X509_REVOKED_free(rvtmp); + goto memerr; + } + } + } + /* TODO: optionally prune deleted entries */ + + if (skey && md && !X509_CRL_sign(crl, skey, md)) + goto memerr; + + return crl; + + memerr: + X509err(X509_F_X509_CRL_DIFF, ERR_R_MALLOC_FAILURE); + if (crl) + X509_CRL_free(crl); + return NULL; +} + int X509_STORE_CTX_get_ex_new_index(long argl, void *argp, CRYPTO_EX_new *new_func, CRYPTO_EX_dup *dup_func, @@ -1874,16 +2133,9 @@ STACK_OF(X509) *X509_STORE_CTX_get_chain(X509_STORE_CTX *ctx) STACK_OF(X509) *X509_STORE_CTX_get1_chain(X509_STORE_CTX *ctx) { - int i; - X509 *x; - STACK_OF(X509) *chain; - if (!ctx->chain || !(chain = sk_X509_dup(ctx->chain))) + if (!ctx->chain) return NULL; - for (i = 0; i < sk_X509_num(chain); i++) { - x = sk_X509_value(chain, i); - CRYPTO_add(&x->references, 1, CRYPTO_LOCK_X509); - } - return chain; + return X509_chain_up_ref(ctx->chain); } X509 *X509_STORE_CTX_get0_current_issuer(X509_STORE_CTX *ctx) diff --git a/crypto/x509/x509_vfy.h b/crypto/x509/x509_vfy.h index aacdf55aa279..bd8613c62ba4 100644 --- a/crypto/x509/x509_vfy.h +++ b/crypto/x509/x509_vfy.h @@ -156,6 +156,8 @@ typedef struct x509_lookup_method_st { X509_OBJECT *ret); } X509_LOOKUP_METHOD; +typedef struct X509_VERIFY_PARAM_ID_st X509_VERIFY_PARAM_ID; + /* * This structure hold all parameters associated with a verify operation by * including an X509_VERIFY_PARAM structure in related structures the @@ -171,6 +173,7 @@ typedef struct X509_VERIFY_PARAM_st { int trust; /* trust setting to check */ int depth; /* Verify depth */ STACK_OF(ASN1_OBJECT) *policies; /* Permissible policies */ + X509_VERIFY_PARAM_ID *id; /* opaque ID data */ } X509_VERIFY_PARAM; DECLARE_STACK_OF(X509_VERIFY_PARAM) @@ -370,6 +373,19 @@ void X509_STORE_CTX_set_depth(X509_STORE_CTX *ctx, int depth); # define X509_V_ERR_UNSUPPORTED_NAME_SYNTAX 53 # define X509_V_ERR_CRL_PATH_VALIDATION_ERROR 54 +/* Suite B mode algorithm violation */ +# define X509_V_ERR_SUITE_B_INVALID_VERSION 56 +# define X509_V_ERR_SUITE_B_INVALID_ALGORITHM 57 +# define X509_V_ERR_SUITE_B_INVALID_CURVE 58 +# define X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM 59 +# define X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED 60 +# define X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256 61 + +/* Host, email and IP check errors */ +# define X509_V_ERR_HOSTNAME_MISMATCH 62 +# define X509_V_ERR_EMAIL_MISMATCH 63 +# define X509_V_ERR_IP_ADDRESS_MISMATCH 64 + /* The application is not happy */ # define X509_V_ERR_APPLICATION_VERIFICATION 50 @@ -405,10 +421,21 @@ void X509_STORE_CTX_set_depth(X509_STORE_CTX *ctx, int depth); # define X509_V_FLAG_USE_DELTAS 0x2000 /* Check selfsigned CA signature */ # define X509_V_FLAG_CHECK_SS_SIGNATURE 0x4000 +/* Use trusted store first */ +# define X509_V_FLAG_TRUSTED_FIRST 0x8000 +/* Suite B 128 bit only mode: not normally used */ +# define X509_V_FLAG_SUITEB_128_LOS_ONLY 0x10000 +/* Suite B 192 bit only mode */ +# define X509_V_FLAG_SUITEB_192_LOS 0x20000 +/* Suite B 128 bit mode allowing 192 bit algorithms */ +# define X509_V_FLAG_SUITEB_128_LOS 0x30000 + +/* Allow partial chains if at least one certificate is in trusted store */ +# define X509_V_FLAG_PARTIAL_CHAIN 0x80000 /* * If the initial chain is not trusted, do not attempt to build an alternative - * chain. Alternate chain checking was introduced in 1.0.1n/1.0.2b. Setting - * this flag will force the behaviour to match that of previous versions. + * chain. Alternate chain checking was introduced in 1.0.2b. Setting this flag + * will force the behaviour to match that of previous versions. */ # define X509_V_FLAG_NO_ALT_CHAINS 0x100000 @@ -445,6 +472,11 @@ int X509_STORE_set1_param(X509_STORE *ctx, X509_VERIFY_PARAM *pm); void X509_STORE_set_verify_cb(X509_STORE *ctx, int (*verify_cb) (int, X509_STORE_CTX *)); +void X509_STORE_set_lookup_crls_cb(X509_STORE *ctx, + STACK_OF(X509_CRL) *(*cb) (X509_STORE_CTX + *ctx, + X509_NAME *nm)); + X509_STORE_CTX *X509_STORE_CTX_new(void); int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x); @@ -455,6 +487,8 @@ int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *store, void X509_STORE_CTX_trusted_stack(X509_STORE_CTX *ctx, STACK_OF(X509) *sk); void X509_STORE_CTX_cleanup(X509_STORE_CTX *ctx); +X509_STORE *X509_STORE_CTX_get0_store(X509_STORE_CTX *ctx); + X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *v, X509_LOOKUP_METHOD *m); X509_LOOKUP_METHOD *X509_LOOKUP_hash_dir(void); @@ -552,9 +586,27 @@ int X509_VERIFY_PARAM_add0_policy(X509_VERIFY_PARAM *param, ASN1_OBJECT *policy); int X509_VERIFY_PARAM_set1_policies(X509_VERIFY_PARAM *param, STACK_OF(ASN1_OBJECT) *policies); + +int X509_VERIFY_PARAM_set1_host(X509_VERIFY_PARAM *param, + const char *name, size_t namelen); +int X509_VERIFY_PARAM_add1_host(X509_VERIFY_PARAM *param, + const char *name, size_t namelen); +void X509_VERIFY_PARAM_set_hostflags(X509_VERIFY_PARAM *param, + unsigned int flags); +char *X509_VERIFY_PARAM_get0_peername(X509_VERIFY_PARAM *); +int X509_VERIFY_PARAM_set1_email(X509_VERIFY_PARAM *param, + const char *email, size_t emaillen); +int X509_VERIFY_PARAM_set1_ip(X509_VERIFY_PARAM *param, + const unsigned char *ip, size_t iplen); +int X509_VERIFY_PARAM_set1_ip_asc(X509_VERIFY_PARAM *param, + const char *ipasc); + int X509_VERIFY_PARAM_get_depth(const X509_VERIFY_PARAM *param); +const char *X509_VERIFY_PARAM_get0_name(const X509_VERIFY_PARAM *param); int X509_VERIFY_PARAM_add0_table(X509_VERIFY_PARAM *param); +int X509_VERIFY_PARAM_get_count(void); +const X509_VERIFY_PARAM *X509_VERIFY_PARAM_get0(int id); const X509_VERIFY_PARAM *X509_VERIFY_PARAM_lookup(const char *name); void X509_VERIFY_PARAM_table_cleanup(void); diff --git a/crypto/x509/x509_vpm.c b/crypto/x509/x509_vpm.c index 6b0bf8a6e69e..1ea0c69f5743 100644 --- a/crypto/x509/x509_vpm.c +++ b/crypto/x509/x509_vpm.c @@ -66,10 +66,73 @@ #include #include +#include "vpm_int.h" + /* X509_VERIFY_PARAM functions */ +#define SET_HOST 0 +#define ADD_HOST 1 + +static char *str_copy(const char *s) +{ + return OPENSSL_strdup(s); +} + +static void str_free(char *s) +{ + OPENSSL_free(s); +} + +#define string_stack_free(sk) sk_OPENSSL_STRING_pop_free(sk, str_free) + +static int int_x509_param_set_hosts(X509_VERIFY_PARAM_ID *id, int mode, + const char *name, size_t namelen) +{ + char *copy; + + /* + * Refuse names with embedded NUL bytes, except perhaps as final byte. + * XXX: Do we need to push an error onto the error stack? + */ + if (namelen == 0) + namelen = name ? strlen(name) : 0; + else if (name && memchr(name, '\0', namelen > 1 ? namelen - 1 : namelen)) + return 0; + if (name && name[namelen - 1] == '\0') + --namelen; + + if (mode == SET_HOST && id->hosts) { + string_stack_free(id->hosts); + id->hosts = NULL; + } + if (name == NULL || namelen == 0) + return 1; + + copy = BUF_strndup(name, namelen); + if (copy == NULL) + return 0; + + if (id->hosts == NULL && + (id->hosts = sk_OPENSSL_STRING_new_null()) == NULL) { + OPENSSL_free(copy); + return 0; + } + + if (!sk_OPENSSL_STRING_push(id->hosts, copy)) { + OPENSSL_free(copy); + if (sk_OPENSSL_STRING_num(id->hosts) == 0) { + sk_OPENSSL_STRING_free(id->hosts); + id->hosts = NULL; + } + return 0; + } + + return 1; +} + static void x509_verify_param_zero(X509_VERIFY_PARAM *param) { + X509_VERIFY_PARAM_ID *paramid; if (!param) return; param->name = NULL; @@ -85,15 +148,42 @@ static void x509_verify_param_zero(X509_VERIFY_PARAM *param) sk_ASN1_OBJECT_pop_free(param->policies, ASN1_OBJECT_free); param->policies = NULL; } + paramid = param->id; + if (paramid->hosts) { + string_stack_free(paramid->hosts); + paramid->hosts = NULL; + } + if (paramid->peername) + OPENSSL_free(paramid->peername); + if (paramid->email) { + OPENSSL_free(paramid->email); + paramid->email = NULL; + paramid->emaillen = 0; + } + if (paramid->ip) { + OPENSSL_free(paramid->ip); + paramid->ip = NULL; + paramid->iplen = 0; + } + } X509_VERIFY_PARAM *X509_VERIFY_PARAM_new(void) { X509_VERIFY_PARAM *param; - param = OPENSSL_malloc(sizeof(X509_VERIFY_PARAM)); + X509_VERIFY_PARAM_ID *paramid; + + param = OPENSSL_malloc(sizeof *param); if (!param) return NULL; - memset(param, 0, sizeof(X509_VERIFY_PARAM)); + paramid = OPENSSL_malloc(sizeof *paramid); + if (!paramid) { + OPENSSL_free(param); + return NULL; + } + memset(param, 0, sizeof *param); + memset(paramid, 0, sizeof *paramid); + param->id = paramid; x509_verify_param_zero(param); return param; } @@ -103,6 +193,7 @@ void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *param) if (param == NULL) return; x509_verify_param_zero(param); + OPENSSL_free(param->id); OPENSSL_free(param); } @@ -144,6 +235,11 @@ void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *param) (to_overwrite || \ ((src->field != def) && (to_default || (dest->field == def)))) +/* As above but for ID fields */ + +#define test_x509_verify_param_copy_id(idf, def) \ + test_x509_verify_param_copy(id->idf, def) + /* Macro to test and copy a field if necessary */ #define x509_verify_param_copy(field, def) \ @@ -155,8 +251,10 @@ int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *dest, { unsigned long inh_flags; int to_default, to_overwrite; + X509_VERIFY_PARAM_ID *id; if (!src) return 1; + id = src->id; inh_flags = dest->inh_flags | src->inh_flags; if (inh_flags & X509_VP_FLAG_ONCE) @@ -197,6 +295,31 @@ int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *dest, return 0; } + /* Copy the host flags if and only if we're copying the host list */ + if (test_x509_verify_param_copy_id(hosts, NULL)) { + if (dest->id->hosts) { + string_stack_free(dest->id->hosts); + dest->id->hosts = NULL; + } + if (id->hosts) { + dest->id->hosts = + sk_OPENSSL_STRING_deep_copy(id->hosts, str_copy, str_free); + if (dest->id->hosts == NULL) + return 0; + dest->id->hostflags = id->hostflags; + } + } + + if (test_x509_verify_param_copy_id(email, NULL)) { + if (!X509_VERIFY_PARAM_set1_email(dest, id->email, id->emaillen)) + return 0; + } + + if (test_x509_verify_param_copy_id(ip, NULL)) { + if (!X509_VERIFY_PARAM_set1_ip(dest, id->ip, id->iplen)) + return 0; + } + return 1; } @@ -211,6 +334,30 @@ int X509_VERIFY_PARAM_set1(X509_VERIFY_PARAM *to, return ret; } +static int int_x509_param_set1(char **pdest, size_t *pdestlen, + const char *src, size_t srclen) +{ + void *tmp; + if (src) { + if (srclen == 0) { + tmp = BUF_strdup(src); + srclen = strlen(src); + } else + tmp = BUF_memdup(src, srclen); + if (!tmp) + return 0; + } else { + tmp = NULL; + srclen = 0; + } + if (*pdest) + OPENSSL_free(*pdest); + *pdest = tmp; + if (pdestlen) + *pdestlen = srclen; + return 1; +} + int X509_VERIFY_PARAM_set1_name(X509_VERIFY_PARAM *param, const char *name) { if (param->name) @@ -308,11 +455,70 @@ int X509_VERIFY_PARAM_set1_policies(X509_VERIFY_PARAM *param, return 1; } +int X509_VERIFY_PARAM_set1_host(X509_VERIFY_PARAM *param, + const char *name, size_t namelen) +{ + return int_x509_param_set_hosts(param->id, SET_HOST, name, namelen); +} + +int X509_VERIFY_PARAM_add1_host(X509_VERIFY_PARAM *param, + const char *name, size_t namelen) +{ + return int_x509_param_set_hosts(param->id, ADD_HOST, name, namelen); +} + +void X509_VERIFY_PARAM_set_hostflags(X509_VERIFY_PARAM *param, + unsigned int flags) +{ + param->id->hostflags = flags; +} + +char *X509_VERIFY_PARAM_get0_peername(X509_VERIFY_PARAM *param) +{ + return param->id->peername; +} + +int X509_VERIFY_PARAM_set1_email(X509_VERIFY_PARAM *param, + const char *email, size_t emaillen) +{ + return int_x509_param_set1(¶m->id->email, ¶m->id->emaillen, + email, emaillen); +} + +int X509_VERIFY_PARAM_set1_ip(X509_VERIFY_PARAM *param, + const unsigned char *ip, size_t iplen) +{ + if (iplen != 0 && iplen != 4 && iplen != 16) + return 0; + return int_x509_param_set1((char **)¶m->id->ip, ¶m->id->iplen, + (char *)ip, iplen); +} + +int X509_VERIFY_PARAM_set1_ip_asc(X509_VERIFY_PARAM *param, const char *ipasc) +{ + unsigned char ipout[16]; + size_t iplen; + + iplen = (size_t)a2i_ipadd(ipout, ipasc); + if (iplen == 0) + return 0; + return X509_VERIFY_PARAM_set1_ip(param, ipout, iplen); +} + int X509_VERIFY_PARAM_get_depth(const X509_VERIFY_PARAM *param) { return param->depth; } +const char *X509_VERIFY_PARAM_get0_name(const X509_VERIFY_PARAM *param) +{ + return param->name; +} + +static X509_VERIFY_PARAM_ID _empty_id = { NULL, 0U, NULL, NULL, 0, NULL, 0 }; + +#define vpm_empty_id (X509_VERIFY_PARAM_ID *)&_empty_id + /* * Default verify parameters: these are used for various applications and can * be overridden by the user specified table. NB: the 'name' field *must* be @@ -328,8 +534,8 @@ static const X509_VERIFY_PARAM default_table[] = { 0, /* purpose */ 0, /* trust */ 100, /* depth */ - NULL /* policies */ - }, + NULL, /* policies */ + vpm_empty_id}, { "pkcs7", /* S/MIME sign parameters */ 0, /* Check time */ @@ -338,8 +544,8 @@ static const X509_VERIFY_PARAM default_table[] = { X509_PURPOSE_SMIME_SIGN, /* purpose */ X509_TRUST_EMAIL, /* trust */ -1, /* depth */ - NULL /* policies */ - }, + NULL, /* policies */ + vpm_empty_id}, { "smime_sign", /* S/MIME sign parameters */ 0, /* Check time */ @@ -348,8 +554,8 @@ static const X509_VERIFY_PARAM default_table[] = { X509_PURPOSE_SMIME_SIGN, /* purpose */ X509_TRUST_EMAIL, /* trust */ -1, /* depth */ - NULL /* policies */ - }, + NULL, /* policies */ + vpm_empty_id}, { "ssl_client", /* SSL/TLS client parameters */ 0, /* Check time */ @@ -358,8 +564,8 @@ static const X509_VERIFY_PARAM default_table[] = { X509_PURPOSE_SSL_CLIENT, /* purpose */ X509_TRUST_SSL_CLIENT, /* trust */ -1, /* depth */ - NULL /* policies */ - }, + NULL, /* policies */ + vpm_empty_id}, { "ssl_server", /* SSL/TLS server parameters */ 0, /* Check time */ @@ -368,8 +574,8 @@ static const X509_VERIFY_PARAM default_table[] = { X509_PURPOSE_SSL_SERVER, /* purpose */ X509_TRUST_SSL_SERVER, /* trust */ -1, /* depth */ - NULL /* policies */ - } + NULL, /* policies */ + vpm_empty_id} }; static STACK_OF(X509_VERIFY_PARAM) *param_table = NULL; @@ -409,6 +615,22 @@ int X509_VERIFY_PARAM_add0_table(X509_VERIFY_PARAM *param) return 1; } +int X509_VERIFY_PARAM_get_count(void) +{ + int num = sizeof(default_table) / sizeof(X509_VERIFY_PARAM); + if (param_table) + num += sk_X509_VERIFY_PARAM_num(param_table); + return num; +} + +const X509_VERIFY_PARAM *X509_VERIFY_PARAM_get0(int id) +{ + int num = sizeof(default_table) / sizeof(X509_VERIFY_PARAM); + if (id < num) + return default_table + id; + return sk_X509_VERIFY_PARAM_value(param_table, id - num); +} + const X509_VERIFY_PARAM *X509_VERIFY_PARAM_lookup(const char *name) { int idx; diff --git a/crypto/x509/x_all.c b/crypto/x509/x_all.c index 43152e933fa1..0f26c546d835 100644 --- a/crypto/x509/x_all.c +++ b/crypto/x509/x_all.c @@ -63,6 +63,7 @@ #include #include #include +#include #ifndef OPENSSL_NO_RSA # include #endif @@ -105,6 +106,12 @@ int X509_sign_ctx(X509 *x, EVP_MD_CTX *ctx) x->sig_alg, x->signature, x->cert_info, ctx); } +int X509_http_nbio(OCSP_REQ_CTX *rctx, X509 **pcert) +{ + return OCSP_REQ_CTX_nbio_d2i(rctx, + (ASN1_VALUE **)pcert, ASN1_ITEM_rptr(X509)); +} + int X509_REQ_sign(X509_REQ *x, EVP_PKEY *pkey, const EVP_MD *md) { return (ASN1_item_sign(ASN1_ITEM_rptr(X509_REQ_INFO), x->sig_alg, NULL, @@ -133,6 +140,13 @@ int X509_CRL_sign_ctx(X509_CRL *x, EVP_MD_CTX *ctx) x->crl, ctx); } +int X509_CRL_http_nbio(OCSP_REQ_CTX *rctx, X509_CRL **pcrl) +{ + return OCSP_REQ_CTX_nbio_d2i(rctx, + (ASN1_VALUE **)pcrl, + ASN1_ITEM_rptr(X509_CRL)); +} + int NETSCAPE_SPKI_sign(NETSCAPE_SPKI *x, EVP_PKEY *pkey, const EVP_MD *md) { return (ASN1_item_sign(ASN1_ITEM_rptr(NETSCAPE_SPKAC), x->sig_algor, NULL, diff --git a/crypto/x509v3/Makefile b/crypto/x509v3/Makefile index 05125aba2782..9791b77a0765 100644 --- a/crypto/x509v3/Makefile +++ b/crypto/x509v3/Makefile @@ -13,7 +13,7 @@ AR= ar r CFLAGS= $(INCLUDES) $(CFLAG) GENERAL=Makefile README -TEST= +TEST=v3nametest.c APPS= LIB=$(TOP)/libcrypto.a @@ -22,13 +22,13 @@ v3_prn.c v3_utl.c v3err.c v3_genn.c v3_alt.c v3_skey.c v3_akey.c v3_pku.c \ v3_int.c v3_enum.c v3_sxnet.c v3_cpols.c v3_crld.c v3_purp.c v3_info.c \ v3_ocsp.c v3_akeya.c v3_pmaps.c v3_pcons.c v3_ncons.c v3_pcia.c v3_pci.c \ pcy_cache.c pcy_node.c pcy_data.c pcy_map.c pcy_tree.c pcy_lib.c \ -v3_asid.c v3_addr.c +v3_asid.c v3_addr.c v3_scts.c LIBOBJ= v3_bcons.o v3_bitst.o v3_conf.o v3_extku.o v3_ia5.o v3_lib.o \ v3_prn.o v3_utl.o v3err.o v3_genn.o v3_alt.o v3_skey.o v3_akey.o v3_pku.o \ v3_int.o v3_enum.o v3_sxnet.o v3_cpols.o v3_crld.o v3_purp.o v3_info.o \ v3_ocsp.o v3_akeya.o v3_pmaps.o v3_pcons.o v3_ncons.o v3_pcia.o v3_pci.o \ pcy_cache.o pcy_node.o pcy_data.o pcy_map.o pcy_tree.o pcy_lib.o \ -v3_asid.o v3_addr.o +v3_asid.o v3_addr.o v3_scts.o SRC= $(LIBSRC) @@ -535,6 +535,20 @@ v3_purp.o: ../../include/openssl/sha.h ../../include/openssl/stack.h v3_purp.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h v3_purp.o: ../../include/openssl/x509_vfy.h ../../include/openssl/x509v3.h v3_purp.o: ../cryptlib.h v3_purp.c +v3_scts.o: ../../e_os.h ../../include/openssl/asn1.h +v3_scts.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h +v3_scts.o: ../../include/openssl/conf.h ../../include/openssl/crypto.h +v3_scts.o: ../../include/openssl/e_os2.h ../../include/openssl/ec.h +v3_scts.o: ../../include/openssl/ecdh.h ../../include/openssl/ecdsa.h +v3_scts.o: ../../include/openssl/err.h ../../include/openssl/evp.h +v3_scts.o: ../../include/openssl/lhash.h ../../include/openssl/obj_mac.h +v3_scts.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h +v3_scts.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h +v3_scts.o: ../../include/openssl/pkcs7.h ../../include/openssl/safestack.h +v3_scts.o: ../../include/openssl/sha.h ../../include/openssl/stack.h +v3_scts.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h +v3_scts.o: ../../include/openssl/x509_vfy.h ../../include/openssl/x509v3.h +v3_scts.o: ../cryptlib.h v3_scts.c v3_skey.o: ../../e_os.h ../../include/openssl/asn1.h v3_skey.o: ../../include/openssl/bio.h ../../include/openssl/buffer.h v3_skey.o: ../../include/openssl/conf.h ../../include/openssl/crypto.h diff --git a/crypto/x509v3/ext_dat.h b/crypto/x509v3/ext_dat.h index 136b1f8ecfba..c3a6fce7524f 100644 --- a/crypto/x509v3/ext_dat.h +++ b/crypto/x509v3/ext_dat.h @@ -69,6 +69,7 @@ extern X509V3_EXT_METHOD v3_crl_hold, v3_pci; extern X509V3_EXT_METHOD v3_policy_mappings, v3_policy_constraints; extern X509V3_EXT_METHOD v3_name_constraints, v3_inhibit_anyp, v3_idp; extern X509V3_EXT_METHOD v3_addr, v3_asid; +extern X509V3_EXT_METHOD v3_ct_scts[]; /* * This table will be searched using OBJ_bsearch so it *must* kept in order @@ -126,6 +127,8 @@ static const X509V3_EXT_METHOD *standard_exts[] = { &v3_idp, &v3_alt[2], &v3_freshest_crl, + &v3_ct_scts[0], + &v3_ct_scts[1], }; /* Number of standard extensions */ diff --git a/crypto/x509v3/v3_lib.c b/crypto/x509v3/v3_lib.c index b5598c9a3e95..8350429aafbe 100644 --- a/crypto/x509v3/v3_lib.c +++ b/crypto/x509v3/v3_lib.c @@ -122,6 +122,28 @@ const X509V3_EXT_METHOD *X509V3_EXT_get(X509_EXTENSION *ext) return X509V3_EXT_get_nid(nid); } +int X509V3_EXT_free(int nid, void *ext_data) +{ + const X509V3_EXT_METHOD *ext_method = X509V3_EXT_get_nid(nid); + if (ext_method == NULL) { + X509V3err(X509V3_F_X509V3_EXT_FREE, + X509V3_R_CANNOT_FIND_FREE_FUNCTION); + return 0; + } + + if (ext_method->it != NULL) + ASN1_item_free(ext_data, ASN1_ITEM_ptr(ext_method->it)); + else if (ext_method->ext_free != NULL) + ext_method->ext_free(ext_data); + else { + X509V3err(X509V3_F_X509V3_EXT_FREE, + X509V3_R_CANNOT_FIND_FREE_FUNCTION); + return 0; + } + + return 1; +} + int X509V3_EXT_add_list(X509V3_EXT_METHOD *extlist) { for (; extlist->ext_nid != -1; extlist++) diff --git a/crypto/x509v3/v3_purp.c b/crypto/x509v3/v3_purp.c index 0d9bc58ce75a..36b0d87a0d8b 100644 --- a/crypto/x509v3/v3_purp.c +++ b/crypto/x509v3/v3_purp.c @@ -395,9 +395,6 @@ static void x509v3_cache_extensions(X509 *x) #ifndef OPENSSL_NO_SHA X509_digest(x, EVP_sha1(), x->sha1_hash, NULL); #endif - /* Does subject name match issuer ? */ - if (!X509_NAME_cmp(X509_get_subject_name(x), X509_get_issuer_name(x))) - x->ex_flags |= EXFLAG_SI; /* V1 should mean no extensions ... */ if (!X509_get_version(x)) x->ex_flags |= EXFLAG_V1; @@ -479,6 +476,10 @@ static void x509v3_cache_extensions(X509 *x) case NID_dvcs: x->ex_xkusage |= XKU_DVCS; break; + + case NID_anyExtendedKeyUsage: + x->ex_xkusage |= XKU_ANYEKU; + break; } } sk_ASN1_OBJECT_pop_free(extusage, ASN1_OBJECT_free); @@ -494,6 +495,13 @@ static void x509v3_cache_extensions(X509 *x) } x->skid = X509_get_ext_d2i(x, NID_subject_key_identifier, NULL, NULL); x->akid = X509_get_ext_d2i(x, NID_authority_key_identifier, NULL, NULL); + /* Does subject name match issuer ? */ + if (!X509_NAME_cmp(X509_get_subject_name(x), X509_get_issuer_name(x))) { + x->ex_flags |= EXFLAG_SI; + /* If SKID matches AKID also indicate self signed */ + if (X509_check_akid(x, x->akid) == X509_V_OK) + x->ex_flags |= EXFLAG_SS; + } x->altname = X509_get_ext_d2i(x, NID_subject_alt_name, NULL, NULL); x->nc = X509_get_ext_d2i(x, NID_name_constraints, &i, NULL); if (!x->nc && (i != -1)) @@ -598,8 +606,8 @@ static int check_purpose_ssl_client(const X509_PURPOSE *xp, const X509 *x, return 0; if (ca) return check_ssl_ca(x); - /* We need to do digital signatures with it */ - if (ku_reject(x, KU_DIGITAL_SIGNATURE)) + /* We need to do digital signatures or key agreement */ + if (ku_reject(x, KU_DIGITAL_SIGNATURE | KU_KEY_AGREEMENT)) return 0; /* nsCertType if present should allow SSL client use */ if (ns_reject(x, NS_SSL_CLIENT)) @@ -607,6 +615,14 @@ static int check_purpose_ssl_client(const X509_PURPOSE *xp, const X509 *x, return 1; } +/* + * Key usage needed for TLS/SSL server: digital signature, encipherment or + * key agreement. The ssl code can check this more thoroughly for individual + * key types. + */ +#define KU_TLS \ + KU_DIGITAL_SIGNATURE|KU_KEY_ENCIPHERMENT|KU_KEY_AGREEMENT + static int check_purpose_ssl_server(const X509_PURPOSE *xp, const X509 *x, int ca) { @@ -617,8 +633,7 @@ static int check_purpose_ssl_server(const X509_PURPOSE *xp, const X509 *x, if (ns_reject(x, NS_SSL_SERVER)) return 0; - /* Now as for keyUsage: we'll at least need to sign OR encipher */ - if (ku_reject(x, KU_DIGITAL_SIGNATURE | KU_KEY_ENCIPHERMENT)) + if (ku_reject(x, KU_TLS)) return 0; return 1; diff --git a/crypto/x509v3/v3_scts.c b/crypto/x509v3/v3_scts.c new file mode 100644 index 000000000000..6e0b8d6844c8 --- /dev/null +++ b/crypto/x509v3/v3_scts.c @@ -0,0 +1,332 @@ +/* v3_scts.c */ +/* + * Written by Rob Stradling (rob@comodo.com) for the OpenSSL project 2014. + */ +/* ==================================================================== + * Copyright (c) 2014 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + +#include +#include "cryptlib.h" +#include +#include + +/* Signature and hash algorithms from RFC 5246 */ +#define TLSEXT_hash_sha256 4 + +#define TLSEXT_signature_rsa 1 +#define TLSEXT_signature_ecdsa 3 + + +#define n2s(c,s) ((s=(((unsigned int)(c[0]))<< 8)| \ + (((unsigned int)(c[1])) )),c+=2) + +#if (defined(_WIN32) || defined(_WIN64)) && !defined(__MINGW32__) +# define SCT_TIMESTAMP unsigned __int64 +#elif defined(__arch64__) +# define SCT_TIMESTAMP unsigned long +#else +# define SCT_TIMESTAMP unsigned long long +#endif + +#define n2l8(c,l) (l =((SCT_TIMESTAMP)(*((c)++)))<<56, \ + l|=((SCT_TIMESTAMP)(*((c)++)))<<48, \ + l|=((SCT_TIMESTAMP)(*((c)++)))<<40, \ + l|=((SCT_TIMESTAMP)(*((c)++)))<<32, \ + l|=((SCT_TIMESTAMP)(*((c)++)))<<24, \ + l|=((SCT_TIMESTAMP)(*((c)++)))<<16, \ + l|=((SCT_TIMESTAMP)(*((c)++)))<< 8, \ + l|=((SCT_TIMESTAMP)(*((c)++)))) + +typedef struct SCT_st { + /* The encoded SCT */ + unsigned char *sct; + unsigned short sctlen; + /* + * Components of the SCT. "logid", "ext" and "sig" point to addresses + * inside "sct". + */ + unsigned char version; + unsigned char *logid; + unsigned short logidlen; + SCT_TIMESTAMP timestamp; + unsigned char *ext; + unsigned short extlen; + unsigned char hash_alg; + unsigned char sig_alg; + unsigned char *sig; + unsigned short siglen; +} SCT; + +DECLARE_STACK_OF(SCT) + +static void SCT_LIST_free(STACK_OF(SCT) *a); +static STACK_OF(SCT) *d2i_SCT_LIST(STACK_OF(SCT) **a, + const unsigned char **pp, long length); +static int i2r_SCT_LIST(X509V3_EXT_METHOD *method, STACK_OF(SCT) *sct_list, + BIO *out, int indent); + +const X509V3_EXT_METHOD v3_ct_scts[] = { + {NID_ct_precert_scts, 0, NULL, + 0, (X509V3_EXT_FREE)SCT_LIST_free, + (X509V3_EXT_D2I)d2i_SCT_LIST, 0, + 0, 0, 0, 0, + (X509V3_EXT_I2R)i2r_SCT_LIST, 0, + NULL}, + + {NID_ct_cert_scts, 0, NULL, + 0, (X509V3_EXT_FREE)SCT_LIST_free, + (X509V3_EXT_D2I)d2i_SCT_LIST, 0, + 0, 0, 0, 0, + (X509V3_EXT_I2R)i2r_SCT_LIST, 0, + NULL}, +}; + +static void tls12_signature_print(BIO *out, const unsigned char hash_alg, + const unsigned char sig_alg) +{ + int nid = NID_undef; + /* RFC6962 only permits two signature algorithms */ + if (hash_alg == TLSEXT_hash_sha256) { + if (sig_alg == TLSEXT_signature_rsa) + nid = NID_sha256WithRSAEncryption; + else if (sig_alg == TLSEXT_signature_ecdsa) + nid = NID_ecdsa_with_SHA256; + } + if (nid == NID_undef) + BIO_printf(out, "%02X%02X", hash_alg, sig_alg); + else + BIO_printf(out, "%s", OBJ_nid2ln(nid)); +} + +static void timestamp_print(BIO *out, SCT_TIMESTAMP timestamp) +{ + ASN1_GENERALIZEDTIME *gen; + char genstr[20]; + gen = ASN1_GENERALIZEDTIME_new(); + ASN1_GENERALIZEDTIME_adj(gen, (time_t)0, + (int)(timestamp / 86400000), + (timestamp % 86400000) / 1000); + /* + * Note GeneralizedTime from ASN1_GENERALIZETIME_adj is always 15 + * characters long with a final Z. Update it with fractional seconds. + */ + BIO_snprintf(genstr, sizeof(genstr), "%.14s.%03dZ", + ASN1_STRING_data(gen), (unsigned int)(timestamp % 1000)); + ASN1_GENERALIZEDTIME_set_string(gen, genstr); + ASN1_GENERALIZEDTIME_print(out, gen); + ASN1_GENERALIZEDTIME_free(gen); +} + +static void SCT_free(SCT *sct) +{ + if (sct) { + if (sct->sct) + OPENSSL_free(sct->sct); + OPENSSL_free(sct); + } +} + +static void SCT_LIST_free(STACK_OF(SCT) *a) +{ + sk_SCT_pop_free(a, SCT_free); +} + +static STACK_OF(SCT) *d2i_SCT_LIST(STACK_OF(SCT) **a, + const unsigned char **pp, long length) +{ + ASN1_OCTET_STRING *oct = NULL; + STACK_OF(SCT) *sk = NULL; + SCT *sct; + unsigned char *p, *p2; + unsigned short listlen, sctlen = 0, fieldlen; + + if (d2i_ASN1_OCTET_STRING(&oct, pp, length) == NULL) + return NULL; + if (oct->length < 2) + goto done; + p = oct->data; + n2s(p, listlen); + if (listlen != oct->length - 2) + goto done; + + if ((sk = sk_SCT_new_null()) == NULL) + goto done; + + while (listlen > 0) { + if (listlen < 2) + goto err; + n2s(p, sctlen); + listlen -= 2; + + if ((sctlen < 1) || (sctlen > listlen)) + goto err; + listlen -= sctlen; + + sct = OPENSSL_malloc(sizeof(SCT)); + if (!sct) + goto err; + if (!sk_SCT_push(sk, sct)) { + OPENSSL_free(sct); + goto err; + } + + sct->sct = OPENSSL_malloc(sctlen); + if (!sct->sct) + goto err; + memcpy(sct->sct, p, sctlen); + sct->sctlen = sctlen; + p += sctlen; + p2 = sct->sct; + + sct->version = *p2++; + if (sct->version == 0) { /* SCT v1 */ + /*- + * Fixed-length header: + * struct { + * (1 byte) Version sct_version; + * (32 bytes) LogID id; + * (8 bytes) uint64 timestamp; + * (2 bytes + ?) CtExtensions extensions; + */ + if (sctlen < 43) + goto err; + sctlen -= 43; + + sct->logid = p2; + sct->logidlen = 32; + p2 += 32; + + n2l8(p2, sct->timestamp); + + n2s(p2, fieldlen); + if (sctlen < fieldlen) + goto err; + sct->ext = p2; + sct->extlen = fieldlen; + p2 += fieldlen; + sctlen -= fieldlen; + + /*- + * digitally-signed struct header: + * (1 byte) Hash algorithm + * (1 byte) Signature algorithm + * (2 bytes + ?) Signature + */ + if (sctlen < 4) + goto err; + sctlen -= 4; + + sct->hash_alg = *p2++; + sct->sig_alg = *p2++; + n2s(p2, fieldlen); + if (sctlen != fieldlen) + goto err; + sct->sig = p2; + sct->siglen = fieldlen; + } + } + + done: + ASN1_OCTET_STRING_free(oct); + return sk; + + err: + SCT_LIST_free(sk); + sk = NULL; + goto done; +} + +static int i2r_SCT_LIST(X509V3_EXT_METHOD *method, STACK_OF(SCT) *sct_list, + BIO *out, int indent) +{ + SCT *sct; + int i; + + for (i = 0; i < sk_SCT_num(sct_list);) { + sct = sk_SCT_value(sct_list, i); + + BIO_printf(out, "%*sSigned Certificate Timestamp:", indent, ""); + BIO_printf(out, "\n%*sVersion : ", indent + 4, ""); + + if (sct->version == 0) { /* SCT v1 */ + BIO_printf(out, "v1(0)"); + + BIO_printf(out, "\n%*sLog ID : ", indent + 4, ""); + BIO_hex_string(out, indent + 16, 16, sct->logid, sct->logidlen); + + BIO_printf(out, "\n%*sTimestamp : ", indent + 4, ""); + timestamp_print(out, sct->timestamp); + + BIO_printf(out, "\n%*sExtensions: ", indent + 4, ""); + if (sct->extlen == 0) + BIO_printf(out, "none"); + else + BIO_hex_string(out, indent + 16, 16, sct->ext, sct->extlen); + + BIO_printf(out, "\n%*sSignature : ", indent + 4, ""); + tls12_signature_print(out, sct->hash_alg, sct->sig_alg); + BIO_printf(out, "\n%*s ", indent + 4, ""); + BIO_hex_string(out, indent + 16, 16, sct->sig, sct->siglen); + } else { /* Unknown version */ + + BIO_printf(out, "unknown\n%*s", indent + 16, ""); + BIO_hex_string(out, indent + 16, 16, sct->sct, sct->sctlen); + } + + if (++i < sk_SCT_num(sct_list)) + BIO_printf(out, "\n"); + } + + return 1; +} diff --git a/crypto/x509v3/v3_utl.c b/crypto/x509v3/v3_utl.c index 94aaebba3e3f..bdd7b95f4570 100644 --- a/crypto/x509v3/v3_utl.c +++ b/crypto/x509v3/v3_utl.c @@ -632,6 +632,433 @@ void X509_email_free(STACK_OF(OPENSSL_STRING) *sk) sk_OPENSSL_STRING_pop_free(sk, str_free); } +typedef int (*equal_fn) (const unsigned char *pattern, size_t pattern_len, + const unsigned char *subject, size_t subject_len, + unsigned int flags); + +/* Skip pattern prefix to match "wildcard" subject */ +static void skip_prefix(const unsigned char **p, size_t *plen, + const unsigned char *subject, size_t subject_len, + unsigned int flags) +{ + const unsigned char *pattern = *p; + size_t pattern_len = *plen; + + /* + * If subject starts with a leading '.' followed by more octets, and + * pattern is longer, compare just an equal-length suffix with the + * full subject (starting at the '.'), provided the prefix contains + * no NULs. + */ + if ((flags & _X509_CHECK_FLAG_DOT_SUBDOMAINS) == 0) + return; + + while (pattern_len > subject_len && *pattern) { + if ((flags & X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS) && + *pattern == '.') + break; + ++pattern; + --pattern_len; + } + + /* Skip if entire prefix acceptable */ + if (pattern_len == subject_len) { + *p = pattern; + *plen = pattern_len; + } +} + +/* Compare while ASCII ignoring case. */ +static int equal_nocase(const unsigned char *pattern, size_t pattern_len, + const unsigned char *subject, size_t subject_len, + unsigned int flags) +{ + skip_prefix(&pattern, &pattern_len, subject, subject_len, flags); + if (pattern_len != subject_len) + return 0; + while (pattern_len) { + unsigned char l = *pattern; + unsigned char r = *subject; + /* The pattern must not contain NUL characters. */ + if (l == 0) + return 0; + if (l != r) { + if ('A' <= l && l <= 'Z') + l = (l - 'A') + 'a'; + if ('A' <= r && r <= 'Z') + r = (r - 'A') + 'a'; + if (l != r) + return 0; + } + ++pattern; + ++subject; + --pattern_len; + } + return 1; +} + +/* Compare using memcmp. */ +static int equal_case(const unsigned char *pattern, size_t pattern_len, + const unsigned char *subject, size_t subject_len, + unsigned int flags) +{ + skip_prefix(&pattern, &pattern_len, subject, subject_len, flags); + if (pattern_len != subject_len) + return 0; + return !memcmp(pattern, subject, pattern_len); +} + +/* + * RFC 5280, section 7.5, requires that only the domain is compared in a + * case-insensitive manner. + */ +static int equal_email(const unsigned char *a, size_t a_len, + const unsigned char *b, size_t b_len, + unsigned int unused_flags) +{ + size_t i = a_len; + if (a_len != b_len) + return 0; + /* + * We search backwards for the '@' character, so that we do not have to + * deal with quoted local-parts. The domain part is compared in a + * case-insensitive manner. + */ + while (i > 0) { + --i; + if (a[i] == '@' || b[i] == '@') { + if (!equal_nocase(a + i, a_len - i, b + i, a_len - i, 0)) + return 0; + break; + } + } + if (i == 0) + i = a_len; + return equal_case(a, i, b, i, 0); +} + +/* + * Compare the prefix and suffix with the subject, and check that the + * characters in-between are valid. + */ +static int wildcard_match(const unsigned char *prefix, size_t prefix_len, + const unsigned char *suffix, size_t suffix_len, + const unsigned char *subject, size_t subject_len, + unsigned int flags) +{ + const unsigned char *wildcard_start; + const unsigned char *wildcard_end; + const unsigned char *p; + int allow_multi = 0; + int allow_idna = 0; + + if (subject_len < prefix_len + suffix_len) + return 0; + if (!equal_nocase(prefix, prefix_len, subject, prefix_len, flags)) + return 0; + wildcard_start = subject + prefix_len; + wildcard_end = subject + (subject_len - suffix_len); + if (!equal_nocase(wildcard_end, suffix_len, suffix, suffix_len, flags)) + return 0; + /* + * If the wildcard makes up the entire first label, it must match at + * least one character. + */ + if (prefix_len == 0 && *suffix == '.') { + if (wildcard_start == wildcard_end) + return 0; + allow_idna = 1; + if (flags & X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS) + allow_multi = 1; + } + /* IDNA labels cannot match partial wildcards */ + if (!allow_idna && + subject_len >= 4 && strncasecmp((char *)subject, "xn--", 4) == 0) + return 0; + /* The wildcard may match a literal '*' */ + if (wildcard_end == wildcard_start + 1 && *wildcard_start == '*') + return 1; + /* + * Check that the part matched by the wildcard contains only + * permitted characters and only matches a single label unless + * allow_multi is set. + */ + for (p = wildcard_start; p != wildcard_end; ++p) + if (!(('0' <= *p && *p <= '9') || + ('A' <= *p && *p <= 'Z') || + ('a' <= *p && *p <= 'z') || + *p == '-' || (allow_multi && *p == '.'))) + return 0; + return 1; +} + +#define LABEL_START (1 << 0) +#define LABEL_END (1 << 1) +#define LABEL_HYPHEN (1 << 2) +#define LABEL_IDNA (1 << 3) + +static const unsigned char *valid_star(const unsigned char *p, size_t len, + unsigned int flags) +{ + const unsigned char *star = 0; + size_t i; + int state = LABEL_START; + int dots = 0; + for (i = 0; i < len; ++i) { + /* + * Locate first and only legal wildcard, either at the start + * or end of a non-IDNA first and not final label. + */ + if (p[i] == '*') { + int atstart = (state & LABEL_START); + int atend = (i == len - 1 || p[i + 1] == '.'); + /*- + * At most one wildcard per pattern. + * No wildcards in IDNA labels. + * No wildcards after the first label. + */ + if (star != NULL || (state & LABEL_IDNA) != 0 || dots) + return NULL; + /* Only full-label '*.example.com' wildcards? */ + if ((flags & X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS) + && (!atstart || !atend)) + return NULL; + /* No 'foo*bar' wildcards */ + if (!atstart && !atend) + return NULL; + star = &p[i]; + state &= ~LABEL_START; + } else if (('a' <= p[i] && p[i] <= 'z') + || ('A' <= p[i] && p[i] <= 'Z') + || ('0' <= p[i] && p[i] <= '9')) { + if ((state & LABEL_START) != 0 + && len - i >= 4 && strncasecmp((char *)&p[i], "xn--", 4) == 0) + state |= LABEL_IDNA; + state &= ~(LABEL_HYPHEN | LABEL_START); + } else if (p[i] == '.') { + if ((state & (LABEL_HYPHEN | LABEL_START)) != 0) + return NULL; + state = LABEL_START; + ++dots; + } else if (p[i] == '-') { + if ((state & LABEL_HYPHEN) != 0) + return NULL; + state |= LABEL_HYPHEN; + } else + return NULL; + } + + /* + * The final label must not end in a hyphen or ".", and + * there must be at least two dots after the star. + */ + if ((state & (LABEL_START | LABEL_HYPHEN)) != 0 || dots < 2) + return NULL; + return star; +} + +/* Compare using wildcards. */ +static int equal_wildcard(const unsigned char *pattern, size_t pattern_len, + const unsigned char *subject, size_t subject_len, + unsigned int flags) +{ + const unsigned char *star = NULL; + + /* + * Subject names starting with '.' can only match a wildcard pattern + * via a subject sub-domain pattern suffix match. + */ + if (!(subject_len > 1 && subject[0] == '.')) + star = valid_star(pattern, pattern_len, flags); + if (star == NULL) + return equal_nocase(pattern, pattern_len, + subject, subject_len, flags); + return wildcard_match(pattern, star - pattern, + star + 1, (pattern + pattern_len) - star - 1, + subject, subject_len, flags); +} + +/* + * Compare an ASN1_STRING to a supplied string. If they match return 1. If + * cmp_type > 0 only compare if string matches the type, otherwise convert it + * to UTF8. + */ + +static int do_check_string(ASN1_STRING *a, int cmp_type, equal_fn equal, + unsigned int flags, const char *b, size_t blen, + char **peername) +{ + int rv = 0; + + if (!a->data || !a->length) + return 0; + if (cmp_type > 0) { + if (cmp_type != a->type) + return 0; + if (cmp_type == V_ASN1_IA5STRING) + rv = equal(a->data, a->length, (unsigned char *)b, blen, flags); + else if (a->length == (int)blen && !memcmp(a->data, b, blen)) + rv = 1; + if (rv > 0 && peername) + *peername = BUF_strndup((char *)a->data, a->length); + } else { + int astrlen; + unsigned char *astr; + astrlen = ASN1_STRING_to_UTF8(&astr, a); + if (astrlen < 0) { + /* + * -1 could be an internal malloc failure or a decoding error from + * malformed input; we can't distinguish. + */ + return -1; + } + rv = equal(astr, astrlen, (unsigned char *)b, blen, flags); + if (rv > 0 && peername) + *peername = BUF_strndup((char *)astr, astrlen); + OPENSSL_free(astr); + } + return rv; +} + +static int do_x509_check(X509 *x, const char *chk, size_t chklen, + unsigned int flags, int check_type, char **peername) +{ + GENERAL_NAMES *gens = NULL; + X509_NAME *name = NULL; + int i; + int cnid; + int alt_type; + int san_present = 0; + int rv = 0; + equal_fn equal; + + /* See below, this flag is internal-only */ + flags &= ~_X509_CHECK_FLAG_DOT_SUBDOMAINS; + if (check_type == GEN_EMAIL) { + cnid = NID_pkcs9_emailAddress; + alt_type = V_ASN1_IA5STRING; + equal = equal_email; + } else if (check_type == GEN_DNS) { + cnid = NID_commonName; + /* Implicit client-side DNS sub-domain pattern */ + if (chklen > 1 && chk[0] == '.') + flags |= _X509_CHECK_FLAG_DOT_SUBDOMAINS; + alt_type = V_ASN1_IA5STRING; + if (flags & X509_CHECK_FLAG_NO_WILDCARDS) + equal = equal_nocase; + else + equal = equal_wildcard; + } else { + cnid = 0; + alt_type = V_ASN1_OCTET_STRING; + equal = equal_case; + } + + if (chklen == 0) + chklen = strlen(chk); + + gens = X509_get_ext_d2i(x, NID_subject_alt_name, NULL, NULL); + if (gens) { + for (i = 0; i < sk_GENERAL_NAME_num(gens); i++) { + GENERAL_NAME *gen; + ASN1_STRING *cstr; + gen = sk_GENERAL_NAME_value(gens, i); + if (gen->type != check_type) + continue; + san_present = 1; + if (check_type == GEN_EMAIL) + cstr = gen->d.rfc822Name; + else if (check_type == GEN_DNS) + cstr = gen->d.dNSName; + else + cstr = gen->d.iPAddress; + /* Positive on success, negative on error! */ + if ((rv = do_check_string(cstr, alt_type, equal, flags, + chk, chklen, peername)) != 0) + break; + } + GENERAL_NAMES_free(gens); + if (rv != 0) + return rv; + if (!cnid + || (san_present + && !(flags & X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT))) + return 0; + } + i = -1; + name = X509_get_subject_name(x); + while ((i = X509_NAME_get_index_by_NID(name, cnid, i)) >= 0) { + X509_NAME_ENTRY *ne; + ASN1_STRING *str; + ne = X509_NAME_get_entry(name, i); + str = X509_NAME_ENTRY_get_data(ne); + /* Positive on success, negative on error! */ + if ((rv = do_check_string(str, -1, equal, flags, + chk, chklen, peername)) != 0) + return rv; + } + return 0; +} + +int X509_check_host(X509 *x, const char *chk, size_t chklen, + unsigned int flags, char **peername) +{ + if (chk == NULL) + return -2; + /* + * Embedded NULs are disallowed, except as the last character of a + * string of length 2 or more (tolerate caller including terminating + * NUL in string length). + */ + if (chklen == 0) + chklen = strlen(chk); + else if (memchr(chk, '\0', chklen > 1 ? chklen - 1 : chklen)) + return -2; + if (chklen > 1 && chk[chklen - 1] == '\0') + --chklen; + return do_x509_check(x, chk, chklen, flags, GEN_DNS, peername); +} + +int X509_check_email(X509 *x, const char *chk, size_t chklen, + unsigned int flags) +{ + if (chk == NULL) + return -2; + /* + * Embedded NULs are disallowed, except as the last character of a + * string of length 2 or more (tolerate caller including terminating + * NUL in string length). + */ + if (chklen == 0) + chklen = strlen((char *)chk); + else if (memchr(chk, '\0', chklen > 1 ? chklen - 1 : chklen)) + return -2; + if (chklen > 1 && chk[chklen - 1] == '\0') + --chklen; + return do_x509_check(x, chk, chklen, flags, GEN_EMAIL, NULL); +} + +int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, + unsigned int flags) +{ + if (chk == NULL) + return -2; + return do_x509_check(x, (char *)chk, chklen, flags, GEN_IPADD, NULL); +} + +int X509_check_ip_asc(X509 *x, const char *ipasc, unsigned int flags) +{ + unsigned char ipout[16]; + size_t iplen; + + if (ipasc == NULL) + return -2; + iplen = (size_t)a2i_ipadd(ipout, ipasc); + if (iplen == 0) + return -2; + return do_x509_check(x, (char *)ipout, iplen, flags, GEN_IPADD, NULL); +} + /* * Convert IP addresses both IPv4 and IPv6 into an OCTET STRING compatible * with RFC3280. diff --git a/crypto/x509v3/v3err.c b/crypto/x509v3/v3err.c index 0138f7a8462a..bcc1be722e42 100644 --- a/crypto/x509v3/v3err.c +++ b/crypto/x509v3/v3err.c @@ -1,6 +1,6 @@ /* crypto/x509v3/v3err.c */ /* ==================================================================== - * Copyright (c) 1999-2007 The OpenSSL Project. All rights reserved. + * Copyright (c) 1999-2014 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -70,7 +70,7 @@ # define ERR_REASON(reason) ERR_PACK(ERR_LIB_X509V3,0,reason) static ERR_STRING_DATA X509V3_str_functs[] = { - {ERR_FUNC(X509V3_F_A2I_GENERAL_NAME), "A2I_GENERAL_NAME"}, + {ERR_FUNC(X509V3_F_A2I_GENERAL_NAME), "a2i_GENERAL_NAME"}, {ERR_FUNC(X509V3_F_ASIDENTIFIERCHOICE_CANONIZE), "ASIDENTIFIERCHOICE_CANONIZE"}, {ERR_FUNC(X509V3_F_ASIDENTIFIERCHOICE_IS_CANONICAL), @@ -132,6 +132,7 @@ static ERR_STRING_DATA X509V3_str_functs[] = { {ERR_FUNC(X509V3_F_X509V3_EXT_ADD), "X509V3_EXT_add"}, {ERR_FUNC(X509V3_F_X509V3_EXT_ADD_ALIAS), "X509V3_EXT_add_alias"}, {ERR_FUNC(X509V3_F_X509V3_EXT_CONF), "X509V3_EXT_conf"}, + {ERR_FUNC(X509V3_F_X509V3_EXT_FREE), "X509V3_EXT_free"}, {ERR_FUNC(X509V3_F_X509V3_EXT_I2D), "X509V3_EXT_i2d"}, {ERR_FUNC(X509V3_F_X509V3_EXT_NCONF), "X509V3_EXT_nconf"}, {ERR_FUNC(X509V3_F_X509V3_GET_SECTION), "X509V3_get_section"}, @@ -149,6 +150,8 @@ static ERR_STRING_DATA X509V3_str_reasons[] = { {ERR_REASON(X509V3_R_BN_DEC2BN_ERROR), "bn dec2bn error"}, {ERR_REASON(X509V3_R_BN_TO_ASN1_INTEGER_ERROR), "bn to asn1 integer error"}, + {ERR_REASON(X509V3_R_CANNOT_FIND_FREE_FUNCTION), + "cannot find free function"}, {ERR_REASON(X509V3_R_DIRNAME_ERROR), "dirname error"}, {ERR_REASON(X509V3_R_DISTPOINT_ALREADY_SET), "distpoint already set"}, {ERR_REASON(X509V3_R_DUPLICATE_ZONE_ID), "duplicate zone id"}, @@ -167,7 +170,6 @@ static ERR_STRING_DATA X509V3_str_reasons[] = { {ERR_REASON(X509V3_R_ILLEGAL_HEX_DIGIT), "illegal hex digit"}, {ERR_REASON(X509V3_R_INCORRECT_POLICY_SYNTAX_TAG), "incorrect policy syntax tag"}, - {ERR_REASON(X509V3_R_INVALID_MULTIPLE_RDNS), "invalid multiple rdns"}, {ERR_REASON(X509V3_R_INVALID_ASNUMBER), "invalid asnumber"}, {ERR_REASON(X509V3_R_INVALID_ASRANGE), "invalid asrange"}, {ERR_REASON(X509V3_R_INVALID_BOOLEAN_STRING), "invalid boolean string"}, @@ -175,6 +177,7 @@ static ERR_STRING_DATA X509V3_str_reasons[] = { "invalid extension string"}, {ERR_REASON(X509V3_R_INVALID_INHERITANCE), "invalid inheritance"}, {ERR_REASON(X509V3_R_INVALID_IPADDRESS), "invalid ipaddress"}, + {ERR_REASON(X509V3_R_INVALID_MULTIPLE_RDNS), "invalid multiple rdns"}, {ERR_REASON(X509V3_R_INVALID_NAME), "invalid name"}, {ERR_REASON(X509V3_R_INVALID_NULL_ARGUMENT), "invalid null argument"}, {ERR_REASON(X509V3_R_INVALID_NULL_NAME), "invalid null name"}, diff --git a/crypto/x509v3/v3nametest.c b/crypto/x509v3/v3nametest.c new file mode 100644 index 000000000000..7b5c1c8e5127 --- /dev/null +++ b/crypto/x509v3/v3nametest.c @@ -0,0 +1,336 @@ +#include +#include +#include "../e_os.h" +#include + +static const char *const names[] = { + "a", "b", ".", "*", "@", + ".a", "a.", ".b", "b.", ".*", "*.", "*@", "@*", "a@", "@a", "b@", "..", + "@@", "**", "*.com", "*com", "*.*.com", "*com", "com*", "*example.com", + "*@example.com", "test@*.example.com", "example.com", "www.example.com", + "test.www.example.com", "*.example.com", "*.www.example.com", + "test.*.example.com", "www.*.com", + ".www.example.com", "*www.example.com", + "example.net", "xn--rger-koa.example.com", + "a.example.com", "b.example.com", + "postmaster@example.com", "Postmaster@example.com", + "postmaster@EXAMPLE.COM", + NULL +}; + +static const char *const exceptions[] = { + "set CN: host: [*.example.com] matches [a.example.com]", + "set CN: host: [*.example.com] matches [b.example.com]", + "set CN: host: [*.example.com] matches [www.example.com]", + "set CN: host: [*.example.com] matches [xn--rger-koa.example.com]", + "set CN: host: [*.www.example.com] matches [test.www.example.com]", + "set CN: host: [*.www.example.com] matches [.www.example.com]", + "set CN: host: [*www.example.com] matches [www.example.com]", + "set CN: host: [test.www.example.com] matches [.www.example.com]", + "set CN: host-no-wildcards: [*.www.example.com] matches [.www.example.com]", + "set CN: host-no-wildcards: [test.www.example.com] matches [.www.example.com]", + "set emailAddress: email: [postmaster@example.com] does not match [Postmaster@example.com]", + "set emailAddress: email: [postmaster@EXAMPLE.COM] does not match [Postmaster@example.com]", + "set emailAddress: email: [Postmaster@example.com] does not match [postmaster@example.com]", + "set emailAddress: email: [Postmaster@example.com] does not match [postmaster@EXAMPLE.COM]", + "set dnsName: host: [*.example.com] matches [www.example.com]", + "set dnsName: host: [*.example.com] matches [a.example.com]", + "set dnsName: host: [*.example.com] matches [b.example.com]", + "set dnsName: host: [*.example.com] matches [xn--rger-koa.example.com]", + "set dnsName: host: [*.www.example.com] matches [test.www.example.com]", + "set dnsName: host-no-wildcards: [*.www.example.com] matches [.www.example.com]", + "set dnsName: host-no-wildcards: [test.www.example.com] matches [.www.example.com]", + "set dnsName: host: [*.www.example.com] matches [.www.example.com]", + "set dnsName: host: [*www.example.com] matches [www.example.com]", + "set dnsName: host: [test.www.example.com] matches [.www.example.com]", + "set rfc822Name: email: [postmaster@example.com] does not match [Postmaster@example.com]", + "set rfc822Name: email: [Postmaster@example.com] does not match [postmaster@example.com]", + "set rfc822Name: email: [Postmaster@example.com] does not match [postmaster@EXAMPLE.COM]", + "set rfc822Name: email: [postmaster@EXAMPLE.COM] does not match [Postmaster@example.com]", + NULL +}; + +static int is_exception(const char *msg) +{ + const char *const *p; + for (p = exceptions; *p; ++p) + if (strcmp(msg, *p) == 0) + return 1; + return 0; +} + +static int set_cn(X509 *crt, ...) +{ + int ret = 0; + X509_NAME *n = NULL; + va_list ap; + va_start(ap, crt); + n = X509_NAME_new(); + if (n == NULL) + goto out; + while (1) { + int nid; + const char *name; + nid = va_arg(ap, int); + if (nid == 0) + break; + name = va_arg(ap, const char *); + if (!X509_NAME_add_entry_by_NID(n, nid, MBSTRING_ASC, + (unsigned char *)name, -1, -1, 1)) + goto out; + } + if (!X509_set_subject_name(crt, n)) + goto out; + ret = 1; + out: + X509_NAME_free(n); + va_end(ap); + return ret; +} + +/*- +int X509_add_ext(X509 *x, X509_EXTENSION *ex, int loc); +X509_EXTENSION *X509_EXTENSION_create_by_NID(X509_EXTENSION **ex, + int nid, int crit, ASN1_OCTET_STRING *data); +int X509_add_ext(X509 *x, X509_EXTENSION *ex, int loc); +*/ + +static int set_altname(X509 *crt, ...) +{ + int ret = 0; + GENERAL_NAMES *gens = NULL; + GENERAL_NAME *gen = NULL; + ASN1_IA5STRING *ia5 = NULL; + va_list ap; + va_start(ap, crt); + gens = sk_GENERAL_NAME_new_null(); + if (gens == NULL) + goto out; + while (1) { + int type; + const char *name; + type = va_arg(ap, int); + if (type == 0) + break; + name = va_arg(ap, const char *); + + gen = GENERAL_NAME_new(); + if (gen == NULL) + goto out; + ia5 = ASN1_IA5STRING_new(); + if (ia5 == NULL) + goto out; + if (!ASN1_STRING_set(ia5, name, -1)) + goto out; + switch (type) { + case GEN_EMAIL: + case GEN_DNS: + GENERAL_NAME_set0_value(gen, type, ia5); + ia5 = NULL; + break; + default: + abort(); + } + sk_GENERAL_NAME_push(gens, gen); + gen = NULL; + } + if (!X509_add1_ext_i2d(crt, NID_subject_alt_name, gens, 0, 0)) + goto out; + ret = 1; + out: + ASN1_IA5STRING_free(ia5); + GENERAL_NAME_free(gen); + GENERAL_NAMES_free(gens); + va_end(ap); + return ret; +} + +static int set_cn1(X509 *crt, const char *name) +{ + return set_cn(crt, NID_commonName, name, 0); +} + +static int set_cn_and_email(X509 *crt, const char *name) +{ + return set_cn(crt, NID_commonName, name, + NID_pkcs9_emailAddress, "dummy@example.com", 0); +} + +static int set_cn2(X509 *crt, const char *name) +{ + return set_cn(crt, NID_commonName, "dummy value", + NID_commonName, name, 0); +} + +static int set_cn3(X509 *crt, const char *name) +{ + return set_cn(crt, NID_commonName, name, + NID_commonName, "dummy value", 0); +} + +static int set_email1(X509 *crt, const char *name) +{ + return set_cn(crt, NID_pkcs9_emailAddress, name, 0); +} + +static int set_email2(X509 *crt, const char *name) +{ + return set_cn(crt, NID_pkcs9_emailAddress, "dummy@example.com", + NID_pkcs9_emailAddress, name, 0); +} + +static int set_email3(X509 *crt, const char *name) +{ + return set_cn(crt, NID_pkcs9_emailAddress, name, + NID_pkcs9_emailAddress, "dummy@example.com", 0); +} + +static int set_email_and_cn(X509 *crt, const char *name) +{ + return set_cn(crt, NID_pkcs9_emailAddress, name, + NID_commonName, "www.example.org", 0); +} + +static int set_altname_dns(X509 *crt, const char *name) +{ + return set_altname(crt, GEN_DNS, name, 0); +} + +static int set_altname_email(X509 *crt, const char *name) +{ + return set_altname(crt, GEN_EMAIL, name, 0); +} + +struct set_name_fn { + int (*fn) (X509 *, const char *); + const char *name; + int host; + int email; +}; + +static const struct set_name_fn name_fns[] = { + {set_cn1, "set CN", 1, 0}, + {set_cn2, "set CN", 1, 0}, + {set_cn3, "set CN", 1, 0}, + {set_cn_and_email, "set CN", 1, 0}, + {set_email1, "set emailAddress", 0, 1}, + {set_email2, "set emailAddress", 0, 1}, + {set_email3, "set emailAddress", 0, 1}, + {set_email_and_cn, "set emailAddress", 0, 1}, + {set_altname_dns, "set dnsName", 1, 0}, + {set_altname_email, "set rfc822Name", 0, 1}, + {NULL, NULL, 0} +}; + +static X509 *make_cert() +{ + X509 *ret = NULL; + X509 *crt = NULL; + X509_NAME *issuer = NULL; + crt = X509_new(); + if (crt == NULL) + goto out; + if (!X509_set_version(crt, 3)) + goto out; + ret = crt; + crt = NULL; + out: + X509_NAME_free(issuer); + return ret; +} + +static int errors; + +static void check_message(const struct set_name_fn *fn, const char *op, + const char *nameincert, int match, const char *name) +{ + char msg[1024]; + if (match < 0) + return; + BIO_snprintf(msg, sizeof(msg), "%s: %s: [%s] %s [%s]", + fn->name, op, nameincert, + match ? "matches" : "does not match", name); + if (is_exception(msg)) + return; + puts(msg); + ++errors; +} + +static void run_cert(X509 *crt, const char *nameincert, + const struct set_name_fn *fn) +{ + const char *const *pname = names; + while (*pname) { + int samename = strcasecmp(nameincert, *pname) == 0; + size_t namelen = strlen(*pname); + char *name = malloc(namelen); + int match, ret; + memcpy(name, *pname, namelen); + + ret = X509_check_host(crt, name, namelen, 0, NULL); + match = -1; + if (ret < 0) { + fprintf(stderr, "internal error in X509_check_host"); + ++errors; + } else if (fn->host) { + if (ret == 1 && !samename) + match = 1; + if (ret == 0 && samename) + match = 0; + } else if (ret == 1) + match = 1; + check_message(fn, "host", nameincert, match, *pname); + + ret = X509_check_host(crt, name, namelen, + X509_CHECK_FLAG_NO_WILDCARDS, NULL); + match = -1; + if (ret < 0) { + fprintf(stderr, "internal error in X509_check_host"); + ++errors; + } else if (fn->host) { + if (ret == 1 && !samename) + match = 1; + if (ret == 0 && samename) + match = 0; + } else if (ret == 1) + match = 1; + check_message(fn, "host-no-wildcards", nameincert, match, *pname); + + ret = X509_check_email(crt, name, namelen, 0); + match = -1; + if (fn->email) { + if (ret && !samename) + match = 1; + if (!ret && samename && strchr(nameincert, '@') != NULL) + match = 0; + } else if (ret) + match = 1; + check_message(fn, "email", nameincert, match, *pname); + ++pname; + free(name); + } +} + +int main(void) +{ + const struct set_name_fn *pfn = name_fns; + while (pfn->name) { + const char *const *pname = names; + while (*pname) { + X509 *crt = make_cert(); + if (crt == NULL) { + fprintf(stderr, "make_cert failed\n"); + return 1; + } + if (!pfn->fn(crt, *pname)) { + fprintf(stderr, "X509 name setting failed\n"); + return 1; + } + run_cert(crt, *pname, pfn); + X509_free(crt); + ++pname; + } + ++pfn; + } + return errors > 0 ? 1 : 0; +} diff --git a/crypto/x509v3/x509v3.h b/crypto/x509v3/x509v3.h index db9c3e8bde6e..f5c61560aa10 100644 --- a/crypto/x509v3/x509v3.h +++ b/crypto/x509v3/x509v3.h @@ -67,6 +67,13 @@ extern "C" { #endif +# ifdef OPENSSL_SYS_WIN32 +/* Under Win32 these are defined in wincrypt.h */ +# undef X509_NAME +# undef X509_CERT_PAIR +# undef X509_EXTENSIONS +# endif + /* Forward reference */ struct v3_ext_method; struct v3_ext_ctx; @@ -405,7 +412,6 @@ struct ISSUING_DIST_POINT_st { # define EXFLAG_CA 0x10 /* Really self issued not necessarily self signed */ # define EXFLAG_SI 0x20 -# define EXFLAG_SS 0x20 # define EXFLAG_V1 0x40 # define EXFLAG_INVALID 0x80 # define EXFLAG_SET 0x100 @@ -414,6 +420,8 @@ struct ISSUING_DIST_POINT_st { # define EXFLAG_INVALID_POLICY 0x800 # define EXFLAG_FRESHEST 0x1000 +/* Self signed */ +# define EXFLAG_SS 0x2000 # define KU_DIGITAL_SIGNATURE 0x0080 # define KU_NON_REPUDIATION 0x0040 @@ -442,6 +450,7 @@ struct ISSUING_DIST_POINT_st { # define XKU_OCSP_SIGN 0x20 # define XKU_TIMESTAMP 0x40 # define XKU_DVCS 0x80 +# define XKU_ANYEKU 0x100 # define X509_PURPOSE_DYNAMIC 0x1 # define X509_PURPOSE_DYNAMIC_NAME 0x2 @@ -665,6 +674,7 @@ STACK_OF(CONF_VALUE) *X509V3_parse_list(const char *line); void *X509V3_EXT_d2i(X509_EXTENSION *ext); void *X509V3_get_d2i(STACK_OF(X509_EXTENSION) *x, int nid, int *crit, int *idx); +int X509V3_EXT_free(int nid, void *ext_data); X509_EXTENSION *X509V3_EXT_i2d(int ext_nid, int crit, void *ext_struc); int X509V3_add1_i2d(STACK_OF(X509_EXTENSION) **x, int nid, void *value, @@ -707,6 +717,34 @@ STACK_OF(OPENSSL_STRING) *X509_get1_email(X509 *x); STACK_OF(OPENSSL_STRING) *X509_REQ_get1_email(X509_REQ *x); void X509_email_free(STACK_OF(OPENSSL_STRING) *sk); STACK_OF(OPENSSL_STRING) *X509_get1_ocsp(X509 *x); +/* Flags for X509_check_* functions */ + +/* + * Always check subject name for host match even if subject alt names present + */ +# define X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT 0x1 +/* Disable wildcard matching for dnsName fields and common name. */ +# define X509_CHECK_FLAG_NO_WILDCARDS 0x2 +/* Wildcards must not match a partial label. */ +# define X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS 0x4 +/* Allow (non-partial) wildcards to match multiple labels. */ +# define X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS 0x8 +/* Constraint verifier subdomain patterns to match a single labels. */ +# define X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS 0x10 +/* + * Match reference identifiers starting with "." to any sub-domain. + * This is a non-public flag, turned on implicitly when the subject + * reference identity is a DNS name. + */ +# define _X509_CHECK_FLAG_DOT_SUBDOMAINS 0x8000 + +int X509_check_host(X509 *x, const char *chk, size_t chklen, + unsigned int flags, char **peername); +int X509_check_email(X509 *x, const char *chk, size_t chklen, + unsigned int flags); +int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, + unsigned int flags); +int X509_check_ip_asc(X509 *x, const char *ipasc, unsigned int flags); ASN1_OCTET_STRING *a2i_IPADDRESS(const char *ipasc); ASN1_OCTET_STRING *a2i_IPADDRESS_NC(const char *ipasc); @@ -930,6 +968,7 @@ void ERR_load_X509V3_strings(void); # define X509V3_F_X509V3_EXT_ADD 104 # define X509V3_F_X509V3_EXT_ADD_ALIAS 106 # define X509V3_F_X509V3_EXT_CONF 107 +# define X509V3_F_X509V3_EXT_FREE 165 # define X509V3_F_X509V3_EXT_I2D 136 # define X509V3_F_X509V3_EXT_NCONF 152 # define X509V3_F_X509V3_GET_SECTION 142 @@ -944,6 +983,7 @@ void ERR_load_X509V3_strings(void); # define X509V3_R_BAD_OBJECT 119 # define X509V3_R_BN_DEC2BN_ERROR 100 # define X509V3_R_BN_TO_ASN1_INTEGER_ERROR 101 +# define X509V3_R_CANNOT_FIND_FREE_FUNCTION 168 # define X509V3_R_DIRNAME_ERROR 149 # define X509V3_R_DISTPOINT_ALREADY_SET 160 # define X509V3_R_DUPLICATE_ZONE_ID 133 @@ -959,13 +999,13 @@ void ERR_load_X509V3_strings(void); # define X509V3_R_ILLEGAL_EMPTY_EXTENSION 151 # define X509V3_R_ILLEGAL_HEX_DIGIT 113 # define X509V3_R_INCORRECT_POLICY_SYNTAX_TAG 152 -# define X509V3_R_INVALID_MULTIPLE_RDNS 161 # define X509V3_R_INVALID_ASNUMBER 162 # define X509V3_R_INVALID_ASRANGE 163 # define X509V3_R_INVALID_BOOLEAN_STRING 104 # define X509V3_R_INVALID_EXTENSION_STRING 105 # define X509V3_R_INVALID_INHERITANCE 165 # define X509V3_R_INVALID_IPADDRESS 166 +# define X509V3_R_INVALID_MULTIPLE_RDNS 161 # define X509V3_R_INVALID_NAME 106 # define X509V3_R_INVALID_NULL_ARGUMENT 107 # define X509V3_R_INVALID_NULL_NAME 108 diff --git a/crypto/x86_64cpuid.pl b/crypto/x86_64cpuid.pl index 6ebfd017ea56..d208d02392e9 100644 --- a/crypto/x86_64cpuid.pl +++ b/crypto/x86_64cpuid.pl @@ -24,7 +24,7 @@ print<<___; call OPENSSL_cpuid_setup .hidden OPENSSL_ia32cap_P -.comm OPENSSL_ia32cap_P,8,4 +.comm OPENSSL_ia32cap_P,16,4 .text @@ -53,12 +53,13 @@ OPENSSL_rdtsc: .size OPENSSL_rdtsc,.-OPENSSL_rdtsc .globl OPENSSL_ia32_cpuid -.type OPENSSL_ia32_cpuid,\@abi-omnipotent +.type OPENSSL_ia32_cpuid,\@function,1 .align 16 OPENSSL_ia32_cpuid: mov %rbx,%r8 # save %rbx xor %eax,%eax + mov %eax,8(%rdi) # clear 3rd word cpuid mov %eax,%r11d # max value for standard query level @@ -126,6 +127,14 @@ OPENSSL_ia32_cpuid: shr \$14,%r10d and \$0xfff,%r10d # number of cores -1 per L1D + cmp \$7,%r11d + jb .Lnocacheinfo + + mov \$7,%eax + xor %ecx,%ecx + cpuid + mov %ebx,8(%rdi) + .Lnocacheinfo: mov \$1,%eax cpuid @@ -165,6 +174,7 @@ OPENSSL_ia32_cpuid: .Lclear_avx: mov \$0xefffe7ff,%eax # ~(1<<28|1<<12|1<<11) and %eax,%r9d # clear AVX, FMA and AMD XOP bits + andl \$0xffffffdf,8(%rdi) # cleax AVX2, ~(1<<5) .Ldone: shl \$32,%r9 mov %r10d,%eax @@ -279,6 +289,21 @@ OPENSSL_ia32_rdrand: cmove %rcx,%rax ret .size OPENSSL_ia32_rdrand,.-OPENSSL_ia32_rdrand + +.globl OPENSSL_ia32_rdseed +.type OPENSSL_ia32_rdseed,\@abi-omnipotent +.align 16 +OPENSSL_ia32_rdseed: + mov \$8,%ecx +.Loop_rdseed: + rdseed %rax + jc .Lbreak_rdseed + loop .Loop_rdseed +.Lbreak_rdseed: + cmp \$0,%rax + cmove %rcx,%rax + ret +.size OPENSSL_ia32_rdseed,.-OPENSSL_ia32_rdseed ___ close STDOUT; # flush diff --git a/crypto/x86cpuid.pl b/crypto/x86cpuid.pl index b270b44337d1..e95f6274f5e0 100644 --- a/crypto/x86cpuid.pl +++ b/crypto/x86cpuid.pl @@ -22,6 +22,8 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); } &xor ("eax","eax"); &bt ("ecx",21); &jnc (&label("nocpuid")); + &mov ("esi",&wparam(0)); + &mov (&DWP(8,"esi"),"eax"); # clear 3rd word &cpuid (); &mov ("edi","eax"); # max value for standard query level @@ -79,6 +81,16 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); } &jmp (&label("generic")); &set_label("intel"); + &cmp ("edi",7); + &jb (&label("cacheinfo")); + + &mov ("esi",&wparam(0)); + &mov ("eax",7); + &xor ("ecx","ecx"); + &cpuid (); + &mov (&DWP(8,"esi"),"ebx"); + +&set_label("cacheinfo"); &cmp ("edi",4); &mov ("edi",-1); &jb (&label("nocacheinfo")); @@ -135,6 +147,8 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); } &and ("esi",0xfeffffff); # clear FXSR &set_label("clear_avx"); &and ("ebp",0xefffe7ff); # clear AVX, FMA and AMD XOP bits + &mov ("edi",&wparam(0)); + &and (&DWP(8,"edi"),0xffffffdf); # clear AVX2 &set_label("done"); &mov ("eax","esi"); &mov ("edx","ebp"); @@ -198,7 +212,7 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); } &function_begin_B("OPENSSL_far_spin"); &pushf (); - &pop ("eax") + &pop ("eax"); &bt ("eax",9); &jnc (&label("nospin")); # interrupts are disabled @@ -353,6 +367,21 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); } &ret (); &function_end_B("OPENSSL_ia32_rdrand"); +&function_begin_B("OPENSSL_ia32_rdseed"); + &mov ("ecx",8); +&set_label("loop"); + &rdseed ("eax"); + &jc (&label("break")); + &loop (&label("loop")); +&set_label("break"); + &cmp ("eax",0); + &cmove ("eax","ecx"); + &ret (); +&function_end_B("OPENSSL_ia32_rdseed"); + &initseg("OPENSSL_cpuid_setup"); +&hidden("OPENSSL_cpuid_setup"); +&hidden("OPENSSL_ia32cap_P"); + &asm_finish(); -- cgit v1.3