diff options
Diffstat (limited to 'crypto/openssl/crypto/sha')
32 files changed, 9534 insertions, 954 deletions
diff --git a/crypto/openssl/crypto/sha/Makefile b/crypto/openssl/crypto/sha/Makefile index f4741b9ee63a..6d191d3936ee 100644 --- a/crypto/openssl/crypto/sha/Makefile +++ b/crypto/openssl/crypto/sha/Makefile @@ -38,25 +38,16 @@ top: all: lib lib: $(LIBOBJ) - $(ARX) $(LIB) $(LIBOBJ) + $(AR) $(LIB) $(LIBOBJ) $(RANLIB) $(LIB) || echo Never mind. @touch lib -# ELF -sx86-elf.s: asm/sha1-586.pl ../perlasm/x86asm.pl - (cd asm; $(PERL) sha1-586.pl elf $(CFLAGS) $(PROCESSOR) > ../$@) -s512sse2-elf.s: asm/sha512-sse2.pl ../perlasm/x86asm.pl - (cd asm; $(PERL) sha512-sse2.pl elf $(CFLAGS) $(PROCESSOR) > ../$@) -# COFF -sx86-cof.s: asm/sha1-586.pl ../perlasm/x86asm.pl - (cd asm; $(PERL) sha1-586.pl coff $(CFLAGS) $(PROCESSOR) > ../$@) -s512sse2-cof.s: asm/sha512-sse2.pl ../perlasm/x86asm.pl - (cd asm; $(PERL) sha512-sse2.pl coff $(CFLAGS) $(PROCESSOR) > ../$@) -# a.out -sx86-out.s: asm/sha1-586.pl ../perlasm/x86asm.pl - (cd asm; $(PERL) sha1-586.pl a.out $(CFLAGS) $(PROCESSOR) > ../$@) -s512sse2-out.s: asm/sha512-sse2.pl ../perlasm/x86asm.pl - (cd asm; $(PERL) sha512-sse2.pl a.out $(CFLAGS) $(PROCESSOR) > ../$@) +sha1-586.s: asm/sha1-586.pl ../perlasm/x86asm.pl + $(PERL) asm/sha1-586.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@ +sha256-586.s: asm/sha256-586.pl ../perlasm/x86asm.pl + $(PERL) asm/sha256-586.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@ +sha512-586.s: asm/sha512-586.pl ../perlasm/x86asm.pl + $(PERL) asm/sha512-586.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@ sha1-ia64.s: asm/sha1-ia64.pl (cd asm; $(PERL) sha1-ia64.pl ../$@ $(CFLAGS)) @@ -65,10 +56,40 @@ sha256-ia64.s: asm/sha512-ia64.pl sha512-ia64.s: asm/sha512-ia64.pl (cd asm; $(PERL) sha512-ia64.pl ../$@ $(CFLAGS)) +sha256-armv4.S: asm/sha256-armv4.pl + $(PERL) $< $(PERLASM_SCHEME) $@ + +sha1-alpha.s: asm/sha1-alpha.pl + $(PERL) $< | $(CC) -E - | tee $@ > /dev/null + # Solaris make has to be explicitly told -sha1-x86_64.s: asm/sha1-x86_64.pl; $(PERL) asm/sha1-x86_64.pl $@ -sha256-x86_64.s:asm/sha512-x86_64.pl; $(PERL) asm/sha512-x86_64.pl $@ -sha512-x86_64.s:asm/sha512-x86_64.pl; $(PERL) asm/sha512-x86_64.pl $@ +sha1-x86_64.s: asm/sha1-x86_64.pl; $(PERL) asm/sha1-x86_64.pl $(PERLASM_SCHEME) > $@ +sha256-x86_64.s:asm/sha512-x86_64.pl; $(PERL) asm/sha512-x86_64.pl $(PERLASM_SCHEME) $@ +sha512-x86_64.s:asm/sha512-x86_64.pl; $(PERL) asm/sha512-x86_64.pl $(PERLASM_SCHEME) $@ +sha1-sparcv9.s: asm/sha1-sparcv9.pl; $(PERL) asm/sha1-sparcv9.pl $@ $(CFLAGS) +sha256-sparcv9.s:asm/sha512-sparcv9.pl; $(PERL) asm/sha512-sparcv9.pl $@ $(CFLAGS) +sha512-sparcv9.s:asm/sha512-sparcv9.pl; $(PERL) asm/sha512-sparcv9.pl $@ $(CFLAGS) + +sha1-ppc.s: asm/sha1-ppc.pl; $(PERL) asm/sha1-ppc.pl $(PERLASM_SCHEME) $@ +sha256-ppc.s: asm/sha512-ppc.pl; $(PERL) asm/sha512-ppc.pl $(PERLASM_SCHEME) $@ +sha512-ppc.s: asm/sha512-ppc.pl; $(PERL) asm/sha512-ppc.pl $(PERLASM_SCHEME) $@ + +sha1-parisc.s: asm/sha1-parisc.pl; $(PERL) asm/sha1-parisc.pl $(PERLASM_SCHEME) $@ +sha256-parisc.s:asm/sha512-parisc.pl; $(PERL) asm/sha512-parisc.pl $(PERLASM_SCHEME) $@ +sha512-parisc.s:asm/sha512-parisc.pl; $(PERL) asm/sha512-parisc.pl $(PERLASM_SCHEME) $@ + +sha1-mips.S: asm/sha1-mips.pl; $(PERL) asm/sha1-mips.pl $(PERLASM_SCHEME) $@ +sha256-mips.S: asm/sha512-mips.pl; $(PERL) asm/sha512-mips.pl $(PERLASM_SCHEME) $@ +sha512-mips.S: asm/sha512-mips.pl; $(PERL) asm/sha512-mips.pl $(PERLASM_SCHEME) $@ + +# GNU make "catch all" +sha1-%.S: asm/sha1-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +sha256-%.S: asm/sha512-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ +sha512-%.S: asm/sha512-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@ + +sha1-armv4-large.o: sha1-armv4-large.S +sha256-armv4.o: sha256-armv4.S +sha512-armv4.o: sha512-armv4.S files: $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO @@ -113,27 +134,26 @@ sha1_one.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h sha1_one.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h sha1_one.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h sha1_one.o: sha1_one.c -sha1dgst.o: ../../include/openssl/e_os2.h ../../include/openssl/fips.h +sha1dgst.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h sha1dgst.o: ../../include/openssl/opensslconf.h -sha1dgst.o: ../../include/openssl/opensslv.h ../../include/openssl/sha.h +sha1dgst.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h +sha1dgst.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h +sha1dgst.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h sha1dgst.o: ../md32_common.h sha1dgst.c sha_locl.h sha256.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h -sha256.o: ../../include/openssl/fips.h ../../include/openssl/opensslconf.h -sha256.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h -sha256.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h -sha256.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h -sha256.o: ../md32_common.h sha256.c +sha256.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h +sha256.o: ../../include/openssl/ossl_typ.h ../../include/openssl/safestack.h +sha256.o: ../../include/openssl/sha.h ../../include/openssl/stack.h +sha256.o: ../../include/openssl/symhacks.h ../md32_common.h sha256.c sha512.o: ../../e_os.h ../../include/openssl/bio.h sha512.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h sha512.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h -sha512.o: ../../include/openssl/fips.h ../../include/openssl/lhash.h -sha512.o: ../../include/openssl/opensslconf.h ../../include/openssl/opensslv.h -sha512.o: ../../include/openssl/ossl_typ.h ../../include/openssl/safestack.h -sha512.o: ../../include/openssl/sha.h ../../include/openssl/stack.h -sha512.o: ../../include/openssl/symhacks.h ../cryptlib.h sha512.c -sha_dgst.o: ../../include/openssl/bio.h ../../include/openssl/crypto.h -sha_dgst.o: ../../include/openssl/e_os2.h ../../include/openssl/err.h -sha_dgst.o: ../../include/openssl/fips.h ../../include/openssl/lhash.h +sha512.o: ../../include/openssl/lhash.h ../../include/openssl/opensslconf.h +sha512.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h +sha512.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h +sha512.o: ../../include/openssl/stack.h ../../include/openssl/symhacks.h +sha512.o: ../cryptlib.h sha512.c +sha_dgst.o: ../../include/openssl/crypto.h ../../include/openssl/e_os2.h sha_dgst.o: ../../include/openssl/opensslconf.h sha_dgst.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h sha_dgst.o: ../../include/openssl/safestack.h ../../include/openssl/sha.h diff --git a/crypto/openssl/crypto/sha/asm/sha1-586.pl b/crypto/openssl/crypto/sha/asm/sha1-586.pl index a787dd37da42..1084d227fe06 100644 --- a/crypto/openssl/crypto/sha/asm/sha1-586.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-586.pl @@ -12,6 +12,8 @@ # commentary below], and in 2006 the rest was rewritten in order to # gain freedom to liberate licensing terms. +# January, September 2004. +# # It was noted that Intel IA-32 C compiler generates code which # performs ~30% *faster* on P4 CPU than original *hand-coded* # SHA1 assembler implementation. To address this problem (and @@ -31,12 +33,92 @@ # ---------------------------------------------------------------- # <appro@fy.chalmers.se> +# August 2009. +# +# George Spelvin has tipped that F_40_59(b,c,d) can be rewritten as +# '(c&d) + (b&(c^d))', which allows to accumulate partial results +# and lighten "pressure" on scratch registers. This resulted in +# >12% performance improvement on contemporary AMD cores (with no +# degradation on other CPUs:-). Also, the code was revised to maximize +# "distance" between instructions producing input to 'lea' instruction +# and the 'lea' instruction itself, which is essential for Intel Atom +# core and resulted in ~15% improvement. + +# October 2010. +# +# Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it +# is to offload message schedule denoted by Wt in NIST specification, +# or Xupdate in OpenSSL source, to SIMD unit. The idea is not novel, +# and in SSE2 context was first explored by Dean Gaudet in 2004, see +# http://arctic.org/~dean/crypto/sha1.html. Since then several things +# have changed that made it interesting again: +# +# a) XMM units became faster and wider; +# b) instruction set became more versatile; +# c) an important observation was made by Max Locktykhin, which made +# it possible to reduce amount of instructions required to perform +# the operation in question, for further details see +# http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/. + +# April 2011. +# +# Add AVX code path, probably most controversial... The thing is that +# switch to AVX alone improves performance by as little as 4% in +# comparison to SSSE3 code path. But below result doesn't look like +# 4% improvement... Trouble is that Sandy Bridge decodes 'ro[rl]' as +# pair of µ-ops, and it's the additional µ-ops, two per round, that +# make it run slower than Core2 and Westmere. But 'sh[rl]d' is decoded +# as single µ-op by Sandy Bridge and it's replacing 'ro[rl]' with +# equivalent 'sh[rl]d' that is responsible for the impressive 5.1 +# cycles per processed byte. But 'sh[rl]d' is not something that used +# to be fast, nor does it appear to be fast in upcoming Bulldozer +# [according to its optimization manual]. Which is why AVX code path +# is guarded by *both* AVX and synthetic bit denoting Intel CPUs. +# One can argue that it's unfair to AMD, but without 'sh[rl]d' it +# makes no sense to keep the AVX code path. If somebody feels that +# strongly, it's probably more appropriate to discuss possibility of +# using vector rotate XOP on AMD... + +###################################################################### +# Current performance is summarized in following table. Numbers are +# CPU clock cycles spent to process single byte (less is better). +# +# x86 SSSE3 AVX +# Pentium 15.7 - +# PIII 11.5 - +# P4 10.6 - +# AMD K8 7.1 - +# Core2 7.3 6.1/+20% - +# Atom 12.5 9.5(*)/+32% - +# Westmere 7.3 5.6/+30% - +# Sandy Bridge 8.8 6.2/+40% 5.1(**)/+70% +# +# (*) Loop is 1056 instructions long and expected result is ~8.25. +# It remains mystery [to me] why ILP is limited to 1.7. +# +# (**) As per above comment, the result is for AVX *plus* sh[rl]d. + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; push(@INC,"${dir}","${dir}../../perlasm"); require "x86asm.pl"; &asm_init($ARGV[0],"sha1-586.pl",$ARGV[$#ARGV] eq "386"); +$xmm=$ymm=0; +for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); } + +$ymm=1 if ($xmm && + `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/ && + $1>=2.19); # first version supporting AVX + +$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ && + $1>=2.03); # first version supporting AVX + +&external_label("OPENSSL_ia32cap_P") if ($xmm); + + $A="eax"; $B="ebx"; $C="ecx"; @@ -47,6 +129,10 @@ $tmp1="ebp"; @V=($A,$B,$C,$D,$E,$T); +$alt=0; # 1 denotes alternative IALU implementation, which performs + # 8% *worse* on P4, same on Westmere and Atom, 2% better on + # Sandy Bridge... + sub BODY_00_15 { local($n,$a,$b,$c,$d,$e,$f)=@_; @@ -59,16 +145,18 @@ sub BODY_00_15 &rotl($tmp1,5); # tmp1=ROTATE(a,5) &xor($f,$d); &add($tmp1,$e); # tmp1+=e; - &and($f,$b); - &mov($e,&swtmp($n%16)); # e becomes volatile and is loaded + &mov($e,&swtmp($n%16)); # e becomes volatile and is loaded # with xi, also note that e becomes # f in next round... - &xor($f,$d); # f holds F_00_19(b,c,d) + &and($f,$b); &rotr($b,2); # b=ROTATE(b,30) - &lea($tmp1,&DWP(0x5a827999,$tmp1,$e)); # tmp1+=K_00_19+xi + &xor($f,$d); # f holds F_00_19(b,c,d) + &lea($tmp1,&DWP(0x5a827999,$tmp1,$e)); # tmp1+=K_00_19+xi - if ($n==15) { &add($f,$tmp1); } # f+=tmp1 + if ($n==15) { &mov($e,&swtmp(($n+1)%16));# pre-fetch f for next round + &add($f,$tmp1); } # f+=tmp1 else { &add($tmp1,$f); } # f becomes a in next round + &mov($tmp1,$a) if ($alt && $n==15); } sub BODY_16_19 @@ -77,22 +165,41 @@ sub BODY_16_19 &comment("16_19 $n"); - &mov($f,&swtmp($n%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) - &mov($tmp1,$c); # tmp1 to hold F_00_19(b,c,d) - &xor($f,&swtmp(($n+2)%16)); - &xor($tmp1,$d); - &xor($f,&swtmp(($n+8)%16)); - &and($tmp1,$b); # tmp1 holds F_00_19(b,c,d) - &rotr($b,2); # b=ROTATE(b,30) +if ($alt) { + &xor($c,$d); + &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) + &and($tmp1,$c); # tmp1 to hold F_00_19(b,c,d), b&=c^d + &xor($f,&swtmp(($n+8)%16)); + &xor($tmp1,$d); # tmp1=F_00_19(b,c,d) + &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd + &rotl($f,1); # f=ROTATE(f,1) + &add($e,$tmp1); # e+=F_00_19(b,c,d) + &xor($c,$d); # restore $c + &mov($tmp1,$a); # b in next round + &rotr($b,$n==16?2:7); # b=ROTATE(b,30) + &mov(&swtmp($n%16),$f); # xi=f + &rotl($a,5); # ROTATE(a,5) + &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e + &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round + &add($f,$a); # f+=ROTATE(a,5) +} else { + &mov($tmp1,$c); # tmp1 to hold F_00_19(b,c,d) + &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) + &xor($tmp1,$d); + &xor($f,&swtmp(($n+8)%16)); + &and($tmp1,$b); &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd &rotl($f,1); # f=ROTATE(f,1) &xor($tmp1,$d); # tmp1=F_00_19(b,c,d) - &mov(&swtmp($n%16),$f); # xi=f - &lea($f,&DWP(0x5a827999,$f,$e));# f+=K_00_19+e - &mov($e,$a); # e becomes volatile - &rotl($e,5); # e=ROTATE(a,5) - &add($f,$tmp1); # f+=F_00_19(b,c,d) - &add($f,$e); # f+=ROTATE(a,5) + &add($e,$tmp1); # e+=F_00_19(b,c,d) + &mov($tmp1,$a); + &rotr($b,2); # b=ROTATE(b,30) + &mov(&swtmp($n%16),$f); # xi=f + &rotl($tmp1,5); # ROTATE(a,5) + &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e + &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round + &add($f,$tmp1); # f+=ROTATE(a,5) +} } sub BODY_20_39 @@ -102,21 +209,41 @@ sub BODY_20_39 &comment("20_39 $n"); +if ($alt) { + &xor($tmp1,$c); # tmp1 to hold F_20_39(b,c,d), b^=c + &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) + &xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d) + &xor($f,&swtmp(($n+8)%16)); + &add($e,$tmp1); # e+=F_20_39(b,c,d) + &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd + &rotl($f,1); # f=ROTATE(f,1) + &mov($tmp1,$a); # b in next round + &rotr($b,7); # b=ROTATE(b,30) + &mov(&swtmp($n%16),$f) if($n<77);# xi=f + &rotl($a,5); # ROTATE(a,5) + &xor($b,$c) if($n==39);# warm up for BODY_40_59 + &and($tmp1,$b) if($n==39); + &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY + &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round + &add($f,$a); # f+=ROTATE(a,5) + &rotr($a,5) if ($n==79); +} else { &mov($tmp1,$b); # tmp1 to hold F_20_39(b,c,d) - &mov($f,&swtmp($n%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) - &rotr($b,2); # b=ROTATE(b,30) - &xor($f,&swtmp(($n+2)%16)); + &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) &xor($tmp1,$c); &xor($f,&swtmp(($n+8)%16)); &xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d) &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd &rotl($f,1); # f=ROTATE(f,1) - &add($tmp1,$e); - &mov(&swtmp($n%16),$f); # xi=f - &mov($e,$a); # e becomes volatile - &rotl($e,5); # e=ROTATE(a,5) - &lea($f,&DWP($K,$f,$tmp1)); # f+=K_20_39+e - &add($f,$e); # f+=ROTATE(a,5) + &add($e,$tmp1); # e+=F_20_39(b,c,d) + &rotr($b,2); # b=ROTATE(b,30) + &mov($tmp1,$a); + &rotl($tmp1,5); # ROTATE(a,5) + &mov(&swtmp($n%16),$f) if($n<77);# xi=f + &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY + &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round + &add($f,$tmp1); # f+=ROTATE(a,5) +} } sub BODY_40_59 @@ -125,41 +252,86 @@ sub BODY_40_59 &comment("40_59 $n"); - &mov($f,&swtmp($n%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) - &mov($tmp1,&swtmp(($n+2)%16)); - &xor($f,$tmp1); - &mov($tmp1,&swtmp(($n+8)%16)); - &xor($f,$tmp1); - &mov($tmp1,&swtmp(($n+13)%16)); - &xor($f,$tmp1); # f holds xa^xb^xc^xd - &mov($tmp1,$b); # tmp1 to hold F_40_59(b,c,d) +if ($alt) { + &add($e,$tmp1); # e+=b&(c^d) + &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) + &mov($tmp1,$d); + &xor($f,&swtmp(($n+8)%16)); + &xor($c,$d); # restore $c + &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd &rotl($f,1); # f=ROTATE(f,1) - &or($tmp1,$c); - &mov(&swtmp($n%16),$f); # xi=f - &and($tmp1,$d); - &lea($f,&DWP(0x8f1bbcdc,$f,$e));# f+=K_40_59+e - &mov($e,$b); # e becomes volatile and is used - # to calculate F_40_59(b,c,d) + &and($tmp1,$c); + &rotr($b,7); # b=ROTATE(b,30) + &add($e,$tmp1); # e+=c&d + &mov($tmp1,$a); # b in next round + &mov(&swtmp($n%16),$f); # xi=f + &rotl($a,5); # ROTATE(a,5) + &xor($b,$c) if ($n<59); + &and($tmp1,$b) if ($n<59);# tmp1 to hold F_40_59(b,c,d) + &lea($f,&DWP(0x8f1bbcdc,$f,$e));# f+=K_40_59+e+(b&(c^d)) + &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round + &add($f,$a); # f+=ROTATE(a,5) +} else { + &mov($tmp1,$c); # tmp1 to hold F_40_59(b,c,d) + &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) + &xor($tmp1,$d); + &xor($f,&swtmp(($n+8)%16)); + &and($tmp1,$b); + &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd + &rotl($f,1); # f=ROTATE(f,1) + &add($tmp1,$e); # b&(c^d)+=e &rotr($b,2); # b=ROTATE(b,30) - &and($e,$c); - &or($tmp1,$e); # tmp1 holds F_40_59(b,c,d) - &mov($e,$a); - &rotl($e,5); # e=ROTATE(a,5) - &add($f,$tmp1); # f+=tmp1; + &mov($e,$a); # e becomes volatile + &rotl($e,5); # ROTATE(a,5) + &mov(&swtmp($n%16),$f); # xi=f + &lea($f,&DWP(0x8f1bbcdc,$f,$tmp1));# f+=K_40_59+e+(b&(c^d)) + &mov($tmp1,$c); &add($f,$e); # f+=ROTATE(a,5) + &and($tmp1,$d); + &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round + &add($f,$tmp1); # f+=c&d +} } &function_begin("sha1_block_data_order"); +if ($xmm) { + &static_label("ssse3_shortcut"); + &static_label("avx_shortcut") if ($ymm); + &static_label("K_XX_XX"); + + &call (&label("pic_point")); # make it PIC! + &set_label("pic_point"); + &blindpop($tmp1); + &picmeup($T,"OPENSSL_ia32cap_P",$tmp1,&label("pic_point")); + &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); + + &mov ($A,&DWP(0,$T)); + &mov ($D,&DWP(4,$T)); + &test ($D,1<<9); # check SSSE3 bit + &jz (&label("x86")); + &test ($A,1<<24); # check FXSR bit + &jz (&label("x86")); + if ($ymm) { + &and ($D,1<<28); # mask AVX bit + &and ($A,1<<30); # mask "Intel CPU" bit + &or ($A,$D); + &cmp ($A,1<<28|1<<30); + &je (&label("avx_shortcut")); + } + &jmp (&label("ssse3_shortcut")); + &set_label("x86",16); +} &mov($tmp1,&wparam(0)); # SHA_CTX *c &mov($T,&wparam(1)); # const void *input &mov($A,&wparam(2)); # size_t num - &stack_push(16); # allocate X[16] + &stack_push(16+3); # allocate X[16] &shl($A,6); &add($A,$T); &mov(&wparam(2),$A); # pointer beyond the end of input &mov($E,&DWP(16,$tmp1));# pre-load E + &jmp(&label("loop")); - &set_label("loop",16); +&set_label("loop",16); # copy input chunk to X, but reversing byte order! for ($i=0; $i<16; $i+=4) @@ -213,7 +385,845 @@ sub BODY_40_59 &mov(&DWP(16,$tmp1),$C); &jb(&label("loop")); - &stack_pop(16); + &stack_pop(16+3); &function_end("sha1_block_data_order"); +if ($xmm) { +###################################################################### +# The SSSE3 implementation. +# +# %xmm[0-7] are used as ring @X[] buffer containing quadruples of last +# 32 elements of the message schedule or Xupdate outputs. First 4 +# quadruples are simply byte-swapped input, next 4 are calculated +# according to method originally suggested by Dean Gaudet (modulo +# being implemented in SSSE3). Once 8 quadruples or 32 elements are +# collected, it switches to routine proposed by Max Locktyukhin. +# +# Calculations inevitably require temporary reqisters, and there are +# no %xmm registers left to spare. For this reason part of the ring +# buffer, X[2..4] to be specific, is offloaded to 3 quadriples ring +# buffer on the stack. Keep in mind that X[2] is alias X[-6], X[3] - +# X[-5], and X[4] - X[-4]... +# +# Another notable optimization is aggressive stack frame compression +# aiming to minimize amount of 9-byte instructions... +# +# Yet another notable optimization is "jumping" $B variable. It means +# that there is no register permanently allocated for $B value. This +# allowed to eliminate one instruction from body_20_39... +# +my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded +my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 +my @V=($A,$B,$C,$D,$E); +my $j=0; # hash round +my @T=($T,$tmp1); +my $inp; + +my $_rol=sub { &rol(@_) }; +my $_ror=sub { &ror(@_) }; + +&function_begin("_sha1_block_data_order_ssse3"); + &call (&label("pic_point")); # make it PIC! + &set_label("pic_point"); + &blindpop($tmp1); + &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); +&set_label("ssse3_shortcut"); + + &movdqa (@X[3],&QWP(0,$tmp1)); # K_00_19 + &movdqa (@X[4],&QWP(16,$tmp1)); # K_20_39 + &movdqa (@X[5],&QWP(32,$tmp1)); # K_40_59 + &movdqa (@X[6],&QWP(48,$tmp1)); # K_60_79 + &movdqa (@X[2],&QWP(64,$tmp1)); # pbswap mask + + &mov ($E,&wparam(0)); # load argument block + &mov ($inp=@T[1],&wparam(1)); + &mov ($D,&wparam(2)); + &mov (@T[0],"esp"); + + # stack frame layout + # + # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area + # X[4]+K X[5]+K X[6]+K X[7]+K + # X[8]+K X[9]+K X[10]+K X[11]+K + # X[12]+K X[13]+K X[14]+K X[15]+K + # + # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area + # X[4] X[5] X[6] X[7] + # X[8] X[9] X[10] X[11] # even borrowed for K_00_19 + # + # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants + # K_40_59 K_40_59 K_40_59 K_40_59 + # K_60_79 K_60_79 K_60_79 K_60_79 + # K_00_19 K_00_19 K_00_19 K_00_19 + # pbswap mask + # + # +192 ctx # argument block + # +196 inp + # +200 end + # +204 esp + &sub ("esp",208); + &and ("esp",-64); + + &movdqa (&QWP(112+0,"esp"),@X[4]); # copy constants + &movdqa (&QWP(112+16,"esp"),@X[5]); + &movdqa (&QWP(112+32,"esp"),@X[6]); + &shl ($D,6); # len*64 + &movdqa (&QWP(112+48,"esp"),@X[3]); + &add ($D,$inp); # end of input + &movdqa (&QWP(112+64,"esp"),@X[2]); + &add ($inp,64); + &mov (&DWP(192+0,"esp"),$E); # save argument block + &mov (&DWP(192+4,"esp"),$inp); + &mov (&DWP(192+8,"esp"),$D); + &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp + + &mov ($A,&DWP(0,$E)); # load context + &mov ($B,&DWP(4,$E)); + &mov ($C,&DWP(8,$E)); + &mov ($D,&DWP(12,$E)); + &mov ($E,&DWP(16,$E)); + &mov (@T[0],$B); # magic seed + + &movdqu (@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3] + &movdqu (@X[-3&7],&QWP(-48,$inp)); + &movdqu (@X[-2&7],&QWP(-32,$inp)); + &movdqu (@X[-1&7],&QWP(-16,$inp)); + &pshufb (@X[-4&7],@X[2]); # byte swap + &pshufb (@X[-3&7],@X[2]); + &pshufb (@X[-2&7],@X[2]); + &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot + &pshufb (@X[-1&7],@X[2]); + &paddd (@X[-4&7],@X[3]); # add K_00_19 + &paddd (@X[-3&7],@X[3]); + &paddd (@X[-2&7],@X[3]); + &movdqa (&QWP(0,"esp"),@X[-4&7]); # X[]+K xfer to IALU + &psubd (@X[-4&7],@X[3]); # restore X[] + &movdqa (&QWP(0+16,"esp"),@X[-3&7]); + &psubd (@X[-3&7],@X[3]); + &movdqa (&QWP(0+32,"esp"),@X[-2&7]); + &psubd (@X[-2&7],@X[3]); + &movdqa (@X[0],@X[-3&7]); + &jmp (&label("loop")); + +###################################################################### +# SSE instruction sequence is first broken to groups of indepentent +# instructions, independent in respect to their inputs and shifter +# (not all architectures have more than one). Then IALU instructions +# are "knitted in" between the SSE groups. Distance is maintained for +# SSE latency of 2 in hope that it fits better upcoming AMD Bulldozer +# [which allegedly also implements SSSE3]... +# +# Temporary registers usage. X[2] is volatile at the entry and at the +# end is restored from backtrace ring buffer. X[3] is expected to +# contain current K_XX_XX constant and is used to caclulate X[-1]+K +# from previous round, it becomes volatile the moment the value is +# saved to stack for transfer to IALU. X[4] becomes volatile whenever +# X[-4] is accumulated and offloaded to backtrace ring buffer, at the +# end it is loaded with next K_XX_XX [which becomes X[3] in next +# round]... +# +sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 40 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + eval(shift(@insns)); + &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]" + &movdqa (@X[2],@X[-1&7]); + eval(shift(@insns)); + eval(shift(@insns)); + + &paddd (@X[3],@X[-1&7]); + &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); + &psrldq (@X[2],4); # "X[-3]", 3 dwords + eval(shift(@insns)); + eval(shift(@insns)); + &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]" + eval(shift(@insns)); + eval(shift(@insns)); + + &pxor (@X[2],@X[-2&7]); # "X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &pxor (@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + + &movdqa (@X[4],@X[0]); + &movdqa (@X[2],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &pslldq (@X[4],12); # "X[0]"<<96, extract one dword + &paddd (@X[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &psrld (@X[2],31); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (@X[3],@X[4]); + eval(shift(@insns)); + eval(shift(@insns)); + + &psrld (@X[4],30); + &por (@X[0],@X[2]); # "X[0]"<<<=1 + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); + + &pslld (@X[3],2); + &pxor (@X[0],@X[4]); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX + eval(shift(@insns)); + eval(shift(@insns)); + + &pxor (@X[0],@X[3]); # "X[0]"^=("X[0]"<<96)<<<2 + &movdqa (@X[1],@X[-2&7]) if ($Xi<7); + eval(shift(@insns)); + eval(shift(@insns)); + + foreach (@insns) { eval; } # remaining instructions [if any] + + $Xi++; push(@X,shift(@X)); # "rotate" X[] +} + +sub Xupdate_ssse3_32_79() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my ($a,$b,$c,$d,$e); + + &movdqa (@X[2],@X[-1&7]) if ($Xi==8); + eval(shift(@insns)); # body_20_39 + &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" + &palignr(@X[2],@X[-2&7],8); # compose "X[-6]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]" + &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); + if ($Xi%5) { + &movdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX... + } else { # ... or load next one + &movdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp")); + } + &paddd (@X[3],@X[-1&7]); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &pxor (@X[0],@X[2]); # "X[0]"^="X[-6]" + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &movdqa (@X[2],@X[0]); + &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &pslld (@X[0],2); + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + &psrld (@X[2],30); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &por (@X[0],@X[2]); # "X[0]"<<<=2 + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + &movdqa (@X[3],@X[0]) if ($Xi<19); + eval(shift(@insns)); + + foreach (@insns) { eval; } # remaining instructions + + $Xi++; push(@X,shift(@X)); # "rotate" X[] +} + +sub Xuplast_ssse3_80() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + &paddd (@X[3],@X[-1&7]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU + + foreach (@insns) { eval; } # remaining instructions + + &mov ($inp=@T[1],&DWP(192+4,"esp")); + &cmp ($inp,&DWP(192+8,"esp")); + &je (&label("done")); + + &movdqa (@X[3],&QWP(112+48,"esp")); # K_00_19 + &movdqa (@X[2],&QWP(112+64,"esp")); # pbswap mask + &movdqu (@X[-4&7],&QWP(0,$inp)); # load input + &movdqu (@X[-3&7],&QWP(16,$inp)); + &movdqu (@X[-2&7],&QWP(32,$inp)); + &movdqu (@X[-1&7],&QWP(48,$inp)); + &add ($inp,64); + &pshufb (@X[-4&7],@X[2]); # byte swap + &mov (&DWP(192+4,"esp"),$inp); + &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot + + $Xi=0; +} + +sub Xloop_ssse3() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + eval(shift(@insns)); + &pshufb (@X[($Xi-3)&7],@X[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@X[($Xi-4)&7],@X[3]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (&QWP(0+16*$Xi,"esp"),@X[($Xi-4)&7]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + &psubd (@X[($Xi-4)&7],@X[3]); + + foreach (@insns) { eval; } + $Xi++; +} + +sub Xtail_ssse3() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + foreach (@insns) { eval; } +} + +sub body_00_19 () { + ( + '($a,$b,$c,$d,$e)=@V;'. + '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer + '&xor ($c,$d);', + '&mov (@T[1],$a);', # $b in next round + '&$_rol ($a,5);', + '&and (@T[0],$c);', # ($b&($c^$d)) + '&xor ($c,$d);', # restore $c + '&xor (@T[0],$d);', + '&add ($e,$a);', + '&$_ror ($b,$j?7:2);', # $b>>>2 + '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} + +sub body_20_39 () { + ( + '($a,$b,$c,$d,$e)=@V;'. + '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer + '&xor (@T[0],$d);', # ($b^$d) + '&mov (@T[1],$a);', # $b in next round + '&$_rol ($a,5);', + '&xor (@T[0],$c);', # ($b^$d^$c) + '&add ($e,$a);', + '&$_ror ($b,7);', # $b>>>2 + '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} + +sub body_40_59 () { + ( + '($a,$b,$c,$d,$e)=@V;'. + '&mov (@T[1],$c);', + '&xor ($c,$d);', + '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer + '&and (@T[1],$d);', + '&and (@T[0],$c);', # ($b&($c^$d)) + '&$_ror ($b,7);', # $b>>>2 + '&add ($e,@T[1]);', + '&mov (@T[1],$a);', # $b in next round + '&$_rol ($a,5);', + '&add ($e,@T[0]);', + '&xor ($c,$d);', # restore $c + '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} + +&set_label("loop",16); + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_32_79(\&body_00_19); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xuplast_ssse3_80(\&body_20_39); # can jump to "done" + + $saved_j=$j; @saved_V=@V; + + &Xloop_ssse3(\&body_20_39); + &Xloop_ssse3(\&body_20_39); + &Xloop_ssse3(\&body_20_39); + + &mov (@T[1],&DWP(192,"esp")); # update context + &add ($A,&DWP(0,@T[1])); + &add (@T[0],&DWP(4,@T[1])); # $b + &add ($C,&DWP(8,@T[1])); + &mov (&DWP(0,@T[1]),$A); + &add ($D,&DWP(12,@T[1])); + &mov (&DWP(4,@T[1]),@T[0]); + &add ($E,&DWP(16,@T[1])); + &mov (&DWP(8,@T[1]),$C); + &mov ($B,@T[0]); + &mov (&DWP(12,@T[1]),$D); + &mov (&DWP(16,@T[1]),$E); + &movdqa (@X[0],@X[-3&7]); + + &jmp (&label("loop")); + +&set_label("done",16); $j=$saved_j; @V=@saved_V; + + &Xtail_ssse3(\&body_20_39); + &Xtail_ssse3(\&body_20_39); + &Xtail_ssse3(\&body_20_39); + + &mov (@T[1],&DWP(192,"esp")); # update context + &add ($A,&DWP(0,@T[1])); + &mov ("esp",&DWP(192+12,"esp")); # restore %esp + &add (@T[0],&DWP(4,@T[1])); # $b + &add ($C,&DWP(8,@T[1])); + &mov (&DWP(0,@T[1]),$A); + &add ($D,&DWP(12,@T[1])); + &mov (&DWP(4,@T[1]),@T[0]); + &add ($E,&DWP(16,@T[1])); + &mov (&DWP(8,@T[1]),$C); + &mov (&DWP(12,@T[1]),$D); + &mov (&DWP(16,@T[1]),$E); + +&function_end("_sha1_block_data_order_ssse3"); + +if ($ymm) { +my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded +my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 +my @V=($A,$B,$C,$D,$E); +my $j=0; # hash round +my @T=($T,$tmp1); +my $inp; + +my $_rol=sub { &shld(@_[0],@_) }; +my $_ror=sub { &shrd(@_[0],@_) }; + +&function_begin("_sha1_block_data_order_avx"); + &call (&label("pic_point")); # make it PIC! + &set_label("pic_point"); + &blindpop($tmp1); + &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); +&set_label("avx_shortcut"); + &vzeroall(); + + &vmovdqa(@X[3],&QWP(0,$tmp1)); # K_00_19 + &vmovdqa(@X[4],&QWP(16,$tmp1)); # K_20_39 + &vmovdqa(@X[5],&QWP(32,$tmp1)); # K_40_59 + &vmovdqa(@X[6],&QWP(48,$tmp1)); # K_60_79 + &vmovdqa(@X[2],&QWP(64,$tmp1)); # pbswap mask + + &mov ($E,&wparam(0)); # load argument block + &mov ($inp=@T[1],&wparam(1)); + &mov ($D,&wparam(2)); + &mov (@T[0],"esp"); + + # stack frame layout + # + # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area + # X[4]+K X[5]+K X[6]+K X[7]+K + # X[8]+K X[9]+K X[10]+K X[11]+K + # X[12]+K X[13]+K X[14]+K X[15]+K + # + # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area + # X[4] X[5] X[6] X[7] + # X[8] X[9] X[10] X[11] # even borrowed for K_00_19 + # + # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants + # K_40_59 K_40_59 K_40_59 K_40_59 + # K_60_79 K_60_79 K_60_79 K_60_79 + # K_00_19 K_00_19 K_00_19 K_00_19 + # pbswap mask + # + # +192 ctx # argument block + # +196 inp + # +200 end + # +204 esp + &sub ("esp",208); + &and ("esp",-64); + + &vmovdqa(&QWP(112+0,"esp"),@X[4]); # copy constants + &vmovdqa(&QWP(112+16,"esp"),@X[5]); + &vmovdqa(&QWP(112+32,"esp"),@X[6]); + &shl ($D,6); # len*64 + &vmovdqa(&QWP(112+48,"esp"),@X[3]); + &add ($D,$inp); # end of input + &vmovdqa(&QWP(112+64,"esp"),@X[2]); + &add ($inp,64); + &mov (&DWP(192+0,"esp"),$E); # save argument block + &mov (&DWP(192+4,"esp"),$inp); + &mov (&DWP(192+8,"esp"),$D); + &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp + + &mov ($A,&DWP(0,$E)); # load context + &mov ($B,&DWP(4,$E)); + &mov ($C,&DWP(8,$E)); + &mov ($D,&DWP(12,$E)); + &mov ($E,&DWP(16,$E)); + &mov (@T[0],$B); # magic seed + + &vmovdqu(@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3] + &vmovdqu(@X[-3&7],&QWP(-48,$inp)); + &vmovdqu(@X[-2&7],&QWP(-32,$inp)); + &vmovdqu(@X[-1&7],&QWP(-16,$inp)); + &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap + &vpshufb(@X[-3&7],@X[-3&7],@X[2]); + &vpshufb(@X[-2&7],@X[-2&7],@X[2]); + &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot + &vpshufb(@X[-1&7],@X[-1&7],@X[2]); + &vpaddd (@X[0],@X[-4&7],@X[3]); # add K_00_19 + &vpaddd (@X[1],@X[-3&7],@X[3]); + &vpaddd (@X[2],@X[-2&7],@X[3]); + &vmovdqa(&QWP(0,"esp"),@X[0]); # X[]+K xfer to IALU + &vmovdqa(&QWP(0+16,"esp"),@X[1]); + &vmovdqa(&QWP(0+32,"esp"),@X[2]); + &jmp (&label("loop")); + +sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 40 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + eval(shift(@insns)); + &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]" + eval(shift(@insns)); + eval(shift(@insns)); + + &vpaddd (@X[3],@X[3],@X[-1&7]); + &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrldq(@X[2],@X[-1&7],4); # "X[-3]", 3 dwords + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[2],@X[2],@X[-2&7]); # "X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrld (@X[2],@X[0],31); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpslldq(@X[4],@X[0],12); # "X[0]"<<96, extract one dword + &vpaddd (@X[0],@X[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrld (@X[3],@X[4],30); + &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=1 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpslld (@X[4],@X[4],2); + &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor (@X[0],@X[0],@X[3]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@X[4]); # "X[0]"^=("X[0]"<<96)<<<2 + eval(shift(@insns)); + eval(shift(@insns)); + &vmovdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX + eval(shift(@insns)); + eval(shift(@insns)); + + foreach (@insns) { eval; } # remaining instructions [if any] + + $Xi++; push(@X,shift(@X)); # "rotate" X[] +} + +sub Xupdate_avx_32_79() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my ($a,$b,$c,$d,$e); + + &vpalignr(@X[2],@X[-1&7],@X[-2&7],8); # compose "X[-6]" + &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" + &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); + if ($Xi%5) { + &vmovdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX... + } else { # ... or load next one + &vmovdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp")); + } + &vpaddd (@X[3],@X[3],@X[-1&7]); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-6]" + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &vpsrld (@X[2],@X[0],30); + &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &vpslld (@X[0],@X[0],2); + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=2 + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + foreach (@insns) { eval; } # remaining instructions + + $Xi++; push(@X,shift(@X)); # "rotate" X[] +} + +sub Xuplast_avx_80() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + &vpaddd (@X[3],@X[3],@X[-1&7]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU + + foreach (@insns) { eval; } # remaining instructions + + &mov ($inp=@T[1],&DWP(192+4,"esp")); + &cmp ($inp,&DWP(192+8,"esp")); + &je (&label("done")); + + &vmovdqa(@X[3],&QWP(112+48,"esp")); # K_00_19 + &vmovdqa(@X[2],&QWP(112+64,"esp")); # pbswap mask + &vmovdqu(@X[-4&7],&QWP(0,$inp)); # load input + &vmovdqu(@X[-3&7],&QWP(16,$inp)); + &vmovdqu(@X[-2&7],&QWP(32,$inp)); + &vmovdqu(@X[-1&7],&QWP(48,$inp)); + &add ($inp,64); + &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap + &mov (&DWP(192+4,"esp"),$inp); + &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot + + $Xi=0; +} + +sub Xloop_avx() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + eval(shift(@insns)); + &vpshufb (@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@X[3]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vmovdqa (&QWP(0+16*$Xi,"esp"),@X[$Xi&7]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + + foreach (@insns) { eval; } + $Xi++; +} + +sub Xtail_avx() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + foreach (@insns) { eval; } +} + +&set_label("loop",16); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_32_79(\&body_00_19); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_20_39); + &Xuplast_avx_80(\&body_20_39); # can jump to "done" + + $saved_j=$j; @saved_V=@V; + + &Xloop_avx(\&body_20_39); + &Xloop_avx(\&body_20_39); + &Xloop_avx(\&body_20_39); + + &mov (@T[1],&DWP(192,"esp")); # update context + &add ($A,&DWP(0,@T[1])); + &add (@T[0],&DWP(4,@T[1])); # $b + &add ($C,&DWP(8,@T[1])); + &mov (&DWP(0,@T[1]),$A); + &add ($D,&DWP(12,@T[1])); + &mov (&DWP(4,@T[1]),@T[0]); + &add ($E,&DWP(16,@T[1])); + &mov (&DWP(8,@T[1]),$C); + &mov ($B,@T[0]); + &mov (&DWP(12,@T[1]),$D); + &mov (&DWP(16,@T[1]),$E); + + &jmp (&label("loop")); + +&set_label("done",16); $j=$saved_j; @V=@saved_V; + + &Xtail_avx(\&body_20_39); + &Xtail_avx(\&body_20_39); + &Xtail_avx(\&body_20_39); + + &vzeroall(); + + &mov (@T[1],&DWP(192,"esp")); # update context + &add ($A,&DWP(0,@T[1])); + &mov ("esp",&DWP(192+12,"esp")); # restore %esp + &add (@T[0],&DWP(4,@T[1])); # $b + &add ($C,&DWP(8,@T[1])); + &mov (&DWP(0,@T[1]),$A); + &add ($D,&DWP(12,@T[1])); + &mov (&DWP(4,@T[1]),@T[0]); + &add ($E,&DWP(16,@T[1])); + &mov (&DWP(8,@T[1]),$C); + &mov (&DWP(12,@T[1]),$D); + &mov (&DWP(16,@T[1]),$E); +&function_end("_sha1_block_data_order_avx"); +} +&set_label("K_XX_XX",64); +&data_word(0x5a827999,0x5a827999,0x5a827999,0x5a827999); # K_00_19 +&data_word(0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1); # K_20_39 +&data_word(0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc); # K_40_59 +&data_word(0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6); # K_60_79 +&data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # pbswap mask +} +&asciz("SHA1 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>"); + &asm_finish(); diff --git a/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl b/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl new file mode 100755 index 000000000000..fe8207f77f8c --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl @@ -0,0 +1,248 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# sha1_block procedure for ARMv4. +# +# January 2007. + +# Size/performance trade-off +# ==================================================================== +# impl size in bytes comp cycles[*] measured performance +# ==================================================================== +# thumb 304 3212 4420 +# armv4-small 392/+29% 1958/+64% 2250/+96% +# armv4-compact 740/+89% 1552/+26% 1840/+22% +# armv4-large 1420/+92% 1307/+19% 1370/+34%[***] +# full unroll ~5100/+260% ~1260/+4% ~1300/+5% +# ==================================================================== +# thumb = same as 'small' but in Thumb instructions[**] and +# with recurring code in two private functions; +# small = detached Xload/update, loops are folded; +# compact = detached Xload/update, 5x unroll; +# large = interleaved Xload/update, 5x unroll; +# full unroll = interleaved Xload/update, full unroll, estimated[!]; +# +# [*] Manually counted instructions in "grand" loop body. Measured +# performance is affected by prologue and epilogue overhead, +# i-cache availability, branch penalties, etc. +# [**] While each Thumb instruction is twice smaller, they are not as +# diverse as ARM ones: e.g., there are only two arithmetic +# instructions with 3 arguments, no [fixed] rotate, addressing +# modes are limited. As result it takes more instructions to do +# the same job in Thumb, therefore the code is never twice as +# small and always slower. +# [***] which is also ~35% better than compiler generated code. Dual- +# issue Cortex A8 core was measured to process input block in +# ~990 cycles. + +# August 2010. +# +# Rescheduling for dual-issue pipeline resulted in 13% improvement on +# Cortex A8 core and in absolute terms ~870 cycles per input block +# [or 13.6 cycles per byte]. + +# February 2011. +# +# Profiler-assisted and platform-specific optimization resulted in 10% +# improvement on Cortex A8 core and 12.2 cycles per byte. + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +$ctx="r0"; +$inp="r1"; +$len="r2"; +$a="r3"; +$b="r4"; +$c="r5"; +$d="r6"; +$e="r7"; +$K="r8"; +$t0="r9"; +$t1="r10"; +$t2="r11"; +$t3="r12"; +$Xi="r14"; +@V=($a,$b,$c,$d,$e); + +sub Xupdate { +my ($a,$b,$c,$d,$e,$opt1,$opt2)=@_; +$code.=<<___; + ldr $t0,[$Xi,#15*4] + ldr $t1,[$Xi,#13*4] + ldr $t2,[$Xi,#7*4] + add $e,$K,$e,ror#2 @ E+=K_xx_xx + ldr $t3,[$Xi,#2*4] + eor $t0,$t0,$t1 + eor $t2,$t2,$t3 @ 1 cycle stall + eor $t1,$c,$d @ F_xx_xx + mov $t0,$t0,ror#31 + add $e,$e,$a,ror#27 @ E+=ROR(A,27) + eor $t0,$t0,$t2,ror#31 + str $t0,[$Xi,#-4]! + $opt1 @ F_xx_xx + $opt2 @ F_xx_xx + add $e,$e,$t0 @ E+=X[i] +___ +} + +sub BODY_00_15 { +my ($a,$b,$c,$d,$e)=@_; +$code.=<<___; +#if __ARM_ARCH__<7 + ldrb $t1,[$inp,#2] + ldrb $t0,[$inp,#3] + ldrb $t2,[$inp,#1] + add $e,$K,$e,ror#2 @ E+=K_00_19 + ldrb $t3,[$inp],#4 + orr $t0,$t0,$t1,lsl#8 + eor $t1,$c,$d @ F_xx_xx + orr $t0,$t0,$t2,lsl#16 + add $e,$e,$a,ror#27 @ E+=ROR(A,27) + orr $t0,$t0,$t3,lsl#24 +#else + ldr $t0,[$inp],#4 @ handles unaligned + add $e,$K,$e,ror#2 @ E+=K_00_19 + eor $t1,$c,$d @ F_xx_xx + add $e,$e,$a,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev $t0,$t0 @ byte swap +#endif +#endif + and $t1,$b,$t1,ror#2 + add $e,$e,$t0 @ E+=X[i] + eor $t1,$t1,$d,ror#2 @ F_00_19(B,C,D) + str $t0,[$Xi,#-4]! + add $e,$e,$t1 @ E+=F_00_19(B,C,D) +___ +} + +sub BODY_16_19 { +my ($a,$b,$c,$d,$e)=@_; + &Xupdate(@_,"and $t1,$b,$t1,ror#2"); +$code.=<<___; + eor $t1,$t1,$d,ror#2 @ F_00_19(B,C,D) + add $e,$e,$t1 @ E+=F_00_19(B,C,D) +___ +} + +sub BODY_20_39 { +my ($a,$b,$c,$d,$e)=@_; + &Xupdate(@_,"eor $t1,$b,$t1,ror#2"); +$code.=<<___; + add $e,$e,$t1 @ E+=F_20_39(B,C,D) +___ +} + +sub BODY_40_59 { +my ($a,$b,$c,$d,$e)=@_; + &Xupdate(@_,"and $t1,$b,$t1,ror#2","and $t2,$c,$d"); +$code.=<<___; + add $e,$e,$t1 @ E+=F_40_59(B,C,D) + add $e,$e,$t2,ror#2 +___ +} + +$code=<<___; +#include "arm_arch.h" + +.text + +.global sha1_block_data_order +.type sha1_block_data_order,%function + +.align 2 +sha1_block_data_order: + stmdb sp!,{r4-r12,lr} + add $len,$inp,$len,lsl#6 @ $len to point at the end of $inp + ldmia $ctx,{$a,$b,$c,$d,$e} +.Lloop: + ldr $K,.LK_00_19 + mov $Xi,sp + sub sp,sp,#15*4 + mov $c,$c,ror#30 + mov $d,$d,ror#30 + mov $e,$e,ror#30 @ [6] +.L_00_15: +___ +for($i=0;$i<5;$i++) { + &BODY_00_15(@V); unshift(@V,pop(@V)); +} +$code.=<<___; + teq $Xi,sp + bne .L_00_15 @ [((11+4)*5+2)*3] +___ + &BODY_00_15(@V); unshift(@V,pop(@V)); + &BODY_16_19(@V); unshift(@V,pop(@V)); + &BODY_16_19(@V); unshift(@V,pop(@V)); + &BODY_16_19(@V); unshift(@V,pop(@V)); + &BODY_16_19(@V); unshift(@V,pop(@V)); +$code.=<<___; + + ldr $K,.LK_20_39 @ [+15+16*4] + sub sp,sp,#25*4 + cmn sp,#0 @ [+3], clear carry to denote 20_39 +.L_20_39_or_60_79: +___ +for($i=0;$i<5;$i++) { + &BODY_20_39(@V); unshift(@V,pop(@V)); +} +$code.=<<___; + teq $Xi,sp @ preserve carry + bne .L_20_39_or_60_79 @ [+((12+3)*5+2)*4] + bcs .L_done @ [+((12+3)*5+2)*4], spare 300 bytes + + ldr $K,.LK_40_59 + sub sp,sp,#20*4 @ [+2] +.L_40_59: +___ +for($i=0;$i<5;$i++) { + &BODY_40_59(@V); unshift(@V,pop(@V)); +} +$code.=<<___; + teq $Xi,sp + bne .L_40_59 @ [+((12+5)*5+2)*4] + + ldr $K,.LK_60_79 + sub sp,sp,#20*4 + cmp sp,#0 @ set carry to denote 60_79 + b .L_20_39_or_60_79 @ [+4], spare 300 bytes +.L_done: + add sp,sp,#80*4 @ "deallocate" stack frame + ldmia $ctx,{$K,$t0,$t1,$t2,$t3} + add $a,$K,$a + add $b,$t0,$b + add $c,$t1,$c,ror#2 + add $d,$t2,$d,ror#2 + add $e,$t3,$e,ror#2 + stmia $ctx,{$a,$b,$c,$d,$e} + teq $inp,$len + bne .Lloop @ [+18], total 1307 + +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.align 2 +.LK_00_19: .word 0x5a827999 +.LK_20_39: .word 0x6ed9eba1 +.LK_40_59: .word 0x8f1bbcdc +.LK_60_79: .word 0xca62c1d6 +.size sha1_block_data_order,.-sha1_block_data_order +.asciz "SHA1 block transform for ARMv4, CRYPTOGAMS by <appro\@openssl.org>" +.align 2 +___ + +$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 +print $code; +close STDOUT; # enforce flush diff --git a/crypto/openssl/crypto/sha/asm/sha1-ia64.pl b/crypto/openssl/crypto/sha/asm/sha1-ia64.pl index 51c4f47ecbdc..db28f0805a11 100644 --- a/crypto/openssl/crypto/sha/asm/sha1-ia64.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-ia64.pl @@ -15,7 +15,7 @@ # is >50% better than HP C and >2x better than gcc. $code=<<___; -.ident \"sha1-ia64.s, version 1.2\" +.ident \"sha1-ia64.s, version 1.3\" .ident \"IA-64 ISA artwork by Andy Polyakov <appro\@fy.chalmers.se>\" .explicit @@ -26,14 +26,10 @@ if ($^O eq "hpux") { $ADDP="addp4"; for (@ARGV) { $ADDP="add" if (/[\+DD|\-mlp]64/); } } else { $ADDP="add"; } -for (@ARGV) { $big_endian=1 if (/\-DB_ENDIAN/); - $big_endian=0 if (/\-DL_ENDIAN/); } -if (!defined($big_endian)) - { $big_endian=(unpack('L',pack('N',1))==1); } #$human=1; if ($human) { # useful for visual code auditing... - ($A,$B,$C,$D,$E,$T) = ("A","B","C","D","E","T"); + ($A,$B,$C,$D,$E) = ("A","B","C","D","E"); ($h0,$h1,$h2,$h3,$h4) = ("h0","h1","h2","h3","h4"); ($K_00_19, $K_20_39, $K_40_59, $K_60_79) = ( "K_00_19","K_20_39","K_40_59","K_60_79" ); @@ -41,47 +37,50 @@ if ($human) { # useful for visual code auditing... "X8", "X9","X10","X11","X12","X13","X14","X15" ); } else { - ($A,$B,$C,$D,$E,$T) = ("loc0","loc1","loc2","loc3","loc4","loc5"); - ($h0,$h1,$h2,$h3,$h4) = ("loc6","loc7","loc8","loc9","loc10"); + ($A,$B,$C,$D,$E) = ("loc0","loc1","loc2","loc3","loc4"); + ($h0,$h1,$h2,$h3,$h4) = ("loc5","loc6","loc7","loc8","loc9"); ($K_00_19, $K_20_39, $K_40_59, $K_60_79) = - ( "r14", "r15", "loc11", "loc12" ); + ( "r14", "r15", "loc10", "loc11" ); @X= ( "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31" ); } sub BODY_00_15 { local *code=shift; -local ($i,$a,$b,$c,$d,$e,$f)=@_; +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +my $Xn=@X[$j%16]; $code.=<<___ if ($i==0); -{ .mmi; ld1 $X[$i&0xf]=[inp],2 // MSB +{ .mmi; ld1 $X[$i]=[inp],2 // MSB ld1 tmp2=[tmp3],2 };; { .mmi; ld1 tmp0=[inp],2 ld1 tmp4=[tmp3],2 // LSB - dep $X[$i&0xf]=$X[$i&0xf],tmp2,8,8 };; + dep $X[$i]=$X[$i],tmp2,8,8 };; ___ if ($i<15) { $code.=<<___; -{ .mmi; ld1 $X[($i+1)&0xf]=[inp],2 // +1 +{ .mmi; ld1 $Xn=[inp],2 // forward Xload + nop.m 0x0 dep tmp1=tmp0,tmp4,8,8 };; -{ .mmi; ld1 tmp2=[tmp3],2 // +1 +{ .mmi; ld1 tmp2=[tmp3],2 // forward Xload and tmp4=$c,$b - dep $X[$i&0xf]=$X[$i&0xf],tmp1,16,16 } //;; -{ .mmi; andcm tmp1=$d,$b - add tmp0=$e,$K_00_19 + dep $X[$i]=$X[$i],tmp1,16,16} //;; +{ .mmi; add $e=$e,$K_00_19 // e+=K_00_19 + andcm tmp1=$d,$b dep.z tmp5=$a,5,27 };; // a<<5 -{ .mmi; or tmp4=tmp4,tmp1 // F_00_19(b,c,d)=(b&c)|(~b&d) - add $f=tmp0,$X[$i&0xf] // f=xi+e+K_00_19 +{ .mmi; add $e=$e,$X[$i] // e+=Xload + or tmp4=tmp4,tmp1 // F_00_19(b,c,d)=(b&c)|(~b&d) extr.u tmp1=$a,27,5 };; // a>>27 -{ .mmi; ld1 tmp0=[inp],2 // +1 - add $f=$f,tmp4 // f+=F_00_19(b,c,d) +{ .mmi; ld1 tmp0=[inp],2 // forward Xload + add $e=$e,tmp4 // e+=F_00_19(b,c,d) shrp $b=tmp6,tmp6,2 } // b=ROTATE(b,30) -{ .mmi; ld1 tmp4=[tmp3],2 // +1 +{ .mmi; ld1 tmp4=[tmp3],2 // forward Xload or tmp5=tmp1,tmp5 // ROTATE(a,5) mux2 tmp6=$a,0x44 };; // see b in next iteration -{ .mii; add $f=$f,tmp5 // f+=ROTATE(a,5) - dep $X[($i+1)&0xf]=$X[($i+1)&0xf],tmp2,8,8 // +1 - mux2 $X[$i&0xf]=$X[$i&0xf],0x44 } //;; +{ .mii; add $e=$e,tmp5 // e+=ROTATE(a,5) + dep $Xn=$Xn,tmp2,8,8 // forward Xload + mux2 $X[$i]=$X[$i],0x44 } //;; ___ } @@ -89,24 +88,24 @@ else { $code.=<<___; { .mii; and tmp3=$c,$b dep tmp1=tmp0,tmp4,8,8;; - dep $X[$i&0xf]=$X[$i&0xf],tmp1,16,16 } //;; -{ .mmi; andcm tmp1=$d,$b - add tmp0=$e,$K_00_19 + dep $X[$i]=$X[$i],tmp1,16,16} //;; +{ .mmi; add $e=$e,$K_00_19 // e+=K_00_19 + andcm tmp1=$d,$b dep.z tmp5=$a,5,27 };; // a<<5 -{ .mmi; or tmp4=tmp3,tmp1 // F_00_19(b,c,d)=(b&c)|(~b&d) - add $f=tmp0,$X[$i&0xf] // f=xi+e+K_00_19 +{ .mmi; add $e=$e,$X[$i] // e+=Xupdate + or tmp4=tmp3,tmp1 // F_00_19(b,c,d)=(b&c)|(~b&d) extr.u tmp1=$a,27,5 } // a>>27 -{ .mmi; xor tmp2=$X[($i+0+1)&0xf],$X[($i+2+1)&0xf] // +1 - xor tmp3=$X[($i+8+1)&0xf],$X[($i+13+1)&0xf] // +1 +{ .mmi; xor $Xn=$Xn,$X[($j+2)%16] // forward Xupdate + xor tmp3=$X[($j+8)%16],$X[($j+13)%16] // forward Xupdate nop.i 0 };; -{ .mmi; add $f=$f,tmp4 // f+=F_00_19(b,c,d) - xor tmp2=tmp2,tmp3 // +1 +{ .mmi; add $e=$e,tmp4 // e+=F_00_19(b,c,d) + xor $Xn=$Xn,tmp3 // forward Xupdate shrp $b=tmp6,tmp6,2 } // b=ROTATE(b,30) { .mmi; or tmp1=tmp1,tmp5 // ROTATE(a,5) mux2 tmp6=$a,0x44 };; // see b in next iteration -{ .mii; add $f=$f,tmp1 // f+=ROTATE(a,5) - shrp $e=tmp2,tmp2,31 // f+1=ROTATE(x[0]^x[2]^x[8]^x[13],1) - mux2 $X[$i&0xf]=$X[$i&0xf],0x44 };; +{ .mii; add $e=$e,tmp1 // e+=ROTATE(a,5) + shrp $Xn=$Xn,$Xn,31 // ROTATE(x[0]^x[2]^x[8]^x[13],1) + mux2 $X[$i]=$X[$i],0x44 };; ___ } @@ -114,27 +113,28 @@ ___ sub BODY_16_19 { local *code=shift; -local ($i,$a,$b,$c,$d,$e,$f)=@_; +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +my $Xn=@X[$j%16]; $code.=<<___; -{ .mmi; mov $X[$i&0xf]=$f // Xupdate - and tmp0=$c,$b +{ .mib; add $e=$e,$K_00_19 // e+=K_00_19 dep.z tmp5=$a,5,27 } // a<<5 -{ .mmi; andcm tmp1=$d,$b - add tmp4=$e,$K_00_19 };; -{ .mmi; or tmp0=tmp0,tmp1 // F_00_19(b,c,d)=(b&c)|(~b&d) - add $f=$f,tmp4 // f+=e+K_00_19 +{ .mib; andcm tmp1=$d,$b + and tmp0=$c,$b };; +{ .mmi; add $e=$e,$X[$i%16] // e+=Xupdate + or tmp0=tmp0,tmp1 // F_00_19(b,c,d)=(b&c)|(~b&d) extr.u tmp1=$a,27,5 } // a>>27 -{ .mmi; xor tmp2=$X[($i+0+1)&0xf],$X[($i+2+1)&0xf] // +1 - xor tmp3=$X[($i+8+1)&0xf],$X[($i+13+1)&0xf] // +1 +{ .mmi; xor $Xn=$Xn,$X[($j+2)%16] // forward Xupdate + xor tmp3=$X[($j+8)%16],$X[($j+13)%16] // forward Xupdate nop.i 0 };; -{ .mmi; add $f=$f,tmp0 // f+=F_00_19(b,c,d) - xor tmp2=tmp2,tmp3 // +1 +{ .mmi; add $e=$e,tmp0 // f+=F_00_19(b,c,d) + xor $Xn=$Xn,tmp3 // forward Xupdate shrp $b=tmp6,tmp6,2 } // b=ROTATE(b,30) { .mmi; or tmp1=tmp1,tmp5 // ROTATE(a,5) mux2 tmp6=$a,0x44 };; // see b in next iteration -{ .mii; add $f=$f,tmp1 // f+=ROTATE(a,5) - shrp $e=tmp2,tmp2,31 // f+1=ROTATE(x[0]^x[2]^x[8]^x[13],1) +{ .mii; add $e=$e,tmp1 // e+=ROTATE(a,5) + shrp $Xn=$Xn,$Xn,31 // ROTATE(x[0]^x[2]^x[8]^x[13],1) nop.i 0 };; ___ @@ -142,49 +142,47 @@ ___ sub BODY_20_39 { local *code=shift; -local ($i,$a,$b,$c,$d,$e,$f,$Konst)=@_; +my ($i,$a,$b,$c,$d,$e,$Konst)=@_; $Konst = $K_20_39 if (!defined($Konst)); +my $j=$i+1; +my $Xn=@X[$j%16]; if ($i<79) { $code.=<<___; -{ .mib; mov $X[$i&0xf]=$f // Xupdate +{ .mib; add $e=$e,$Konst // e+=K_XX_XX dep.z tmp5=$a,5,27 } // a<<5 { .mib; xor tmp0=$c,$b - add tmp4=$e,$Konst };; -{ .mmi; xor tmp0=tmp0,$d // F_20_39(b,c,d)=b^c^d - add $f=$f,tmp4 // f+=e+K_20_39 + xor $Xn=$Xn,$X[($j+2)%16] };; // forward Xupdate +{ .mib; add $e=$e,$X[$i%16] // e+=Xupdate extr.u tmp1=$a,27,5 } // a>>27 -{ .mmi; xor tmp2=$X[($i+0+1)&0xf],$X[($i+2+1)&0xf] // +1 - xor tmp3=$X[($i+8+1)&0xf],$X[($i+13+1)&0xf] // +1 - nop.i 0 };; -{ .mmi; add $f=$f,tmp0 // f+=F_20_39(b,c,d) - xor tmp2=tmp2,tmp3 // +1 +{ .mib; xor tmp0=tmp0,$d // F_20_39(b,c,d)=b^c^d + xor $Xn=$Xn,$X[($j+8)%16] };; // forward Xupdate +{ .mmi; add $e=$e,tmp0 // e+=F_20_39(b,c,d) + xor $Xn=$Xn,$X[($j+13)%16] // forward Xupdate shrp $b=tmp6,tmp6,2 } // b=ROTATE(b,30) { .mmi; or tmp1=tmp1,tmp5 // ROTATE(a,5) mux2 tmp6=$a,0x44 };; // see b in next iteration -{ .mii; add $f=$f,tmp1 // f+=ROTATE(a,5) - shrp $e=tmp2,tmp2,31 // f+1=ROTATE(x[0]^x[2]^x[8]^x[13],1) +{ .mii; add $e=$e,tmp1 // e+=ROTATE(a,5) + shrp $Xn=$Xn,$Xn,31 // ROTATE(x[0]^x[2]^x[8]^x[13],1) nop.i 0 };; ___ } else { $code.=<<___; -{ .mib; mov $X[$i&0xf]=$f // Xupdate +{ .mib; add $e=$e,$Konst // e+=K_60_79 dep.z tmp5=$a,5,27 } // a<<5 { .mib; xor tmp0=$c,$b - add tmp4=$e,$Konst };; -{ .mib; xor tmp0=tmp0,$d // F_20_39(b,c,d)=b^c^d - extr.u tmp1=$a,27,5 } // a>>27 -{ .mib; add $f=$f,tmp4 // f+=e+K_20_39 add $h1=$h1,$a };; // wrap up -{ .mmi; add $f=$f,tmp0 // f+=F_20_39(b,c,d) - shrp $b=tmp6,tmp6,2 } // b=ROTATE(b,30) ;;? -{ .mmi; or tmp1=tmp1,tmp5 // ROTATE(a,5) +{ .mib; add $e=$e,$X[$i%16] // e+=Xupdate + extr.u tmp1=$a,27,5 } // a>>27 +{ .mib; xor tmp0=tmp0,$d // F_20_39(b,c,d)=b^c^d add $h3=$h3,$c };; // wrap up -{ .mib; add tmp3=1,inp // used in unaligned codepath - add $f=$f,tmp1 } // f+=ROTATE(a,5) -{ .mib; add $h2=$h2,$b // wrap up +{ .mmi; add $e=$e,tmp0 // e+=F_20_39(b,c,d) + or tmp1=tmp1,tmp5 // ROTATE(a,5) + shrp $b=tmp6,tmp6,2 };; // b=ROTATE(b,30) ;;? +{ .mmi; add $e=$e,tmp1 // e+=ROTATE(a,5) + add tmp3=1,inp // used in unaligned codepath add $h4=$h4,$d };; // wrap up ___ @@ -193,29 +191,29 @@ ___ sub BODY_40_59 { local *code=shift; -local ($i,$a,$b,$c,$d,$e,$f)=@_; +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +my $Xn=@X[$j%16]; $code.=<<___; -{ .mmi; mov $X[$i&0xf]=$f // Xupdate - and tmp0=$c,$b +{ .mib; add $e=$e,$K_40_59 // e+=K_40_59 dep.z tmp5=$a,5,27 } // a<<5 -{ .mmi; and tmp1=$d,$b - add tmp4=$e,$K_40_59 };; -{ .mmi; or tmp0=tmp0,tmp1 // (b&c)|(b&d) - add $f=$f,tmp4 // f+=e+K_40_59 +{ .mib; and tmp1=$c,$d + xor tmp0=$c,$d };; +{ .mmi; add $e=$e,$X[$i%16] // e+=Xupdate + add tmp5=tmp5,tmp1 // a<<5+(c&d) extr.u tmp1=$a,27,5 } // a>>27 -{ .mmi; and tmp4=$c,$d - xor tmp2=$X[($i+0+1)&0xf],$X[($i+2+1)&0xf] // +1 - xor tmp3=$X[($i+8+1)&0xf],$X[($i+13+1)&0xf] // +1 - };; -{ .mmi; or tmp1=tmp1,tmp5 // ROTATE(a,5) - xor tmp2=tmp2,tmp3 // +1 +{ .mmi; and tmp0=tmp0,$b + xor $Xn=$Xn,$X[($j+2)%16] // forward Xupdate + xor tmp3=$X[($j+8)%16],$X[($j+13)%16] };; // forward Xupdate +{ .mmi; add $e=$e,tmp0 // e+=b&(c^d) + add tmp5=tmp5,tmp1 // ROTATE(a,5)+(c&d) shrp $b=tmp6,tmp6,2 } // b=ROTATE(b,30) -{ .mmi; or tmp0=tmp0,tmp4 // F_40_59(b,c,d)=(b&c)|(b&d)|(c&d) +{ .mmi; xor $Xn=$Xn,tmp3 mux2 tmp6=$a,0x44 };; // see b in next iteration -{ .mii; add $f=$f,tmp0 // f+=F_40_59(b,c,d) - shrp $e=tmp2,tmp2,31;; // f+1=ROTATE(x[0]^x[2]^x[8]^x[13],1) - add $f=$f,tmp1 };; // f+=ROTATE(a,5) +{ .mii; add $e=$e,tmp5 // e+=ROTATE(a,5)+(c&d) + shrp $Xn=$Xn,$Xn,31 // ROTATE(x[0]^x[2]^x[8]^x[13],1) + nop.i 0x0 };; ___ } @@ -237,7 +235,7 @@ inp=r33; // in1 .align 32 sha1_block_data_order: .prologue -{ .mmi; alloc tmp1=ar.pfs,3,15,0,0 +{ .mmi; alloc tmp1=ar.pfs,3,14,0,0 $ADDP tmp0=4,ctx .save ar.lc,r3 mov r3=ar.lc } @@ -245,8 +243,8 @@ sha1_block_data_order: $ADDP inp=0,inp mov r2=pr };; tmp4=in2; -tmp5=loc13; -tmp6=loc14; +tmp5=loc12; +tmp6=loc13; .body { .mlx; ld4 $h0=[ctx],8 movl $K_00_19=0x5a827999 } @@ -273,7 +271,7 @@ tmp6=loc14; ___ -{ my $i,@V=($A,$B,$C,$D,$E,$T); +{ my $i,@V=($A,$B,$C,$D,$E); for($i=0;$i<16;$i++) { &BODY_00_15(\$code,$i,@V); unshift(@V,pop(@V)); } for(;$i<20;$i++) { &BODY_16_19(\$code,$i,@V); unshift(@V,pop(@V)); } @@ -281,12 +279,12 @@ ___ for(;$i<60;$i++) { &BODY_40_59(\$code,$i,@V); unshift(@V,pop(@V)); } for(;$i<80;$i++) { &BODY_60_79(\$code,$i,@V); unshift(@V,pop(@V)); } - (($V[5] eq $D) and ($V[0] eq $E)) or die; # double-check + (($V[0] eq $A) and ($V[4] eq $E)) or die; # double-check } $code.=<<___; -{ .mmb; add $h0=$h0,$E - nop.m 0 +{ .mmb; add $h0=$h0,$A + add $h2=$h2,$C br.ctop.dptk.many .Ldtop };; .Ldend: { .mmi; add tmp0=4,ctx diff --git a/crypto/openssl/crypto/sha/asm/sha1-mips.pl b/crypto/openssl/crypto/sha/asm/sha1-mips.pl new file mode 100755 index 000000000000..f1a702f38f53 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-mips.pl @@ -0,0 +1,354 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA1 block procedure for MIPS. + +# Performance improvement is 30% on unaligned input. The "secret" is +# to deploy lwl/lwr pair to load unaligned input. One could have +# vectorized Xupdate on MIPSIII/IV, but the goal was to code MIPS32- +# compatible subroutine. There is room for minor optimization on +# little-endian platforms... + +###################################################################### +# There is a number of MIPS ABI in use, O32 and N32/64 are most +# widely used. Then there is a new contender: NUBI. It appears that if +# one picks the latter, it's possible to arrange code in ABI neutral +# manner. Therefore let's stick to NUBI register layout: +# +($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25)); +($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23)); +($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31)); +# +# The return value is placed in $a0. Following coding rules facilitate +# interoperability: +# +# - never ever touch $tp, "thread pointer", former $gp; +# - copy return value to $t0, former $v0 [or to $a0 if you're adapting +# old code]; +# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary; +# +# For reference here is register layout for N32/64 MIPS ABIs: +# +# ($zero,$at,$v0,$v1)=map("\$$_",(0..3)); +# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25)); +# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); +# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); +# +$flavour = shift; # supported flavours are o32,n32,64,nubi32,nubi64 + +if ($flavour =~ /64|n32/i) { + $PTR_ADD="dadd"; # incidentally works even on n32 + $PTR_SUB="dsub"; # incidentally works even on n32 + $REG_S="sd"; + $REG_L="ld"; + $PTR_SLL="dsll"; # incidentally works even on n32 + $SZREG=8; +} else { + $PTR_ADD="add"; + $PTR_SUB="sub"; + $REG_S="sw"; + $REG_L="lw"; + $PTR_SLL="sll"; + $SZREG=4; +} +# +# <appro@openssl.org> +# +###################################################################### + +$big_endian=(`echo MIPSEL | $ENV{CC} -E -P -`=~/MIPSEL/)?1:0; + +for (@ARGV) { $output=$_ if (/^\w[\w\-]*\.\w+$/); } +open STDOUT,">$output"; + +if (!defined($big_endian)) + { $big_endian=(unpack('L',pack('N',1))==1); } + +# offsets of the Most and Least Significant Bytes +$MSB=$big_endian?0:3; +$LSB=3&~$MSB; + +@X=map("\$$_",(8..23)); # a4-a7,s0-s11 + +$ctx=$a0; +$inp=$a1; +$num=$a2; +$A="\$1"; +$B="\$2"; +$C="\$3"; +$D="\$7"; +$E="\$24"; @V=($A,$B,$C,$D,$E); +$t0="\$25"; +$t1=$num; # $num is offloaded to stack +$t2="\$30"; # fp +$K="\$31"; # ra + +sub BODY_00_14 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +$code.=<<___ if (!$big_endian); + srl $t0,@X[$i],24 # byte swap($i) + srl $t1,@X[$i],8 + andi $t2,@X[$i],0xFF00 + sll @X[$i],@X[$i],24 + andi $t1,0xFF00 + sll $t2,$t2,8 + or @X[$i],$t0 + or $t1,$t2 + or @X[$i],$t1 +___ +$code.=<<___; + lwl @X[$j],$j*4+$MSB($inp) + sll $t0,$a,5 # $i + addu $e,$K + lwr @X[$j],$j*4+$LSB($inp) + srl $t1,$a,27 + addu $e,$t0 + xor $t0,$c,$d + addu $e,$t1 + sll $t2,$b,30 + and $t0,$b + srl $b,$b,2 + xor $t0,$d + addu $e,@X[$i] + or $b,$t2 + addu $e,$t0 +___ +} + +sub BODY_15_19 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; + +$code.=<<___ if (!$big_endian && $i==15); + srl $t0,@X[$i],24 # byte swap($i) + srl $t1,@X[$i],8 + andi $t2,@X[$i],0xFF00 + sll @X[$i],@X[$i],24 + andi $t1,0xFF00 + sll $t2,$t2,8 + or @X[$i],$t0 + or @X[$i],$t1 + or @X[$i],$t2 +___ +$code.=<<___; + xor @X[$j%16],@X[($j+2)%16] + sll $t0,$a,5 # $i + addu $e,$K + srl $t1,$a,27 + addu $e,$t0 + xor @X[$j%16],@X[($j+8)%16] + xor $t0,$c,$d + addu $e,$t1 + xor @X[$j%16],@X[($j+13)%16] + sll $t2,$b,30 + and $t0,$b + srl $t1,@X[$j%16],31 + addu @X[$j%16],@X[$j%16] + srl $b,$b,2 + xor $t0,$d + or @X[$j%16],$t1 + addu $e,@X[$i%16] + or $b,$t2 + addu $e,$t0 +___ +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +$code.=<<___ if ($i<79); + xor @X[$j%16],@X[($j+2)%16] + sll $t0,$a,5 # $i + addu $e,$K + srl $t1,$a,27 + addu $e,$t0 + xor @X[$j%16],@X[($j+8)%16] + xor $t0,$c,$d + addu $e,$t1 + xor @X[$j%16],@X[($j+13)%16] + sll $t2,$b,30 + xor $t0,$b + srl $t1,@X[$j%16],31 + addu @X[$j%16],@X[$j%16] + srl $b,$b,2 + addu $e,@X[$i%16] + or @X[$j%16],$t1 + or $b,$t2 + addu $e,$t0 +___ +$code.=<<___ if ($i==79); + lw @X[0],0($ctx) + sll $t0,$a,5 # $i + addu $e,$K + lw @X[1],4($ctx) + srl $t1,$a,27 + addu $e,$t0 + lw @X[2],8($ctx) + xor $t0,$c,$d + addu $e,$t1 + lw @X[3],12($ctx) + sll $t2,$b,30 + xor $t0,$b + lw @X[4],16($ctx) + srl $b,$b,2 + addu $e,@X[$i%16] + or $b,$t2 + addu $e,$t0 +___ +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +$code.=<<___ if ($i<79); + xor @X[$j%16],@X[($j+2)%16] + sll $t0,$a,5 # $i + addu $e,$K + srl $t1,$a,27 + addu $e,$t0 + xor @X[$j%16],@X[($j+8)%16] + and $t0,$c,$d + addu $e,$t1 + xor @X[$j%16],@X[($j+13)%16] + sll $t2,$b,30 + addu $e,$t0 + srl $t1,@X[$j%16],31 + xor $t0,$c,$d + addu @X[$j%16],@X[$j%16] + and $t0,$b + srl $b,$b,2 + or @X[$j%16],$t1 + addu $e,@X[$i%16] + or $b,$t2 + addu $e,$t0 +___ +} + +$FRAMESIZE=16; # large enough to accomodate NUBI saved registers +$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? 0xc0fff008 : 0xc0ff0000; + +$code=<<___; +#ifdef OPENSSL_FIPSCANISTER +# include <openssl/fipssyms.h> +#endif + +.text + +.set noat +.set noreorder +.align 5 +.globl sha1_block_data_order +.ent sha1_block_data_order +sha1_block_data_order: + .frame $sp,$FRAMESIZE*$SZREG,$ra + .mask $SAVED_REGS_MASK,-$SZREG + .set noreorder + $PTR_SUB $sp,$FRAMESIZE*$SZREG + $REG_S $ra,($FRAMESIZE-1)*$SZREG($sp) + $REG_S $fp,($FRAMESIZE-2)*$SZREG($sp) + $REG_S $s11,($FRAMESIZE-3)*$SZREG($sp) + $REG_S $s10,($FRAMESIZE-4)*$SZREG($sp) + $REG_S $s9,($FRAMESIZE-5)*$SZREG($sp) + $REG_S $s8,($FRAMESIZE-6)*$SZREG($sp) + $REG_S $s7,($FRAMESIZE-7)*$SZREG($sp) + $REG_S $s6,($FRAMESIZE-8)*$SZREG($sp) + $REG_S $s5,($FRAMESIZE-9)*$SZREG($sp) + $REG_S $s4,($FRAMESIZE-10)*$SZREG($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi prologue + $REG_S $s3,($FRAMESIZE-11)*$SZREG($sp) + $REG_S $s2,($FRAMESIZE-12)*$SZREG($sp) + $REG_S $s1,($FRAMESIZE-13)*$SZREG($sp) + $REG_S $s0,($FRAMESIZE-14)*$SZREG($sp) + $REG_S $gp,($FRAMESIZE-15)*$SZREG($sp) +___ +$code.=<<___; + $PTR_SLL $num,6 + $PTR_ADD $num,$inp + $REG_S $num,0($sp) + lw $A,0($ctx) + lw $B,4($ctx) + lw $C,8($ctx) + lw $D,12($ctx) + b .Loop + lw $E,16($ctx) +.align 4 +.Loop: + .set reorder + lwl @X[0],$MSB($inp) + lui $K,0x5a82 + lwr @X[0],$LSB($inp) + ori $K,0x7999 # K_00_19 +___ +for ($i=0;$i<15;$i++) { &BODY_00_14($i,@V); unshift(@V,pop(@V)); } +for (;$i<20;$i++) { &BODY_15_19($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + lui $K,0x6ed9 + ori $K,0xeba1 # K_20_39 +___ +for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + lui $K,0x8f1b + ori $K,0xbcdc # K_40_59 +___ +for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + lui $K,0xca62 + ori $K,0xc1d6 # K_60_79 +___ +for (;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + $PTR_ADD $inp,64 + $REG_L $num,0($sp) + + addu $A,$X[0] + addu $B,$X[1] + sw $A,0($ctx) + addu $C,$X[2] + addu $D,$X[3] + sw $B,4($ctx) + addu $E,$X[4] + sw $C,8($ctx) + sw $D,12($ctx) + sw $E,16($ctx) + .set noreorder + bne $inp,$num,.Loop + nop + + .set noreorder + $REG_L $ra,($FRAMESIZE-1)*$SZREG($sp) + $REG_L $fp,($FRAMESIZE-2)*$SZREG($sp) + $REG_L $s11,($FRAMESIZE-3)*$SZREG($sp) + $REG_L $s10,($FRAMESIZE-4)*$SZREG($sp) + $REG_L $s9,($FRAMESIZE-5)*$SZREG($sp) + $REG_L $s8,($FRAMESIZE-6)*$SZREG($sp) + $REG_L $s7,($FRAMESIZE-7)*$SZREG($sp) + $REG_L $s6,($FRAMESIZE-8)*$SZREG($sp) + $REG_L $s5,($FRAMESIZE-9)*$SZREG($sp) + $REG_L $s4,($FRAMESIZE-10)*$SZREG($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); + $REG_L $s3,($FRAMESIZE-11)*$SZREG($sp) + $REG_L $s2,($FRAMESIZE-12)*$SZREG($sp) + $REG_L $s1,($FRAMESIZE-13)*$SZREG($sp) + $REG_L $s0,($FRAMESIZE-14)*$SZREG($sp) + $REG_L $gp,($FRAMESIZE-15)*$SZREG($sp) +___ +$code.=<<___; + jr $ra + $PTR_ADD $sp,$FRAMESIZE*$SZREG +.end sha1_block_data_order +.rdata +.asciiz "SHA1 for MIPS, CRYPTOGAMS by <appro\@openssl.org>" +___ +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha1-parisc.pl b/crypto/openssl/crypto/sha/asm/sha1-parisc.pl new file mode 100755 index 000000000000..6d7bf495b20a --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-parisc.pl @@ -0,0 +1,259 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA1 block procedure for PA-RISC. + +# June 2009. +# +# On PA-7100LC performance is >30% better than gcc 3.2 generated code +# for aligned input and >50% better for unaligned. Compared to vendor +# compiler on PA-8600 it's almost 60% faster in 64-bit build and just +# few percent faster in 32-bit one (this for aligned input, data for +# unaligned input is not available). +# +# Special thanks to polarhome.com for providing HP-UX account. + +$flavour = shift; +$output = shift; +open STDOUT,">$output"; + +if ($flavour =~ /64/) { + $LEVEL ="2.0W"; + $SIZE_T =8; + $FRAME_MARKER =80; + $SAVED_RP =16; + $PUSH ="std"; + $PUSHMA ="std,ma"; + $POP ="ldd"; + $POPMB ="ldd,mb"; +} else { + $LEVEL ="1.0"; + $SIZE_T =4; + $FRAME_MARKER =48; + $SAVED_RP =20; + $PUSH ="stw"; + $PUSHMA ="stwm"; + $POP ="ldw"; + $POPMB ="ldwm"; +} + +$FRAME=14*$SIZE_T+$FRAME_MARKER;# 14 saved regs + frame marker + # [+ argument transfer] +$ctx="%r26"; # arg0 +$inp="%r25"; # arg1 +$num="%r24"; # arg2 + +$t0="%r28"; +$t1="%r29"; +$K="%r31"; + +@X=("%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", "%r8", + "%r9", "%r10","%r11","%r12","%r13","%r14","%r15","%r16",$t0); + +@V=($A,$B,$C,$D,$E)=("%r19","%r20","%r21","%r22","%r23"); + +sub BODY_00_19 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +$code.=<<___ if ($i<15); + addl $K,$e,$e ; $i + shd $a,$a,27,$t1 + addl @X[$i],$e,$e + and $c,$b,$t0 + addl $t1,$e,$e + andcm $d,$b,$t1 + shd $b,$b,2,$b + or $t1,$t0,$t0 + addl $t0,$e,$e +___ +$code.=<<___ if ($i>=15); # with forward Xupdate + addl $K,$e,$e ; $i + shd $a,$a,27,$t1 + xor @X[($j+2)%16],@X[$j%16],@X[$j%16] + addl @X[$i%16],$e,$e + and $c,$b,$t0 + xor @X[($j+8)%16],@X[$j%16],@X[$j%16] + addl $t1,$e,$e + andcm $d,$b,$t1 + shd $b,$b,2,$b + or $t1,$t0,$t0 + xor @X[($j+13)%16],@X[$j%16],@X[$j%16] + add $t0,$e,$e + shd @X[$j%16],@X[$j%16],31,@X[$j%16] +___ +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +$code.=<<___ if ($i<79); + xor @X[($j+2)%16],@X[$j%16],@X[$j%16] ; $i + addl $K,$e,$e + shd $a,$a,27,$t1 + xor @X[($j+8)%16],@X[$j%16],@X[$j%16] + addl @X[$i%16],$e,$e + xor $b,$c,$t0 + xor @X[($j+13)%16],@X[$j%16],@X[$j%16] + addl $t1,$e,$e + shd $b,$b,2,$b + xor $d,$t0,$t0 + shd @X[$j%16],@X[$j%16],31,@X[$j%16] + addl $t0,$e,$e +___ +$code.=<<___ if ($i==79); # with context load + ldw 0($ctx),@X[0] ; $i + addl $K,$e,$e + shd $a,$a,27,$t1 + ldw 4($ctx),@X[1] + addl @X[$i%16],$e,$e + xor $b,$c,$t0 + ldw 8($ctx),@X[2] + addl $t1,$e,$e + shd $b,$b,2,$b + xor $d,$t0,$t0 + ldw 12($ctx),@X[3] + addl $t0,$e,$e + ldw 16($ctx),@X[4] +___ +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i+1; +$code.=<<___; + shd $a,$a,27,$t1 ; $i + addl $K,$e,$e + xor @X[($j+2)%16],@X[$j%16],@X[$j%16] + xor $d,$c,$t0 + addl @X[$i%16],$e,$e + xor @X[($j+8)%16],@X[$j%16],@X[$j%16] + and $b,$t0,$t0 + addl $t1,$e,$e + shd $b,$b,2,$b + xor @X[($j+13)%16],@X[$j%16],@X[$j%16] + addl $t0,$e,$e + and $d,$c,$t1 + shd @X[$j%16],@X[$j%16],31,@X[$j%16] + addl $t1,$e,$e +___ +} + +$code=<<___; + .LEVEL $LEVEL + .SPACE \$TEXT\$ + .SUBSPA \$CODE\$,QUAD=0,ALIGN=8,ACCESS=0x2C,CODE_ONLY + + .EXPORT sha1_block_data_order,ENTRY,ARGW0=GR,ARGW1=GR,ARGW2=GR +sha1_block_data_order + .PROC + .CALLINFO FRAME=`$FRAME-14*$SIZE_T`,NO_CALLS,SAVE_RP,ENTRY_GR=16 + .ENTRY + $PUSH %r2,-$SAVED_RP(%sp) ; standard prologue + $PUSHMA %r3,$FRAME(%sp) + $PUSH %r4,`-$FRAME+1*$SIZE_T`(%sp) + $PUSH %r5,`-$FRAME+2*$SIZE_T`(%sp) + $PUSH %r6,`-$FRAME+3*$SIZE_T`(%sp) + $PUSH %r7,`-$FRAME+4*$SIZE_T`(%sp) + $PUSH %r8,`-$FRAME+5*$SIZE_T`(%sp) + $PUSH %r9,`-$FRAME+6*$SIZE_T`(%sp) + $PUSH %r10,`-$FRAME+7*$SIZE_T`(%sp) + $PUSH %r11,`-$FRAME+8*$SIZE_T`(%sp) + $PUSH %r12,`-$FRAME+9*$SIZE_T`(%sp) + $PUSH %r13,`-$FRAME+10*$SIZE_T`(%sp) + $PUSH %r14,`-$FRAME+11*$SIZE_T`(%sp) + $PUSH %r15,`-$FRAME+12*$SIZE_T`(%sp) + $PUSH %r16,`-$FRAME+13*$SIZE_T`(%sp) + + ldw 0($ctx),$A + ldw 4($ctx),$B + ldw 8($ctx),$C + ldw 12($ctx),$D + ldw 16($ctx),$E + + extru $inp,31,2,$t0 ; t0=inp&3; + sh3addl $t0,%r0,$t0 ; t0*=8; + subi 32,$t0,$t0 ; t0=32-t0; + mtctl $t0,%cr11 ; %sar=t0; + +L\$oop + ldi 3,$t0 + andcm $inp,$t0,$t0 ; 64-bit neutral +___ + for ($i=0;$i<15;$i++) { # load input block + $code.="\tldw `4*$i`($t0),@X[$i]\n"; } +$code.=<<___; + cmpb,*= $inp,$t0,L\$aligned + ldw 60($t0),@X[15] + ldw 64($t0),@X[16] +___ + for ($i=0;$i<16;$i++) { # align input + $code.="\tvshd @X[$i],@X[$i+1],@X[$i]\n"; } +$code.=<<___; +L\$aligned + ldil L'0x5a827000,$K ; K_00_19 + ldo 0x999($K),$K +___ +for ($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + ldil L'0x6ed9e000,$K ; K_20_39 + ldo 0xba1($K),$K +___ + +for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + ldil L'0x8f1bb000,$K ; K_40_59 + ldo 0xcdc($K),$K +___ + +for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + ldil L'0xca62c000,$K ; K_60_79 + ldo 0x1d6($K),$K +___ +for (;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } + +$code.=<<___; + addl @X[0],$A,$A + addl @X[1],$B,$B + addl @X[2],$C,$C + addl @X[3],$D,$D + addl @X[4],$E,$E + stw $A,0($ctx) + stw $B,4($ctx) + stw $C,8($ctx) + stw $D,12($ctx) + stw $E,16($ctx) + addib,*<> -1,$num,L\$oop + ldo 64($inp),$inp + + $POP `-$FRAME-$SAVED_RP`(%sp),%r2 ; standard epilogue + $POP `-$FRAME+1*$SIZE_T`(%sp),%r4 + $POP `-$FRAME+2*$SIZE_T`(%sp),%r5 + $POP `-$FRAME+3*$SIZE_T`(%sp),%r6 + $POP `-$FRAME+4*$SIZE_T`(%sp),%r7 + $POP `-$FRAME+5*$SIZE_T`(%sp),%r8 + $POP `-$FRAME+6*$SIZE_T`(%sp),%r9 + $POP `-$FRAME+7*$SIZE_T`(%sp),%r10 + $POP `-$FRAME+8*$SIZE_T`(%sp),%r11 + $POP `-$FRAME+9*$SIZE_T`(%sp),%r12 + $POP `-$FRAME+10*$SIZE_T`(%sp),%r13 + $POP `-$FRAME+11*$SIZE_T`(%sp),%r14 + $POP `-$FRAME+12*$SIZE_T`(%sp),%r15 + $POP `-$FRAME+13*$SIZE_T`(%sp),%r16 + bv (%r2) + .EXIT + $POPMB -$FRAME(%sp),%r3 + .PROCEND + .STRINGZ "SHA1 block transform for PA-RISC, CRYPTOGAMS by <appro\@openssl.org>" +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +$code =~ s/,\*/,/gm if ($SIZE_T==4); +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha1-ppc.pl b/crypto/openssl/crypto/sha/asm/sha1-ppc.pl new file mode 100755 index 000000000000..2140dd2f8dd6 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-ppc.pl @@ -0,0 +1,326 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# I let hardware handle unaligned input(*), except on page boundaries +# (see below for details). Otherwise straightforward implementation +# with X vector in register bank. The module is big-endian [which is +# not big deal as there're no little-endian targets left around]. +# +# (*) this means that this module is inappropriate for PPC403? Does +# anybody know if pre-POWER3 can sustain unaligned load? + +# -m64 -m32 +# ---------------------------------- +# PPC970,gcc-4.0.0 +76% +59% +# Power6,xlc-7 +68% +33% + +$flavour = shift; + +if ($flavour =~ /64/) { + $SIZE_T =8; + $LRSAVE =2*$SIZE_T; + $UCMP ="cmpld"; + $STU ="stdu"; + $POP ="ld"; + $PUSH ="std"; +} elsif ($flavour =~ /32/) { + $SIZE_T =4; + $LRSAVE =$SIZE_T; + $UCMP ="cmplw"; + $STU ="stwu"; + $POP ="lwz"; + $PUSH ="stw"; +} else { die "nonsense $flavour"; } + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!"; + +$FRAME=24*$SIZE_T+64; +$LOCALS=6*$SIZE_T; + +$K ="r0"; +$sp ="r1"; +$toc="r2"; +$ctx="r3"; +$inp="r4"; +$num="r5"; +$t0 ="r15"; +$t1 ="r6"; + +$A ="r7"; +$B ="r8"; +$C ="r9"; +$D ="r10"; +$E ="r11"; +$T ="r12"; + +@V=($A,$B,$C,$D,$E,$T); +@X=("r16","r17","r18","r19","r20","r21","r22","r23", + "r24","r25","r26","r27","r28","r29","r30","r31"); + +sub BODY_00_19 { +my ($i,$a,$b,$c,$d,$e,$f)=@_; +my $j=$i+1; +$code.=<<___ if ($i==0); + lwz @X[$i],`$i*4`($inp) +___ +$code.=<<___ if ($i<15); + lwz @X[$j],`$j*4`($inp) + add $f,$K,$e + rotlwi $e,$a,5 + add $f,$f,@X[$i] + and $t0,$c,$b + add $f,$f,$e + andc $t1,$d,$b + rotlwi $b,$b,30 + or $t0,$t0,$t1 + add $f,$f,$t0 +___ +$code.=<<___ if ($i>=15); + add $f,$K,$e + rotlwi $e,$a,5 + xor @X[$j%16],@X[$j%16],@X[($j+2)%16] + add $f,$f,@X[$i%16] + and $t0,$c,$b + xor @X[$j%16],@X[$j%16],@X[($j+8)%16] + add $f,$f,$e + andc $t1,$d,$b + rotlwi $b,$b,30 + or $t0,$t0,$t1 + xor @X[$j%16],@X[$j%16],@X[($j+13)%16] + add $f,$f,$t0 + rotlwi @X[$j%16],@X[$j%16],1 +___ +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e,$f)=@_; +my $j=$i+1; +$code.=<<___ if ($i<79); + add $f,$K,$e + rotlwi $e,$a,5 + xor @X[$j%16],@X[$j%16],@X[($j+2)%16] + add $f,$f,@X[$i%16] + xor $t0,$b,$c + xor @X[$j%16],@X[$j%16],@X[($j+8)%16] + add $f,$f,$e + rotlwi $b,$b,30 + xor $t0,$t0,$d + xor @X[$j%16],@X[$j%16],@X[($j+13)%16] + add $f,$f,$t0 + rotlwi @X[$j%16],@X[$j%16],1 +___ +$code.=<<___ if ($i==79); + add $f,$K,$e + rotlwi $e,$a,5 + lwz r16,0($ctx) + add $f,$f,@X[$i%16] + xor $t0,$b,$c + lwz r17,4($ctx) + add $f,$f,$e + rotlwi $b,$b,30 + lwz r18,8($ctx) + xor $t0,$t0,$d + lwz r19,12($ctx) + add $f,$f,$t0 + lwz r20,16($ctx) +___ +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e,$f)=@_; +my $j=$i+1; +$code.=<<___; + add $f,$K,$e + rotlwi $e,$a,5 + xor @X[$j%16],@X[$j%16],@X[($j+2)%16] + add $f,$f,@X[$i%16] + and $t0,$b,$c + xor @X[$j%16],@X[$j%16],@X[($j+8)%16] + add $f,$f,$e + or $t1,$b,$c + rotlwi $b,$b,30 + xor @X[$j%16],@X[$j%16],@X[($j+13)%16] + and $t1,$t1,$d + or $t0,$t0,$t1 + rotlwi @X[$j%16],@X[$j%16],1 + add $f,$f,$t0 +___ +} + +$code=<<___; +.machine "any" +.text + +.globl .sha1_block_data_order +.align 4 +.sha1_block_data_order: + $STU $sp,-$FRAME($sp) + mflr r0 + $PUSH r15,`$FRAME-$SIZE_T*17`($sp) + $PUSH r16,`$FRAME-$SIZE_T*16`($sp) + $PUSH r17,`$FRAME-$SIZE_T*15`($sp) + $PUSH r18,`$FRAME-$SIZE_T*14`($sp) + $PUSH r19,`$FRAME-$SIZE_T*13`($sp) + $PUSH r20,`$FRAME-$SIZE_T*12`($sp) + $PUSH r21,`$FRAME-$SIZE_T*11`($sp) + $PUSH r22,`$FRAME-$SIZE_T*10`($sp) + $PUSH r23,`$FRAME-$SIZE_T*9`($sp) + $PUSH r24,`$FRAME-$SIZE_T*8`($sp) + $PUSH r25,`$FRAME-$SIZE_T*7`($sp) + $PUSH r26,`$FRAME-$SIZE_T*6`($sp) + $PUSH r27,`$FRAME-$SIZE_T*5`($sp) + $PUSH r28,`$FRAME-$SIZE_T*4`($sp) + $PUSH r29,`$FRAME-$SIZE_T*3`($sp) + $PUSH r30,`$FRAME-$SIZE_T*2`($sp) + $PUSH r31,`$FRAME-$SIZE_T*1`($sp) + $PUSH r0,`$FRAME+$LRSAVE`($sp) + lwz $A,0($ctx) + lwz $B,4($ctx) + lwz $C,8($ctx) + lwz $D,12($ctx) + lwz $E,16($ctx) + andi. r0,$inp,3 + bne Lunaligned +Laligned: + mtctr $num + bl Lsha1_block_private + b Ldone + +; PowerPC specification allows an implementation to be ill-behaved +; upon unaligned access which crosses page boundary. "Better safe +; than sorry" principle makes me treat it specially. But I don't +; look for particular offending word, but rather for 64-byte input +; block which crosses the boundary. Once found that block is aligned +; and hashed separately... +.align 4 +Lunaligned: + subfic $t1,$inp,4096 + andi. $t1,$t1,4095 ; distance to closest page boundary + srwi. $t1,$t1,6 ; t1/=64 + beq Lcross_page + $UCMP $num,$t1 + ble- Laligned ; didn't cross the page boundary + mtctr $t1 + subfc $num,$t1,$num + bl Lsha1_block_private +Lcross_page: + li $t1,16 + mtctr $t1 + addi r20,$sp,$LOCALS ; spot within the frame +Lmemcpy: + lbz r16,0($inp) + lbz r17,1($inp) + lbz r18,2($inp) + lbz r19,3($inp) + addi $inp,$inp,4 + stb r16,0(r20) + stb r17,1(r20) + stb r18,2(r20) + stb r19,3(r20) + addi r20,r20,4 + bdnz Lmemcpy + + $PUSH $inp,`$FRAME-$SIZE_T*18`($sp) + li $t1,1 + addi $inp,$sp,$LOCALS + mtctr $t1 + bl Lsha1_block_private + $POP $inp,`$FRAME-$SIZE_T*18`($sp) + addic. $num,$num,-1 + bne- Lunaligned + +Ldone: + $POP r0,`$FRAME+$LRSAVE`($sp) + $POP r15,`$FRAME-$SIZE_T*17`($sp) + $POP r16,`$FRAME-$SIZE_T*16`($sp) + $POP r17,`$FRAME-$SIZE_T*15`($sp) + $POP r18,`$FRAME-$SIZE_T*14`($sp) + $POP r19,`$FRAME-$SIZE_T*13`($sp) + $POP r20,`$FRAME-$SIZE_T*12`($sp) + $POP r21,`$FRAME-$SIZE_T*11`($sp) + $POP r22,`$FRAME-$SIZE_T*10`($sp) + $POP r23,`$FRAME-$SIZE_T*9`($sp) + $POP r24,`$FRAME-$SIZE_T*8`($sp) + $POP r25,`$FRAME-$SIZE_T*7`($sp) + $POP r26,`$FRAME-$SIZE_T*6`($sp) + $POP r27,`$FRAME-$SIZE_T*5`($sp) + $POP r28,`$FRAME-$SIZE_T*4`($sp) + $POP r29,`$FRAME-$SIZE_T*3`($sp) + $POP r30,`$FRAME-$SIZE_T*2`($sp) + $POP r31,`$FRAME-$SIZE_T*1`($sp) + mtlr r0 + addi $sp,$sp,$FRAME + blr + .long 0 + .byte 0,12,4,1,0x80,18,3,0 + .long 0 +___ + +# This is private block function, which uses tailored calling +# interface, namely upon entry SHA_CTX is pre-loaded to given +# registers and counter register contains amount of chunks to +# digest... +$code.=<<___; +.align 4 +Lsha1_block_private: +___ +$code.=<<___; # load K_00_19 + lis $K,0x5a82 + ori $K,$K,0x7999 +___ +for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; # load K_20_39 + lis $K,0x6ed9 + ori $K,$K,0xeba1 +___ +for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; # load K_40_59 + lis $K,0x8f1b + ori $K,$K,0xbcdc +___ +for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; # load K_60_79 + lis $K,0xca62 + ori $K,$K,0xc1d6 +___ +for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + add r16,r16,$E + add r17,r17,$T + add r18,r18,$A + add r19,r19,$B + add r20,r20,$C + stw r16,0($ctx) + mr $A,r16 + stw r17,4($ctx) + mr $B,r17 + stw r18,8($ctx) + mr $C,r18 + stw r19,12($ctx) + mr $D,r19 + stw r20,16($ctx) + mr $E,r20 + addi $inp,$inp,`16*4` + bdnz- Lsha1_block_private + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +___ +$code.=<<___; +.asciz "SHA1 block transform for PPC, CRYPTOGAMS by <appro\@fy.chalmers.se>" +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha1-s390x.pl b/crypto/openssl/crypto/sha/asm/sha1-s390x.pl new file mode 100755 index 000000000000..9193dda45eff --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-s390x.pl @@ -0,0 +1,246 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA1 block procedure for s390x. + +# April 2007. +# +# Performance is >30% better than gcc 3.3 generated code. But the real +# twist is that SHA1 hardware support is detected and utilized. In +# which case performance can reach further >4.5x for larger chunks. + +# January 2009. +# +# Optimize Xupdate for amount of memory references and reschedule +# instructions to favour dual-issue z10 pipeline. On z10 hardware is +# "only" ~2.3x faster than software. + +# November 2010. +# +# Adapt for -m31 build. If kernel supports what's called "highgprs" +# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit +# instructions and achieve "64-bit" performance even in 31-bit legacy +# application context. The feature is not specific to any particular +# processor, as long as it's "z-CPU". Latter implies that the code +# remains z/Architecture specific. + +$kimdfunc=1; # magic function code for kimd instruction + +$flavour = shift; + +if ($flavour =~ /3[12]/) { + $SIZE_T=4; + $g=""; +} else { + $SIZE_T=8; + $g="g"; +} + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +$K_00_39="%r0"; $K=$K_00_39; +$K_40_79="%r1"; +$ctx="%r2"; $prefetch="%r2"; +$inp="%r3"; +$len="%r4"; + +$A="%r5"; +$B="%r6"; +$C="%r7"; +$D="%r8"; +$E="%r9"; @V=($A,$B,$C,$D,$E); +$t0="%r10"; +$t1="%r11"; +@X=("%r12","%r13","%r14"); +$sp="%r15"; + +$stdframe=16*$SIZE_T+4*8; +$frame=$stdframe+16*4; + +sub Xupdate { +my $i=shift; + +$code.=<<___ if ($i==15); + lg $prefetch,$stdframe($sp) ### Xupdate(16) warm-up + lr $X[0],$X[2] +___ +return if ($i&1); # Xupdate is vectorized and executed every 2nd cycle +$code.=<<___ if ($i<16); + lg $X[0],`$i*4`($inp) ### Xload($i) + rllg $X[1],$X[0],32 +___ +$code.=<<___ if ($i>=16); + xgr $X[0],$prefetch ### Xupdate($i) + lg $prefetch,`$stdframe+4*(($i+2)%16)`($sp) + xg $X[0],`$stdframe+4*(($i+8)%16)`($sp) + xgr $X[0],$prefetch + rll $X[0],$X[0],1 + rllg $X[1],$X[0],32 + rll $X[1],$X[1],1 + rllg $X[0],$X[1],32 + lr $X[2],$X[1] # feedback +___ +$code.=<<___ if ($i<=70); + stg $X[0],`$stdframe+4*($i%16)`($sp) +___ +unshift(@X,pop(@X)); +} + +sub BODY_00_19 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $xi=$X[1]; + + &Xupdate($i); +$code.=<<___; + alr $e,$K ### $i + rll $t1,$a,5 + lr $t0,$d + xr $t0,$c + alr $e,$t1 + nr $t0,$b + alr $e,$xi + xr $t0,$d + rll $b,$b,30 + alr $e,$t0 +___ +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $xi=$X[1]; + + &Xupdate($i); +$code.=<<___; + alr $e,$K ### $i + rll $t1,$a,5 + lr $t0,$b + alr $e,$t1 + xr $t0,$c + alr $e,$xi + xr $t0,$d + rll $b,$b,30 + alr $e,$t0 +___ +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $xi=$X[1]; + + &Xupdate($i); +$code.=<<___; + alr $e,$K ### $i + rll $t1,$a,5 + lr $t0,$b + alr $e,$t1 + or $t0,$c + lr $t1,$b + nr $t0,$d + nr $t1,$c + alr $e,$xi + or $t0,$t1 + rll $b,$b,30 + alr $e,$t0 +___ +} + +$code.=<<___; +.text +.align 64 +.type Ktable,\@object +Ktable: .long 0x5a827999,0x6ed9eba1,0x8f1bbcdc,0xca62c1d6 + .skip 48 #.long 0,0,0,0,0,0,0,0,0,0,0,0 +.size Ktable,.-Ktable +.globl sha1_block_data_order +.type sha1_block_data_order,\@function +sha1_block_data_order: +___ +$code.=<<___ if ($kimdfunc); + larl %r1,OPENSSL_s390xcap_P + lg %r0,0(%r1) + tmhl %r0,0x4000 # check for message-security assist + jz .Lsoftware + lghi %r0,0 + la %r1,`2*$SIZE_T`($sp) + .long 0xb93e0002 # kimd %r0,%r2 + lg %r0,`2*$SIZE_T`($sp) + tmhh %r0,`0x8000>>$kimdfunc` + jz .Lsoftware + lghi %r0,$kimdfunc + lgr %r1,$ctx + lgr %r2,$inp + sllg %r3,$len,6 + .long 0xb93e0002 # kimd %r0,%r2 + brc 1,.-4 # pay attention to "partial completion" + br %r14 +.align 16 +.Lsoftware: +___ +$code.=<<___; + lghi %r1,-$frame + st${g} $ctx,`2*$SIZE_T`($sp) + stm${g} %r6,%r15,`6*$SIZE_T`($sp) + lgr %r0,$sp + la $sp,0(%r1,$sp) + st${g} %r0,0($sp) + + larl $t0,Ktable + llgf $A,0($ctx) + llgf $B,4($ctx) + llgf $C,8($ctx) + llgf $D,12($ctx) + llgf $E,16($ctx) + + lg $K_00_39,0($t0) + lg $K_40_79,8($t0) + +.Lloop: + rllg $K_00_39,$K_00_39,32 +___ +for ($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + rllg $K_00_39,$K_00_39,32 +___ +for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; $K=$K_40_79; + rllg $K_40_79,$K_40_79,32 +___ +for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + rllg $K_40_79,$K_40_79,32 +___ +for (;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + + l${g} $ctx,`$frame+2*$SIZE_T`($sp) + la $inp,64($inp) + al $A,0($ctx) + al $B,4($ctx) + al $C,8($ctx) + al $D,12($ctx) + al $E,16($ctx) + st $A,0($ctx) + st $B,4($ctx) + st $C,8($ctx) + st $D,12($ctx) + st $E,16($ctx) + brct${g} $len,.Lloop + + lm${g} %r6,%r15,`$frame+6*$SIZE_T`($sp) + br %r14 +.size sha1_block_data_order,.-sha1_block_data_order +.string "SHA1 block transform for s390x, CRYPTOGAMS by <appro\@openssl.org>" +.comm OPENSSL_s390xcap_P,16,8 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; + +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha1-sparcv9.pl b/crypto/openssl/crypto/sha/asm/sha1-sparcv9.pl new file mode 100755 index 000000000000..5c161cecd696 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-sparcv9.pl @@ -0,0 +1,284 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# Performance improvement is not really impressive on pre-T1 CPU: +8% +# over Sun C and +25% over gcc [3.3]. While on T1, a.k.a. Niagara, it +# turned to be 40% faster than 64-bit code generated by Sun C 5.8 and +# >2x than 64-bit code generated by gcc 3.4. And there is a gimmick. +# X[16] vector is packed to 8 64-bit registers and as result nothing +# is spilled on stack. In addition input data is loaded in compact +# instruction sequence, thus minimizing the window when the code is +# subject to [inter-thread] cache-thrashing hazard. The goal is to +# ensure scalability on UltraSPARC T1, or rather to avoid decay when +# amount of active threads exceeds the number of physical cores. + +$bits=32; +for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } +if ($bits==64) { $bias=2047; $frame=192; } +else { $bias=0; $frame=112; } + +$output=shift; +open STDOUT,">$output"; + +@X=("%o0","%o1","%o2","%o3","%o4","%o5","%g1","%o7"); +$rot1m="%g2"; +$tmp64="%g3"; +$Xi="%g4"; +$A="%l0"; +$B="%l1"; +$C="%l2"; +$D="%l3"; +$E="%l4"; +@V=($A,$B,$C,$D,$E); +$K_00_19="%l5"; +$K_20_39="%l6"; +$K_40_59="%l7"; +$K_60_79="%g5"; +@K=($K_00_19,$K_20_39,$K_40_59,$K_60_79); + +$ctx="%i0"; +$inp="%i1"; +$len="%i2"; +$tmp0="%i3"; +$tmp1="%i4"; +$tmp2="%i5"; + +sub BODY_00_15 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $xi=($i&1)?@X[($i/2)%8]:$Xi; + +$code.=<<___; + sll $a,5,$tmp0 !! $i + add @K[$i/20],$e,$e + srl $a,27,$tmp1 + add $tmp0,$e,$e + and $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + andn $d,$b,$tmp1 + srl $b,2,$b + or $tmp1,$tmp0,$tmp1 + or $tmp2,$b,$b + add $xi,$e,$e +___ +if ($i&1 && $i<15) { + $code.= + " srlx @X[(($i+1)/2)%8],32,$Xi\n"; +} +$code.=<<___; + add $tmp1,$e,$e +___ +} + +sub Xupdate { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i/2; + +if ($i&1) { +$code.=<<___; + sll $a,5,$tmp0 !! $i + add @K[$i/20],$e,$e + srl $a,27,$tmp1 +___ +} else { +$code.=<<___; + sllx @X[($j+6)%8],32,$Xi ! Xupdate($i) + xor @X[($j+1)%8],@X[$j%8],@X[$j%8] + srlx @X[($j+7)%8],32,$tmp1 + xor @X[($j+4)%8],@X[$j%8],@X[$j%8] + sll $a,5,$tmp0 !! $i + or $tmp1,$Xi,$Xi + add @K[$i/20],$e,$e !! + xor $Xi,@X[$j%8],@X[$j%8] + srlx @X[$j%8],31,$Xi + add @X[$j%8],@X[$j%8],@X[$j%8] + and $Xi,$rot1m,$Xi + andn @X[$j%8],$rot1m,@X[$j%8] + srl $a,27,$tmp1 !! + or $Xi,@X[$j%8],@X[$j%8] +___ +} +} + +sub BODY_16_19 { +my ($i,$a,$b,$c,$d,$e)=@_; + + &Xupdate(@_); + if ($i&1) { + $xi=@X[($i/2)%8]; + } else { + $xi=$Xi; + $code.="\tsrlx @X[($i/2)%8],32,$xi\n"; + } +$code.=<<___; + add $tmp0,$e,$e !! + and $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + add $xi,$e,$e + andn $d,$b,$tmp1 + srl $b,2,$b + or $tmp1,$tmp0,$tmp1 + or $tmp2,$b,$b + add $tmp1,$e,$e +___ +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $xi; + &Xupdate(@_); + if ($i&1) { + $xi=@X[($i/2)%8]; + } else { + $xi=$Xi; + $code.="\tsrlx @X[($i/2)%8],32,$xi\n"; + } +$code.=<<___; + add $tmp0,$e,$e !! + xor $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + srl $b,2,$b + add $tmp1,$e,$e + or $tmp2,$b,$b + add $xi,$e,$e +___ +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $xi; + &Xupdate(@_); + if ($i&1) { + $xi=@X[($i/2)%8]; + } else { + $xi=$Xi; + $code.="\tsrlx @X[($i/2)%8],32,$xi\n"; + } +$code.=<<___; + add $tmp0,$e,$e !! + and $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + or $c,$b,$tmp1 + srl $b,2,$b + and $d,$tmp1,$tmp1 + add $xi,$e,$e + or $tmp1,$tmp0,$tmp1 + or $tmp2,$b,$b + add $tmp1,$e,$e +___ +} + +$code.=<<___ if ($bits==64); +.register %g2,#scratch +.register %g3,#scratch +___ +$code.=<<___; +.section ".text",#alloc,#execinstr + +.align 32 +.globl sha1_block_data_order +sha1_block_data_order: + save %sp,-$frame,%sp + sllx $len,6,$len + add $inp,$len,$len + + or %g0,1,$rot1m + sllx $rot1m,32,$rot1m + or $rot1m,1,$rot1m + + ld [$ctx+0],$A + ld [$ctx+4],$B + ld [$ctx+8],$C + ld [$ctx+12],$D + ld [$ctx+16],$E + andn $inp,7,$tmp0 + + sethi %hi(0x5a827999),$K_00_19 + or $K_00_19,%lo(0x5a827999),$K_00_19 + sethi %hi(0x6ed9eba1),$K_20_39 + or $K_20_39,%lo(0x6ed9eba1),$K_20_39 + sethi %hi(0x8f1bbcdc),$K_40_59 + or $K_40_59,%lo(0x8f1bbcdc),$K_40_59 + sethi %hi(0xca62c1d6),$K_60_79 + or $K_60_79,%lo(0xca62c1d6),$K_60_79 + +.Lloop: + ldx [$tmp0+0],@X[0] + ldx [$tmp0+16],@X[2] + ldx [$tmp0+32],@X[4] + ldx [$tmp0+48],@X[6] + and $inp,7,$tmp1 + ldx [$tmp0+8],@X[1] + sll $tmp1,3,$tmp1 + ldx [$tmp0+24],@X[3] + subcc %g0,$tmp1,$tmp2 ! should be 64-$tmp1, but -$tmp1 works too + ldx [$tmp0+40],@X[5] + bz,pt %icc,.Laligned + ldx [$tmp0+56],@X[7] + + sllx @X[0],$tmp1,@X[0] + ldx [$tmp0+64],$tmp64 +___ +for($i=0;$i<7;$i++) +{ $code.=<<___; + srlx @X[$i+1],$tmp2,$Xi + sllx @X[$i+1],$tmp1,@X[$i+1] + or $Xi,@X[$i],@X[$i] +___ +} +$code.=<<___; + srlx $tmp64,$tmp2,$tmp64 + or $tmp64,@X[7],@X[7] +.Laligned: + srlx @X[0],32,$Xi +___ +for ($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } +for (;$i<20;$i++) { &BODY_16_19($i,@V); unshift(@V,pop(@V)); } +for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +for (;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + + ld [$ctx+0],@X[0] + ld [$ctx+4],@X[1] + ld [$ctx+8],@X[2] + ld [$ctx+12],@X[3] + add $inp,64,$inp + ld [$ctx+16],@X[4] + cmp $inp,$len + + add $A,@X[0],$A + st $A,[$ctx+0] + add $B,@X[1],$B + st $B,[$ctx+4] + add $C,@X[2],$C + st $C,[$ctx+8] + add $D,@X[3],$D + st $D,[$ctx+12] + add $E,@X[4],$E + st $E,[$ctx+16] + + bne `$bits==64?"%xcc":"%icc"`,.Lloop + andn $inp,7,$tmp0 + + ret + restore +.type sha1_block_data_order,#function +.size sha1_block_data_order,(.-sha1_block_data_order) +.asciz "SHA1 block transform for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>" +.align 4 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha1-sparcv9a.pl b/crypto/openssl/crypto/sha/asm/sha1-sparcv9a.pl new file mode 100755 index 000000000000..85e8d68086bb --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-sparcv9a.pl @@ -0,0 +1,601 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# January 2009 +# +# Provided that UltraSPARC VIS instructions are pipe-lined(*) and +# pairable(*) with IALU ones, offloading of Xupdate to the UltraSPARC +# Graphic Unit would make it possible to achieve higher instruction- +# level parallelism, ILP, and thus higher performance. It should be +# explicitly noted that ILP is the keyword, and it means that this +# code would be unsuitable for cores like UltraSPARC-Tx. The idea is +# not really novel, Sun had VIS-powered implementation for a while. +# Unlike Sun's implementation this one can process multiple unaligned +# input blocks, and as such works as drop-in replacement for OpenSSL +# sha1_block_data_order. Performance improvement was measured to be +# 40% over pure IALU sha1-sparcv9.pl on UltraSPARC-IIi, but 12% on +# UltraSPARC-III. See below for discussion... +# +# The module does not present direct interest for OpenSSL, because +# it doesn't provide better performance on contemporary SPARCv9 CPUs, +# UltraSPARC-Tx and SPARC64-V[II] to be specific. Those who feel they +# absolutely must score on UltraSPARC-I-IV can simply replace +# crypto/sha/asm/sha1-sparcv9.pl with this module. +# +# (*) "Pipe-lined" means that even if it takes several cycles to +# complete, next instruction using same functional unit [but not +# depending on the result of the current instruction] can start +# execution without having to wait for the unit. "Pairable" +# means that two [or more] independent instructions can be +# issued at the very same time. + +$bits=32; +for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } +if ($bits==64) { $bias=2047; $frame=192; } +else { $bias=0; $frame=112; } + +$output=shift; +open STDOUT,">$output"; + +$ctx="%i0"; +$inp="%i1"; +$len="%i2"; +$tmp0="%i3"; +$tmp1="%i4"; +$tmp2="%i5"; +$tmp3="%g5"; + +$base="%g1"; +$align="%g4"; +$Xfer="%o5"; +$nXfer=$tmp3; +$Xi="%o7"; + +$A="%l0"; +$B="%l1"; +$C="%l2"; +$D="%l3"; +$E="%l4"; +@V=($A,$B,$C,$D,$E); + +$Actx="%o0"; +$Bctx="%o1"; +$Cctx="%o2"; +$Dctx="%o3"; +$Ectx="%o4"; + +$fmul="%f32"; +$VK_00_19="%f34"; +$VK_20_39="%f36"; +$VK_40_59="%f38"; +$VK_60_79="%f40"; +@VK=($VK_00_19,$VK_20_39,$VK_40_59,$VK_60_79); +@X=("%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7", + "%f8", "%f9","%f10","%f11","%f12","%f13","%f14","%f15","%f16"); + +# This is reference 2x-parallelized VIS-powered Xupdate procedure. It +# covers even K_NN_MM addition... +sub Xupdate { +my ($i)=@_; +my $K=@VK[($i+16)/20]; +my $j=($i+16)%16; + +# [ provided that GSR.alignaddr_offset is 5, $mul contains +# 0x100ULL<<32|0x100 value and K_NN_MM are pre-loaded to +# chosen registers... ] +$code.=<<___; + fxors @X[($j+13)%16],@X[$j],@X[$j] !-1/-1/-1:X[0]^=X[13] + fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14] + fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9] + fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9] + faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24 + fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1 + fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1 + ![fxors %f15,%f2,%f2] + for %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp + ![fxors %f0,%f3,%f3] !10/17/12:X[0] dependency + fpadd32 $K,@X[$j],%f20 + std %f20,[$Xfer+`4*$j`] +___ +# The numbers delimited with slash are the earliest possible dispatch +# cycles for given instruction assuming 1 cycle latency for simple VIS +# instructions, such as on UltraSPARC-I&II, 3 cycles latency, such as +# on UltraSPARC-III&IV, and 2 cycles latency(*), respectively. Being +# 2x-parallelized the procedure is "worth" 5, 8.5 or 6 ticks per SHA1 +# round. As [long as] FPU/VIS instructions are perfectly pairable with +# IALU ones, the round timing is defined by the maximum between VIS +# and IALU timings. The latter varies from round to round and averages +# out at 6.25 ticks. This means that USI&II should operate at IALU +# rate, while USIII&IV - at VIS rate. This explains why performance +# improvement varies among processors. Well, given that pure IALU +# sha1-sparcv9.pl module exhibits virtually uniform performance of +# ~9.3 cycles per SHA1 round. Timings mentioned above are theoretical +# lower limits. Real-life performance was measured to be 6.6 cycles +# per SHA1 round on USIIi and 8.3 on USIII. The latter is lower than +# half-round VIS timing, because there are 16 Xupdate-free rounds, +# which "push down" average theoretical timing to 8 cycles... + +# (*) SPARC64-V[II] was originally believed to have 2 cycles VIS +# latency. Well, it might have, but it doesn't have dedicated +# VIS-unit. Instead, VIS instructions are executed by other +# functional units, ones used here - by IALU. This doesn't +# improve effective ILP... +} + +# The reference Xupdate procedure is then "strained" over *pairs* of +# BODY_NN_MM and kind of modulo-scheduled in respect to X[n]^=X[n+13] +# and K_NN_MM addition. It's "running" 15 rounds ahead, which leaves +# plenty of room to amortize for read-after-write hazard, as well as +# to fetch and align input for the next spin. The VIS instructions are +# scheduled for latency of 2 cycles, because there are not enough IALU +# instructions to schedule for latency of 3, while scheduling for 1 +# would give no gain on USI&II anyway. + +sub BODY_00_19 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i&~1; +my $k=($j+16+2)%16; # ahead reference +my $l=($j+16-2)%16; # behind reference +my $K=@VK[($j+16-2)/20]; + +$j=($j+16)%16; + +$code.=<<___ if (!($i&1)); + sll $a,5,$tmp0 !! $i + and $c,$b,$tmp3 + ld [$Xfer+`4*($i%16)`],$Xi + fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14] + srl $a,27,$tmp1 + add $tmp0,$e,$e + fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9] + sll $b,30,$tmp2 + add $tmp1,$e,$e + andn $d,$b,$tmp1 + add $Xi,$e,$e + fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9] + srl $b,2,$b + or $tmp1,$tmp3,$tmp1 + or $tmp2,$b,$b + add $tmp1,$e,$e + faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24 +___ +$code.=<<___ if ($i&1); + sll $a,5,$tmp0 !! $i + and $c,$b,$tmp3 + ld [$Xfer+`4*($i%16)`],$Xi + fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1 + srl $a,27,$tmp1 + add $tmp0,$e,$e + fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1 + sll $b,30,$tmp2 + add $tmp1,$e,$e + fpadd32 $K,@X[$l],%f20 ! + andn $d,$b,$tmp1 + add $Xi,$e,$e + fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13] + srl $b,2,$b + or $tmp1,$tmp3,$tmp1 + fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp + or $tmp2,$b,$b + add $tmp1,$e,$e +___ +$code.=<<___ if ($i&1 && $i>=2); + std %f20,[$Xfer+`4*$l`] ! +___ +} + +sub BODY_20_39 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i&~1; +my $k=($j+16+2)%16; # ahead reference +my $l=($j+16-2)%16; # behind reference +my $K=@VK[($j+16-2)/20]; + +$j=($j+16)%16; + +$code.=<<___ if (!($i&1) && $i<64); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14] + srl $a,27,$tmp1 + add $tmp0,$e,$e + fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9] + xor $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9] + srl $b,2,$b + add $tmp1,$e,$e + or $tmp2,$b,$b + add $Xi,$e,$e + faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24 +___ +$code.=<<___ if ($i&1 && $i<64); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1 + srl $a,27,$tmp1 + add $tmp0,$e,$e + fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1 + xor $c,$b,$tmp0 + add $tmp1,$e,$e + fpadd32 $K,@X[$l],%f20 ! + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13] + srl $b,2,$b + add $tmp1,$e,$e + fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp + or $tmp2,$b,$b + add $Xi,$e,$e + std %f20,[$Xfer+`4*$l`] ! +___ +$code.=<<___ if ($i==64); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + fpadd32 $K,@X[$l],%f20 + srl $a,27,$tmp1 + add $tmp0,$e,$e + xor $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + std %f20,[$Xfer+`4*$l`] + srl $b,2,$b + add $tmp1,$e,$e + or $tmp2,$b,$b + add $Xi,$e,$e +___ +$code.=<<___ if ($i>64); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + srl $a,27,$tmp1 + add $tmp0,$e,$e + xor $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + srl $b,2,$b + add $tmp1,$e,$e + or $tmp2,$b,$b + add $Xi,$e,$e +___ +} + +sub BODY_40_59 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i&~1; +my $k=($j+16+2)%16; # ahead reference +my $l=($j+16-2)%16; # behind reference +my $K=@VK[($j+16-2)/20]; + +$j=($j+16)%16; + +$code.=<<___ if (!($i&1)); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14] + srl $a,27,$tmp1 + add $tmp0,$e,$e + fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9] + and $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + or $c,$b,$tmp1 + fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9] + srl $b,2,$b + and $d,$tmp1,$tmp1 + add $Xi,$e,$e + or $tmp1,$tmp0,$tmp1 + faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24 + or $tmp2,$b,$b + add $tmp1,$e,$e + fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1 +___ +$code.=<<___ if ($i&1); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + srl $a,27,$tmp1 + add $tmp0,$e,$e + fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1 + and $c,$b,$tmp0 + add $tmp1,$e,$e + fpadd32 $K,@X[$l],%f20 ! + sll $b,30,$tmp2 + or $c,$b,$tmp1 + fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13] + srl $b,2,$b + and $d,$tmp1,$tmp1 + fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp + add $Xi,$e,$e + or $tmp1,$tmp0,$tmp1 + or $tmp2,$b,$b + add $tmp1,$e,$e + std %f20,[$Xfer+`4*$l`] ! +___ +} + +# If there is more data to process, then we pre-fetch the data for +# next iteration in last ten rounds... +sub BODY_70_79 { +my ($i,$a,$b,$c,$d,$e)=@_; +my $j=$i&~1; +my $m=($i%8)*2; + +$j=($j+16)%16; + +$code.=<<___ if ($i==70); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + srl $a,27,$tmp1 + add $tmp0,$e,$e + ldd [$inp+64],@X[0] + xor $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + srl $b,2,$b + add $tmp1,$e,$e + or $tmp2,$b,$b + add $Xi,$e,$e + + and $inp,-64,$nXfer + inc 64,$inp + and $nXfer,255,$nXfer + alignaddr %g0,$align,%g0 + add $base,$nXfer,$nXfer +___ +$code.=<<___ if ($i==71); + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + srl $a,27,$tmp1 + add $tmp0,$e,$e + xor $c,$b,$tmp0 + add $tmp1,$e,$e + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + srl $b,2,$b + add $tmp1,$e,$e + or $tmp2,$b,$b + add $Xi,$e,$e +___ +$code.=<<___ if ($i>=72); + faligndata @X[$m],@X[$m+2],@X[$m] + sll $a,5,$tmp0 !! $i + ld [$Xfer+`4*($i%16)`],$Xi + srl $a,27,$tmp1 + add $tmp0,$e,$e + xor $c,$b,$tmp0 + add $tmp1,$e,$e + fpadd32 $VK_00_19,@X[$m],%f20 + sll $b,30,$tmp2 + xor $d,$tmp0,$tmp1 + srl $b,2,$b + add $tmp1,$e,$e + or $tmp2,$b,$b + add $Xi,$e,$e +___ +$code.=<<___ if ($i<77); + ldd [$inp+`8*($i+1-70)`],@X[2*($i+1-70)] +___ +$code.=<<___ if ($i==77); # redundant if $inp was aligned + add $align,63,$tmp0 + and $tmp0,-8,$tmp0 + ldd [$inp+$tmp0],@X[16] +___ +$code.=<<___ if ($i>=72); + std %f20,[$nXfer+`4*$m`] +___ +} + +$code.=<<___; +.section ".text",#alloc,#execinstr + +.align 64 +vis_const: +.long 0x5a827999,0x5a827999 ! K_00_19 +.long 0x6ed9eba1,0x6ed9eba1 ! K_20_39 +.long 0x8f1bbcdc,0x8f1bbcdc ! K_40_59 +.long 0xca62c1d6,0xca62c1d6 ! K_60_79 +.long 0x00000100,0x00000100 +.align 64 +.type vis_const,#object +.size vis_const,(.-vis_const) + +.globl sha1_block_data_order +sha1_block_data_order: + save %sp,-$frame,%sp + add %fp,$bias-256,$base + +1: call .+8 + add %o7,vis_const-1b,$tmp0 + + ldd [$tmp0+0],$VK_00_19 + ldd [$tmp0+8],$VK_20_39 + ldd [$tmp0+16],$VK_40_59 + ldd [$tmp0+24],$VK_60_79 + ldd [$tmp0+32],$fmul + + ld [$ctx+0],$Actx + and $base,-256,$base + ld [$ctx+4],$Bctx + sub $base,$bias+$frame,%sp + ld [$ctx+8],$Cctx + and $inp,7,$align + ld [$ctx+12],$Dctx + and $inp,-8,$inp + ld [$ctx+16],$Ectx + + ! X[16] is maintained in FP register bank + alignaddr %g0,$align,%g0 + ldd [$inp+0],@X[0] + sub $inp,-64,$Xfer + ldd [$inp+8],@X[2] + and $Xfer,-64,$Xfer + ldd [$inp+16],@X[4] + and $Xfer,255,$Xfer + ldd [$inp+24],@X[6] + add $base,$Xfer,$Xfer + ldd [$inp+32],@X[8] + ldd [$inp+40],@X[10] + ldd [$inp+48],@X[12] + brz,pt $align,.Laligned + ldd [$inp+56],@X[14] + + ldd [$inp+64],@X[16] + faligndata @X[0],@X[2],@X[0] + faligndata @X[2],@X[4],@X[2] + faligndata @X[4],@X[6],@X[4] + faligndata @X[6],@X[8],@X[6] + faligndata @X[8],@X[10],@X[8] + faligndata @X[10],@X[12],@X[10] + faligndata @X[12],@X[14],@X[12] + faligndata @X[14],@X[16],@X[14] + +.Laligned: + mov 5,$tmp0 + dec 1,$len + alignaddr %g0,$tmp0,%g0 + fpadd32 $VK_00_19,@X[0],%f16 + fpadd32 $VK_00_19,@X[2],%f18 + fpadd32 $VK_00_19,@X[4],%f20 + fpadd32 $VK_00_19,@X[6],%f22 + fpadd32 $VK_00_19,@X[8],%f24 + fpadd32 $VK_00_19,@X[10],%f26 + fpadd32 $VK_00_19,@X[12],%f28 + fpadd32 $VK_00_19,@X[14],%f30 + std %f16,[$Xfer+0] + mov $Actx,$A + std %f18,[$Xfer+8] + mov $Bctx,$B + std %f20,[$Xfer+16] + mov $Cctx,$C + std %f22,[$Xfer+24] + mov $Dctx,$D + std %f24,[$Xfer+32] + mov $Ectx,$E + std %f26,[$Xfer+40] + fxors @X[13],@X[0],@X[0] + std %f28,[$Xfer+48] + ba .Loop + std %f30,[$Xfer+56] +.align 32 +.Loop: +___ +for ($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } +for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } +for (;$i<70;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + tst $len + bz,pn `$bits==32?"%icc":"%xcc"`,.Ltail + nop +___ +for (;$i<80;$i++) { &BODY_70_79($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + add $A,$Actx,$Actx + add $B,$Bctx,$Bctx + add $C,$Cctx,$Cctx + add $D,$Dctx,$Dctx + add $E,$Ectx,$Ectx + mov 5,$tmp0 + fxors @X[13],@X[0],@X[0] + mov $Actx,$A + mov $Bctx,$B + mov $Cctx,$C + mov $Dctx,$D + mov $Ectx,$E + alignaddr %g0,$tmp0,%g0 + dec 1,$len + ba .Loop + mov $nXfer,$Xfer + +.align 32 +.Ltail: +___ +for($i=70;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + add $A,$Actx,$Actx + add $B,$Bctx,$Bctx + add $C,$Cctx,$Cctx + add $D,$Dctx,$Dctx + add $E,$Ectx,$Ectx + + st $Actx,[$ctx+0] + st $Bctx,[$ctx+4] + st $Cctx,[$ctx+8] + st $Dctx,[$ctx+12] + st $Ectx,[$ctx+16] + + ret + restore +.type sha1_block_data_order,#function +.size sha1_block_data_order,(.-sha1_block_data_order) +.asciz "SHA1 block transform for SPARCv9a, CRYPTOGAMS by <appro\@openssl.org>" +.align 4 +___ + +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my $ref,$opf; +my %visopf = ( "fmul8ulx16" => 0x037, + "faligndata" => 0x048, + "fpadd32" => 0x052, + "fxor" => 0x06c, + "fxors" => 0x06d ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} +sub unalignaddr { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my $ref="$mnemonic\t$rs1,$rs2,$rd"; + + foreach ($rs1,$rs2,$rd) { + if (/%([goli])([0-7])/) { $_=$bias{$1}+$2; } + else { return $ref; } + } + return sprintf ".word\t0x%08x !%s", + 0x81b00300|$rd<<25|$rs1<<14|$rs2, + $ref; +} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +$code =~ s/\b(f[^\s]*)\s+(%f[0-9]{1,2}),(%f[0-9]{1,2}),(%f[0-9]{1,2})/ + &unvis($1,$2,$3,$4) + /gem; +$code =~ s/\b(alignaddr)\s+(%[goli][0-7]),(%[goli][0-7]),(%[goli][0-7])/ + &unalignaddr($1,$2,$3,$4) + /gem; +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha1-thumb.pl b/crypto/openssl/crypto/sha/asm/sha1-thumb.pl new file mode 100755 index 000000000000..7c9ea9b0296c --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha1-thumb.pl @@ -0,0 +1,259 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# sha1_block for Thumb. +# +# January 2007. +# +# The code does not present direct interest to OpenSSL, because of low +# performance. Its purpose is to establish _size_ benchmark. Pretty +# useless one I must say, because 30% or 88 bytes larger ARMv4 code +# [avialable on demand] is almost _twice_ as fast. It should also be +# noted that in-lining of .Lcommon and .Lrotate improves performance +# by over 40%, while code increases by only 10% or 32 bytes. But once +# again, the goal was to establish _size_ benchmark, not performance. + +$output=shift; +open STDOUT,">$output"; + +$inline=0; +#$cheat_on_binutils=1; + +$t0="r0"; +$t1="r1"; +$t2="r2"; +$a="r3"; +$b="r4"; +$c="r5"; +$d="r6"; +$e="r7"; +$K="r8"; # "upper" registers can be used in add/sub and mov insns +$ctx="r9"; +$inp="r10"; +$len="r11"; +$Xi="r12"; + +sub common { +<<___; + sub $t0,#4 + ldr $t1,[$t0] + add $e,$K @ E+=K_xx_xx + lsl $t2,$a,#5 + add $t2,$e + lsr $e,$a,#27 + add $t2,$e @ E+=ROR(A,27) + add $t2,$t1 @ E+=X[i] +___ +} +sub rotate { +<<___; + mov $e,$d @ E=D + mov $d,$c @ D=C + lsl $c,$b,#30 + lsr $b,$b,#2 + orr $c,$b @ C=ROR(B,2) + mov $b,$a @ B=A + add $a,$t2,$t1 @ A=E+F_xx_xx(B,C,D) +___ +} + +sub BODY_00_19 { +$code.=$inline?&common():"\tbl .Lcommon\n"; +$code.=<<___; + mov $t1,$c + eor $t1,$d + and $t1,$b + eor $t1,$d @ F_00_19(B,C,D) +___ +$code.=$inline?&rotate():"\tbl .Lrotate\n"; +} + +sub BODY_20_39 { +$code.=$inline?&common():"\tbl .Lcommon\n"; +$code.=<<___; + mov $t1,$b + eor $t1,$c + eor $t1,$d @ F_20_39(B,C,D) +___ +$code.=$inline?&rotate():"\tbl .Lrotate\n"; +} + +sub BODY_40_59 { +$code.=$inline?&common():"\tbl .Lcommon\n"; +$code.=<<___; + mov $t1,$b + and $t1,$c + mov $e,$b + orr $e,$c + and $e,$d + orr $t1,$e @ F_40_59(B,C,D) +___ +$code.=$inline?&rotate():"\tbl .Lrotate\n"; +} + +$code=<<___; +.text +.code 16 + +.global sha1_block_data_order +.type sha1_block_data_order,%function + +.align 2 +sha1_block_data_order: +___ +if ($cheat_on_binutils) { +$code.=<<___; +.code 32 + add r3,pc,#1 + bx r3 @ switch to Thumb ISA +.code 16 +___ +} +$code.=<<___; + push {r4-r7} + mov r3,r8 + mov r4,r9 + mov r5,r10 + mov r6,r11 + mov r7,r12 + push {r3-r7,lr} + lsl r2,#6 + mov $ctx,r0 @ save context + mov $inp,r1 @ save inp + mov $len,r2 @ save len + add $len,$inp @ $len to point at inp end + +.Lloop: + mov $Xi,sp + mov $t2,sp + sub $t2,#16*4 @ [3] +.LXload: + ldrb $a,[$t1,#0] @ $t1 is r1 and holds inp + ldrb $b,[$t1,#1] + ldrb $c,[$t1,#2] + ldrb $d,[$t1,#3] + lsl $a,#24 + lsl $b,#16 + lsl $c,#8 + orr $a,$b + orr $a,$c + orr $a,$d + add $t1,#4 + push {$a} + cmp sp,$t2 + bne .LXload @ [+14*16] + + mov $inp,$t1 @ update $inp + sub $t2,#32*4 + sub $t2,#32*4 + mov $e,#31 @ [+4] +.LXupdate: + ldr $a,[sp,#15*4] + ldr $b,[sp,#13*4] + ldr $c,[sp,#7*4] + ldr $d,[sp,#2*4] + eor $a,$b + eor $a,$c + eor $a,$d + ror $a,$e + push {$a} + cmp sp,$t2 + bne .LXupdate @ [+(11+1)*64] + + ldmia $t0!,{$a,$b,$c,$d,$e} @ $t0 is r0 and holds ctx + mov $t0,$Xi + + ldr $t2,.LK_00_19 + mov $t1,$t0 + sub $t1,#20*4 + mov $Xi,$t1 + mov $K,$t2 @ [+7+4] +.L_00_19: +___ + &BODY_00_19(); +$code.=<<___; + cmp $Xi,$t0 + bne .L_00_19 @ [+(2+9+4+2+8+2)*20] + + ldr $t2,.LK_20_39 + mov $t1,$t0 + sub $t1,#20*4 + mov $Xi,$t1 + mov $K,$t2 @ [+5] +.L_20_39_or_60_79: +___ + &BODY_20_39(); +$code.=<<___; + cmp $Xi,$t0 + bne .L_20_39_or_60_79 @ [+(2+9+3+2+8+2)*20*2] + cmp sp,$t0 + beq .Ldone @ [+2] + + ldr $t2,.LK_40_59 + mov $t1,$t0 + sub $t1,#20*4 + mov $Xi,$t1 + mov $K,$t2 @ [+5] +.L_40_59: +___ + &BODY_40_59(); +$code.=<<___; + cmp $Xi,$t0 + bne .L_40_59 @ [+(2+9+6+2+8+2)*20] + + ldr $t2,.LK_60_79 + mov $Xi,sp + mov $K,$t2 + b .L_20_39_or_60_79 @ [+4] +.Ldone: + mov $t0,$ctx + ldr $t1,[$t0,#0] + ldr $t2,[$t0,#4] + add $a,$t1 + ldr $t1,[$t0,#8] + add $b,$t2 + ldr $t2,[$t0,#12] + add $c,$t1 + ldr $t1,[$t0,#16] + add $d,$t2 + add $e,$t1 + stmia $t0!,{$a,$b,$c,$d,$e} @ [+20] + + add sp,#80*4 @ deallocate stack frame + mov $t0,$ctx @ restore ctx + mov $t1,$inp @ restore inp + cmp $t1,$len + beq .Lexit + b .Lloop @ [+6] total 3212 cycles +.Lexit: + pop {r2-r7} + mov r8,r2 + mov r9,r3 + mov r10,r4 + mov r11,r5 + mov r12,r6 + mov lr,r7 + pop {r4-r7} + bx lr +.align 2 +___ +$code.=".Lcommon:\n".&common()."\tmov pc,lr\n" if (!$inline); +$code.=".Lrotate:\n".&rotate()."\tmov pc,lr\n" if (!$inline); +$code.=<<___; +.align 2 +.LK_00_19: .word 0x5a827999 +.LK_20_39: .word 0x6ed9eba1 +.LK_40_59: .word 0x8f1bbcdc +.LK_60_79: .word 0xca62c1d6 +.size sha1_block_data_order,.-sha1_block_data_order +.asciz "SHA1 block transform for Thumb, CRYPTOGAMS by <appro\@openssl.org>" +___ + +print $code; +close STDOUT; # enforce flush diff --git a/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl index f7ed67a72609..f27c1e3fb031 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl @@ -16,7 +16,7 @@ # There was suggestion to mechanically translate 32-bit code, but I # dismissed it, reasoning that x86_64 offers enough register bank # capacity to fully utilize SHA-1 parallelism. Therefore this fresh -# implementation:-) However! While 64-bit code does performs better +# implementation:-) However! While 64-bit code does perform better # on Opteron, I failed to beat 32-bit assembler on EM64T core. Well, # x86_64 does offer larger *addressable* bank, but out-of-order core # reaches for even more registers through dynamic aliasing, and EM64T @@ -29,14 +29,60 @@ # Xeon P4 +65% +0% 9.9 # Core2 +60% +10% 7.0 -$output=shift; +# August 2009. +# +# The code was revised to minimize code size and to maximize +# "distance" between instructions producing input to 'lea' +# instruction and the 'lea' instruction itself, which is essential +# for Intel Atom core. + +# October 2010. +# +# Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it +# is to offload message schedule denoted by Wt in NIST specification, +# or Xupdate in OpenSSL source, to SIMD unit. See sha1-586.pl module +# for background and implementation details. The only difference from +# 32-bit code is that 64-bit code doesn't have to spill @X[] elements +# to free temporary registers. + +# April 2011. +# +# Add AVX code path. See sha1-586.pl for further information. + +###################################################################### +# Current performance is summarized in following table. Numbers are +# CPU clock cycles spent to process single byte (less is better). +# +# x86_64 SSSE3 AVX +# P4 9.8 - +# Opteron 6.6 - +# Core2 6.7 6.1/+10% - +# Atom 11.0 9.7/+13% - +# Westmere 7.1 5.6/+27% - +# Sandy Bridge 7.9 6.3/+25% 5.2/+51% + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or die "can't locate x86_64-xlate.pl"; -open STDOUT,"| $^X $xlate $output"; +$avx=1 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/ && + $1>=2.19); +$avx=1 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ && + $1>=2.09); +$avx=1 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./ && + $1>=10); + +open STDOUT,"| $^X $xlate $flavour $output"; $ctx="%rdi"; # 1st arg $inp="%rsi"; # 2nd arg @@ -47,193 +93,1165 @@ $ctx="%r8"; $inp="%r9"; $num="%r10"; -$xi="%eax"; -$t0="%ebx"; -$t1="%ecx"; -$A="%edx"; -$B="%esi"; -$C="%edi"; -$D="%ebp"; -$E="%r11d"; -$T="%r12d"; - -@V=($A,$B,$C,$D,$E,$T); +$t0="%eax"; +$t1="%ebx"; +$t2="%ecx"; +@xi=("%edx","%ebp"); +$A="%esi"; +$B="%edi"; +$C="%r11d"; +$D="%r12d"; +$E="%r13d"; -sub PROLOGUE { -my $func=shift; -$code.=<<___; -.globl $func -.type $func,\@function,3 -.align 16 -$func: - push %rbx - push %rbp - push %r12 - mov %rsp,%rax - mov %rdi,$ctx # reassigned argument - sub \$`8+16*4`,%rsp - mov %rsi,$inp # reassigned argument - and \$-64,%rsp - mov %rdx,$num # reassigned argument - mov %rax,`16*4`(%rsp) - - mov 0($ctx),$A - mov 4($ctx),$B - mov 8($ctx),$C - mov 12($ctx),$D - mov 16($ctx),$E -___ -} - -sub EPILOGUE { -my $func=shift; -$code.=<<___; - mov `16*4`(%rsp),%rsp - pop %r12 - pop %rbp - pop %rbx - ret -.size $func,.-$func -___ -} +@V=($A,$B,$C,$D,$E); sub BODY_00_19 { -my ($i,$a,$b,$c,$d,$e,$f,$host)=@_; +my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; $code.=<<___ if ($i==0); - mov `4*$i`($inp),$xi - `"bswap $xi" if(!defined($host))` - mov $xi,`4*$i`(%rsp) + mov `4*$i`($inp),$xi[0] + bswap $xi[0] + mov $xi[0],`4*$i`(%rsp) ___ $code.=<<___ if ($i<15); - lea 0x5a827999($xi,$e),$f mov $c,$t0 - mov `4*$j`($inp),$xi - mov $a,$e + mov `4*$j`($inp),$xi[1] + mov $a,$t2 xor $d,$t0 - `"bswap $xi" if(!defined($host))` - rol \$5,$e + bswap $xi[1] + rol \$5,$t2 + lea 0x5a827999($xi[0],$e),$e and $b,$t0 - mov $xi,`4*$j`(%rsp) - add $e,$f + mov $xi[1],`4*$j`(%rsp) + add $t2,$e xor $d,$t0 rol \$30,$b - add $t0,$f + add $t0,$e ___ $code.=<<___ if ($i>=15); - lea 0x5a827999($xi,$e),$f - mov `4*($j%16)`(%rsp),$xi + mov `4*($j%16)`(%rsp),$xi[1] mov $c,$t0 - mov $a,$e - xor `4*(($j+2)%16)`(%rsp),$xi + mov $a,$t2 + xor `4*(($j+2)%16)`(%rsp),$xi[1] xor $d,$t0 - rol \$5,$e - xor `4*(($j+8)%16)`(%rsp),$xi + rol \$5,$t2 + xor `4*(($j+8)%16)`(%rsp),$xi[1] and $b,$t0 - add $e,$f - xor `4*(($j+13)%16)`(%rsp),$xi + lea 0x5a827999($xi[0],$e),$e + xor `4*(($j+13)%16)`(%rsp),$xi[1] xor $d,$t0 + rol \$1,$xi[1] + add $t2,$e rol \$30,$b - add $t0,$f - rol \$1,$xi - mov $xi,`4*($j%16)`(%rsp) + mov $xi[1],`4*($j%16)`(%rsp) + add $t0,$e ___ +unshift(@xi,pop(@xi)); } sub BODY_20_39 { -my ($i,$a,$b,$c,$d,$e,$f)=@_; +my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; my $K=($i<40)?0x6ed9eba1:0xca62c1d6; $code.=<<___ if ($i<79); - lea $K($xi,$e),$f - mov `4*($j%16)`(%rsp),$xi + mov `4*($j%16)`(%rsp),$xi[1] mov $c,$t0 - mov $a,$e - xor `4*(($j+2)%16)`(%rsp),$xi + mov $a,$t2 + xor `4*(($j+2)%16)`(%rsp),$xi[1] xor $b,$t0 - rol \$5,$e - xor `4*(($j+8)%16)`(%rsp),$xi + rol \$5,$t2 + lea $K($xi[0],$e),$e + xor `4*(($j+8)%16)`(%rsp),$xi[1] xor $d,$t0 - add $e,$f - xor `4*(($j+13)%16)`(%rsp),$xi + add $t2,$e + xor `4*(($j+13)%16)`(%rsp),$xi[1] rol \$30,$b - add $t0,$f - rol \$1,$xi + add $t0,$e + rol \$1,$xi[1] ___ $code.=<<___ if ($i<76); - mov $xi,`4*($j%16)`(%rsp) + mov $xi[1],`4*($j%16)`(%rsp) ___ $code.=<<___ if ($i==79); - lea $K($xi,$e),$f mov $c,$t0 - mov $a,$e + mov $a,$t2 xor $b,$t0 - rol \$5,$e + lea $K($xi[0],$e),$e + rol \$5,$t2 xor $d,$t0 - add $e,$f + add $t2,$e rol \$30,$b - add $t0,$f + add $t0,$e ___ +unshift(@xi,pop(@xi)); } sub BODY_40_59 { -my ($i,$a,$b,$c,$d,$e,$f)=@_; +my ($i,$a,$b,$c,$d,$e)=@_; my $j=$i+1; $code.=<<___; - lea 0x8f1bbcdc($xi,$e),$f - mov `4*($j%16)`(%rsp),$xi - mov $b,$t0 - mov $b,$t1 - xor `4*(($j+2)%16)`(%rsp),$xi - mov $a,$e - and $c,$t0 - xor `4*(($j+8)%16)`(%rsp),$xi - or $c,$t1 - rol \$5,$e - xor `4*(($j+13)%16)`(%rsp),$xi - and $d,$t1 - add $e,$f - rol \$1,$xi - or $t1,$t0 + mov `4*($j%16)`(%rsp),$xi[1] + mov $c,$t0 + mov $c,$t1 + xor `4*(($j+2)%16)`(%rsp),$xi[1] + and $d,$t0 + mov $a,$t2 + xor `4*(($j+8)%16)`(%rsp),$xi[1] + xor $d,$t1 + lea 0x8f1bbcdc($xi[0],$e),$e + rol \$5,$t2 + xor `4*(($j+13)%16)`(%rsp),$xi[1] + add $t0,$e + and $b,$t1 + rol \$1,$xi[1] + add $t1,$e rol \$30,$b - mov $xi,`4*($j%16)`(%rsp) - add $t0,$f + mov $xi[1],`4*($j%16)`(%rsp) + add $t2,$e ___ +unshift(@xi,pop(@xi)); } -$code=".text\n"; +$code.=<<___; +.text +.extern OPENSSL_ia32cap_P + +.globl sha1_block_data_order +.type sha1_block_data_order,\@function,3 +.align 16 +sha1_block_data_order: + mov OPENSSL_ia32cap_P+0(%rip),%r9d + mov OPENSSL_ia32cap_P+4(%rip),%r8d + test \$`1<<9`,%r8d # check SSSE3 bit + jz .Lialu +___ +$code.=<<___ if ($avx); + and \$`1<<28`,%r8d # mask AVX bit + and \$`1<<30`,%r9d # mask "Intel CPU" bit + or %r9d,%r8d + cmp \$`1<<28|1<<30`,%r8d + je _avx_shortcut +___ +$code.=<<___; + jmp _ssse3_shortcut -&PROLOGUE("sha1_block_data_order"); -$code.=".align 4\n.Lloop:\n"; +.align 16 +.Lialu: + push %rbx + push %rbp + push %r12 + push %r13 + mov %rsp,%r11 + mov %rdi,$ctx # reassigned argument + sub \$`8+16*4`,%rsp + mov %rsi,$inp # reassigned argument + and \$-64,%rsp + mov %rdx,$num # reassigned argument + mov %r11,`16*4`(%rsp) +.Lprologue: + + mov 0($ctx),$A + mov 4($ctx),$B + mov 8($ctx),$C + mov 12($ctx),$D + mov 16($ctx),$E + jmp .Lloop + +.align 16 +.Lloop: +___ for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } $code.=<<___; - add 0($ctx),$E - add 4($ctx),$T - add 8($ctx),$A - add 12($ctx),$B - add 16($ctx),$C - mov $E,0($ctx) - mov $T,4($ctx) - mov $A,8($ctx) - mov $B,12($ctx) - mov $C,16($ctx) - - xchg $E,$A # mov $E,$A - xchg $T,$B # mov $T,$B - xchg $E,$C # mov $A,$C - xchg $T,$D # mov $B,$D - # mov $C,$E - lea `16*4`($inp),$inp + add 0($ctx),$A + add 4($ctx),$B + add 8($ctx),$C + add 12($ctx),$D + add 16($ctx),$E + mov $A,0($ctx) + mov $B,4($ctx) + mov $C,8($ctx) + mov $D,12($ctx) + mov $E,16($ctx) + sub \$1,$num + lea `16*4`($inp),$inp jnz .Lloop + + mov `16*4`(%rsp),%rsi + mov (%rsi),%r13 + mov 8(%rsi),%r12 + mov 16(%rsi),%rbp + mov 24(%rsi),%rbx + lea 32(%rsi),%rsp +.Lepilogue: + ret +.size sha1_block_data_order,.-sha1_block_data_order +___ +{{{ +my $Xi=4; +my @X=map("%xmm$_",(4..7,0..3)); +my @Tx=map("%xmm$_",(8..10)); +my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization +my @T=("%esi","%edi"); +my $j=0; +my $K_XX_XX="%r11"; + +my $_rol=sub { &rol(@_) }; +my $_ror=sub { &ror(@_) }; + +$code.=<<___; +.type sha1_block_data_order_ssse3,\@function,3 +.align 16 +sha1_block_data_order_ssse3: +_ssse3_shortcut: + push %rbx + push %rbp + push %r12 + lea `-64-($win64?5*16:0)`(%rsp),%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,64+0(%rsp) + movaps %xmm7,64+16(%rsp) + movaps %xmm8,64+32(%rsp) + movaps %xmm9,64+48(%rsp) + movaps %xmm10,64+64(%rsp) +.Lprologue_ssse3: +___ +$code.=<<___; + mov %rdi,$ctx # reassigned argument + mov %rsi,$inp # reassigned argument + mov %rdx,$num # reassigned argument + + shl \$6,$num + add $inp,$num + lea K_XX_XX(%rip),$K_XX_XX + + mov 0($ctx),$A # load context + mov 4($ctx),$B + mov 8($ctx),$C + mov 12($ctx),$D + mov $B,@T[0] # magic seed + mov 16($ctx),$E + + movdqa 64($K_XX_XX),@X[2] # pbswap mask + movdqa 0($K_XX_XX),@Tx[1] # K_00_19 + movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] + movdqu 16($inp),@X[-3&7] + movdqu 32($inp),@X[-2&7] + movdqu 48($inp),@X[-1&7] + pshufb @X[2],@X[-4&7] # byte swap + add \$64,$inp + pshufb @X[2],@X[-3&7] + pshufb @X[2],@X[-2&7] + pshufb @X[2],@X[-1&7] + paddd @Tx[1],@X[-4&7] # add K_00_19 + paddd @Tx[1],@X[-3&7] + paddd @Tx[1],@X[-2&7] + movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU + psubd @Tx[1],@X[-4&7] # restore X[] + movdqa @X[-3&7],16(%rsp) + psubd @Tx[1],@X[-3&7] + movdqa @X[-2&7],32(%rsp) + psubd @Tx[1],@X[-2&7] + jmp .Loop_ssse3 +___ + +sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; + my $arg = pop; + $arg = "\$$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; +} + +sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4 +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 40 instructions + my ($a,$b,$c,$d,$e); + + &movdqa (@X[0],@X[-3&7]); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (@Tx[0],@X[-1&7]); + &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]" + eval(shift(@insns)); + eval(shift(@insns)); + + &paddd (@Tx[1],@X[-1&7]); + eval(shift(@insns)); + eval(shift(@insns)); + &psrldq (@Tx[0],4); # "X[-3]", 3 dwords + eval(shift(@insns)); + eval(shift(@insns)); + &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]" + eval(shift(@insns)); + eval(shift(@insns)); + + &pxor (@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + + &movdqa (@Tx[2],@X[0]); + &movdqa (@Tx[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &pslldq (@Tx[2],12); # "X[0]"<<96, extract one dword + &paddd (@X[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &psrld (@Tx[0],31); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (@Tx[1],@Tx[2]); + eval(shift(@insns)); + eval(shift(@insns)); + + &psrld (@Tx[2],30); + &por (@X[0],@Tx[0]); # "X[0]"<<<=1 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &pslld (@Tx[1],2); + &pxor (@X[0],@Tx[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX + eval(shift(@insns)); + eval(shift(@insns)); + + &pxor (@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2 + + foreach (@insns) { eval; } # remaining instructions [if any] + + $Xi++; push(@X,shift(@X)); # "rotate" X[] + push(@Tx,shift(@Tx)); +} + +sub Xupdate_ssse3_32_79() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my ($a,$b,$c,$d,$e); + + &movdqa (@Tx[0],@X[-1&7]) if ($Xi==8); + eval(shift(@insns)); # body_20_39 + &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" + &palignr(@Tx[0],@X[-2&7],8); # compose "X[-6]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]" + eval(shift(@insns)); + eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/); + if ($Xi%5) { + &movdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX... + } else { # ... or load next one + &movdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)"); + } + &paddd (@Tx[1],@X[-1&7]); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-6]" + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &movdqa (@Tx[0],@X[0]); + &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &pslld (@X[0],2); + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + &psrld (@Tx[0],30); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &por (@X[0],@Tx[0]); # "X[0]"<<<=2 + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + &movdqa (@Tx[1],@X[0]) if ($Xi<19); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + + foreach (@insns) { eval; } # remaining instructions + + $Xi++; push(@X,shift(@X)); # "rotate" X[] + push(@Tx,shift(@Tx)); +} + +sub Xuplast_ssse3_80() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + &paddd (@Tx[1],@X[-1&7]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU + + foreach (@insns) { eval; } # remaining instructions + + &cmp ($inp,$num); + &je (".Ldone_ssse3"); + + unshift(@Tx,pop(@Tx)); + + &movdqa (@X[2],"64($K_XX_XX)"); # pbswap mask + &movdqa (@Tx[1],"0($K_XX_XX)"); # K_00_19 + &movdqu (@X[-4&7],"0($inp)"); # load input + &movdqu (@X[-3&7],"16($inp)"); + &movdqu (@X[-2&7],"32($inp)"); + &movdqu (@X[-1&7],"48($inp)"); + &pshufb (@X[-4&7],@X[2]); # byte swap + &add ($inp,64); + + $Xi=0; +} + +sub Xloop_ssse3() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + eval(shift(@insns)); + &pshufb (@X[($Xi-3)&7],@X[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &paddd (@X[($Xi-4)&7],@Tx[1]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &movdqa (eval(16*$Xi)."(%rsp)",@X[($Xi-4)&7]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + &psubd (@X[($Xi-4)&7],@Tx[1]); + + foreach (@insns) { eval; } + $Xi++; +} + +sub Xtail_ssse3() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + foreach (@insns) { eval; } +} + +sub body_00_19 () { + ( + '($a,$b,$c,$d,$e)=@V;'. + '&add ($e,eval(4*($j&15))."(%rsp)");', # X[]+K xfer + '&xor ($c,$d);', + '&mov (@T[1],$a);', # $b in next round + '&$_rol ($a,5);', + '&and (@T[0],$c);', # ($b&($c^$d)) + '&xor ($c,$d);', # restore $c + '&xor (@T[0],$d);', + '&add ($e,$a);', + '&$_ror ($b,$j?7:2);', # $b>>>2 + '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} + +sub body_20_39 () { + ( + '($a,$b,$c,$d,$e)=@V;'. + '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer + '&xor (@T[0],$d);', # ($b^$d) + '&mov (@T[1],$a);', # $b in next round + '&$_rol ($a,5);', + '&xor (@T[0],$c);', # ($b^$d^$c) + '&add ($e,$a);', + '&$_ror ($b,7);', # $b>>>2 + '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} + +sub body_40_59 () { + ( + '($a,$b,$c,$d,$e)=@V;'. + '&mov (@T[1],$c);', + '&xor ($c,$d);', + '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer + '&and (@T[1],$d);', + '&and (@T[0],$c);', # ($b&($c^$d)) + '&$_ror ($b,7);', # $b>>>2 + '&add ($e,@T[1]);', + '&mov (@T[1],$a);', # $b in next round + '&$_rol ($a,5);', + '&add ($e,@T[0]);', + '&xor ($c,$d);', # restore $c + '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));' + ); +} +$code.=<<___; +.align 16 +.Loop_ssse3: +___ + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_16_31(\&body_00_19); + &Xupdate_ssse3_32_79(\&body_00_19); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_40_59); + &Xupdate_ssse3_32_79(\&body_20_39); + &Xuplast_ssse3_80(\&body_20_39); # can jump to "done" + + $saved_j=$j; @saved_V=@V; + + &Xloop_ssse3(\&body_20_39); + &Xloop_ssse3(\&body_20_39); + &Xloop_ssse3(\&body_20_39); + +$code.=<<___; + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + add 12($ctx),$D + mov $A,0($ctx) + add 16($ctx),$E + mov @T[0],4($ctx) + mov @T[0],$B # magic seed + mov $C,8($ctx) + mov $D,12($ctx) + mov $E,16($ctx) + jmp .Loop_ssse3 + +.align 16 +.Ldone_ssse3: +___ + $j=$saved_j; @V=@saved_V; + + &Xtail_ssse3(\&body_20_39); + &Xtail_ssse3(\&body_20_39); + &Xtail_ssse3(\&body_20_39); + +$code.=<<___; + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + mov $A,0($ctx) + add 12($ctx),$D + mov @T[0],4($ctx) + add 16($ctx),$E + mov $C,8($ctx) + mov $D,12($ctx) + mov $E,16($ctx) +___ +$code.=<<___ if ($win64); + movaps 64+0(%rsp),%xmm6 + movaps 64+16(%rsp),%xmm7 + movaps 64+32(%rsp),%xmm8 + movaps 64+48(%rsp),%xmm9 + movaps 64+64(%rsp),%xmm10 +___ +$code.=<<___; + lea `64+($win64?5*16:0)`(%rsp),%rsi + mov 0(%rsi),%r12 + mov 8(%rsi),%rbp + mov 16(%rsi),%rbx + lea 24(%rsi),%rsp +.Lepilogue_ssse3: + ret +.size sha1_block_data_order_ssse3,.-sha1_block_data_order_ssse3 +___ + +if ($avx) { +my $Xi=4; +my @X=map("%xmm$_",(4..7,0..3)); +my @Tx=map("%xmm$_",(8..10)); +my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization +my @T=("%esi","%edi"); +my $j=0; +my $K_XX_XX="%r11"; + +my $_rol=sub { &shld(@_[0],@_) }; +my $_ror=sub { &shrd(@_[0],@_) }; + +$code.=<<___; +.type sha1_block_data_order_avx,\@function,3 +.align 16 +sha1_block_data_order_avx: +_avx_shortcut: + push %rbx + push %rbp + push %r12 + lea `-64-($win64?5*16:0)`(%rsp),%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,64+0(%rsp) + movaps %xmm7,64+16(%rsp) + movaps %xmm8,64+32(%rsp) + movaps %xmm9,64+48(%rsp) + movaps %xmm10,64+64(%rsp) +.Lprologue_avx: +___ +$code.=<<___; + mov %rdi,$ctx # reassigned argument + mov %rsi,$inp # reassigned argument + mov %rdx,$num # reassigned argument + vzeroall + + shl \$6,$num + add $inp,$num + lea K_XX_XX(%rip),$K_XX_XX + + mov 0($ctx),$A # load context + mov 4($ctx),$B + mov 8($ctx),$C + mov 12($ctx),$D + mov $B,@T[0] # magic seed + mov 16($ctx),$E + + vmovdqa 64($K_XX_XX),@X[2] # pbswap mask + vmovdqa 0($K_XX_XX),@Tx[1] # K_00_19 + vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3] + vmovdqu 16($inp),@X[-3&7] + vmovdqu 32($inp),@X[-2&7] + vmovdqu 48($inp),@X[-1&7] + vpshufb @X[2],@X[-4&7],@X[-4&7] # byte swap + add \$64,$inp + vpshufb @X[2],@X[-3&7],@X[-3&7] + vpshufb @X[2],@X[-2&7],@X[-2&7] + vpshufb @X[2],@X[-1&7],@X[-1&7] + vpaddd @Tx[1],@X[-4&7],@X[0] # add K_00_19 + vpaddd @Tx[1],@X[-3&7],@X[1] + vpaddd @Tx[1],@X[-2&7],@X[2] + vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU + vmovdqa @X[1],16(%rsp) + vmovdqa @X[2],32(%rsp) + jmp .Loop_avx +___ + +sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4 +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 40 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + eval(shift(@insns)); + &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]" + eval(shift(@insns)); + eval(shift(@insns)); + + &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords + eval(shift(@insns)); + eval(shift(@insns)); + &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]" + eval(shift(@insns)); + eval(shift(@insns)); + &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrld (@Tx[0],@X[0],31); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword + &vpaddd (@X[0],@X[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpsrld (@Tx[1],@Tx[2],30); + &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpslld (@Tx[2],@Tx[2],2); + &vpxor (@X[0],@X[0],@Tx[1]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2 + eval(shift(@insns)); + eval(shift(@insns)); + &vmovdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX + eval(shift(@insns)); + eval(shift(@insns)); + + + foreach (@insns) { eval; } # remaining instructions [if any] + + $Xi++; push(@X,shift(@X)); # "rotate" X[] + push(@Tx,shift(@Tx)); +} + +sub Xupdate_avx_32_79() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions + my ($a,$b,$c,$d,$e); + + &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8); # compose "X[-6]" + &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" + eval(shift(@insns)); + eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/); + if ($Xi%5) { + &vmovdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX... + } else { # ... or load next one + &vmovdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)"); + } + &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-6]" + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + + &vpsrld (@Tx[0],@X[0],30); + &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &vpslld (@X[0],@X[0],2); + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # ror + eval(shift(@insns)); + + &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2 + eval(shift(@insns)); # body_20_39 + eval(shift(@insns)); + &vmovdqa (@Tx[1],@X[0]) if ($Xi<19); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); # rol + eval(shift(@insns)); + + foreach (@insns) { eval; } # remaining instructions + + $Xi++; push(@X,shift(@X)); # "rotate" X[] + push(@Tx,shift(@Tx)); +} + +sub Xuplast_avx_80() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + &vpaddd (@Tx[1],@Tx[1],@X[-1&7]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + + &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU + + foreach (@insns) { eval; } # remaining instructions + + &cmp ($inp,$num); + &je (".Ldone_avx"); + + unshift(@Tx,pop(@Tx)); + + &vmovdqa(@X[2],"64($K_XX_XX)"); # pbswap mask + &vmovdqa(@Tx[1],"0($K_XX_XX)"); # K_00_19 + &vmovdqu(@X[-4&7],"0($inp)"); # load input + &vmovdqu(@X[-3&7],"16($inp)"); + &vmovdqu(@X[-2&7],"32($inp)"); + &vmovdqu(@X[-1&7],"48($inp)"); + &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap + &add ($inp,64); + + $Xi=0; +} + +sub Xloop_avx() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + eval(shift(@insns)); + eval(shift(@insns)); + &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@Tx[1]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vmovdqa(eval(16*$Xi)."(%rsp)",@X[$Xi&7]); # X[]+K xfer to IALU + eval(shift(@insns)); + eval(shift(@insns)); + + foreach (@insns) { eval; } + $Xi++; +} + +sub Xtail_avx() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); # 32 instructions + my ($a,$b,$c,$d,$e); + + foreach (@insns) { eval; } +} + +$code.=<<___; +.align 16 +.Loop_avx: ___ -&EPILOGUE("sha1_block_data_order"); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_16_31(\&body_00_19); + &Xupdate_avx_32_79(\&body_00_19); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_20_39); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_40_59); + &Xupdate_avx_32_79(\&body_20_39); + &Xuplast_avx_80(\&body_20_39); # can jump to "done" + + $saved_j=$j; @saved_V=@V; + + &Xloop_avx(\&body_20_39); + &Xloop_avx(\&body_20_39); + &Xloop_avx(\&body_20_39); + +$code.=<<___; + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + add 12($ctx),$D + mov $A,0($ctx) + add 16($ctx),$E + mov @T[0],4($ctx) + mov @T[0],$B # magic seed + mov $C,8($ctx) + mov $D,12($ctx) + mov $E,16($ctx) + jmp .Loop_avx + +.align 16 +.Ldone_avx: +___ + $j=$saved_j; @V=@saved_V; + + &Xtail_avx(\&body_20_39); + &Xtail_avx(\&body_20_39); + &Xtail_avx(\&body_20_39); + +$code.=<<___; + vzeroall + + add 0($ctx),$A # update context + add 4($ctx),@T[0] + add 8($ctx),$C + mov $A,0($ctx) + add 12($ctx),$D + mov @T[0],4($ctx) + add 16($ctx),$E + mov $C,8($ctx) + mov $D,12($ctx) + mov $E,16($ctx) +___ +$code.=<<___ if ($win64); + movaps 64+0(%rsp),%xmm6 + movaps 64+16(%rsp),%xmm7 + movaps 64+32(%rsp),%xmm8 + movaps 64+48(%rsp),%xmm9 + movaps 64+64(%rsp),%xmm10 +___ +$code.=<<___; + lea `64+($win64?5*16:0)`(%rsp),%rsi + mov 0(%rsi),%r12 + mov 8(%rsi),%rbp + mov 16(%rsi),%rbx + lea 24(%rsi),%rsp +.Lepilogue_avx: + ret +.size sha1_block_data_order_avx,.-sha1_block_data_order_avx +___ +} +$code.=<<___; +.align 64 +K_XX_XX: +.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 +.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39 +.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 +.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask +___ +}}} $code.=<<___; .asciz "SHA1 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>" +.align 64 +___ + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + lea .Lprologue(%rip),%r10 + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea .Lepilogue(%rip),%r10 + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + mov `16*4`(%rax),%rax # pull saved stack pointer + lea 32(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + + jmp .Lcommon_seh_tail +.size se_handler,.-se_handler + +.type ssse3_handler,\@abi-omnipotent +.align 16 +ssse3_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<prologue label + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + lea 64(%rax),%rsi + lea 512($context),%rdi # &context.Xmm6 + mov \$10,%ecx + .long 0xa548f3fc # cld; rep movsq + lea `24+64+5*16`(%rax),%rax # adjust stack pointer + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore cotnext->R12 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size ssse3_handler,.-ssse3_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_sha1_block_data_order + .rva .LSEH_end_sha1_block_data_order + .rva .LSEH_info_sha1_block_data_order + .rva .LSEH_begin_sha1_block_data_order_ssse3 + .rva .LSEH_end_sha1_block_data_order_ssse3 + .rva .LSEH_info_sha1_block_data_order_ssse3 +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_sha1_block_data_order_avx + .rva .LSEH_end_sha1_block_data_order_avx + .rva .LSEH_info_sha1_block_data_order_avx ___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_sha1_block_data_order: + .byte 9,0,0,0 + .rva se_handler +.LSEH_info_sha1_block_data_order_ssse3: + .byte 9,0,0,0 + .rva ssse3_handler + .rva .Lprologue_ssse3,.Lepilogue_ssse3 # HandlerData[] +___ +$code.=<<___ if ($avx); +.LSEH_info_sha1_block_data_order_avx: + .byte 9,0,0,0 + .rva ssse3_handler + .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[] +___ +} #################################################################### diff --git a/crypto/openssl/crypto/sha/asm/sha256-586.pl b/crypto/openssl/crypto/sha/asm/sha256-586.pl new file mode 100755 index 000000000000..928ec53123bf --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha256-586.pl @@ -0,0 +1,249 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# SHA256 block transform for x86. September 2007. +# +# Performance in clock cycles per processed byte (less is better): +# +# Pentium PIII P4 AMD K8 Core2 +# gcc 46 36 41 27 26 +# icc 57 33 38 25 23 +# x86 asm 40 30 33 20 18 +# x86_64 asm(*) - - 21 16 16 +# +# (*) x86_64 assembler performance is presented for reference +# purposes. +# +# Performance improvement over compiler generated code varies from +# 10% to 40% [see above]. Not very impressive on some µ-archs, but +# it's 5 times smaller and optimizies amount of writes. + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +push(@INC,"${dir}","${dir}../../perlasm"); +require "x86asm.pl"; + +&asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386"); + +$A="eax"; +$E="edx"; +$T="ebx"; +$Aoff=&DWP(0,"esp"); +$Boff=&DWP(4,"esp"); +$Coff=&DWP(8,"esp"); +$Doff=&DWP(12,"esp"); +$Eoff=&DWP(16,"esp"); +$Foff=&DWP(20,"esp"); +$Goff=&DWP(24,"esp"); +$Hoff=&DWP(28,"esp"); +$Xoff=&DWP(32,"esp"); +$K256="ebp"; + +sub BODY_00_15() { + my $in_16_63=shift; + + &mov ("ecx",$E); + &add ($T,"edi") if ($in_16_63); # T += sigma1(X[-2]) + &ror ("ecx",25-11); + &mov ("esi",$Foff); + &xor ("ecx",$E); + &ror ("ecx",11-6); + &mov (&DWP(4*(8+15),"esp"),$T) if ($in_16_63); # save X[0] + &xor ("ecx",$E); + &ror ("ecx",6); # Sigma1(e) + &mov ("edi",$Goff); + &add ($T,"ecx"); # T += Sigma1(e) + + &xor ("esi","edi"); + &mov ($Eoff,$E); # modulo-scheduled + &mov ("ecx",$A); + &and ("esi",$E); + &mov ($E,$Doff); # e becomes d, which is e in next iteration + &xor ("esi","edi"); # Ch(e,f,g) + &mov ("edi",$A); + &add ($T,"esi"); # T += Ch(e,f,g) + + &ror ("ecx",22-13); + &add ($T,$Hoff); # T += h + &xor ("ecx",$A); + &ror ("ecx",13-2); + &mov ("esi",$Boff); + &xor ("ecx",$A); + &ror ("ecx",2); # Sigma0(a) + &add ($E,$T); # d += T + &mov ("edi",$Coff); + + &add ($T,"ecx"); # T += Sigma0(a) + &mov ($Aoff,$A); # modulo-scheduled + + &mov ("ecx",$A); + &sub ("esp",4); + &or ($A,"esi"); # a becomes h, which is a in next iteration + &and ("ecx","esi"); + &and ($A,"edi"); + &mov ("esi",&DWP(0,$K256)); + &or ($A,"ecx"); # h=Maj(a,b,c) + + &add ($K256,4); + &add ($A,$T); # h += T + &mov ($T,&DWP(4*(8+15+16-1),"esp")) if ($in_16_63); # preload T + &add ($E,"esi"); # d += K256[i] + &add ($A,"esi"); # h += K256[i] +} + +&function_begin("sha256_block_data_order"); + &mov ("esi",wparam(0)); # ctx + &mov ("edi",wparam(1)); # inp + &mov ("eax",wparam(2)); # num + &mov ("ebx","esp"); # saved sp + + &call (&label("pic_point")); # make it PIC! +&set_label("pic_point"); + &blindpop($K256); + &lea ($K256,&DWP(&label("K256")."-".&label("pic_point"),$K256)); + + &sub ("esp",16); + &and ("esp",-64); + + &shl ("eax",6); + &add ("eax","edi"); + &mov (&DWP(0,"esp"),"esi"); # ctx + &mov (&DWP(4,"esp"),"edi"); # inp + &mov (&DWP(8,"esp"),"eax"); # inp+num*128 + &mov (&DWP(12,"esp"),"ebx"); # saved sp + +&set_label("loop",16); + # copy input block to stack reversing byte and dword order + for($i=0;$i<4;$i++) { + &mov ("eax",&DWP($i*16+0,"edi")); + &mov ("ebx",&DWP($i*16+4,"edi")); + &mov ("ecx",&DWP($i*16+8,"edi")); + &mov ("edx",&DWP($i*16+12,"edi")); + &bswap ("eax"); + &bswap ("ebx"); + &bswap ("ecx"); + &bswap ("edx"); + &push ("eax"); + &push ("ebx"); + &push ("ecx"); + &push ("edx"); + } + &add ("edi",64); + &sub ("esp",4*8); # place for A,B,C,D,E,F,G,H + &mov (&DWP(4*(8+16)+4,"esp"),"edi"); + + # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack + &mov ($A,&DWP(0,"esi")); + &mov ("ebx",&DWP(4,"esi")); + &mov ("ecx",&DWP(8,"esi")); + &mov ("edi",&DWP(12,"esi")); + # &mov ($Aoff,$A); + &mov ($Boff,"ebx"); + &mov ($Coff,"ecx"); + &mov ($Doff,"edi"); + &mov ($E,&DWP(16,"esi")); + &mov ("ebx",&DWP(20,"esi")); + &mov ("ecx",&DWP(24,"esi")); + &mov ("edi",&DWP(28,"esi")); + # &mov ($Eoff,$E); + &mov ($Foff,"ebx"); + &mov ($Goff,"ecx"); + &mov ($Hoff,"edi"); + +&set_label("00_15",16); + &mov ($T,&DWP(4*(8+15),"esp")); + + &BODY_00_15(); + + &cmp ("esi",0xc19bf174); + &jne (&label("00_15")); + + &mov ($T,&DWP(4*(8+15+16-1),"esp")); # preloaded in BODY_00_15(1) +&set_label("16_63",16); + &mov ("esi",$T); + &mov ("ecx",&DWP(4*(8+15+16-14),"esp")); + &ror ("esi",18-7); + &mov ("edi","ecx"); + &xor ("esi",$T); + &ror ("esi",7); + &shr ($T,3); + + &ror ("edi",19-17); + &xor ($T,"esi"); # T = sigma0(X[-15]) + &xor ("edi","ecx"); + &ror ("edi",17); + &shr ("ecx",10); + &add ($T,&DWP(4*(8+15+16),"esp")); # T += X[-16] + &xor ("edi","ecx"); # sigma1(X[-2]) + + &add ($T,&DWP(4*(8+15+16-9),"esp")); # T += X[-7] + # &add ($T,"edi"); # T += sigma1(X[-2]) + # &mov (&DWP(4*(8+15),"esp"),$T); # save X[0] + + &BODY_00_15(1); + + &cmp ("esi",0xc67178f2); + &jne (&label("16_63")); + + &mov ("esi",&DWP(4*(8+16+64)+0,"esp"));#ctx + # &mov ($A,$Aoff); + &mov ("ebx",$Boff); + &mov ("ecx",$Coff); + &mov ("edi",$Doff); + &add ($A,&DWP(0,"esi")); + &add ("ebx",&DWP(4,"esi")); + &add ("ecx",&DWP(8,"esi")); + &add ("edi",&DWP(12,"esi")); + &mov (&DWP(0,"esi"),$A); + &mov (&DWP(4,"esi"),"ebx"); + &mov (&DWP(8,"esi"),"ecx"); + &mov (&DWP(12,"esi"),"edi"); + # &mov ($E,$Eoff); + &mov ("eax",$Foff); + &mov ("ebx",$Goff); + &mov ("ecx",$Hoff); + &mov ("edi",&DWP(4*(8+16+64)+4,"esp"));#inp + &add ($E,&DWP(16,"esi")); + &add ("eax",&DWP(20,"esi")); + &add ("ebx",&DWP(24,"esi")); + &add ("ecx",&DWP(28,"esi")); + &mov (&DWP(16,"esi"),$E); + &mov (&DWP(20,"esi"),"eax"); + &mov (&DWP(24,"esi"),"ebx"); + &mov (&DWP(28,"esi"),"ecx"); + + &add ("esp",4*(8+16+64)); # destroy frame + &sub ($K256,4*64); # rewind K + + &cmp ("edi",&DWP(8,"esp")); # are we done yet? + &jb (&label("loop")); + + &mov ("esp",&DWP(12,"esp")); # restore sp +&function_end_A(); + +&set_label("K256",64); # Yes! I keep it in the code segment! + &data_word(0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5); + &data_word(0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5); + &data_word(0xd807aa98,0x12835b01,0x243185be,0x550c7dc3); + &data_word(0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174); + &data_word(0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc); + &data_word(0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da); + &data_word(0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7); + &data_word(0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967); + &data_word(0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13); + &data_word(0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85); + &data_word(0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3); + &data_word(0xd192e819,0xd6990624,0xf40e3585,0x106aa070); + &data_word(0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5); + &data_word(0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3); + &data_word(0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208); + &data_word(0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2); +&function_end_B("sha256_block_data_order"); +&asciz("SHA256 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>"); + +&asm_finish(); diff --git a/crypto/openssl/crypto/sha/asm/sha256-armv4.pl b/crypto/openssl/crypto/sha/asm/sha256-armv4.pl new file mode 100755 index 000000000000..9c84e8d93c30 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha256-armv4.pl @@ -0,0 +1,211 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA256 block procedure for ARMv4. May 2007. + +# Performance is ~2x better than gcc 3.4 generated code and in "abso- +# lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per +# byte [on single-issue Xscale PXA250 core]. + +# July 2010. +# +# Rescheduling for dual-issue pipeline resulted in 22% improvement on +# Cortex A8 core and ~20 cycles per processed byte. + +# February 2011. +# +# Profiler-assisted and platform-specific optimization resulted in 16% +# improvement on Cortex A8 core and ~17 cycles per processed byte. + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +$ctx="r0"; $t0="r0"; +$inp="r1"; $t3="r1"; +$len="r2"; $t1="r2"; +$T1="r3"; +$A="r4"; +$B="r5"; +$C="r6"; +$D="r7"; +$E="r8"; +$F="r9"; +$G="r10"; +$H="r11"; +@V=($A,$B,$C,$D,$E,$F,$G,$H); +$t2="r12"; +$Ktbl="r14"; + +@Sigma0=( 2,13,22); +@Sigma1=( 6,11,25); +@sigma0=( 7,18, 3); +@sigma1=(17,19,10); + +sub BODY_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; + +$code.=<<___ if ($i<16); +#if __ARM_ARCH__>=7 + ldr $T1,[$inp],#4 +#else + ldrb $T1,[$inp,#3] @ $i + ldrb $t2,[$inp,#2] + ldrb $t1,[$inp,#1] + ldrb $t0,[$inp],#4 + orr $T1,$T1,$t2,lsl#8 + orr $T1,$T1,$t1,lsl#16 + orr $T1,$T1,$t0,lsl#24 +#endif +___ +$code.=<<___; + mov $t0,$e,ror#$Sigma1[0] + ldr $t2,[$Ktbl],#4 @ *K256++ + eor $t0,$t0,$e,ror#$Sigma1[1] + eor $t1,$f,$g +#if $i>=16 + add $T1,$T1,$t3 @ from BODY_16_xx +#elif __ARM_ARCH__>=7 && defined(__ARMEL__) + rev $T1,$T1 +#endif +#if $i==15 + str $inp,[sp,#17*4] @ leave room for $t3 +#endif + eor $t0,$t0,$e,ror#$Sigma1[2] @ Sigma1(e) + and $t1,$t1,$e + str $T1,[sp,#`$i%16`*4] + add $T1,$T1,$t0 + eor $t1,$t1,$g @ Ch(e,f,g) + add $T1,$T1,$h + mov $h,$a,ror#$Sigma0[0] + add $T1,$T1,$t1 + eor $h,$h,$a,ror#$Sigma0[1] + add $T1,$T1,$t2 + eor $h,$h,$a,ror#$Sigma0[2] @ Sigma0(a) +#if $i>=15 + ldr $t3,[sp,#`($i+2)%16`*4] @ from BODY_16_xx +#endif + orr $t0,$a,$b + and $t1,$a,$b + and $t0,$t0,$c + add $h,$h,$T1 + orr $t0,$t0,$t1 @ Maj(a,b,c) + add $d,$d,$T1 + add $h,$h,$t0 +___ +} + +sub BODY_16_XX { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; + +$code.=<<___; + @ ldr $t3,[sp,#`($i+1)%16`*4] @ $i + ldr $t2,[sp,#`($i+14)%16`*4] + mov $t0,$t3,ror#$sigma0[0] + ldr $T1,[sp,#`($i+0)%16`*4] + eor $t0,$t0,$t3,ror#$sigma0[1] + ldr $t1,[sp,#`($i+9)%16`*4] + eor $t0,$t0,$t3,lsr#$sigma0[2] @ sigma0(X[i+1]) + mov $t3,$t2,ror#$sigma1[0] + add $T1,$T1,$t0 + eor $t3,$t3,$t2,ror#$sigma1[1] + add $T1,$T1,$t1 + eor $t3,$t3,$t2,lsr#$sigma1[2] @ sigma1(X[i+14]) + @ add $T1,$T1,$t3 +___ + &BODY_00_15(@_); +} + +$code=<<___; +#include "arm_arch.h" + +.text +.code 32 + +.type K256,%object +.align 5 +K256: +.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.size K256,.-K256 + +.global sha256_block_data_order +.type sha256_block_data_order,%function +sha256_block_data_order: + sub r3,pc,#8 @ sha256_block_data_order + add $len,$inp,$len,lsl#6 @ len to point at the end of inp + stmdb sp!,{$ctx,$inp,$len,r4-r11,lr} + ldmia $ctx,{$A,$B,$C,$D,$E,$F,$G,$H} + sub $Ktbl,r3,#256 @ K256 + sub sp,sp,#16*4 @ alloca(X[16]) +.Loop: +___ +for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } +$code.=".Lrounds_16_xx:\n"; +for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + and $t2,$t2,#0xff + cmp $t2,#0xf2 + bne .Lrounds_16_xx + + ldr $T1,[sp,#16*4] @ pull ctx + ldr $t0,[$T1,#0] + ldr $t1,[$T1,#4] + ldr $t2,[$T1,#8] + add $A,$A,$t0 + ldr $t0,[$T1,#12] + add $B,$B,$t1 + ldr $t1,[$T1,#16] + add $C,$C,$t2 + ldr $t2,[$T1,#20] + add $D,$D,$t0 + ldr $t0,[$T1,#24] + add $E,$E,$t1 + ldr $t1,[$T1,#28] + add $F,$F,$t2 + ldr $inp,[sp,#17*4] @ pull inp + ldr $t2,[sp,#18*4] @ pull inp+len + add $G,$G,$t0 + add $H,$H,$t1 + stmia $T1,{$A,$B,$C,$D,$E,$F,$G,$H} + cmp $inp,$t2 + sub $Ktbl,$Ktbl,#256 @ rewind Ktbl + bne .Loop + + add sp,sp,#`16+3`*4 @ destroy frame +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else + ldmia sp!,{r4-r11,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size sha256_block_data_order,.-sha256_block_data_order +.asciz "SHA256 block transform for ARMv4, CRYPTOGAMS by <appro\@openssl.org>" +.align 2 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 +print $code; +close STDOUT; # enforce flush diff --git a/crypto/openssl/crypto/sha/asm/sha512-586.pl b/crypto/openssl/crypto/sha/asm/sha512-586.pl new file mode 100755 index 000000000000..5b9f3337add3 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-586.pl @@ -0,0 +1,644 @@ +#!/usr/bin/env perl +# +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# SHA512 block transform for x86. September 2007. +# +# Performance in clock cycles per processed byte (less is better): +# +# Pentium PIII P4 AMD K8 Core2 +# gcc 100 75 116 54 66 +# icc 97 77 95 55 57 +# x86 asm 61 56 82 36 40 +# SSE2 asm - - 38 24 20 +# x86_64 asm(*) - - 30 10.0 10.5 +# +# (*) x86_64 assembler performance is presented for reference +# purposes. +# +# IALU code-path is optimized for elder Pentiums. On vanilla Pentium +# performance improvement over compiler generated code reaches ~60%, +# while on PIII - ~35%. On newer µ-archs improvement varies from 15% +# to 50%, but it's less important as they are expected to execute SSE2 +# code-path, which is commonly ~2-3x faster [than compiler generated +# code]. SSE2 code-path is as fast as original sha512-sse2.pl, even +# though it does not use 128-bit operations. The latter means that +# SSE2-aware kernel is no longer required to execute the code. Another +# difference is that new code optimizes amount of writes, but at the +# cost of increased data cache "footprint" by 1/2KB. + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +push(@INC,"${dir}","${dir}../../perlasm"); +require "x86asm.pl"; + +&asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386"); + +$sse2=0; +for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); } + +&external_label("OPENSSL_ia32cap_P") if ($sse2); + +$Tlo=&DWP(0,"esp"); $Thi=&DWP(4,"esp"); +$Alo=&DWP(8,"esp"); $Ahi=&DWP(8+4,"esp"); +$Blo=&DWP(16,"esp"); $Bhi=&DWP(16+4,"esp"); +$Clo=&DWP(24,"esp"); $Chi=&DWP(24+4,"esp"); +$Dlo=&DWP(32,"esp"); $Dhi=&DWP(32+4,"esp"); +$Elo=&DWP(40,"esp"); $Ehi=&DWP(40+4,"esp"); +$Flo=&DWP(48,"esp"); $Fhi=&DWP(48+4,"esp"); +$Glo=&DWP(56,"esp"); $Ghi=&DWP(56+4,"esp"); +$Hlo=&DWP(64,"esp"); $Hhi=&DWP(64+4,"esp"); +$K512="ebp"; + +$Asse2=&QWP(0,"esp"); +$Bsse2=&QWP(8,"esp"); +$Csse2=&QWP(16,"esp"); +$Dsse2=&QWP(24,"esp"); +$Esse2=&QWP(32,"esp"); +$Fsse2=&QWP(40,"esp"); +$Gsse2=&QWP(48,"esp"); +$Hsse2=&QWP(56,"esp"); + +$A="mm0"; # B-D and +$E="mm4"; # F-H are commonly loaded to respectively mm1-mm3 and + # mm5-mm7, but it's done on on-demand basis... + +sub BODY_00_15_sse2 { + my $prefetch=shift; + + &movq ("mm5",$Fsse2); # load f + &movq ("mm6",$Gsse2); # load g + &movq ("mm7",$Hsse2); # load h + + &movq ("mm1",$E); # %mm1 is sliding right + &movq ("mm2",$E); # %mm2 is sliding left + &psrlq ("mm1",14); + &movq ($Esse2,$E); # modulo-scheduled save e + &psllq ("mm2",23); + &movq ("mm3","mm1"); # %mm3 is T1 + &psrlq ("mm1",4); + &pxor ("mm3","mm2"); + &psllq ("mm2",23); + &pxor ("mm3","mm1"); + &psrlq ("mm1",23); + &pxor ("mm3","mm2"); + &psllq ("mm2",4); + &pxor ("mm3","mm1"); + &paddq ("mm7",QWP(0,$K512)); # h+=K512[i] + &pxor ("mm3","mm2"); # T1=Sigma1_512(e) + + &pxor ("mm5","mm6"); # f^=g + &movq ("mm1",$Bsse2); # load b + &pand ("mm5",$E); # f&=e + &movq ("mm2",$Csse2); # load c + &pxor ("mm5","mm6"); # f^=g + &movq ($E,$Dsse2); # e = load d + &paddq ("mm3","mm5"); # T1+=Ch(e,f,g) + &movq (&QWP(0,"esp"),$A); # modulo-scheduled save a + &paddq ("mm3","mm7"); # T1+=h + + &movq ("mm5",$A); # %mm5 is sliding right + &movq ("mm6",$A); # %mm6 is sliding left + &paddq ("mm3",&QWP(8*9,"esp")); # T1+=X[0] + &psrlq ("mm5",28); + &paddq ($E,"mm3"); # e += T1 + &psllq ("mm6",25); + &movq ("mm7","mm5"); # %mm7 is T2 + &psrlq ("mm5",6); + &pxor ("mm7","mm6"); + &psllq ("mm6",5); + &pxor ("mm7","mm5"); + &psrlq ("mm5",5); + &pxor ("mm7","mm6"); + &psllq ("mm6",6); + &pxor ("mm7","mm5"); + &sub ("esp",8); + &pxor ("mm7","mm6"); # T2=Sigma0_512(a) + + &movq ("mm5",$A); # %mm5=a + &por ($A,"mm2"); # a=a|c + &movq ("mm6",&QWP(8*(9+16-14),"esp")) if ($prefetch); + &pand ("mm5","mm2"); # %mm5=a&c + &pand ($A,"mm1"); # a=(a|c)&b + &movq ("mm2",&QWP(8*(9+16-1),"esp")) if ($prefetch); + &por ("mm5",$A); # %mm5=(a&c)|((a|c)&b) + &paddq ("mm7","mm5"); # T2+=Maj(a,b,c) + &movq ($A,"mm3"); # a=T1 + + &mov (&LB("edx"),&BP(0,$K512)); + &paddq ($A,"mm7"); # a+=T2 + &add ($K512,8); +} + +sub BODY_00_15_x86 { + #define Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41)) + # LO lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23 + # HI hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23 + &mov ("ecx",$Elo); + &mov ("edx",$Ehi); + &mov ("esi","ecx"); + + &shr ("ecx",9) # lo>>9 + &mov ("edi","edx"); + &shr ("edx",9) # hi>>9 + &mov ("ebx","ecx"); + &shl ("esi",14); # lo<<14 + &mov ("eax","edx"); + &shl ("edi",14); # hi<<14 + &xor ("ebx","esi"); + + &shr ("ecx",14-9); # lo>>14 + &xor ("eax","edi"); + &shr ("edx",14-9); # hi>>14 + &xor ("eax","ecx"); + &shl ("esi",18-14); # lo<<18 + &xor ("ebx","edx"); + &shl ("edi",18-14); # hi<<18 + &xor ("ebx","esi"); + + &shr ("ecx",18-14); # lo>>18 + &xor ("eax","edi"); + &shr ("edx",18-14); # hi>>18 + &xor ("eax","ecx"); + &shl ("esi",23-18); # lo<<23 + &xor ("ebx","edx"); + &shl ("edi",23-18); # hi<<23 + &xor ("eax","esi"); + &xor ("ebx","edi"); # T1 = Sigma1(e) + + &mov ("ecx",$Flo); + &mov ("edx",$Fhi); + &mov ("esi",$Glo); + &mov ("edi",$Ghi); + &add ("eax",$Hlo); + &adc ("ebx",$Hhi); # T1 += h + &xor ("ecx","esi"); + &xor ("edx","edi"); + &and ("ecx",$Elo); + &and ("edx",$Ehi); + &add ("eax",&DWP(8*(9+15)+0,"esp")); + &adc ("ebx",&DWP(8*(9+15)+4,"esp")); # T1 += X[0] + &xor ("ecx","esi"); + &xor ("edx","edi"); # Ch(e,f,g) = (f^g)&e)^g + + &mov ("esi",&DWP(0,$K512)); + &mov ("edi",&DWP(4,$K512)); # K[i] + &add ("eax","ecx"); + &adc ("ebx","edx"); # T1 += Ch(e,f,g) + &mov ("ecx",$Dlo); + &mov ("edx",$Dhi); + &add ("eax","esi"); + &adc ("ebx","edi"); # T1 += K[i] + &mov ($Tlo,"eax"); + &mov ($Thi,"ebx"); # put T1 away + &add ("eax","ecx"); + &adc ("ebx","edx"); # d += T1 + + #define Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39)) + # LO lo>>28^hi<<4 ^ hi>>2^lo<<30 ^ hi>>7^lo<<25 + # HI hi>>28^lo<<4 ^ lo>>2^hi<<30 ^ lo>>7^hi<<25 + &mov ("ecx",$Alo); + &mov ("edx",$Ahi); + &mov ($Dlo,"eax"); + &mov ($Dhi,"ebx"); + &mov ("esi","ecx"); + + &shr ("ecx",2) # lo>>2 + &mov ("edi","edx"); + &shr ("edx",2) # hi>>2 + &mov ("ebx","ecx"); + &shl ("esi",4); # lo<<4 + &mov ("eax","edx"); + &shl ("edi",4); # hi<<4 + &xor ("ebx","esi"); + + &shr ("ecx",7-2); # lo>>7 + &xor ("eax","edi"); + &shr ("edx",7-2); # hi>>7 + &xor ("ebx","ecx"); + &shl ("esi",25-4); # lo<<25 + &xor ("eax","edx"); + &shl ("edi",25-4); # hi<<25 + &xor ("eax","esi"); + + &shr ("ecx",28-7); # lo>>28 + &xor ("ebx","edi"); + &shr ("edx",28-7); # hi>>28 + &xor ("eax","ecx"); + &shl ("esi",30-25); # lo<<30 + &xor ("ebx","edx"); + &shl ("edi",30-25); # hi<<30 + &xor ("eax","esi"); + &xor ("ebx","edi"); # Sigma0(a) + + &mov ("ecx",$Alo); + &mov ("edx",$Ahi); + &mov ("esi",$Blo); + &mov ("edi",$Bhi); + &add ("eax",$Tlo); + &adc ("ebx",$Thi); # T1 = Sigma0(a)+T1 + &or ("ecx","esi"); + &or ("edx","edi"); + &and ("ecx",$Clo); + &and ("edx",$Chi); + &and ("esi",$Alo); + &and ("edi",$Ahi); + &or ("ecx","esi"); + &or ("edx","edi"); # Maj(a,b,c) = ((a|b)&c)|(a&b) + + &add ("eax","ecx"); + &adc ("ebx","edx"); # T1 += Maj(a,b,c) + &mov ($Tlo,"eax"); + &mov ($Thi,"ebx"); + + &mov (&LB("edx"),&BP(0,$K512)); # pre-fetch LSB of *K + &sub ("esp",8); + &lea ($K512,&DWP(8,$K512)); # K++ +} + + +&function_begin("sha512_block_data_order"); + &mov ("esi",wparam(0)); # ctx + &mov ("edi",wparam(1)); # inp + &mov ("eax",wparam(2)); # num + &mov ("ebx","esp"); # saved sp + + &call (&label("pic_point")); # make it PIC! +&set_label("pic_point"); + &blindpop($K512); + &lea ($K512,&DWP(&label("K512")."-".&label("pic_point"),$K512)); + + &sub ("esp",16); + &and ("esp",-64); + + &shl ("eax",7); + &add ("eax","edi"); + &mov (&DWP(0,"esp"),"esi"); # ctx + &mov (&DWP(4,"esp"),"edi"); # inp + &mov (&DWP(8,"esp"),"eax"); # inp+num*128 + &mov (&DWP(12,"esp"),"ebx"); # saved sp + +if ($sse2) { + &picmeup("edx","OPENSSL_ia32cap_P",$K512,&label("K512")); + &bt (&DWP(0,"edx"),26); + &jnc (&label("loop_x86")); + + # load ctx->h[0-7] + &movq ($A,&QWP(0,"esi")); + &movq ("mm1",&QWP(8,"esi")); + &movq ("mm2",&QWP(16,"esi")); + &movq ("mm3",&QWP(24,"esi")); + &movq ($E,&QWP(32,"esi")); + &movq ("mm5",&QWP(40,"esi")); + &movq ("mm6",&QWP(48,"esi")); + &movq ("mm7",&QWP(56,"esi")); + &sub ("esp",8*10); + +&set_label("loop_sse2",16); + # &movq ($Asse2,$A); + &movq ($Bsse2,"mm1"); + &movq ($Csse2,"mm2"); + &movq ($Dsse2,"mm3"); + # &movq ($Esse2,$E); + &movq ($Fsse2,"mm5"); + &movq ($Gsse2,"mm6"); + &movq ($Hsse2,"mm7"); + + &mov ("ecx",&DWP(0,"edi")); + &mov ("edx",&DWP(4,"edi")); + &add ("edi",8); + &bswap ("ecx"); + &bswap ("edx"); + &mov (&DWP(8*9+4,"esp"),"ecx"); + &mov (&DWP(8*9+0,"esp"),"edx"); + +&set_label("00_14_sse2",16); + &mov ("eax",&DWP(0,"edi")); + &mov ("ebx",&DWP(4,"edi")); + &add ("edi",8); + &bswap ("eax"); + &bswap ("ebx"); + &mov (&DWP(8*8+4,"esp"),"eax"); + &mov (&DWP(8*8+0,"esp"),"ebx"); + + &BODY_00_15_sse2(); + + &cmp (&LB("edx"),0x35); + &jne (&label("00_14_sse2")); + + &BODY_00_15_sse2(1); + +&set_label("16_79_sse2",16); + #&movq ("mm2",&QWP(8*(9+16-1),"esp")); #prefetched in BODY_00_15 + #&movq ("mm6",&QWP(8*(9+16-14),"esp")); + &movq ("mm1","mm2"); + + &psrlq ("mm2",1); + &movq ("mm7","mm6"); + &psrlq ("mm6",6); + &movq ("mm3","mm2"); + + &psrlq ("mm2",7-1); + &movq ("mm5","mm6"); + &psrlq ("mm6",19-6); + &pxor ("mm3","mm2"); + + &psrlq ("mm2",8-7); + &pxor ("mm5","mm6"); + &psrlq ("mm6",61-19); + &pxor ("mm3","mm2"); + + &movq ("mm2",&QWP(8*(9+16),"esp")); + + &psllq ("mm1",56); + &pxor ("mm5","mm6"); + &psllq ("mm7",3); + &pxor ("mm3","mm1"); + + &paddq ("mm2",&QWP(8*(9+16-9),"esp")); + + &psllq ("mm1",63-56); + &pxor ("mm5","mm7"); + &psllq ("mm7",45-3); + &pxor ("mm3","mm1"); + &pxor ("mm5","mm7"); + + &paddq ("mm3","mm5"); + &paddq ("mm3","mm2"); + &movq (&QWP(8*9,"esp"),"mm3"); + + &BODY_00_15_sse2(1); + + &cmp (&LB("edx"),0x17); + &jne (&label("16_79_sse2")); + + # &movq ($A,$Asse2); + &movq ("mm1",$Bsse2); + &movq ("mm2",$Csse2); + &movq ("mm3",$Dsse2); + # &movq ($E,$Esse2); + &movq ("mm5",$Fsse2); + &movq ("mm6",$Gsse2); + &movq ("mm7",$Hsse2); + + &paddq ($A,&QWP(0,"esi")); + &paddq ("mm1",&QWP(8,"esi")); + &paddq ("mm2",&QWP(16,"esi")); + &paddq ("mm3",&QWP(24,"esi")); + &paddq ($E,&QWP(32,"esi")); + &paddq ("mm5",&QWP(40,"esi")); + &paddq ("mm6",&QWP(48,"esi")); + &paddq ("mm7",&QWP(56,"esi")); + + &movq (&QWP(0,"esi"),$A); + &movq (&QWP(8,"esi"),"mm1"); + &movq (&QWP(16,"esi"),"mm2"); + &movq (&QWP(24,"esi"),"mm3"); + &movq (&QWP(32,"esi"),$E); + &movq (&QWP(40,"esi"),"mm5"); + &movq (&QWP(48,"esi"),"mm6"); + &movq (&QWP(56,"esi"),"mm7"); + + &add ("esp",8*80); # destroy frame + &sub ($K512,8*80); # rewind K + + &cmp ("edi",&DWP(8*10+8,"esp")); # are we done yet? + &jb (&label("loop_sse2")); + + &emms (); + &mov ("esp",&DWP(8*10+12,"esp")); # restore sp +&function_end_A(); +} +&set_label("loop_x86",16); + # copy input block to stack reversing byte and qword order + for ($i=0;$i<8;$i++) { + &mov ("eax",&DWP($i*16+0,"edi")); + &mov ("ebx",&DWP($i*16+4,"edi")); + &mov ("ecx",&DWP($i*16+8,"edi")); + &mov ("edx",&DWP($i*16+12,"edi")); + &bswap ("eax"); + &bswap ("ebx"); + &bswap ("ecx"); + &bswap ("edx"); + &push ("eax"); + &push ("ebx"); + &push ("ecx"); + &push ("edx"); + } + &add ("edi",128); + &sub ("esp",9*8); # place for T,A,B,C,D,E,F,G,H + &mov (&DWP(8*(9+16)+4,"esp"),"edi"); + + # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack + &lea ("edi",&DWP(8,"esp")); + &mov ("ecx",16); + &data_word(0xA5F3F689); # rep movsd + +&set_label("00_15_x86",16); + &BODY_00_15_x86(); + + &cmp (&LB("edx"),0x94); + &jne (&label("00_15_x86")); + +&set_label("16_79_x86",16); + #define sigma0(x) (ROTR((x),1) ^ ROTR((x),8) ^ ((x)>>7)) + # LO lo>>1^hi<<31 ^ lo>>8^hi<<24 ^ lo>>7^hi<<25 + # HI hi>>1^lo<<31 ^ hi>>8^lo<<24 ^ hi>>7 + &mov ("ecx",&DWP(8*(9+15+16-1)+0,"esp")); + &mov ("edx",&DWP(8*(9+15+16-1)+4,"esp")); + &mov ("esi","ecx"); + + &shr ("ecx",1) # lo>>1 + &mov ("edi","edx"); + &shr ("edx",1) # hi>>1 + &mov ("eax","ecx"); + &shl ("esi",24); # lo<<24 + &mov ("ebx","edx"); + &shl ("edi",24); # hi<<24 + &xor ("ebx","esi"); + + &shr ("ecx",7-1); # lo>>7 + &xor ("eax","edi"); + &shr ("edx",7-1); # hi>>7 + &xor ("eax","ecx"); + &shl ("esi",31-24); # lo<<31 + &xor ("ebx","edx"); + &shl ("edi",25-24); # hi<<25 + &xor ("ebx","esi"); + + &shr ("ecx",8-7); # lo>>8 + &xor ("eax","edi"); + &shr ("edx",8-7); # hi>>8 + &xor ("eax","ecx"); + &shl ("edi",31-25); # hi<<31 + &xor ("ebx","edx"); + &xor ("eax","edi"); # T1 = sigma0(X[-15]) + + &mov (&DWP(0,"esp"),"eax"); + &mov (&DWP(4,"esp"),"ebx"); # put T1 away + + #define sigma1(x) (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6)) + # LO lo>>19^hi<<13 ^ hi>>29^lo<<3 ^ lo>>6^hi<<26 + # HI hi>>19^lo<<13 ^ lo>>29^hi<<3 ^ hi>>6 + &mov ("ecx",&DWP(8*(9+15+16-14)+0,"esp")); + &mov ("edx",&DWP(8*(9+15+16-14)+4,"esp")); + &mov ("esi","ecx"); + + &shr ("ecx",6) # lo>>6 + &mov ("edi","edx"); + &shr ("edx",6) # hi>>6 + &mov ("eax","ecx"); + &shl ("esi",3); # lo<<3 + &mov ("ebx","edx"); + &shl ("edi",3); # hi<<3 + &xor ("eax","esi"); + + &shr ("ecx",19-6); # lo>>19 + &xor ("ebx","edi"); + &shr ("edx",19-6); # hi>>19 + &xor ("eax","ecx"); + &shl ("esi",13-3); # lo<<13 + &xor ("ebx","edx"); + &shl ("edi",13-3); # hi<<13 + &xor ("ebx","esi"); + + &shr ("ecx",29-19); # lo>>29 + &xor ("eax","edi"); + &shr ("edx",29-19); # hi>>29 + &xor ("ebx","ecx"); + &shl ("edi",26-13); # hi<<26 + &xor ("eax","edx"); + &xor ("eax","edi"); # sigma1(X[-2]) + + &mov ("ecx",&DWP(8*(9+15+16)+0,"esp")); + &mov ("edx",&DWP(8*(9+15+16)+4,"esp")); + &add ("eax",&DWP(0,"esp")); + &adc ("ebx",&DWP(4,"esp")); # T1 = sigma1(X[-2])+T1 + &mov ("esi",&DWP(8*(9+15+16-9)+0,"esp")); + &mov ("edi",&DWP(8*(9+15+16-9)+4,"esp")); + &add ("eax","ecx"); + &adc ("ebx","edx"); # T1 += X[-16] + &add ("eax","esi"); + &adc ("ebx","edi"); # T1 += X[-7] + &mov (&DWP(8*(9+15)+0,"esp"),"eax"); + &mov (&DWP(8*(9+15)+4,"esp"),"ebx"); # save X[0] + + &BODY_00_15_x86(); + + &cmp (&LB("edx"),0x17); + &jne (&label("16_79_x86")); + + &mov ("esi",&DWP(8*(9+16+80)+0,"esp"));# ctx + &mov ("edi",&DWP(8*(9+16+80)+4,"esp"));# inp + for($i=0;$i<4;$i++) { + &mov ("eax",&DWP($i*16+0,"esi")); + &mov ("ebx",&DWP($i*16+4,"esi")); + &mov ("ecx",&DWP($i*16+8,"esi")); + &mov ("edx",&DWP($i*16+12,"esi")); + &add ("eax",&DWP(8+($i*16)+0,"esp")); + &adc ("ebx",&DWP(8+($i*16)+4,"esp")); + &mov (&DWP($i*16+0,"esi"),"eax"); + &mov (&DWP($i*16+4,"esi"),"ebx"); + &add ("ecx",&DWP(8+($i*16)+8,"esp")); + &adc ("edx",&DWP(8+($i*16)+12,"esp")); + &mov (&DWP($i*16+8,"esi"),"ecx"); + &mov (&DWP($i*16+12,"esi"),"edx"); + } + &add ("esp",8*(9+16+80)); # destroy frame + &sub ($K512,8*80); # rewind K + + &cmp ("edi",&DWP(8,"esp")); # are we done yet? + &jb (&label("loop_x86")); + + &mov ("esp",&DWP(12,"esp")); # restore sp +&function_end_A(); + +&set_label("K512",64); # Yes! I keep it in the code segment! + &data_word(0xd728ae22,0x428a2f98); # u64 + &data_word(0x23ef65cd,0x71374491); # u64 + &data_word(0xec4d3b2f,0xb5c0fbcf); # u64 + &data_word(0x8189dbbc,0xe9b5dba5); # u64 + &data_word(0xf348b538,0x3956c25b); # u64 + &data_word(0xb605d019,0x59f111f1); # u64 + &data_word(0xaf194f9b,0x923f82a4); # u64 + &data_word(0xda6d8118,0xab1c5ed5); # u64 + &data_word(0xa3030242,0xd807aa98); # u64 + &data_word(0x45706fbe,0x12835b01); # u64 + &data_word(0x4ee4b28c,0x243185be); # u64 + &data_word(0xd5ffb4e2,0x550c7dc3); # u64 + &data_word(0xf27b896f,0x72be5d74); # u64 + &data_word(0x3b1696b1,0x80deb1fe); # u64 + &data_word(0x25c71235,0x9bdc06a7); # u64 + &data_word(0xcf692694,0xc19bf174); # u64 + &data_word(0x9ef14ad2,0xe49b69c1); # u64 + &data_word(0x384f25e3,0xefbe4786); # u64 + &data_word(0x8b8cd5b5,0x0fc19dc6); # u64 + &data_word(0x77ac9c65,0x240ca1cc); # u64 + &data_word(0x592b0275,0x2de92c6f); # u64 + &data_word(0x6ea6e483,0x4a7484aa); # u64 + &data_word(0xbd41fbd4,0x5cb0a9dc); # u64 + &data_word(0x831153b5,0x76f988da); # u64 + &data_word(0xee66dfab,0x983e5152); # u64 + &data_word(0x2db43210,0xa831c66d); # u64 + &data_word(0x98fb213f,0xb00327c8); # u64 + &data_word(0xbeef0ee4,0xbf597fc7); # u64 + &data_word(0x3da88fc2,0xc6e00bf3); # u64 + &data_word(0x930aa725,0xd5a79147); # u64 + &data_word(0xe003826f,0x06ca6351); # u64 + &data_word(0x0a0e6e70,0x14292967); # u64 + &data_word(0x46d22ffc,0x27b70a85); # u64 + &data_word(0x5c26c926,0x2e1b2138); # u64 + &data_word(0x5ac42aed,0x4d2c6dfc); # u64 + &data_word(0x9d95b3df,0x53380d13); # u64 + &data_word(0x8baf63de,0x650a7354); # u64 + &data_word(0x3c77b2a8,0x766a0abb); # u64 + &data_word(0x47edaee6,0x81c2c92e); # u64 + &data_word(0x1482353b,0x92722c85); # u64 + &data_word(0x4cf10364,0xa2bfe8a1); # u64 + &data_word(0xbc423001,0xa81a664b); # u64 + &data_word(0xd0f89791,0xc24b8b70); # u64 + &data_word(0x0654be30,0xc76c51a3); # u64 + &data_word(0xd6ef5218,0xd192e819); # u64 + &data_word(0x5565a910,0xd6990624); # u64 + &data_word(0x5771202a,0xf40e3585); # u64 + &data_word(0x32bbd1b8,0x106aa070); # u64 + &data_word(0xb8d2d0c8,0x19a4c116); # u64 + &data_word(0x5141ab53,0x1e376c08); # u64 + &data_word(0xdf8eeb99,0x2748774c); # u64 + &data_word(0xe19b48a8,0x34b0bcb5); # u64 + &data_word(0xc5c95a63,0x391c0cb3); # u64 + &data_word(0xe3418acb,0x4ed8aa4a); # u64 + &data_word(0x7763e373,0x5b9cca4f); # u64 + &data_word(0xd6b2b8a3,0x682e6ff3); # u64 + &data_word(0x5defb2fc,0x748f82ee); # u64 + &data_word(0x43172f60,0x78a5636f); # u64 + &data_word(0xa1f0ab72,0x84c87814); # u64 + &data_word(0x1a6439ec,0x8cc70208); # u64 + &data_word(0x23631e28,0x90befffa); # u64 + &data_word(0xde82bde9,0xa4506ceb); # u64 + &data_word(0xb2c67915,0xbef9a3f7); # u64 + &data_word(0xe372532b,0xc67178f2); # u64 + &data_word(0xea26619c,0xca273ece); # u64 + &data_word(0x21c0c207,0xd186b8c7); # u64 + &data_word(0xcde0eb1e,0xeada7dd6); # u64 + &data_word(0xee6ed178,0xf57d4f7f); # u64 + &data_word(0x72176fba,0x06f067aa); # u64 + &data_word(0xa2c898a6,0x0a637dc5); # u64 + &data_word(0xbef90dae,0x113f9804); # u64 + &data_word(0x131c471b,0x1b710b35); # u64 + &data_word(0x23047d84,0x28db77f5); # u64 + &data_word(0x40c72493,0x32caab7b); # u64 + &data_word(0x15c9bebc,0x3c9ebe0a); # u64 + &data_word(0x9c100d4c,0x431d67c4); # u64 + &data_word(0xcb3e42b6,0x4cc5d4be); # u64 + &data_word(0xfc657e2a,0x597f299c); # u64 + &data_word(0x3ad6faec,0x5fcb6fab); # u64 + &data_word(0x4a475817,0x6c44198c); # u64 +&function_end_B("sha512_block_data_order"); +&asciz("SHA512 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>"); + +&asm_finish(); diff --git a/crypto/openssl/crypto/sha/asm/sha512-armv4.pl b/crypto/openssl/crypto/sha/asm/sha512-armv4.pl new file mode 100755 index 000000000000..7faf37b14790 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-armv4.pl @@ -0,0 +1,582 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA512 block procedure for ARMv4. September 2007. + +# This code is ~4.5 (four and a half) times faster than code generated +# by gcc 3.4 and it spends ~72 clock cycles per byte [on single-issue +# Xscale PXA250 core]. +# +# July 2010. +# +# Rescheduling for dual-issue pipeline resulted in 6% improvement on +# Cortex A8 core and ~40 cycles per processed byte. + +# February 2011. +# +# Profiler-assisted and platform-specific optimization resulted in 7% +# improvement on Coxtex A8 core and ~38 cycles per byte. + +# March 2011. +# +# Add NEON implementation. On Cortex A8 it was measured to process +# one byte in 25.5 cycles or 47% faster than integer-only code. + +# Byte order [in]dependence. ========================================= +# +# Originally caller was expected to maintain specific *dword* order in +# h[0-7], namely with most significant dword at *lower* address, which +# was reflected in below two parameters as 0 and 4. Now caller is +# expected to maintain native byte order for whole 64-bit values. +$hi="HI"; +$lo="LO"; +# ==================================================================== + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +$ctx="r0"; # parameter block +$inp="r1"; +$len="r2"; + +$Tlo="r3"; +$Thi="r4"; +$Alo="r5"; +$Ahi="r6"; +$Elo="r7"; +$Ehi="r8"; +$t0="r9"; +$t1="r10"; +$t2="r11"; +$t3="r12"; +############ r13 is stack pointer +$Ktbl="r14"; +############ r15 is program counter + +$Aoff=8*0; +$Boff=8*1; +$Coff=8*2; +$Doff=8*3; +$Eoff=8*4; +$Foff=8*5; +$Goff=8*6; +$Hoff=8*7; +$Xoff=8*8; + +sub BODY_00_15() { +my $magic = shift; +$code.=<<___; + @ Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41)) + @ LO lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23 + @ HI hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23 + mov $t0,$Elo,lsr#14 + str $Tlo,[sp,#$Xoff+0] + mov $t1,$Ehi,lsr#14 + str $Thi,[sp,#$Xoff+4] + eor $t0,$t0,$Ehi,lsl#18 + ldr $t2,[sp,#$Hoff+0] @ h.lo + eor $t1,$t1,$Elo,lsl#18 + ldr $t3,[sp,#$Hoff+4] @ h.hi + eor $t0,$t0,$Elo,lsr#18 + eor $t1,$t1,$Ehi,lsr#18 + eor $t0,$t0,$Ehi,lsl#14 + eor $t1,$t1,$Elo,lsl#14 + eor $t0,$t0,$Ehi,lsr#9 + eor $t1,$t1,$Elo,lsr#9 + eor $t0,$t0,$Elo,lsl#23 + eor $t1,$t1,$Ehi,lsl#23 @ Sigma1(e) + adds $Tlo,$Tlo,$t0 + ldr $t0,[sp,#$Foff+0] @ f.lo + adc $Thi,$Thi,$t1 @ T += Sigma1(e) + ldr $t1,[sp,#$Foff+4] @ f.hi + adds $Tlo,$Tlo,$t2 + ldr $t2,[sp,#$Goff+0] @ g.lo + adc $Thi,$Thi,$t3 @ T += h + ldr $t3,[sp,#$Goff+4] @ g.hi + + eor $t0,$t0,$t2 + str $Elo,[sp,#$Eoff+0] + eor $t1,$t1,$t3 + str $Ehi,[sp,#$Eoff+4] + and $t0,$t0,$Elo + str $Alo,[sp,#$Aoff+0] + and $t1,$t1,$Ehi + str $Ahi,[sp,#$Aoff+4] + eor $t0,$t0,$t2 + ldr $t2,[$Ktbl,#$lo] @ K[i].lo + eor $t1,$t1,$t3 @ Ch(e,f,g) + ldr $t3,[$Ktbl,#$hi] @ K[i].hi + + adds $Tlo,$Tlo,$t0 + ldr $Elo,[sp,#$Doff+0] @ d.lo + adc $Thi,$Thi,$t1 @ T += Ch(e,f,g) + ldr $Ehi,[sp,#$Doff+4] @ d.hi + adds $Tlo,$Tlo,$t2 + and $t0,$t2,#0xff + adc $Thi,$Thi,$t3 @ T += K[i] + adds $Elo,$Elo,$Tlo + ldr $t2,[sp,#$Boff+0] @ b.lo + adc $Ehi,$Ehi,$Thi @ d += T + teq $t0,#$magic + + ldr $t3,[sp,#$Coff+0] @ c.lo + orreq $Ktbl,$Ktbl,#1 + @ Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39)) + @ LO lo>>28^hi<<4 ^ hi>>2^lo<<30 ^ hi>>7^lo<<25 + @ HI hi>>28^lo<<4 ^ lo>>2^hi<<30 ^ lo>>7^hi<<25 + mov $t0,$Alo,lsr#28 + mov $t1,$Ahi,lsr#28 + eor $t0,$t0,$Ahi,lsl#4 + eor $t1,$t1,$Alo,lsl#4 + eor $t0,$t0,$Ahi,lsr#2 + eor $t1,$t1,$Alo,lsr#2 + eor $t0,$t0,$Alo,lsl#30 + eor $t1,$t1,$Ahi,lsl#30 + eor $t0,$t0,$Ahi,lsr#7 + eor $t1,$t1,$Alo,lsr#7 + eor $t0,$t0,$Alo,lsl#25 + eor $t1,$t1,$Ahi,lsl#25 @ Sigma0(a) + adds $Tlo,$Tlo,$t0 + and $t0,$Alo,$t2 + adc $Thi,$Thi,$t1 @ T += Sigma0(a) + + ldr $t1,[sp,#$Boff+4] @ b.hi + orr $Alo,$Alo,$t2 + ldr $t2,[sp,#$Coff+4] @ c.hi + and $Alo,$Alo,$t3 + and $t3,$Ahi,$t1 + orr $Ahi,$Ahi,$t1 + orr $Alo,$Alo,$t0 @ Maj(a,b,c).lo + and $Ahi,$Ahi,$t2 + adds $Alo,$Alo,$Tlo + orr $Ahi,$Ahi,$t3 @ Maj(a,b,c).hi + sub sp,sp,#8 + adc $Ahi,$Ahi,$Thi @ h += T + tst $Ktbl,#1 + add $Ktbl,$Ktbl,#8 +___ +} +$code=<<___; +#include "arm_arch.h" +#ifdef __ARMEL__ +# define LO 0 +# define HI 4 +# define WORD64(hi0,lo0,hi1,lo1) .word lo0,hi0, lo1,hi1 +#else +# define HI 0 +# define LO 4 +# define WORD64(hi0,lo0,hi1,lo1) .word hi0,lo0, hi1,lo1 +#endif + +.text +.code 32 +.type K512,%object +.align 5 +K512: +WORD64(0x428a2f98,0xd728ae22, 0x71374491,0x23ef65cd) +WORD64(0xb5c0fbcf,0xec4d3b2f, 0xe9b5dba5,0x8189dbbc) +WORD64(0x3956c25b,0xf348b538, 0x59f111f1,0xb605d019) +WORD64(0x923f82a4,0xaf194f9b, 0xab1c5ed5,0xda6d8118) +WORD64(0xd807aa98,0xa3030242, 0x12835b01,0x45706fbe) +WORD64(0x243185be,0x4ee4b28c, 0x550c7dc3,0xd5ffb4e2) +WORD64(0x72be5d74,0xf27b896f, 0x80deb1fe,0x3b1696b1) +WORD64(0x9bdc06a7,0x25c71235, 0xc19bf174,0xcf692694) +WORD64(0xe49b69c1,0x9ef14ad2, 0xefbe4786,0x384f25e3) +WORD64(0x0fc19dc6,0x8b8cd5b5, 0x240ca1cc,0x77ac9c65) +WORD64(0x2de92c6f,0x592b0275, 0x4a7484aa,0x6ea6e483) +WORD64(0x5cb0a9dc,0xbd41fbd4, 0x76f988da,0x831153b5) +WORD64(0x983e5152,0xee66dfab, 0xa831c66d,0x2db43210) +WORD64(0xb00327c8,0x98fb213f, 0xbf597fc7,0xbeef0ee4) +WORD64(0xc6e00bf3,0x3da88fc2, 0xd5a79147,0x930aa725) +WORD64(0x06ca6351,0xe003826f, 0x14292967,0x0a0e6e70) +WORD64(0x27b70a85,0x46d22ffc, 0x2e1b2138,0x5c26c926) +WORD64(0x4d2c6dfc,0x5ac42aed, 0x53380d13,0x9d95b3df) +WORD64(0x650a7354,0x8baf63de, 0x766a0abb,0x3c77b2a8) +WORD64(0x81c2c92e,0x47edaee6, 0x92722c85,0x1482353b) +WORD64(0xa2bfe8a1,0x4cf10364, 0xa81a664b,0xbc423001) +WORD64(0xc24b8b70,0xd0f89791, 0xc76c51a3,0x0654be30) +WORD64(0xd192e819,0xd6ef5218, 0xd6990624,0x5565a910) +WORD64(0xf40e3585,0x5771202a, 0x106aa070,0x32bbd1b8) +WORD64(0x19a4c116,0xb8d2d0c8, 0x1e376c08,0x5141ab53) +WORD64(0x2748774c,0xdf8eeb99, 0x34b0bcb5,0xe19b48a8) +WORD64(0x391c0cb3,0xc5c95a63, 0x4ed8aa4a,0xe3418acb) +WORD64(0x5b9cca4f,0x7763e373, 0x682e6ff3,0xd6b2b8a3) +WORD64(0x748f82ee,0x5defb2fc, 0x78a5636f,0x43172f60) +WORD64(0x84c87814,0xa1f0ab72, 0x8cc70208,0x1a6439ec) +WORD64(0x90befffa,0x23631e28, 0xa4506ceb,0xde82bde9) +WORD64(0xbef9a3f7,0xb2c67915, 0xc67178f2,0xe372532b) +WORD64(0xca273ece,0xea26619c, 0xd186b8c7,0x21c0c207) +WORD64(0xeada7dd6,0xcde0eb1e, 0xf57d4f7f,0xee6ed178) +WORD64(0x06f067aa,0x72176fba, 0x0a637dc5,0xa2c898a6) +WORD64(0x113f9804,0xbef90dae, 0x1b710b35,0x131c471b) +WORD64(0x28db77f5,0x23047d84, 0x32caab7b,0x40c72493) +WORD64(0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c) +WORD64(0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a) +WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817) +.size K512,.-K512 +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-sha512_block_data_order +.skip 32-4 + +.global sha512_block_data_order +.type sha512_block_data_order,%function +sha512_block_data_order: + sub r3,pc,#8 @ sha512_block_data_order + add $len,$inp,$len,lsl#7 @ len to point at the end of inp +#if __ARM_ARCH__>=7 + ldr r12,.LOPENSSL_armcap + ldr r12,[r3,r12] @ OPENSSL_armcap_P + tst r12,#1 + bne .LNEON +#endif + stmdb sp!,{r4-r12,lr} + sub $Ktbl,r3,#672 @ K512 + sub sp,sp,#9*8 + + ldr $Elo,[$ctx,#$Eoff+$lo] + ldr $Ehi,[$ctx,#$Eoff+$hi] + ldr $t0, [$ctx,#$Goff+$lo] + ldr $t1, [$ctx,#$Goff+$hi] + ldr $t2, [$ctx,#$Hoff+$lo] + ldr $t3, [$ctx,#$Hoff+$hi] +.Loop: + str $t0, [sp,#$Goff+0] + str $t1, [sp,#$Goff+4] + str $t2, [sp,#$Hoff+0] + str $t3, [sp,#$Hoff+4] + ldr $Alo,[$ctx,#$Aoff+$lo] + ldr $Ahi,[$ctx,#$Aoff+$hi] + ldr $Tlo,[$ctx,#$Boff+$lo] + ldr $Thi,[$ctx,#$Boff+$hi] + ldr $t0, [$ctx,#$Coff+$lo] + ldr $t1, [$ctx,#$Coff+$hi] + ldr $t2, [$ctx,#$Doff+$lo] + ldr $t3, [$ctx,#$Doff+$hi] + str $Tlo,[sp,#$Boff+0] + str $Thi,[sp,#$Boff+4] + str $t0, [sp,#$Coff+0] + str $t1, [sp,#$Coff+4] + str $t2, [sp,#$Doff+0] + str $t3, [sp,#$Doff+4] + ldr $Tlo,[$ctx,#$Foff+$lo] + ldr $Thi,[$ctx,#$Foff+$hi] + str $Tlo,[sp,#$Foff+0] + str $Thi,[sp,#$Foff+4] + +.L00_15: +#if __ARM_ARCH__<7 + ldrb $Tlo,[$inp,#7] + ldrb $t0, [$inp,#6] + ldrb $t1, [$inp,#5] + ldrb $t2, [$inp,#4] + ldrb $Thi,[$inp,#3] + ldrb $t3, [$inp,#2] + orr $Tlo,$Tlo,$t0,lsl#8 + ldrb $t0, [$inp,#1] + orr $Tlo,$Tlo,$t1,lsl#16 + ldrb $t1, [$inp],#8 + orr $Tlo,$Tlo,$t2,lsl#24 + orr $Thi,$Thi,$t3,lsl#8 + orr $Thi,$Thi,$t0,lsl#16 + orr $Thi,$Thi,$t1,lsl#24 +#else + ldr $Tlo,[$inp,#4] + ldr $Thi,[$inp],#8 +#ifdef __ARMEL__ + rev $Tlo,$Tlo + rev $Thi,$Thi +#endif +#endif +___ + &BODY_00_15(0x94); +$code.=<<___; + tst $Ktbl,#1 + beq .L00_15 + ldr $t0,[sp,#`$Xoff+8*(16-1)`+0] + ldr $t1,[sp,#`$Xoff+8*(16-1)`+4] + bic $Ktbl,$Ktbl,#1 +.L16_79: + @ sigma0(x) (ROTR((x),1) ^ ROTR((x),8) ^ ((x)>>7)) + @ LO lo>>1^hi<<31 ^ lo>>8^hi<<24 ^ lo>>7^hi<<25 + @ HI hi>>1^lo<<31 ^ hi>>8^lo<<24 ^ hi>>7 + mov $Tlo,$t0,lsr#1 + ldr $t2,[sp,#`$Xoff+8*(16-14)`+0] + mov $Thi,$t1,lsr#1 + ldr $t3,[sp,#`$Xoff+8*(16-14)`+4] + eor $Tlo,$Tlo,$t1,lsl#31 + eor $Thi,$Thi,$t0,lsl#31 + eor $Tlo,$Tlo,$t0,lsr#8 + eor $Thi,$Thi,$t1,lsr#8 + eor $Tlo,$Tlo,$t1,lsl#24 + eor $Thi,$Thi,$t0,lsl#24 + eor $Tlo,$Tlo,$t0,lsr#7 + eor $Thi,$Thi,$t1,lsr#7 + eor $Tlo,$Tlo,$t1,lsl#25 + + @ sigma1(x) (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6)) + @ LO lo>>19^hi<<13 ^ hi>>29^lo<<3 ^ lo>>6^hi<<26 + @ HI hi>>19^lo<<13 ^ lo>>29^hi<<3 ^ hi>>6 + mov $t0,$t2,lsr#19 + mov $t1,$t3,lsr#19 + eor $t0,$t0,$t3,lsl#13 + eor $t1,$t1,$t2,lsl#13 + eor $t0,$t0,$t3,lsr#29 + eor $t1,$t1,$t2,lsr#29 + eor $t0,$t0,$t2,lsl#3 + eor $t1,$t1,$t3,lsl#3 + eor $t0,$t0,$t2,lsr#6 + eor $t1,$t1,$t3,lsr#6 + ldr $t2,[sp,#`$Xoff+8*(16-9)`+0] + eor $t0,$t0,$t3,lsl#26 + + ldr $t3,[sp,#`$Xoff+8*(16-9)`+4] + adds $Tlo,$Tlo,$t0 + ldr $t0,[sp,#`$Xoff+8*16`+0] + adc $Thi,$Thi,$t1 + + ldr $t1,[sp,#`$Xoff+8*16`+4] + adds $Tlo,$Tlo,$t2 + adc $Thi,$Thi,$t3 + adds $Tlo,$Tlo,$t0 + adc $Thi,$Thi,$t1 +___ + &BODY_00_15(0x17); +$code.=<<___; + ldreq $t0,[sp,#`$Xoff+8*(16-1)`+0] + ldreq $t1,[sp,#`$Xoff+8*(16-1)`+4] + beq .L16_79 + bic $Ktbl,$Ktbl,#1 + + ldr $Tlo,[sp,#$Boff+0] + ldr $Thi,[sp,#$Boff+4] + ldr $t0, [$ctx,#$Aoff+$lo] + ldr $t1, [$ctx,#$Aoff+$hi] + ldr $t2, [$ctx,#$Boff+$lo] + ldr $t3, [$ctx,#$Boff+$hi] + adds $t0,$Alo,$t0 + str $t0, [$ctx,#$Aoff+$lo] + adc $t1,$Ahi,$t1 + str $t1, [$ctx,#$Aoff+$hi] + adds $t2,$Tlo,$t2 + str $t2, [$ctx,#$Boff+$lo] + adc $t3,$Thi,$t3 + str $t3, [$ctx,#$Boff+$hi] + + ldr $Alo,[sp,#$Coff+0] + ldr $Ahi,[sp,#$Coff+4] + ldr $Tlo,[sp,#$Doff+0] + ldr $Thi,[sp,#$Doff+4] + ldr $t0, [$ctx,#$Coff+$lo] + ldr $t1, [$ctx,#$Coff+$hi] + ldr $t2, [$ctx,#$Doff+$lo] + ldr $t3, [$ctx,#$Doff+$hi] + adds $t0,$Alo,$t0 + str $t0, [$ctx,#$Coff+$lo] + adc $t1,$Ahi,$t1 + str $t1, [$ctx,#$Coff+$hi] + adds $t2,$Tlo,$t2 + str $t2, [$ctx,#$Doff+$lo] + adc $t3,$Thi,$t3 + str $t3, [$ctx,#$Doff+$hi] + + ldr $Tlo,[sp,#$Foff+0] + ldr $Thi,[sp,#$Foff+4] + ldr $t0, [$ctx,#$Eoff+$lo] + ldr $t1, [$ctx,#$Eoff+$hi] + ldr $t2, [$ctx,#$Foff+$lo] + ldr $t3, [$ctx,#$Foff+$hi] + adds $Elo,$Elo,$t0 + str $Elo,[$ctx,#$Eoff+$lo] + adc $Ehi,$Ehi,$t1 + str $Ehi,[$ctx,#$Eoff+$hi] + adds $t2,$Tlo,$t2 + str $t2, [$ctx,#$Foff+$lo] + adc $t3,$Thi,$t3 + str $t3, [$ctx,#$Foff+$hi] + + ldr $Alo,[sp,#$Goff+0] + ldr $Ahi,[sp,#$Goff+4] + ldr $Tlo,[sp,#$Hoff+0] + ldr $Thi,[sp,#$Hoff+4] + ldr $t0, [$ctx,#$Goff+$lo] + ldr $t1, [$ctx,#$Goff+$hi] + ldr $t2, [$ctx,#$Hoff+$lo] + ldr $t3, [$ctx,#$Hoff+$hi] + adds $t0,$Alo,$t0 + str $t0, [$ctx,#$Goff+$lo] + adc $t1,$Ahi,$t1 + str $t1, [$ctx,#$Goff+$hi] + adds $t2,$Tlo,$t2 + str $t2, [$ctx,#$Hoff+$lo] + adc $t3,$Thi,$t3 + str $t3, [$ctx,#$Hoff+$hi] + + add sp,sp,#640 + sub $Ktbl,$Ktbl,#640 + + teq $inp,$len + bne .Loop + + add sp,sp,#8*9 @ destroy frame +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +___ + +{ +my @Sigma0=(28,34,39); +my @Sigma1=(14,18,41); +my @sigma0=(1, 8, 7); +my @sigma1=(19,61,6); + +my $Ktbl="r3"; +my $cnt="r12"; # volatile register known as ip, intra-procedure-call scratch + +my @X=map("d$_",(0..15)); +my @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("d$_",(16..23)); + +sub NEON_00_15() { +my $i=shift; +my ($a,$b,$c,$d,$e,$f,$g,$h)=@_; +my ($t0,$t1,$t2,$T1,$K,$Ch,$Maj)=map("d$_",(24..31)); # temps + +$code.=<<___ if ($i<16 || $i&1); + vshr.u64 $t0,$e,#@Sigma1[0] @ $i +#if $i<16 + vld1.64 {@X[$i%16]},[$inp]! @ handles unaligned +#endif + vshr.u64 $t1,$e,#@Sigma1[1] + vshr.u64 $t2,$e,#@Sigma1[2] +___ +$code.=<<___; + vld1.64 {$K},[$Ktbl,:64]! @ K[i++] + vsli.64 $t0,$e,#`64-@Sigma1[0]` + vsli.64 $t1,$e,#`64-@Sigma1[1]` + vsli.64 $t2,$e,#`64-@Sigma1[2]` +#if $i<16 && defined(__ARMEL__) + vrev64.8 @X[$i],@X[$i] +#endif + vadd.i64 $T1,$K,$h + veor $Ch,$f,$g + veor $t0,$t1 + vand $Ch,$e + veor $t0,$t2 @ Sigma1(e) + veor $Ch,$g @ Ch(e,f,g) + vadd.i64 $T1,$t0 + vshr.u64 $t0,$a,#@Sigma0[0] + vadd.i64 $T1,$Ch + vshr.u64 $t1,$a,#@Sigma0[1] + vshr.u64 $t2,$a,#@Sigma0[2] + vsli.64 $t0,$a,#`64-@Sigma0[0]` + vsli.64 $t1,$a,#`64-@Sigma0[1]` + vsli.64 $t2,$a,#`64-@Sigma0[2]` + vadd.i64 $T1,@X[$i%16] + vorr $Maj,$a,$c + vand $Ch,$a,$c + veor $h,$t0,$t1 + vand $Maj,$b + veor $h,$t2 @ Sigma0(a) + vorr $Maj,$Ch @ Maj(a,b,c) + vadd.i64 $h,$T1 + vadd.i64 $d,$T1 + vadd.i64 $h,$Maj +___ +} + +sub NEON_16_79() { +my $i=shift; + +if ($i&1) { &NEON_00_15($i,@_); return; } + +# 2x-vectorized, therefore runs every 2nd round +my @X=map("q$_",(0..7)); # view @X as 128-bit vector +my ($t0,$t1,$s0,$s1) = map("q$_",(12..15)); # temps +my ($d0,$d1,$d2) = map("d$_",(24..26)); # temps from NEON_00_15 +my $e=@_[4]; # $e from NEON_00_15 +$i /= 2; +$code.=<<___; + vshr.u64 $t0,@X[($i+7)%8],#@sigma1[0] + vshr.u64 $t1,@X[($i+7)%8],#@sigma1[1] + vshr.u64 $s1,@X[($i+7)%8],#@sigma1[2] + vsli.64 $t0,@X[($i+7)%8],#`64-@sigma1[0]` + vext.8 $s0,@X[$i%8],@X[($i+1)%8],#8 @ X[i+1] + vsli.64 $t1,@X[($i+7)%8],#`64-@sigma1[1]` + veor $s1,$t0 + vshr.u64 $t0,$s0,#@sigma0[0] + veor $s1,$t1 @ sigma1(X[i+14]) + vshr.u64 $t1,$s0,#@sigma0[1] + vadd.i64 @X[$i%8],$s1 + vshr.u64 $s1,$s0,#@sigma0[2] + vsli.64 $t0,$s0,#`64-@sigma0[0]` + vsli.64 $t1,$s0,#`64-@sigma0[1]` + vext.8 $s0,@X[($i+4)%8],@X[($i+5)%8],#8 @ X[i+9] + veor $s1,$t0 + vshr.u64 $d0,$e,#@Sigma1[0] @ from NEON_00_15 + vadd.i64 @X[$i%8],$s0 + vshr.u64 $d1,$e,#@Sigma1[1] @ from NEON_00_15 + veor $s1,$t1 @ sigma0(X[i+1]) + vshr.u64 $d2,$e,#@Sigma1[2] @ from NEON_00_15 + vadd.i64 @X[$i%8],$s1 +___ + &NEON_00_15(2*$i,@_); +} + +$code.=<<___; +#if __ARM_ARCH__>=7 +.fpu neon + +.align 4 +.LNEON: + dmb @ errata #451034 on early Cortex A8 + vstmdb sp!,{d8-d15} @ ABI specification says so + sub $Ktbl,r3,#672 @ K512 + vldmia $ctx,{$A-$H} @ load context +.Loop_neon: +___ +for($i=0;$i<16;$i++) { &NEON_00_15($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + mov $cnt,#4 +.L16_79_neon: + subs $cnt,#1 +___ +for(;$i<32;$i++) { &NEON_16_79($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + bne .L16_79_neon + + vldmia $ctx,{d24-d31} @ load context to temp + vadd.i64 q8,q12 @ vectorized accumulate + vadd.i64 q9,q13 + vadd.i64 q10,q14 + vadd.i64 q11,q15 + vstmia $ctx,{$A-$H} @ save context + teq $inp,$len + sub $Ktbl,#640 @ rewind K512 + bne .Loop_neon + + vldmia sp!,{d8-d15} @ epilogue + bx lr +#endif +___ +} +$code.=<<___; +.size sha512_block_data_order,.-sha512_block_data_order +.asciz "SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>" +.align 2 +.comm OPENSSL_armcap_P,4,4 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 +print $code; +close STDOUT; # enforce flush diff --git a/crypto/openssl/crypto/sha/asm/sha512-mips.pl b/crypto/openssl/crypto/sha/asm/sha512-mips.pl new file mode 100755 index 000000000000..ba5b250890e0 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-mips.pl @@ -0,0 +1,455 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA2 block procedures for MIPS. + +# October 2010. +# +# SHA256 performance improvement on MIPS R5000 CPU is ~27% over gcc- +# generated code in o32 build and ~55% in n32/64 build. SHA512 [which +# for now can only be compiled for MIPS64 ISA] improvement is modest +# ~17%, but it comes for free, because it's same instruction sequence. +# Improvement coefficients are for aligned input. + +###################################################################### +# There is a number of MIPS ABI in use, O32 and N32/64 are most +# widely used. Then there is a new contender: NUBI. It appears that if +# one picks the latter, it's possible to arrange code in ABI neutral +# manner. Therefore let's stick to NUBI register layout: +# +($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25)); +($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23)); +($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31)); +# +# The return value is placed in $a0. Following coding rules facilitate +# interoperability: +# +# - never ever touch $tp, "thread pointer", former $gp [o32 can be +# excluded from the rule, because it's specified volatile]; +# - copy return value to $t0, former $v0 [or to $a0 if you're adapting +# old code]; +# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary; +# +# For reference here is register layout for N32/64 MIPS ABIs: +# +# ($zero,$at,$v0,$v1)=map("\$$_",(0..3)); +# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25)); +# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); +# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); +# +$flavour = shift; # supported flavours are o32,n32,64,nubi32,nubi64 + +if ($flavour =~ /64|n32/i) { + $PTR_ADD="dadd"; # incidentally works even on n32 + $PTR_SUB="dsub"; # incidentally works even on n32 + $REG_S="sd"; + $REG_L="ld"; + $PTR_SLL="dsll"; # incidentally works even on n32 + $SZREG=8; +} else { + $PTR_ADD="add"; + $PTR_SUB="sub"; + $REG_S="sw"; + $REG_L="lw"; + $PTR_SLL="sll"; + $SZREG=4; +} +$pf = ($flavour =~ /nubi/i) ? $t0 : $t2; +# +# <appro@openssl.org> +# +###################################################################### + +$big_endian=(`echo MIPSEL | $ENV{CC} -E -P -`=~/MIPSEL/)?1:0; + +for (@ARGV) { $output=$_ if (/^\w[\w\-]*\.\w+$/); } +open STDOUT,">$output"; + +if (!defined($big_endian)) { $big_endian=(unpack('L',pack('N',1))==1); } + +if ($output =~ /512/) { + $label="512"; + $SZ=8; + $LD="ld"; # load from memory + $ST="sd"; # store to memory + $SLL="dsll"; # shift left logical + $SRL="dsrl"; # shift right logical + $ADDU="daddu"; + @Sigma0=(28,34,39); + @Sigma1=(14,18,41); + @sigma0=( 7, 1, 8); # right shift first + @sigma1=( 6,19,61); # right shift first + $lastK=0x817; + $rounds=80; +} else { + $label="256"; + $SZ=4; + $LD="lw"; # load from memory + $ST="sw"; # store to memory + $SLL="sll"; # shift left logical + $SRL="srl"; # shift right logical + $ADDU="addu"; + @Sigma0=( 2,13,22); + @Sigma1=( 6,11,25); + @sigma0=( 3, 7,18); # right shift first + @sigma1=(10,17,19); # right shift first + $lastK=0x8f2; + $rounds=64; +} + +$MSB = $big_endian ? 0 : ($SZ-1); +$LSB = ($SZ-1)&~$MSB; + +@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("\$$_",(1,2,3,7,24,25,30,31)); +@X=map("\$$_",(8..23)); + +$ctx=$a0; +$inp=$a1; +$len=$a2; $Ktbl=$len; + +sub BODY_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +my ($T1,$tmp0,$tmp1,$tmp2)=(@X[4],@X[5],@X[6],@X[7]); + +$code.=<<___ if ($i<15); + ${LD}l @X[1],`($i+1)*$SZ+$MSB`($inp) + ${LD}r @X[1],`($i+1)*$SZ+$LSB`($inp) +___ +$code.=<<___ if (!$big_endian && $i<16 && $SZ==4); + srl $tmp0,@X[0],24 # byte swap($i) + srl $tmp1,@X[0],8 + andi $tmp2,@X[0],0xFF00 + sll @X[0],@X[0],24 + andi $tmp1,0xFF00 + sll $tmp2,$tmp2,8 + or @X[0],$tmp0 + or $tmp1,$tmp2 + or @X[0],$tmp1 +___ +$code.=<<___ if (!$big_endian && $i<16 && $SZ==8); + ori $tmp0,$zero,0xFF + dsll $tmp2,$tmp0,32 + or $tmp0,$tmp2 # 0x000000FF000000FF + and $tmp1,@X[0],$tmp0 # byte swap($i) + dsrl $tmp2,@X[0],24 + dsll $tmp1,24 + and $tmp2,$tmp0 + dsll $tmp0,8 # 0x0000FF000000FF00 + or $tmp1,$tmp2 + and $tmp2,@X[0],$tmp0 + dsrl @X[0],8 + dsll $tmp2,8 + and @X[0],$tmp0 + or $tmp1,$tmp2 + or @X[0],$tmp1 + dsrl $tmp1,@X[0],32 + dsll @X[0],32 + or @X[0],$tmp1 +___ +$code.=<<___; + $ADDU $T1,$X[0],$h # $i + $SRL $h,$e,@Sigma1[0] + xor $tmp2,$f,$g + $SLL $tmp1,$e,`$SZ*8-@Sigma1[2]` + and $tmp2,$e + $SRL $tmp0,$e,@Sigma1[1] + xor $h,$tmp1 + $SLL $tmp1,$e,`$SZ*8-@Sigma1[1]` + xor $h,$tmp0 + $SRL $tmp0,$e,@Sigma1[2] + xor $h,$tmp1 + $SLL $tmp1,$e,`$SZ*8-@Sigma1[0]` + xor $h,$tmp0 + xor $tmp2,$g # Ch(e,f,g) + xor $tmp0,$tmp1,$h # Sigma1(e) + + $SRL $h,$a,@Sigma0[0] + $ADDU $T1,$tmp2 + $LD $tmp2,`$i*$SZ`($Ktbl) # K[$i] + $SLL $tmp1,$a,`$SZ*8-@Sigma0[2]` + $ADDU $T1,$tmp0 + $SRL $tmp0,$a,@Sigma0[1] + xor $h,$tmp1 + $SLL $tmp1,$a,`$SZ*8-@Sigma0[1]` + xor $h,$tmp0 + $SRL $tmp0,$a,@Sigma0[2] + xor $h,$tmp1 + $SLL $tmp1,$a,`$SZ*8-@Sigma0[0]` + xor $h,$tmp0 + $ST @X[0],`($i%16)*$SZ`($sp) # offload to ring buffer + xor $h,$tmp1 # Sigma0(a) + + or $tmp0,$a,$b + and $tmp1,$a,$b + and $tmp0,$c + or $tmp1,$tmp0 # Maj(a,b,c) + $ADDU $T1,$tmp2 # +=K[$i] + $ADDU $h,$tmp1 + + $ADDU $d,$T1 + $ADDU $h,$T1 +___ +$code.=<<___ if ($i>=13); + $LD @X[3],`(($i+3)%16)*$SZ`($sp) # prefetch from ring buffer +___ +} + +sub BODY_16_XX { +my $i=@_[0]; +my ($tmp0,$tmp1,$tmp2,$tmp3)=(@X[4],@X[5],@X[6],@X[7]); + +$code.=<<___; + $SRL $tmp2,@X[1],@sigma0[0] # Xupdate($i) + $ADDU @X[0],@X[9] # +=X[i+9] + $SLL $tmp1,@X[1],`$SZ*8-@sigma0[2]` + $SRL $tmp0,@X[1],@sigma0[1] + xor $tmp2,$tmp1 + $SLL $tmp1,`@sigma0[2]-@sigma0[1]` + xor $tmp2,$tmp0 + $SRL $tmp0,@X[1],@sigma0[2] + xor $tmp2,$tmp1 + + $SRL $tmp3,@X[14],@sigma1[0] + xor $tmp2,$tmp0 # sigma0(X[i+1]) + $SLL $tmp1,@X[14],`$SZ*8-@sigma1[2]` + $ADDU @X[0],$tmp2 + $SRL $tmp0,@X[14],@sigma1[1] + xor $tmp3,$tmp1 + $SLL $tmp1,`@sigma1[2]-@sigma1[1]` + xor $tmp3,$tmp0 + $SRL $tmp0,@X[14],@sigma1[2] + xor $tmp3,$tmp1 + + xor $tmp3,$tmp0 # sigma1(X[i+14]) + $ADDU @X[0],$tmp3 +___ + &BODY_00_15(@_); +} + +$FRAMESIZE=16*$SZ+16*$SZREG; +$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? 0xc0fff008 : 0xc0ff0000; + +$code.=<<___; +#ifdef OPENSSL_FIPSCANISTER +# include <openssl/fipssyms.h> +#endif + +.text +.set noat +#if !defined(__vxworks) || defined(__pic__) +.option pic2 +#endif + +.align 5 +.globl sha${label}_block_data_order +.ent sha${label}_block_data_order +sha${label}_block_data_order: + .frame $sp,$FRAMESIZE,$ra + .mask $SAVED_REGS_MASK,-$SZREG + .set noreorder +___ +$code.=<<___ if ($flavour =~ /o32/i); # o32 PIC-ification + .cpload $pf +___ +$code.=<<___; + $PTR_SUB $sp,$FRAMESIZE + $REG_S $ra,$FRAMESIZE-1*$SZREG($sp) + $REG_S $fp,$FRAMESIZE-2*$SZREG($sp) + $REG_S $s11,$FRAMESIZE-3*$SZREG($sp) + $REG_S $s10,$FRAMESIZE-4*$SZREG($sp) + $REG_S $s9,$FRAMESIZE-5*$SZREG($sp) + $REG_S $s8,$FRAMESIZE-6*$SZREG($sp) + $REG_S $s7,$FRAMESIZE-7*$SZREG($sp) + $REG_S $s6,$FRAMESIZE-8*$SZREG($sp) + $REG_S $s5,$FRAMESIZE-9*$SZREG($sp) + $REG_S $s4,$FRAMESIZE-10*$SZREG($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi prologue + $REG_S $s3,$FRAMESIZE-11*$SZREG($sp) + $REG_S $s2,$FRAMESIZE-12*$SZREG($sp) + $REG_S $s1,$FRAMESIZE-13*$SZREG($sp) + $REG_S $s0,$FRAMESIZE-14*$SZREG($sp) + $REG_S $gp,$FRAMESIZE-15*$SZREG($sp) +___ +$code.=<<___; + $PTR_SLL @X[15],$len,`log(16*$SZ)/log(2)` +___ +$code.=<<___ if ($flavour !~ /o32/i); # non-o32 PIC-ification + .cplocal $Ktbl + .cpsetup $pf,$zero,sha${label}_block_data_order +___ +$code.=<<___; + .set reorder + la $Ktbl,K${label} # PIC-ified 'load address' + + $LD $A,0*$SZ($ctx) # load context + $LD $B,1*$SZ($ctx) + $LD $C,2*$SZ($ctx) + $LD $D,3*$SZ($ctx) + $LD $E,4*$SZ($ctx) + $LD $F,5*$SZ($ctx) + $LD $G,6*$SZ($ctx) + $LD $H,7*$SZ($ctx) + + $PTR_ADD @X[15],$inp # pointer to the end of input + $REG_S @X[15],16*$SZ($sp) + b .Loop + +.align 5 +.Loop: + ${LD}l @X[0],$MSB($inp) + ${LD}r @X[0],$LSB($inp) +___ +for ($i=0;$i<16;$i++) +{ &BODY_00_15($i,@V); unshift(@V,pop(@V)); push(@X,shift(@X)); } +$code.=<<___; + b .L16_xx +.align 4 +.L16_xx: +___ +for (;$i<32;$i++) +{ &BODY_16_XX($i,@V); unshift(@V,pop(@V)); push(@X,shift(@X)); } +$code.=<<___; + and @X[6],0xfff + li @X[7],$lastK + .set noreorder + bne @X[6],@X[7],.L16_xx + $PTR_ADD $Ktbl,16*$SZ # Ktbl+=16 + + $REG_L @X[15],16*$SZ($sp) # restore pointer to the end of input + $LD @X[0],0*$SZ($ctx) + $LD @X[1],1*$SZ($ctx) + $LD @X[2],2*$SZ($ctx) + $PTR_ADD $inp,16*$SZ + $LD @X[3],3*$SZ($ctx) + $ADDU $A,@X[0] + $LD @X[4],4*$SZ($ctx) + $ADDU $B,@X[1] + $LD @X[5],5*$SZ($ctx) + $ADDU $C,@X[2] + $LD @X[6],6*$SZ($ctx) + $ADDU $D,@X[3] + $LD @X[7],7*$SZ($ctx) + $ADDU $E,@X[4] + $ST $A,0*$SZ($ctx) + $ADDU $F,@X[5] + $ST $B,1*$SZ($ctx) + $ADDU $G,@X[6] + $ST $C,2*$SZ($ctx) + $ADDU $H,@X[7] + $ST $D,3*$SZ($ctx) + $ST $E,4*$SZ($ctx) + $ST $F,5*$SZ($ctx) + $ST $G,6*$SZ($ctx) + $ST $H,7*$SZ($ctx) + + bnel $inp,@X[15],.Loop + $PTR_SUB $Ktbl,`($rounds-16)*$SZ` # rewind $Ktbl + + $REG_L $ra,$FRAMESIZE-1*$SZREG($sp) + $REG_L $fp,$FRAMESIZE-2*$SZREG($sp) + $REG_L $s11,$FRAMESIZE-3*$SZREG($sp) + $REG_L $s10,$FRAMESIZE-4*$SZREG($sp) + $REG_L $s9,$FRAMESIZE-5*$SZREG($sp) + $REG_L $s8,$FRAMESIZE-6*$SZREG($sp) + $REG_L $s7,$FRAMESIZE-7*$SZREG($sp) + $REG_L $s6,$FRAMESIZE-8*$SZREG($sp) + $REG_L $s5,$FRAMESIZE-9*$SZREG($sp) + $REG_L $s4,$FRAMESIZE-10*$SZREG($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); + $REG_L $s3,$FRAMESIZE-11*$SZREG($sp) + $REG_L $s2,$FRAMESIZE-12*$SZREG($sp) + $REG_L $s1,$FRAMESIZE-13*$SZREG($sp) + $REG_L $s0,$FRAMESIZE-14*$SZREG($sp) + $REG_L $gp,$FRAMESIZE-15*$SZREG($sp) +___ +$code.=<<___; + jr $ra + $PTR_ADD $sp,$FRAMESIZE +.end sha${label}_block_data_order + +.rdata +.align 5 +K${label}: +___ +if ($SZ==4) { +$code.=<<___; + .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 + .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 + .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 + .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 + .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc + .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da + .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 + .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 + .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 + .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 + .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 + .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 + .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 + .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 + .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 + .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +___ +} else { +$code.=<<___; + .dword 0x428a2f98d728ae22, 0x7137449123ef65cd + .dword 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc + .dword 0x3956c25bf348b538, 0x59f111f1b605d019 + .dword 0x923f82a4af194f9b, 0xab1c5ed5da6d8118 + .dword 0xd807aa98a3030242, 0x12835b0145706fbe + .dword 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2 + .dword 0x72be5d74f27b896f, 0x80deb1fe3b1696b1 + .dword 0x9bdc06a725c71235, 0xc19bf174cf692694 + .dword 0xe49b69c19ef14ad2, 0xefbe4786384f25e3 + .dword 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65 + .dword 0x2de92c6f592b0275, 0x4a7484aa6ea6e483 + .dword 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5 + .dword 0x983e5152ee66dfab, 0xa831c66d2db43210 + .dword 0xb00327c898fb213f, 0xbf597fc7beef0ee4 + .dword 0xc6e00bf33da88fc2, 0xd5a79147930aa725 + .dword 0x06ca6351e003826f, 0x142929670a0e6e70 + .dword 0x27b70a8546d22ffc, 0x2e1b21385c26c926 + .dword 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df + .dword 0x650a73548baf63de, 0x766a0abb3c77b2a8 + .dword 0x81c2c92e47edaee6, 0x92722c851482353b + .dword 0xa2bfe8a14cf10364, 0xa81a664bbc423001 + .dword 0xc24b8b70d0f89791, 0xc76c51a30654be30 + .dword 0xd192e819d6ef5218, 0xd69906245565a910 + .dword 0xf40e35855771202a, 0x106aa07032bbd1b8 + .dword 0x19a4c116b8d2d0c8, 0x1e376c085141ab53 + .dword 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8 + .dword 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb + .dword 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3 + .dword 0x748f82ee5defb2fc, 0x78a5636f43172f60 + .dword 0x84c87814a1f0ab72, 0x8cc702081a6439ec + .dword 0x90befffa23631e28, 0xa4506cebde82bde9 + .dword 0xbef9a3f7b2c67915, 0xc67178f2e372532b + .dword 0xca273eceea26619c, 0xd186b8c721c0c207 + .dword 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178 + .dword 0x06f067aa72176fba, 0x0a637dc5a2c898a6 + .dword 0x113f9804bef90dae, 0x1b710b35131c471b + .dword 0x28db77f523047d84, 0x32caab7b40c72493 + .dword 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c + .dword 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a + .dword 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 +___ +} +$code.=<<___; +.asciiz "SHA${label} for MIPS, CRYPTOGAMS by <appro\@openssl.org>" +.align 5 + +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha512-parisc.pl b/crypto/openssl/crypto/sha/asm/sha512-parisc.pl new file mode 100755 index 000000000000..e24ee58ae97f --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-parisc.pl @@ -0,0 +1,791 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA256/512 block procedure for PA-RISC. + +# June 2009. +# +# SHA256 performance is >75% better than gcc 3.2 generated code on +# PA-7100LC. Compared to code generated by vendor compiler this +# implementation is almost 70% faster in 64-bit build, but delivers +# virtually same performance in 32-bit build on PA-8600. +# +# SHA512 performance is >2.9x better than gcc 3.2 generated code on +# PA-7100LC, PA-RISC 1.1 processor. Then implementation detects if the +# code is executed on PA-RISC 2.0 processor and switches to 64-bit +# code path delivering adequate peformance even in "blended" 32-bit +# build. Though 64-bit code is not any faster than code generated by +# vendor compiler on PA-8600... +# +# Special thanks to polarhome.com for providing HP-UX account. + +$flavour = shift; +$output = shift; +open STDOUT,">$output"; + +if ($flavour =~ /64/) { + $LEVEL ="2.0W"; + $SIZE_T =8; + $FRAME_MARKER =80; + $SAVED_RP =16; + $PUSH ="std"; + $PUSHMA ="std,ma"; + $POP ="ldd"; + $POPMB ="ldd,mb"; +} else { + $LEVEL ="1.0"; + $SIZE_T =4; + $FRAME_MARKER =48; + $SAVED_RP =20; + $PUSH ="stw"; + $PUSHMA ="stwm"; + $POP ="ldw"; + $POPMB ="ldwm"; +} + +if ($output =~ /512/) { + $func="sha512_block_data_order"; + $SZ=8; + @Sigma0=(28,34,39); + @Sigma1=(14,18,41); + @sigma0=(1, 8, 7); + @sigma1=(19,61, 6); + $rounds=80; + $LAST10BITS=0x017; + $LD="ldd"; + $LDM="ldd,ma"; + $ST="std"; +} else { + $func="sha256_block_data_order"; + $SZ=4; + @Sigma0=( 2,13,22); + @Sigma1=( 6,11,25); + @sigma0=( 7,18, 3); + @sigma1=(17,19,10); + $rounds=64; + $LAST10BITS=0x0f2; + $LD="ldw"; + $LDM="ldwm"; + $ST="stw"; +} + +$FRAME=16*$SIZE_T+$FRAME_MARKER;# 16 saved regs + frame marker + # [+ argument transfer] +$XOFF=16*$SZ+32; # local variables +$FRAME+=$XOFF; +$XOFF+=$FRAME_MARKER; # distance between %sp and local variables + +$ctx="%r26"; # zapped by $a0 +$inp="%r25"; # zapped by $a1 +$num="%r24"; # zapped by $t0 + +$a0 ="%r26"; +$a1 ="%r25"; +$t0 ="%r24"; +$t1 ="%r29"; +$Tbl="%r31"; + +@V=($A,$B,$C,$D,$E,$F,$G,$H)=("%r17","%r18","%r19","%r20","%r21","%r22","%r23","%r28"); + +@X=("%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", "%r8", + "%r9", "%r10","%r11","%r12","%r13","%r14","%r15","%r16",$inp); + +sub ROUND_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +$code.=<<___; + _ror $e,$Sigma1[0],$a0 + and $f,$e,$t0 + _ror $e,$Sigma1[1],$a1 + addl $t1,$h,$h + andcm $g,$e,$t1 + xor $a1,$a0,$a0 + _ror $a1,`$Sigma1[2]-$Sigma1[1]`,$a1 + or $t0,$t1,$t1 ; Ch(e,f,g) + addl @X[$i%16],$h,$h + xor $a0,$a1,$a1 ; Sigma1(e) + addl $t1,$h,$h + _ror $a,$Sigma0[0],$a0 + addl $a1,$h,$h + + _ror $a,$Sigma0[1],$a1 + and $a,$b,$t0 + and $a,$c,$t1 + xor $a1,$a0,$a0 + _ror $a1,`$Sigma0[2]-$Sigma0[1]`,$a1 + xor $t1,$t0,$t0 + and $b,$c,$t1 + xor $a0,$a1,$a1 ; Sigma0(a) + addl $h,$d,$d + xor $t1,$t0,$t0 ; Maj(a,b,c) + `"$LDM $SZ($Tbl),$t1" if ($i<15)` + addl $a1,$h,$h + addl $t0,$h,$h + +___ +} + +sub ROUND_16_xx { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +$i-=16; +$code.=<<___; + _ror @X[($i+1)%16],$sigma0[0],$a0 + _ror @X[($i+1)%16],$sigma0[1],$a1 + addl @X[($i+9)%16],@X[$i],@X[$i] + _ror @X[($i+14)%16],$sigma1[0],$t0 + _ror @X[($i+14)%16],$sigma1[1],$t1 + xor $a1,$a0,$a0 + _shr @X[($i+1)%16],$sigma0[2],$a1 + xor $t1,$t0,$t0 + _shr @X[($i+14)%16],$sigma1[2],$t1 + xor $a1,$a0,$a0 ; sigma0(X[(i+1)&0x0f]) + xor $t1,$t0,$t0 ; sigma1(X[(i+14)&0x0f]) + $LDM $SZ($Tbl),$t1 + addl $a0,@X[$i],@X[$i] + addl $t0,@X[$i],@X[$i] +___ +$code.=<<___ if ($i==15); + extru $t1,31,10,$a1 + comiclr,<> $LAST10BITS,$a1,%r0 + ldo 1($Tbl),$Tbl ; signal end of $Tbl +___ +&ROUND_00_15($i+16,$a,$b,$c,$d,$e,$f,$g,$h); +} + +$code=<<___; + .LEVEL $LEVEL + .SPACE \$TEXT\$ + .SUBSPA \$CODE\$,QUAD=0,ALIGN=8,ACCESS=0x2C,CODE_ONLY + + .ALIGN 64 +L\$table +___ +$code.=<<___ if ($SZ==8); + .WORD 0x428a2f98,0xd728ae22,0x71374491,0x23ef65cd + .WORD 0xb5c0fbcf,0xec4d3b2f,0xe9b5dba5,0x8189dbbc + .WORD 0x3956c25b,0xf348b538,0x59f111f1,0xb605d019 + .WORD 0x923f82a4,0xaf194f9b,0xab1c5ed5,0xda6d8118 + .WORD 0xd807aa98,0xa3030242,0x12835b01,0x45706fbe + .WORD 0x243185be,0x4ee4b28c,0x550c7dc3,0xd5ffb4e2 + .WORD 0x72be5d74,0xf27b896f,0x80deb1fe,0x3b1696b1 + .WORD 0x9bdc06a7,0x25c71235,0xc19bf174,0xcf692694 + .WORD 0xe49b69c1,0x9ef14ad2,0xefbe4786,0x384f25e3 + .WORD 0x0fc19dc6,0x8b8cd5b5,0x240ca1cc,0x77ac9c65 + .WORD 0x2de92c6f,0x592b0275,0x4a7484aa,0x6ea6e483 + .WORD 0x5cb0a9dc,0xbd41fbd4,0x76f988da,0x831153b5 + .WORD 0x983e5152,0xee66dfab,0xa831c66d,0x2db43210 + .WORD 0xb00327c8,0x98fb213f,0xbf597fc7,0xbeef0ee4 + .WORD 0xc6e00bf3,0x3da88fc2,0xd5a79147,0x930aa725 + .WORD 0x06ca6351,0xe003826f,0x14292967,0x0a0e6e70 + .WORD 0x27b70a85,0x46d22ffc,0x2e1b2138,0x5c26c926 + .WORD 0x4d2c6dfc,0x5ac42aed,0x53380d13,0x9d95b3df + .WORD 0x650a7354,0x8baf63de,0x766a0abb,0x3c77b2a8 + .WORD 0x81c2c92e,0x47edaee6,0x92722c85,0x1482353b + .WORD 0xa2bfe8a1,0x4cf10364,0xa81a664b,0xbc423001 + .WORD 0xc24b8b70,0xd0f89791,0xc76c51a3,0x0654be30 + .WORD 0xd192e819,0xd6ef5218,0xd6990624,0x5565a910 + .WORD 0xf40e3585,0x5771202a,0x106aa070,0x32bbd1b8 + .WORD 0x19a4c116,0xb8d2d0c8,0x1e376c08,0x5141ab53 + .WORD 0x2748774c,0xdf8eeb99,0x34b0bcb5,0xe19b48a8 + .WORD 0x391c0cb3,0xc5c95a63,0x4ed8aa4a,0xe3418acb + .WORD 0x5b9cca4f,0x7763e373,0x682e6ff3,0xd6b2b8a3 + .WORD 0x748f82ee,0x5defb2fc,0x78a5636f,0x43172f60 + .WORD 0x84c87814,0xa1f0ab72,0x8cc70208,0x1a6439ec + .WORD 0x90befffa,0x23631e28,0xa4506ceb,0xde82bde9 + .WORD 0xbef9a3f7,0xb2c67915,0xc67178f2,0xe372532b + .WORD 0xca273ece,0xea26619c,0xd186b8c7,0x21c0c207 + .WORD 0xeada7dd6,0xcde0eb1e,0xf57d4f7f,0xee6ed178 + .WORD 0x06f067aa,0x72176fba,0x0a637dc5,0xa2c898a6 + .WORD 0x113f9804,0xbef90dae,0x1b710b35,0x131c471b + .WORD 0x28db77f5,0x23047d84,0x32caab7b,0x40c72493 + .WORD 0x3c9ebe0a,0x15c9bebc,0x431d67c4,0x9c100d4c + .WORD 0x4cc5d4be,0xcb3e42b6,0x597f299c,0xfc657e2a + .WORD 0x5fcb6fab,0x3ad6faec,0x6c44198c,0x4a475817 +___ +$code.=<<___ if ($SZ==4); + .WORD 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .WORD 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .WORD 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .WORD 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .WORD 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .WORD 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .WORD 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .WORD 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .WORD 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .WORD 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .WORD 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .WORD 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .WORD 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .WORD 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .WORD 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .WORD 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +___ +$code.=<<___; + + .EXPORT $func,ENTRY,ARGW0=GR,ARGW1=GR,ARGW2=GR + .ALIGN 64 +$func + .PROC + .CALLINFO FRAME=`$FRAME-16*$SIZE_T`,NO_CALLS,SAVE_RP,ENTRY_GR=18 + .ENTRY + $PUSH %r2,-$SAVED_RP(%sp) ; standard prologue + $PUSHMA %r3,$FRAME(%sp) + $PUSH %r4,`-$FRAME+1*$SIZE_T`(%sp) + $PUSH %r5,`-$FRAME+2*$SIZE_T`(%sp) + $PUSH %r6,`-$FRAME+3*$SIZE_T`(%sp) + $PUSH %r7,`-$FRAME+4*$SIZE_T`(%sp) + $PUSH %r8,`-$FRAME+5*$SIZE_T`(%sp) + $PUSH %r9,`-$FRAME+6*$SIZE_T`(%sp) + $PUSH %r10,`-$FRAME+7*$SIZE_T`(%sp) + $PUSH %r11,`-$FRAME+8*$SIZE_T`(%sp) + $PUSH %r12,`-$FRAME+9*$SIZE_T`(%sp) + $PUSH %r13,`-$FRAME+10*$SIZE_T`(%sp) + $PUSH %r14,`-$FRAME+11*$SIZE_T`(%sp) + $PUSH %r15,`-$FRAME+12*$SIZE_T`(%sp) + $PUSH %r16,`-$FRAME+13*$SIZE_T`(%sp) + $PUSH %r17,`-$FRAME+14*$SIZE_T`(%sp) + $PUSH %r18,`-$FRAME+15*$SIZE_T`(%sp) + + _shl $num,`log(16*$SZ)/log(2)`,$num + addl $inp,$num,$num ; $num to point at the end of $inp + + $PUSH $num,`-$FRAME_MARKER-4*$SIZE_T`(%sp) ; save arguments + $PUSH $inp,`-$FRAME_MARKER-3*$SIZE_T`(%sp) + $PUSH $ctx,`-$FRAME_MARKER-2*$SIZE_T`(%sp) + + blr %r0,$Tbl + ldi 3,$t1 +L\$pic + andcm $Tbl,$t1,$Tbl ; wipe privilege level + ldo L\$table-L\$pic($Tbl),$Tbl +___ +$code.=<<___ if ($SZ==8 && $SIZE_T==4); + ldi 31,$t1 + mtctl $t1,%cr11 + extrd,u,*= $t1,%sar,1,$t1 ; executes on PA-RISC 1.0 + b L\$parisc1 + nop +___ +$code.=<<___; + $LD `0*$SZ`($ctx),$A ; load context + $LD `1*$SZ`($ctx),$B + $LD `2*$SZ`($ctx),$C + $LD `3*$SZ`($ctx),$D + $LD `4*$SZ`($ctx),$E + $LD `5*$SZ`($ctx),$F + $LD `6*$SZ`($ctx),$G + $LD `7*$SZ`($ctx),$H + + extru $inp,31,`log($SZ)/log(2)`,$t0 + sh3addl $t0,%r0,$t0 + subi `8*$SZ`,$t0,$t0 + mtctl $t0,%cr11 ; load %sar with align factor + +L\$oop + ldi `$SZ-1`,$t0 + $LDM $SZ($Tbl),$t1 + andcm $inp,$t0,$t0 ; align $inp +___ + for ($i=0;$i<15;$i++) { # load input block + $code.="\t$LD `$SZ*$i`($t0),@X[$i]\n"; } +$code.=<<___; + cmpb,*= $inp,$t0,L\$aligned + $LD `$SZ*15`($t0),@X[15] + $LD `$SZ*16`($t0),@X[16] +___ + for ($i=0;$i<16;$i++) { # align data + $code.="\t_align @X[$i],@X[$i+1],@X[$i]\n"; } +$code.=<<___; +L\$aligned + nop ; otherwise /usr/ccs/bin/as is confused by below .WORD +___ + +for($i=0;$i<16;$i++) { &ROUND_00_15($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; +L\$rounds + nop ; otherwise /usr/ccs/bin/as is confused by below .WORD +___ +for(;$i<32;$i++) { &ROUND_16_xx($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + bb,>= $Tbl,31,L\$rounds ; end of $Tbl signalled? + nop + + $POP `-$FRAME_MARKER-2*$SIZE_T`(%sp),$ctx ; restore arguments + $POP `-$FRAME_MARKER-3*$SIZE_T`(%sp),$inp + $POP `-$FRAME_MARKER-4*$SIZE_T`(%sp),$num + ldo `-$rounds*$SZ-1`($Tbl),$Tbl ; rewind $Tbl + + $LD `0*$SZ`($ctx),@X[0] ; load context + $LD `1*$SZ`($ctx),@X[1] + $LD `2*$SZ`($ctx),@X[2] + $LD `3*$SZ`($ctx),@X[3] + $LD `4*$SZ`($ctx),@X[4] + $LD `5*$SZ`($ctx),@X[5] + addl @X[0],$A,$A + $LD `6*$SZ`($ctx),@X[6] + addl @X[1],$B,$B + $LD `7*$SZ`($ctx),@X[7] + ldo `16*$SZ`($inp),$inp ; advance $inp + + $ST $A,`0*$SZ`($ctx) ; save context + addl @X[2],$C,$C + $ST $B,`1*$SZ`($ctx) + addl @X[3],$D,$D + $ST $C,`2*$SZ`($ctx) + addl @X[4],$E,$E + $ST $D,`3*$SZ`($ctx) + addl @X[5],$F,$F + $ST $E,`4*$SZ`($ctx) + addl @X[6],$G,$G + $ST $F,`5*$SZ`($ctx) + addl @X[7],$H,$H + $ST $G,`6*$SZ`($ctx) + $ST $H,`7*$SZ`($ctx) + + cmpb,*<>,n $inp,$num,L\$oop + $PUSH $inp,`-$FRAME_MARKER-3*$SIZE_T`(%sp) ; save $inp +___ +if ($SZ==8 && $SIZE_T==4) # SHA512 for 32-bit PA-RISC 1.0 +{{ +$code.=<<___; + b L\$done + nop + + .ALIGN 64 +L\$parisc1 +___ + +@V=( $Ahi, $Alo, $Bhi, $Blo, $Chi, $Clo, $Dhi, $Dlo, + $Ehi, $Elo, $Fhi, $Flo, $Ghi, $Glo, $Hhi, $Hlo) = + ( "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", "%r8", + "%r9","%r10","%r11","%r12","%r13","%r14","%r15","%r16"); +$a0 ="%r17"; +$a1 ="%r18"; +$a2 ="%r19"; +$a3 ="%r20"; +$t0 ="%r21"; +$t1 ="%r22"; +$t2 ="%r28"; +$t3 ="%r29"; +$Tbl="%r31"; + +@X=("%r23","%r24","%r25","%r26"); # zaps $num,$inp,$ctx + +sub ROUND_00_15_pa1 { +my ($i,$ahi,$alo,$bhi,$blo,$chi,$clo,$dhi,$dlo, + $ehi,$elo,$fhi,$flo,$ghi,$glo,$hhi,$hlo,$flag)=@_; +my ($Xhi,$Xlo,$Xnhi,$Xnlo) = @X; + +$code.=<<___ if (!$flag); + ldw `-$XOFF+8*(($i+1)%16)`(%sp),$Xnhi + ldw `-$XOFF+8*(($i+1)%16)+4`(%sp),$Xnlo ; load X[i+1] +___ +$code.=<<___; + shd $ehi,$elo,$Sigma1[0],$t0 + add $Xlo,$hlo,$hlo + shd $elo,$ehi,$Sigma1[0],$t1 + addc $Xhi,$hhi,$hhi ; h += X[i] + shd $ehi,$elo,$Sigma1[1],$t2 + ldwm 8($Tbl),$Xhi + shd $elo,$ehi,$Sigma1[1],$t3 + ldw -4($Tbl),$Xlo ; load K[i] + xor $t2,$t0,$t0 + xor $t3,$t1,$t1 + and $flo,$elo,$a0 + and $fhi,$ehi,$a1 + shd $ehi,$elo,$Sigma1[2],$t2 + andcm $glo,$elo,$a2 + shd $elo,$ehi,$Sigma1[2],$t3 + andcm $ghi,$ehi,$a3 + xor $t2,$t0,$t0 + xor $t3,$t1,$t1 ; Sigma1(e) + add $Xlo,$hlo,$hlo + xor $a2,$a0,$a0 + addc $Xhi,$hhi,$hhi ; h += K[i] + xor $a3,$a1,$a1 ; Ch(e,f,g) + + add $t0,$hlo,$hlo + shd $ahi,$alo,$Sigma0[0],$t0 + addc $t1,$hhi,$hhi ; h += Sigma1(e) + shd $alo,$ahi,$Sigma0[0],$t1 + add $a0,$hlo,$hlo + shd $ahi,$alo,$Sigma0[1],$t2 + addc $a1,$hhi,$hhi ; h += Ch(e,f,g) + shd $alo,$ahi,$Sigma0[1],$t3 + + xor $t2,$t0,$t0 + xor $t3,$t1,$t1 + shd $ahi,$alo,$Sigma0[2],$t2 + and $alo,$blo,$a0 + shd $alo,$ahi,$Sigma0[2],$t3 + and $ahi,$bhi,$a1 + xor $t2,$t0,$t0 + xor $t3,$t1,$t1 ; Sigma0(a) + + and $alo,$clo,$a2 + and $ahi,$chi,$a3 + xor $a2,$a0,$a0 + add $hlo,$dlo,$dlo + xor $a3,$a1,$a1 + addc $hhi,$dhi,$dhi ; d += h + and $blo,$clo,$a2 + add $t0,$hlo,$hlo + and $bhi,$chi,$a3 + addc $t1,$hhi,$hhi ; h += Sigma0(a) + xor $a2,$a0,$a0 + add $a0,$hlo,$hlo + xor $a3,$a1,$a1 ; Maj(a,b,c) + addc $a1,$hhi,$hhi ; h += Maj(a,b,c) + +___ +$code.=<<___ if ($i==15 && $flag); + extru $Xlo,31,10,$Xlo + comiclr,= $LAST10BITS,$Xlo,%r0 + b L\$rounds_pa1 + nop +___ +push(@X,shift(@X)); push(@X,shift(@X)); +} + +sub ROUND_16_xx_pa1 { +my ($Xhi,$Xlo,$Xnhi,$Xnlo) = @X; +my ($i)=shift; +$i-=16; +$code.=<<___; + ldw `-$XOFF+8*(($i+1)%16)`(%sp),$Xnhi + ldw `-$XOFF+8*(($i+1)%16)+4`(%sp),$Xnlo ; load X[i+1] + ldw `-$XOFF+8*(($i+9)%16)`(%sp),$a1 + ldw `-$XOFF+8*(($i+9)%16)+4`(%sp),$a0 ; load X[i+9] + ldw `-$XOFF+8*(($i+14)%16)`(%sp),$a3 + ldw `-$XOFF+8*(($i+14)%16)+4`(%sp),$a2 ; load X[i+14] + shd $Xnhi,$Xnlo,$sigma0[0],$t0 + shd $Xnlo,$Xnhi,$sigma0[0],$t1 + add $a0,$Xlo,$Xlo + shd $Xnhi,$Xnlo,$sigma0[1],$t2 + addc $a1,$Xhi,$Xhi + shd $Xnlo,$Xnhi,$sigma0[1],$t3 + xor $t2,$t0,$t0 + shd $Xnhi,$Xnlo,$sigma0[2],$t2 + xor $t3,$t1,$t1 + extru $Xnhi,`31-$sigma0[2]`,`32-$sigma0[2]`,$t3 + xor $t2,$t0,$t0 + shd $a3,$a2,$sigma1[0],$a0 + xor $t3,$t1,$t1 ; sigma0(X[i+1)&0x0f]) + shd $a2,$a3,$sigma1[0],$a1 + add $t0,$Xlo,$Xlo + shd $a3,$a2,$sigma1[1],$t2 + addc $t1,$Xhi,$Xhi + shd $a2,$a3,$sigma1[1],$t3 + xor $t2,$a0,$a0 + shd $a3,$a2,$sigma1[2],$t2 + xor $t3,$a1,$a1 + extru $a3,`31-$sigma1[2]`,`32-$sigma1[2]`,$t3 + xor $t2,$a0,$a0 + xor $t3,$a1,$a1 ; sigma0(X[i+14)&0x0f]) + add $a0,$Xlo,$Xlo + addc $a1,$Xhi,$Xhi + + stw $Xhi,`-$XOFF+8*($i%16)`(%sp) + stw $Xlo,`-$XOFF+8*($i%16)+4`(%sp) +___ +&ROUND_00_15_pa1($i,@_,1); +} +$code.=<<___; + ldw `0*4`($ctx),$Ahi ; load context + ldw `1*4`($ctx),$Alo + ldw `2*4`($ctx),$Bhi + ldw `3*4`($ctx),$Blo + ldw `4*4`($ctx),$Chi + ldw `5*4`($ctx),$Clo + ldw `6*4`($ctx),$Dhi + ldw `7*4`($ctx),$Dlo + ldw `8*4`($ctx),$Ehi + ldw `9*4`($ctx),$Elo + ldw `10*4`($ctx),$Fhi + ldw `11*4`($ctx),$Flo + ldw `12*4`($ctx),$Ghi + ldw `13*4`($ctx),$Glo + ldw `14*4`($ctx),$Hhi + ldw `15*4`($ctx),$Hlo + + extru $inp,31,2,$t0 + sh3addl $t0,%r0,$t0 + subi 32,$t0,$t0 + mtctl $t0,%cr11 ; load %sar with align factor + +L\$oop_pa1 + extru $inp,31,2,$a3 + comib,= 0,$a3,L\$aligned_pa1 + sub $inp,$a3,$inp + + ldw `0*4`($inp),$X[0] + ldw `1*4`($inp),$X[1] + ldw `2*4`($inp),$t2 + ldw `3*4`($inp),$t3 + ldw `4*4`($inp),$a0 + ldw `5*4`($inp),$a1 + ldw `6*4`($inp),$a2 + ldw `7*4`($inp),$a3 + vshd $X[0],$X[1],$X[0] + vshd $X[1],$t2,$X[1] + stw $X[0],`-$XOFF+0*4`(%sp) + ldw `8*4`($inp),$t0 + vshd $t2,$t3,$t2 + stw $X[1],`-$XOFF+1*4`(%sp) + ldw `9*4`($inp),$t1 + vshd $t3,$a0,$t3 +___ +{ +my @t=($t2,$t3,$a0,$a1,$a2,$a3,$t0,$t1); +for ($i=2;$i<=(128/4-8);$i++) { +$code.=<<___; + stw $t[0],`-$XOFF+$i*4`(%sp) + ldw `(8+$i)*4`($inp),$t[0] + vshd $t[1],$t[2],$t[1] +___ +push(@t,shift(@t)); +} +for (;$i<(128/4-1);$i++) { +$code.=<<___; + stw $t[0],`-$XOFF+$i*4`(%sp) + vshd $t[1],$t[2],$t[1] +___ +push(@t,shift(@t)); +} +$code.=<<___; + b L\$collected_pa1 + stw $t[0],`-$XOFF+$i*4`(%sp) + +___ +} +$code.=<<___; +L\$aligned_pa1 + ldw `0*4`($inp),$X[0] + ldw `1*4`($inp),$X[1] + ldw `2*4`($inp),$t2 + ldw `3*4`($inp),$t3 + ldw `4*4`($inp),$a0 + ldw `5*4`($inp),$a1 + ldw `6*4`($inp),$a2 + ldw `7*4`($inp),$a3 + stw $X[0],`-$XOFF+0*4`(%sp) + ldw `8*4`($inp),$t0 + stw $X[1],`-$XOFF+1*4`(%sp) + ldw `9*4`($inp),$t1 +___ +{ +my @t=($t2,$t3,$a0,$a1,$a2,$a3,$t0,$t1); +for ($i=2;$i<(128/4-8);$i++) { +$code.=<<___; + stw $t[0],`-$XOFF+$i*4`(%sp) + ldw `(8+$i)*4`($inp),$t[0] +___ +push(@t,shift(@t)); +} +for (;$i<128/4;$i++) { +$code.=<<___; + stw $t[0],`-$XOFF+$i*4`(%sp) +___ +push(@t,shift(@t)); +} +$code.="L\$collected_pa1\n"; +} + +for($i=0;$i<16;$i++) { &ROUND_00_15_pa1($i,@V); unshift(@V,pop(@V)); unshift(@V,pop(@V)); } +$code.="L\$rounds_pa1\n"; +for(;$i<32;$i++) { &ROUND_16_xx_pa1($i,@V); unshift(@V,pop(@V)); unshift(@V,pop(@V)); } + +$code.=<<___; + $POP `-$FRAME_MARKER-2*$SIZE_T`(%sp),$ctx ; restore arguments + $POP `-$FRAME_MARKER-3*$SIZE_T`(%sp),$inp + $POP `-$FRAME_MARKER-4*$SIZE_T`(%sp),$num + ldo `-$rounds*$SZ`($Tbl),$Tbl ; rewind $Tbl + + ldw `0*4`($ctx),$t1 ; update context + ldw `1*4`($ctx),$t0 + ldw `2*4`($ctx),$t3 + ldw `3*4`($ctx),$t2 + ldw `4*4`($ctx),$a1 + ldw `5*4`($ctx),$a0 + ldw `6*4`($ctx),$a3 + add $t0,$Alo,$Alo + ldw `7*4`($ctx),$a2 + addc $t1,$Ahi,$Ahi + ldw `8*4`($ctx),$t1 + add $t2,$Blo,$Blo + ldw `9*4`($ctx),$t0 + addc $t3,$Bhi,$Bhi + ldw `10*4`($ctx),$t3 + add $a0,$Clo,$Clo + ldw `11*4`($ctx),$t2 + addc $a1,$Chi,$Chi + ldw `12*4`($ctx),$a1 + add $a2,$Dlo,$Dlo + ldw `13*4`($ctx),$a0 + addc $a3,$Dhi,$Dhi + ldw `14*4`($ctx),$a3 + add $t0,$Elo,$Elo + ldw `15*4`($ctx),$a2 + addc $t1,$Ehi,$Ehi + stw $Ahi,`0*4`($ctx) + add $t2,$Flo,$Flo + stw $Alo,`1*4`($ctx) + addc $t3,$Fhi,$Fhi + stw $Bhi,`2*4`($ctx) + add $a0,$Glo,$Glo + stw $Blo,`3*4`($ctx) + addc $a1,$Ghi,$Ghi + stw $Chi,`4*4`($ctx) + add $a2,$Hlo,$Hlo + stw $Clo,`5*4`($ctx) + addc $a3,$Hhi,$Hhi + stw $Dhi,`6*4`($ctx) + ldo `16*$SZ`($inp),$inp ; advance $inp + stw $Dlo,`7*4`($ctx) + stw $Ehi,`8*4`($ctx) + stw $Elo,`9*4`($ctx) + stw $Fhi,`10*4`($ctx) + stw $Flo,`11*4`($ctx) + stw $Ghi,`12*4`($ctx) + stw $Glo,`13*4`($ctx) + stw $Hhi,`14*4`($ctx) + comb,= $inp,$num,L\$done + stw $Hlo,`15*4`($ctx) + b L\$oop_pa1 + $PUSH $inp,`-$FRAME_MARKER-3*$SIZE_T`(%sp) ; save $inp +L\$done +___ +}} +$code.=<<___; + $POP `-$FRAME-$SAVED_RP`(%sp),%r2 ; standard epilogue + $POP `-$FRAME+1*$SIZE_T`(%sp),%r4 + $POP `-$FRAME+2*$SIZE_T`(%sp),%r5 + $POP `-$FRAME+3*$SIZE_T`(%sp),%r6 + $POP `-$FRAME+4*$SIZE_T`(%sp),%r7 + $POP `-$FRAME+5*$SIZE_T`(%sp),%r8 + $POP `-$FRAME+6*$SIZE_T`(%sp),%r9 + $POP `-$FRAME+7*$SIZE_T`(%sp),%r10 + $POP `-$FRAME+8*$SIZE_T`(%sp),%r11 + $POP `-$FRAME+9*$SIZE_T`(%sp),%r12 + $POP `-$FRAME+10*$SIZE_T`(%sp),%r13 + $POP `-$FRAME+11*$SIZE_T`(%sp),%r14 + $POP `-$FRAME+12*$SIZE_T`(%sp),%r15 + $POP `-$FRAME+13*$SIZE_T`(%sp),%r16 + $POP `-$FRAME+14*$SIZE_T`(%sp),%r17 + $POP `-$FRAME+15*$SIZE_T`(%sp),%r18 + bv (%r2) + .EXIT + $POPMB -$FRAME(%sp),%r3 + .PROCEND + .STRINGZ "SHA`64*$SZ` block transform for PA-RISC, CRYPTOGAMS by <appro\@openssl.org>" +___ + +# Explicitly encode PA-RISC 2.0 instructions used in this module, so +# that it can be compiled with .LEVEL 1.0. It should be noted that I +# wouldn't have to do this, if GNU assembler understood .ALLOW 2.0 +# directive... + +my $ldd = sub { + my ($mod,$args) = @_; + my $orig = "ldd$mod\t$args"; + + if ($args =~ /(\-?[0-9]+)\(%r([0-9]+)\),%r([0-9]+)/) # format 3 suffices + { my $opcode=(0x14<<26)|($2<<21)|($3<<16)|(($1&0x1FF8)<<1)|(($1>>13)&1); + $opcode|=(1<<3) if ($mod =~ /^,m/); + $opcode|=(1<<2) if ($mod =~ /^,mb/); + sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig; + } + else { "\t".$orig; } +}; + +my $std = sub { + my ($mod,$args) = @_; + my $orig = "std$mod\t$args"; + + if ($args =~ /%r([0-9]+),(\-?[0-9]+)\(%r([0-9]+)\)/) # format 3 suffices + { my $opcode=(0x1c<<26)|($3<<21)|($1<<16)|(($2&0x1FF8)<<1)|(($2>>13)&1); + sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig; + } + else { "\t".$orig; } +}; + +my $extrd = sub { + my ($mod,$args) = @_; + my $orig = "extrd$mod\t$args"; + + # I only have ",u" completer, it's implicitly encoded... + if ($args =~ /%r([0-9]+),([0-9]+),([0-9]+),%r([0-9]+)/) # format 15 + { my $opcode=(0x36<<26)|($1<<21)|($4<<16); + my $len=32-$3; + $opcode |= (($2&0x20)<<6)|(($2&0x1f)<<5); # encode pos + $opcode |= (($len&0x20)<<7)|($len&0x1f); # encode len + sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig; + } + elsif ($args =~ /%r([0-9]+),%sar,([0-9]+),%r([0-9]+)/) # format 12 + { my $opcode=(0x34<<26)|($1<<21)|($3<<16)|(2<<11)|(1<<9); + my $len=32-$2; + $opcode |= (($len&0x20)<<3)|($len&0x1f); # encode len + $opcode |= (1<<13) if ($mod =~ /,\**=/); + sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig; + } + else { "\t".$orig; } +}; + +my $shrpd = sub { + my ($mod,$args) = @_; + my $orig = "shrpd$mod\t$args"; + + if ($args =~ /%r([0-9]+),%r([0-9]+),([0-9]+),%r([0-9]+)/) # format 14 + { my $opcode=(0x34<<26)|($2<<21)|($1<<16)|(1<<10)|$4; + my $cpos=63-$3; + $opcode |= (($cpos&0x20)<<6)|(($cpos&0x1f)<<5); # encode sa + sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig; + } + elsif ($args =~ /%r([0-9]+),%r([0-9]+),%sar,%r([0-9]+)/) # format 11 + { sprintf "\t.WORD\t0x%08x\t; %s", + (0x34<<26)|($2<<21)|($1<<16)|(1<<9)|$3,$orig; + } + else { "\t".$orig; } +}; + +sub assemble { + my ($mnemonic,$mod,$args)=@_; + my $opcode = eval("\$$mnemonic"); + + ref($opcode) eq 'CODE' ? &$opcode($mod,$args) : "\t$mnemonic$mod\t$args"; +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/shd\s+(%r[0-9]+),(%r[0-9]+),([0-9]+)/ + $3>31 ? sprintf("shd\t%$2,%$1,%d",$3-32) # rotation for >=32 + : sprintf("shd\t%$1,%$2,%d",$3)/e or + # translate made up instructons: _ror, _shr, _align, _shl + s/_ror(\s+)(%r[0-9]+),/ + ($SZ==4 ? "shd" : "shrpd")."$1$2,$2,"/e or + + s/_shr(\s+%r[0-9]+),([0-9]+),/ + $SZ==4 ? sprintf("extru%s,%d,%d,",$1,31-$2,32-$2) + : sprintf("extrd,u%s,%d,%d,",$1,63-$2,64-$2)/e or + + s/_align(\s+%r[0-9]+,%r[0-9]+),/ + ($SZ==4 ? "vshd$1," : "shrpd$1,%sar,")/e or + + s/_shl(\s+%r[0-9]+),([0-9]+),/ + $SIZE_T==4 ? sprintf("zdep%s,%d,%d,",$1,31-$2,32-$2) + : sprintf("depd,z%s,%d,%d,",$1,63-$2,64-$2)/e; + + s/^\s+([a-z]+)([\S]*)\s+([\S]*)/&assemble($1,$2,$3)/e if ($SIZE_T==4); + + s/cmpb,\*/comb,/ if ($SIZE_T==4); + + print $_,"\n"; +} + +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha512-ppc.pl b/crypto/openssl/crypto/sha/asm/sha512-ppc.pl new file mode 100755 index 000000000000..6b44a68e599e --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-ppc.pl @@ -0,0 +1,460 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# I let hardware handle unaligned input, except on page boundaries +# (see below for details). Otherwise straightforward implementation +# with X vector in register bank. The module is big-endian [which is +# not big deal as there're no little-endian targets left around]. + +# sha256 | sha512 +# -m64 -m32 | -m64 -m32 +# --------------------------------------+----------------------- +# PPC970,gcc-4.0.0 +50% +38% | +40% +410%(*) +# Power6,xlc-7 +150% +90% | +100% +430%(*) +# +# (*) 64-bit code in 32-bit application context, which actually is +# on TODO list. It should be noted that for safe deployment in +# 32-bit *mutli-threaded* context asyncronous signals should be +# blocked upon entry to SHA512 block routine. This is because +# 32-bit signaling procedure invalidates upper halves of GPRs. +# Context switch procedure preserves them, but not signaling:-( + +# Second version is true multi-thread safe. Trouble with the original +# version was that it was using thread local storage pointer register. +# Well, it scrupulously preserved it, but the problem would arise the +# moment asynchronous signal was delivered and signal handler would +# dereference the TLS pointer. While it's never the case in openssl +# application or test suite, we have to respect this scenario and not +# use TLS pointer register. Alternative would be to require caller to +# block signals prior calling this routine. For the record, in 32-bit +# context R2 serves as TLS pointer, while in 64-bit context - R13. + +$flavour=shift; +$output =shift; + +if ($flavour =~ /64/) { + $SIZE_T=8; + $LRSAVE=2*$SIZE_T; + $STU="stdu"; + $UCMP="cmpld"; + $SHL="sldi"; + $POP="ld"; + $PUSH="std"; +} elsif ($flavour =~ /32/) { + $SIZE_T=4; + $LRSAVE=$SIZE_T; + $STU="stwu"; + $UCMP="cmplw"; + $SHL="slwi"; + $POP="lwz"; + $PUSH="stw"; +} else { die "nonsense $flavour"; } + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; + +if ($output =~ /512/) { + $func="sha512_block_data_order"; + $SZ=8; + @Sigma0=(28,34,39); + @Sigma1=(14,18,41); + @sigma0=(1, 8, 7); + @sigma1=(19,61, 6); + $rounds=80; + $LD="ld"; + $ST="std"; + $ROR="rotrdi"; + $SHR="srdi"; +} else { + $func="sha256_block_data_order"; + $SZ=4; + @Sigma0=( 2,13,22); + @Sigma1=( 6,11,25); + @sigma0=( 7,18, 3); + @sigma1=(17,19,10); + $rounds=64; + $LD="lwz"; + $ST="stw"; + $ROR="rotrwi"; + $SHR="srwi"; +} + +$FRAME=32*$SIZE_T+16*$SZ; +$LOCALS=6*$SIZE_T; + +$sp ="r1"; +$toc="r2"; +$ctx="r3"; # zapped by $a0 +$inp="r4"; # zapped by $a1 +$num="r5"; # zapped by $t0 + +$T ="r0"; +$a0 ="r3"; +$a1 ="r4"; +$t0 ="r5"; +$t1 ="r6"; +$Tbl="r7"; + +$A ="r8"; +$B ="r9"; +$C ="r10"; +$D ="r11"; +$E ="r12"; +$F ="r13"; $F="r2" if ($SIZE_T==8);# reassigned to exempt TLS pointer +$G ="r14"; +$H ="r15"; + +@V=($A,$B,$C,$D,$E,$F,$G,$H); +@X=("r16","r17","r18","r19","r20","r21","r22","r23", + "r24","r25","r26","r27","r28","r29","r30","r31"); + +$inp="r31"; # reassigned $inp! aliases with @X[15] + +sub ROUND_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +$code.=<<___; + $LD $T,`$i*$SZ`($Tbl) + $ROR $a0,$e,$Sigma1[0] + $ROR $a1,$e,$Sigma1[1] + and $t0,$f,$e + andc $t1,$g,$e + add $T,$T,$h + xor $a0,$a0,$a1 + $ROR $a1,$a1,`$Sigma1[2]-$Sigma1[1]` + or $t0,$t0,$t1 ; Ch(e,f,g) + add $T,$T,@X[$i] + xor $a0,$a0,$a1 ; Sigma1(e) + add $T,$T,$t0 + add $T,$T,$a0 + + $ROR $a0,$a,$Sigma0[0] + $ROR $a1,$a,$Sigma0[1] + and $t0,$a,$b + and $t1,$a,$c + xor $a0,$a0,$a1 + $ROR $a1,$a1,`$Sigma0[2]-$Sigma0[1]` + xor $t0,$t0,$t1 + and $t1,$b,$c + xor $a0,$a0,$a1 ; Sigma0(a) + add $d,$d,$T + xor $t0,$t0,$t1 ; Maj(a,b,c) + add $h,$T,$a0 + add $h,$h,$t0 + +___ +} + +sub ROUND_16_xx { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +$i-=16; +$code.=<<___; + $ROR $a0,@X[($i+1)%16],$sigma0[0] + $ROR $a1,@X[($i+1)%16],$sigma0[1] + $ROR $t0,@X[($i+14)%16],$sigma1[0] + $ROR $t1,@X[($i+14)%16],$sigma1[1] + xor $a0,$a0,$a1 + $SHR $a1,@X[($i+1)%16],$sigma0[2] + xor $t0,$t0,$t1 + $SHR $t1,@X[($i+14)%16],$sigma1[2] + add @X[$i],@X[$i],@X[($i+9)%16] + xor $a0,$a0,$a1 ; sigma0(X[(i+1)&0x0f]) + xor $t0,$t0,$t1 ; sigma1(X[(i+14)&0x0f]) + add @X[$i],@X[$i],$a0 + add @X[$i],@X[$i],$t0 +___ +&ROUND_00_15($i,$a,$b,$c,$d,$e,$f,$g,$h); +} + +$code=<<___; +.machine "any" +.text + +.globl $func +.align 6 +$func: + $STU $sp,-$FRAME($sp) + mflr r0 + $SHL $num,$num,`log(16*$SZ)/log(2)` + + $PUSH $ctx,`$FRAME-$SIZE_T*22`($sp) + + $PUSH $toc,`$FRAME-$SIZE_T*20`($sp) + $PUSH r13,`$FRAME-$SIZE_T*19`($sp) + $PUSH r14,`$FRAME-$SIZE_T*18`($sp) + $PUSH r15,`$FRAME-$SIZE_T*17`($sp) + $PUSH r16,`$FRAME-$SIZE_T*16`($sp) + $PUSH r17,`$FRAME-$SIZE_T*15`($sp) + $PUSH r18,`$FRAME-$SIZE_T*14`($sp) + $PUSH r19,`$FRAME-$SIZE_T*13`($sp) + $PUSH r20,`$FRAME-$SIZE_T*12`($sp) + $PUSH r21,`$FRAME-$SIZE_T*11`($sp) + $PUSH r22,`$FRAME-$SIZE_T*10`($sp) + $PUSH r23,`$FRAME-$SIZE_T*9`($sp) + $PUSH r24,`$FRAME-$SIZE_T*8`($sp) + $PUSH r25,`$FRAME-$SIZE_T*7`($sp) + $PUSH r26,`$FRAME-$SIZE_T*6`($sp) + $PUSH r27,`$FRAME-$SIZE_T*5`($sp) + $PUSH r28,`$FRAME-$SIZE_T*4`($sp) + $PUSH r29,`$FRAME-$SIZE_T*3`($sp) + $PUSH r30,`$FRAME-$SIZE_T*2`($sp) + $PUSH r31,`$FRAME-$SIZE_T*1`($sp) + $PUSH r0,`$FRAME+$LRSAVE`($sp) + + $LD $A,`0*$SZ`($ctx) + mr $inp,r4 ; incarnate $inp + $LD $B,`1*$SZ`($ctx) + $LD $C,`2*$SZ`($ctx) + $LD $D,`3*$SZ`($ctx) + $LD $E,`4*$SZ`($ctx) + $LD $F,`5*$SZ`($ctx) + $LD $G,`6*$SZ`($ctx) + $LD $H,`7*$SZ`($ctx) + + bl LPICmeup +LPICedup: + andi. r0,$inp,3 + bne Lunaligned +Laligned: + add $num,$inp,$num + $PUSH $num,`$FRAME-$SIZE_T*24`($sp) ; end pointer + $PUSH $inp,`$FRAME-$SIZE_T*23`($sp) ; inp pointer + bl Lsha2_block_private + b Ldone + +; PowerPC specification allows an implementation to be ill-behaved +; upon unaligned access which crosses page boundary. "Better safe +; than sorry" principle makes me treat it specially. But I don't +; look for particular offending word, but rather for the input +; block which crosses the boundary. Once found that block is aligned +; and hashed separately... +.align 4 +Lunaligned: + subfic $t1,$inp,4096 + andi. $t1,$t1,`4096-16*$SZ` ; distance to closest page boundary + beq Lcross_page + $UCMP $num,$t1 + ble- Laligned ; didn't cross the page boundary + subfc $num,$t1,$num + add $t1,$inp,$t1 + $PUSH $num,`$FRAME-$SIZE_T*25`($sp) ; save real remaining num + $PUSH $t1,`$FRAME-$SIZE_T*24`($sp) ; intermediate end pointer + $PUSH $inp,`$FRAME-$SIZE_T*23`($sp) ; inp pointer + bl Lsha2_block_private + ; $inp equals to the intermediate end pointer here + $POP $num,`$FRAME-$SIZE_T*25`($sp) ; restore real remaining num +Lcross_page: + li $t1,`16*$SZ/4` + mtctr $t1 + addi r20,$sp,$LOCALS ; aligned spot below the frame +Lmemcpy: + lbz r16,0($inp) + lbz r17,1($inp) + lbz r18,2($inp) + lbz r19,3($inp) + addi $inp,$inp,4 + stb r16,0(r20) + stb r17,1(r20) + stb r18,2(r20) + stb r19,3(r20) + addi r20,r20,4 + bdnz Lmemcpy + + $PUSH $inp,`$FRAME-$SIZE_T*26`($sp) ; save real inp + addi $t1,$sp,`$LOCALS+16*$SZ` ; fictitious end pointer + addi $inp,$sp,$LOCALS ; fictitious inp pointer + $PUSH $num,`$FRAME-$SIZE_T*25`($sp) ; save real num + $PUSH $t1,`$FRAME-$SIZE_T*24`($sp) ; end pointer + $PUSH $inp,`$FRAME-$SIZE_T*23`($sp) ; inp pointer + bl Lsha2_block_private + $POP $inp,`$FRAME-$SIZE_T*26`($sp) ; restore real inp + $POP $num,`$FRAME-$SIZE_T*25`($sp) ; restore real num + addic. $num,$num,`-16*$SZ` ; num-- + bne- Lunaligned + +Ldone: + $POP r0,`$FRAME+$LRSAVE`($sp) + $POP $toc,`$FRAME-$SIZE_T*20`($sp) + $POP r13,`$FRAME-$SIZE_T*19`($sp) + $POP r14,`$FRAME-$SIZE_T*18`($sp) + $POP r15,`$FRAME-$SIZE_T*17`($sp) + $POP r16,`$FRAME-$SIZE_T*16`($sp) + $POP r17,`$FRAME-$SIZE_T*15`($sp) + $POP r18,`$FRAME-$SIZE_T*14`($sp) + $POP r19,`$FRAME-$SIZE_T*13`($sp) + $POP r20,`$FRAME-$SIZE_T*12`($sp) + $POP r21,`$FRAME-$SIZE_T*11`($sp) + $POP r22,`$FRAME-$SIZE_T*10`($sp) + $POP r23,`$FRAME-$SIZE_T*9`($sp) + $POP r24,`$FRAME-$SIZE_T*8`($sp) + $POP r25,`$FRAME-$SIZE_T*7`($sp) + $POP r26,`$FRAME-$SIZE_T*6`($sp) + $POP r27,`$FRAME-$SIZE_T*5`($sp) + $POP r28,`$FRAME-$SIZE_T*4`($sp) + $POP r29,`$FRAME-$SIZE_T*3`($sp) + $POP r30,`$FRAME-$SIZE_T*2`($sp) + $POP r31,`$FRAME-$SIZE_T*1`($sp) + mtlr r0 + addi $sp,$sp,$FRAME + blr + .long 0 + .byte 0,12,4,1,0x80,18,3,0 + .long 0 + +.align 4 +Lsha2_block_private: +___ +for($i=0;$i<16;$i++) { +$code.=<<___ if ($SZ==4); + lwz @X[$i],`$i*$SZ`($inp) +___ +# 64-bit loads are split to 2x32-bit ones, as CPU can't handle +# unaligned 64-bit loads, only 32-bit ones... +$code.=<<___ if ($SZ==8); + lwz $t0,`$i*$SZ`($inp) + lwz @X[$i],`$i*$SZ+4`($inp) + insrdi @X[$i],$t0,32,0 +___ + &ROUND_00_15($i,@V); + unshift(@V,pop(@V)); +} +$code.=<<___; + li $T,`$rounds/16-1` + mtctr $T +.align 4 +Lrounds: + addi $Tbl,$Tbl,`16*$SZ` +___ +for(;$i<32;$i++) { + &ROUND_16_xx($i,@V); + unshift(@V,pop(@V)); +} +$code.=<<___; + bdnz- Lrounds + + $POP $ctx,`$FRAME-$SIZE_T*22`($sp) + $POP $inp,`$FRAME-$SIZE_T*23`($sp) ; inp pointer + $POP $num,`$FRAME-$SIZE_T*24`($sp) ; end pointer + subi $Tbl,$Tbl,`($rounds-16)*$SZ` ; rewind Tbl + + $LD r16,`0*$SZ`($ctx) + $LD r17,`1*$SZ`($ctx) + $LD r18,`2*$SZ`($ctx) + $LD r19,`3*$SZ`($ctx) + $LD r20,`4*$SZ`($ctx) + $LD r21,`5*$SZ`($ctx) + $LD r22,`6*$SZ`($ctx) + addi $inp,$inp,`16*$SZ` ; advance inp + $LD r23,`7*$SZ`($ctx) + add $A,$A,r16 + add $B,$B,r17 + $PUSH $inp,`$FRAME-$SIZE_T*23`($sp) + add $C,$C,r18 + $ST $A,`0*$SZ`($ctx) + add $D,$D,r19 + $ST $B,`1*$SZ`($ctx) + add $E,$E,r20 + $ST $C,`2*$SZ`($ctx) + add $F,$F,r21 + $ST $D,`3*$SZ`($ctx) + add $G,$G,r22 + $ST $E,`4*$SZ`($ctx) + add $H,$H,r23 + $ST $F,`5*$SZ`($ctx) + $ST $G,`6*$SZ`($ctx) + $UCMP $inp,$num + $ST $H,`7*$SZ`($ctx) + bne Lsha2_block_private + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +___ + +# Ugly hack here, because PPC assembler syntax seem to vary too +# much from platforms to platform... +$code.=<<___; +.align 6 +LPICmeup: + mflr r0 + bcl 20,31,\$+4 + mflr $Tbl ; vvvvvv "distance" between . and 1st data entry + addi $Tbl,$Tbl,`64-8` + mtlr r0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + .space `64-9*4` +___ +$code.=<<___ if ($SZ==8); + .long 0x428a2f98,0xd728ae22,0x71374491,0x23ef65cd + .long 0xb5c0fbcf,0xec4d3b2f,0xe9b5dba5,0x8189dbbc + .long 0x3956c25b,0xf348b538,0x59f111f1,0xb605d019 + .long 0x923f82a4,0xaf194f9b,0xab1c5ed5,0xda6d8118 + .long 0xd807aa98,0xa3030242,0x12835b01,0x45706fbe + .long 0x243185be,0x4ee4b28c,0x550c7dc3,0xd5ffb4e2 + .long 0x72be5d74,0xf27b896f,0x80deb1fe,0x3b1696b1 + .long 0x9bdc06a7,0x25c71235,0xc19bf174,0xcf692694 + .long 0xe49b69c1,0x9ef14ad2,0xefbe4786,0x384f25e3 + .long 0x0fc19dc6,0x8b8cd5b5,0x240ca1cc,0x77ac9c65 + .long 0x2de92c6f,0x592b0275,0x4a7484aa,0x6ea6e483 + .long 0x5cb0a9dc,0xbd41fbd4,0x76f988da,0x831153b5 + .long 0x983e5152,0xee66dfab,0xa831c66d,0x2db43210 + .long 0xb00327c8,0x98fb213f,0xbf597fc7,0xbeef0ee4 + .long 0xc6e00bf3,0x3da88fc2,0xd5a79147,0x930aa725 + .long 0x06ca6351,0xe003826f,0x14292967,0x0a0e6e70 + .long 0x27b70a85,0x46d22ffc,0x2e1b2138,0x5c26c926 + .long 0x4d2c6dfc,0x5ac42aed,0x53380d13,0x9d95b3df + .long 0x650a7354,0x8baf63de,0x766a0abb,0x3c77b2a8 + .long 0x81c2c92e,0x47edaee6,0x92722c85,0x1482353b + .long 0xa2bfe8a1,0x4cf10364,0xa81a664b,0xbc423001 + .long 0xc24b8b70,0xd0f89791,0xc76c51a3,0x0654be30 + .long 0xd192e819,0xd6ef5218,0xd6990624,0x5565a910 + .long 0xf40e3585,0x5771202a,0x106aa070,0x32bbd1b8 + .long 0x19a4c116,0xb8d2d0c8,0x1e376c08,0x5141ab53 + .long 0x2748774c,0xdf8eeb99,0x34b0bcb5,0xe19b48a8 + .long 0x391c0cb3,0xc5c95a63,0x4ed8aa4a,0xe3418acb + .long 0x5b9cca4f,0x7763e373,0x682e6ff3,0xd6b2b8a3 + .long 0x748f82ee,0x5defb2fc,0x78a5636f,0x43172f60 + .long 0x84c87814,0xa1f0ab72,0x8cc70208,0x1a6439ec + .long 0x90befffa,0x23631e28,0xa4506ceb,0xde82bde9 + .long 0xbef9a3f7,0xb2c67915,0xc67178f2,0xe372532b + .long 0xca273ece,0xea26619c,0xd186b8c7,0x21c0c207 + .long 0xeada7dd6,0xcde0eb1e,0xf57d4f7f,0xee6ed178 + .long 0x06f067aa,0x72176fba,0x0a637dc5,0xa2c898a6 + .long 0x113f9804,0xbef90dae,0x1b710b35,0x131c471b + .long 0x28db77f5,0x23047d84,0x32caab7b,0x40c72493 + .long 0x3c9ebe0a,0x15c9bebc,0x431d67c4,0x9c100d4c + .long 0x4cc5d4be,0xcb3e42b6,0x597f299c,0xfc657e2a + .long 0x5fcb6fab,0x3ad6faec,0x6c44198c,0x4a475817 +___ +$code.=<<___ if ($SZ==4); + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha512-s390x.pl b/crypto/openssl/crypto/sha/asm/sha512-s390x.pl new file mode 100755 index 000000000000..079a3fc78ab4 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-s390x.pl @@ -0,0 +1,322 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA256/512 block procedures for s390x. + +# April 2007. +# +# sha256_block_data_order is reportedly >3 times faster than gcc 3.3 +# generated code (must be a bug in compiler, as improvement is +# "pathologically" high, in particular in comparison to other SHA +# modules). But the real twist is that it detects if hardware support +# for SHA256 is available and in such case utilizes it. Then the +# performance can reach >6.5x of assembler one for larger chunks. +# +# sha512_block_data_order is ~70% faster than gcc 3.3 generated code. + +# January 2009. +# +# Add support for hardware SHA512 and reschedule instructions to +# favour dual-issue z10 pipeline. Hardware SHA256/512 is ~4.7x faster +# than software. + +# November 2010. +# +# Adapt for -m31 build. If kernel supports what's called "highgprs" +# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit +# instructions and achieve "64-bit" performance even in 31-bit legacy +# application context. The feature is not specific to any particular +# processor, as long as it's "z-CPU". Latter implies that the code +# remains z/Architecture specific. On z900 SHA256 was measured to +# perform 2.4x and SHA512 - 13x better than code generated by gcc 4.3. + +$flavour = shift; + +if ($flavour =~ /3[12]/) { + $SIZE_T=4; + $g=""; +} else { + $SIZE_T=8; + $g="g"; +} + +$t0="%r0"; +$t1="%r1"; +$ctx="%r2"; $t2="%r2"; +$inp="%r3"; +$len="%r4"; # used as index in inner loop + +$A="%r5"; +$B="%r6"; +$C="%r7"; +$D="%r8"; +$E="%r9"; +$F="%r10"; +$G="%r11"; +$H="%r12"; @V=($A,$B,$C,$D,$E,$F,$G,$H); +$tbl="%r13"; +$T1="%r14"; +$sp="%r15"; + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +if ($output =~ /512/) { + $label="512"; + $SZ=8; + $LD="lg"; # load from memory + $ST="stg"; # store to memory + $ADD="alg"; # add with memory operand + $ROT="rllg"; # rotate left + $SHR="srlg"; # logical right shift [see even at the end] + @Sigma0=(25,30,36); + @Sigma1=(23,46,50); + @sigma0=(56,63, 7); + @sigma1=( 3,45, 6); + $rounds=80; + $kimdfunc=3; # 0 means unknown/unsupported/unimplemented/disabled +} else { + $label="256"; + $SZ=4; + $LD="llgf"; # load from memory + $ST="st"; # store to memory + $ADD="al"; # add with memory operand + $ROT="rll"; # rotate left + $SHR="srl"; # logical right shift + @Sigma0=(10,19,30); + @Sigma1=( 7,21,26); + @sigma0=(14,25, 3); + @sigma1=(13,15,10); + $rounds=64; + $kimdfunc=2; # magic function code for kimd instruction +} +$Func="sha${label}_block_data_order"; +$Table="K${label}"; +$stdframe=16*$SIZE_T+4*8; +$frame=$stdframe+16*$SZ; + +sub BODY_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; + +$code.=<<___ if ($i<16); + $LD $T1,`$i*$SZ`($inp) ### $i +___ +$code.=<<___; + $ROT $t0,$e,$Sigma1[0] + $ROT $t1,$e,$Sigma1[1] + lgr $t2,$f + xgr $t0,$t1 + $ROT $t1,$t1,`$Sigma1[2]-$Sigma1[1]` + xgr $t2,$g + $ST $T1,`$stdframe+$SZ*($i%16)`($sp) + xgr $t0,$t1 # Sigma1(e) + algr $T1,$h # T1+=h + ngr $t2,$e + lgr $t1,$a + algr $T1,$t0 # T1+=Sigma1(e) + $ROT $h,$a,$Sigma0[0] + xgr $t2,$g # Ch(e,f,g) + $ADD $T1,`$i*$SZ`($len,$tbl) # T1+=K[i] + $ROT $t0,$a,$Sigma0[1] + algr $T1,$t2 # T1+=Ch(e,f,g) + ogr $t1,$b + xgr $h,$t0 + lgr $t2,$a + ngr $t1,$c + $ROT $t0,$t0,`$Sigma0[2]-$Sigma0[1]` + xgr $h,$t0 # h=Sigma0(a) + ngr $t2,$b + algr $h,$T1 # h+=T1 + ogr $t2,$t1 # Maj(a,b,c) + algr $d,$T1 # d+=T1 + algr $h,$t2 # h+=Maj(a,b,c) +___ +} + +sub BODY_16_XX { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; + +$code.=<<___; + $LD $T1,`$stdframe+$SZ*(($i+1)%16)`($sp) ### $i + $LD $t1,`$stdframe+$SZ*(($i+14)%16)`($sp) + $ROT $t0,$T1,$sigma0[0] + $SHR $T1,$sigma0[2] + $ROT $t2,$t0,`$sigma0[1]-$sigma0[0]` + xgr $T1,$t0 + $ROT $t0,$t1,$sigma1[0] + xgr $T1,$t2 # sigma0(X[i+1]) + $SHR $t1,$sigma1[2] + $ADD $T1,`$stdframe+$SZ*($i%16)`($sp) # +=X[i] + xgr $t1,$t0 + $ROT $t0,$t0,`$sigma1[1]-$sigma1[0]` + $ADD $T1,`$stdframe+$SZ*(($i+9)%16)`($sp) # +=X[i+9] + xgr $t1,$t0 # sigma1(X[i+14]) + algr $T1,$t1 # +=sigma1(X[i+14]) +___ + &BODY_00_15(@_); +} + +$code.=<<___; +.text +.align 64 +.type $Table,\@object +$Table: +___ +$code.=<<___ if ($SZ==4); + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +___ +$code.=<<___ if ($SZ==8); + .quad 0x428a2f98d728ae22,0x7137449123ef65cd + .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc + .quad 0x3956c25bf348b538,0x59f111f1b605d019 + .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 + .quad 0xd807aa98a3030242,0x12835b0145706fbe + .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 + .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 + .quad 0x9bdc06a725c71235,0xc19bf174cf692694 + .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 + .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 + .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 + .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 + .quad 0x983e5152ee66dfab,0xa831c66d2db43210 + .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 + .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 + .quad 0x06ca6351e003826f,0x142929670a0e6e70 + .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 + .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df + .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 + .quad 0x81c2c92e47edaee6,0x92722c851482353b + .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 + .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 + .quad 0xd192e819d6ef5218,0xd69906245565a910 + .quad 0xf40e35855771202a,0x106aa07032bbd1b8 + .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 + .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 + .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb + .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 + .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 + .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec + .quad 0x90befffa23631e28,0xa4506cebde82bde9 + .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b + .quad 0xca273eceea26619c,0xd186b8c721c0c207 + .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 + .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 + .quad 0x113f9804bef90dae,0x1b710b35131c471b + .quad 0x28db77f523047d84,0x32caab7b40c72493 + .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c + .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a + .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +___ +$code.=<<___; +.size $Table,.-$Table +.globl $Func +.type $Func,\@function +$Func: + sllg $len,$len,`log(16*$SZ)/log(2)` +___ +$code.=<<___ if ($kimdfunc); + larl %r1,OPENSSL_s390xcap_P + lg %r0,0(%r1) + tmhl %r0,0x4000 # check for message-security assist + jz .Lsoftware + lghi %r0,0 + la %r1,`2*$SIZE_T`($sp) + .long 0xb93e0002 # kimd %r0,%r2 + lg %r0,`2*$SIZE_T`($sp) + tmhh %r0,`0x8000>>$kimdfunc` + jz .Lsoftware + lghi %r0,$kimdfunc + lgr %r1,$ctx + lgr %r2,$inp + lgr %r3,$len + .long 0xb93e0002 # kimd %r0,%r2 + brc 1,.-4 # pay attention to "partial completion" + br %r14 +.align 16 +.Lsoftware: +___ +$code.=<<___; + lghi %r1,-$frame + la $len,0($len,$inp) + stm${g} $ctx,%r15,`2*$SIZE_T`($sp) + lgr %r0,$sp + la $sp,0(%r1,$sp) + st${g} %r0,0($sp) + + larl $tbl,$Table + $LD $A,`0*$SZ`($ctx) + $LD $B,`1*$SZ`($ctx) + $LD $C,`2*$SZ`($ctx) + $LD $D,`3*$SZ`($ctx) + $LD $E,`4*$SZ`($ctx) + $LD $F,`5*$SZ`($ctx) + $LD $G,`6*$SZ`($ctx) + $LD $H,`7*$SZ`($ctx) + +.Lloop: + lghi $len,0 +___ +for ($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } +$code.=".Lrounds_16_xx:\n"; +for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + aghi $len,`16*$SZ` + lghi $t0,`($rounds-16)*$SZ` + clgr $len,$t0 + jne .Lrounds_16_xx + + l${g} $ctx,`$frame+2*$SIZE_T`($sp) + la $inp,`16*$SZ`($inp) + $ADD $A,`0*$SZ`($ctx) + $ADD $B,`1*$SZ`($ctx) + $ADD $C,`2*$SZ`($ctx) + $ADD $D,`3*$SZ`($ctx) + $ADD $E,`4*$SZ`($ctx) + $ADD $F,`5*$SZ`($ctx) + $ADD $G,`6*$SZ`($ctx) + $ADD $H,`7*$SZ`($ctx) + $ST $A,`0*$SZ`($ctx) + $ST $B,`1*$SZ`($ctx) + $ST $C,`2*$SZ`($ctx) + $ST $D,`3*$SZ`($ctx) + $ST $E,`4*$SZ`($ctx) + $ST $F,`5*$SZ`($ctx) + $ST $G,`6*$SZ`($ctx) + $ST $H,`7*$SZ`($ctx) + cl${g} $inp,`$frame+4*$SIZE_T`($sp) + jne .Lloop + + lm${g} %r6,%r15,`$frame+6*$SIZE_T`($sp) + br %r14 +.size $Func,.-$Func +.string "SHA${label} block transform for s390x, CRYPTOGAMS by <appro\@openssl.org>" +.comm OPENSSL_s390xcap_P,16,8 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +# unlike 32-bit shift 64-bit one takes three arguments +$code =~ s/(srlg\s+)(%r[0-9]+),/$1$2,$2,/gm; + +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha512-sparcv9.pl b/crypto/openssl/crypto/sha/asm/sha512-sparcv9.pl new file mode 100755 index 000000000000..585740789e63 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-sparcv9.pl @@ -0,0 +1,594 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== + +# SHA256 performance improvement over compiler generated code varies +# from 40% for Sun C [32-bit build] to 70% for gcc [3.3, 64-bit +# build]. Just like in SHA1 module I aim to ensure scalability on +# UltraSPARC T1 by packing X[16] to 8 64-bit registers. + +# SHA512 on pre-T1 UltraSPARC. +# +# Performance is >75% better than 64-bit code generated by Sun C and +# over 2x than 32-bit code. X[16] resides on stack, but access to it +# is scheduled for L2 latency and staged through 32 least significant +# bits of %l0-%l7. The latter is done to achieve 32-/64-bit ABI +# duality. Nevetheless it's ~40% faster than SHA256, which is pretty +# good [optimal coefficient is 50%]. +# +# SHA512 on UltraSPARC T1. +# +# It's not any faster than 64-bit code generated by Sun C 5.8. This is +# because 64-bit code generator has the advantage of using 64-bit +# loads(*) to access X[16], which I consciously traded for 32-/64-bit +# ABI duality [as per above]. But it surpasses 32-bit Sun C generated +# code by 60%, not to mention that it doesn't suffer from severe decay +# when running 4 times physical cores threads and that it leaves gcc +# [3.4] behind by over 4x factor! If compared to SHA256, single thread +# performance is only 10% better, but overall throughput for maximum +# amount of threads for given CPU exceeds corresponding one of SHA256 +# by 30% [again, optimal coefficient is 50%]. +# +# (*) Unlike pre-T1 UltraSPARC loads on T1 are executed strictly +# in-order, i.e. load instruction has to complete prior next +# instruction in given thread is executed, even if the latter is +# not dependent on load result! This means that on T1 two 32-bit +# loads are always slower than one 64-bit load. Once again this +# is unlike pre-T1 UltraSPARC, where, if scheduled appropriately, +# 2x32-bit loads can be as fast as 1x64-bit ones. + +$bits=32; +for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } +if ($bits==64) { $bias=2047; $frame=192; } +else { $bias=0; $frame=112; } + +$output=shift; +open STDOUT,">$output"; + +if ($output =~ /512/) { + $label="512"; + $SZ=8; + $LD="ldx"; # load from memory + $ST="stx"; # store to memory + $SLL="sllx"; # shift left logical + $SRL="srlx"; # shift right logical + @Sigma0=(28,34,39); + @Sigma1=(14,18,41); + @sigma0=( 7, 1, 8); # right shift first + @sigma1=( 6,19,61); # right shift first + $lastK=0x817; + $rounds=80; + $align=4; + + $locals=16*$SZ; # X[16] + + $A="%o0"; + $B="%o1"; + $C="%o2"; + $D="%o3"; + $E="%o4"; + $F="%o5"; + $G="%g1"; + $H="%o7"; + @V=($A,$B,$C,$D,$E,$F,$G,$H); +} else { + $label="256"; + $SZ=4; + $LD="ld"; # load from memory + $ST="st"; # store to memory + $SLL="sll"; # shift left logical + $SRL="srl"; # shift right logical + @Sigma0=( 2,13,22); + @Sigma1=( 6,11,25); + @sigma0=( 3, 7,18); # right shift first + @sigma1=(10,17,19); # right shift first + $lastK=0x8f2; + $rounds=64; + $align=8; + + $locals=0; # X[16] is register resident + @X=("%o0","%o1","%o2","%o3","%o4","%o5","%g1","%o7"); + + $A="%l0"; + $B="%l1"; + $C="%l2"; + $D="%l3"; + $E="%l4"; + $F="%l5"; + $G="%l6"; + $H="%l7"; + @V=($A,$B,$C,$D,$E,$F,$G,$H); +} +$T1="%g2"; +$tmp0="%g3"; +$tmp1="%g4"; +$tmp2="%g5"; + +$ctx="%i0"; +$inp="%i1"; +$len="%i2"; +$Ktbl="%i3"; +$tmp31="%i4"; +$tmp32="%i5"; + +########### SHA256 +$Xload = sub { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; + + if ($i==0) { +$code.=<<___; + ldx [$inp+0],@X[0] + ldx [$inp+16],@X[2] + ldx [$inp+32],@X[4] + ldx [$inp+48],@X[6] + ldx [$inp+8],@X[1] + ldx [$inp+24],@X[3] + subcc %g0,$tmp31,$tmp32 ! should be 64-$tmp31, but -$tmp31 works too + ldx [$inp+40],@X[5] + bz,pt %icc,.Laligned + ldx [$inp+56],@X[7] + + sllx @X[0],$tmp31,@X[0] + ldx [$inp+64],$T1 +___ +for($j=0;$j<7;$j++) +{ $code.=<<___; + srlx @X[$j+1],$tmp32,$tmp1 + sllx @X[$j+1],$tmp31,@X[$j+1] + or $tmp1,@X[$j],@X[$j] +___ +} +$code.=<<___; + srlx $T1,$tmp32,$T1 + or $T1,@X[7],@X[7] +.Laligned: +___ + } + + if ($i&1) { + $code.="\tadd @X[$i/2],$h,$T1\n"; + } else { + $code.="\tsrlx @X[$i/2],32,$T1\n\tadd $h,$T1,$T1\n"; + } +} if ($SZ==4); + +########### SHA512 +$Xload = sub { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; +my @pair=("%l".eval(($i*2)%8),"%l".eval(($i*2)%8+1),"%l".eval((($i+1)*2)%8)); + +$code.=<<___ if ($i==0); + ld [$inp+0],%l0 + ld [$inp+4],%l1 + ld [$inp+8],%l2 + ld [$inp+12],%l3 + ld [$inp+16],%l4 + ld [$inp+20],%l5 + ld [$inp+24],%l6 + ld [$inp+28],%l7 +___ +$code.=<<___ if ($i<15); + sllx @pair[1],$tmp31,$tmp2 ! Xload($i) + add $tmp31,32,$tmp0 + sllx @pair[0],$tmp0,$tmp1 + `"ld [$inp+".eval(32+0+$i*8)."],@pair[0]" if ($i<12)` + srlx @pair[2],$tmp32,@pair[1] + or $tmp1,$tmp2,$tmp2 + or @pair[1],$tmp2,$tmp2 + `"ld [$inp+".eval(32+4+$i*8)."],@pair[1]" if ($i<12)` + add $h,$tmp2,$T1 + $ST $tmp2,[%sp+`$bias+$frame+$i*$SZ`] +___ +$code.=<<___ if ($i==12); + brnz,a $tmp31,.+8 + ld [$inp+128],%l0 +___ +$code.=<<___ if ($i==15); + ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+0`],%l2 + sllx @pair[1],$tmp31,$tmp2 ! Xload($i) + add $tmp31,32,$tmp0 + ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+4`],%l3 + sllx @pair[0],$tmp0,$tmp1 + ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+0`],%l4 + srlx @pair[2],$tmp32,@pair[1] + or $tmp1,$tmp2,$tmp2 + ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+4`],%l5 + or @pair[1],$tmp2,$tmp2 + ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+0`],%l6 + add $h,$tmp2,$T1 + $ST $tmp2,[%sp+`$bias+$frame+$i*$SZ`] + ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+4`],%l7 + ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+0`],%l0 + ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+4`],%l1 +___ +} if ($SZ==8); + +########### common +sub BODY_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_; + + if ($i<16) { + &$Xload(@_); + } else { + $code.="\tadd $h,$T1,$T1\n"; + } + +$code.=<<___; + $SRL $e,@Sigma1[0],$h !! $i + xor $f,$g,$tmp2 + $SLL $e,`$SZ*8-@Sigma1[2]`,$tmp1 + and $e,$tmp2,$tmp2 + $SRL $e,@Sigma1[1],$tmp0 + xor $tmp1,$h,$h + $SLL $e,`$SZ*8-@Sigma1[1]`,$tmp1 + xor $tmp0,$h,$h + $SRL $e,@Sigma1[2],$tmp0 + xor $tmp1,$h,$h + $SLL $e,`$SZ*8-@Sigma1[0]`,$tmp1 + xor $tmp0,$h,$h + xor $g,$tmp2,$tmp2 ! Ch(e,f,g) + xor $tmp1,$h,$tmp0 ! Sigma1(e) + + $SRL $a,@Sigma0[0],$h + add $tmp2,$T1,$T1 + $LD [$Ktbl+`$i*$SZ`],$tmp2 ! K[$i] + $SLL $a,`$SZ*8-@Sigma0[2]`,$tmp1 + add $tmp0,$T1,$T1 + $SRL $a,@Sigma0[1],$tmp0 + xor $tmp1,$h,$h + $SLL $a,`$SZ*8-@Sigma0[1]`,$tmp1 + xor $tmp0,$h,$h + $SRL $a,@Sigma0[2],$tmp0 + xor $tmp1,$h,$h + $SLL $a,`$SZ*8-@Sigma0[0]`,$tmp1 + xor $tmp0,$h,$h + xor $tmp1,$h,$h ! Sigma0(a) + + or $a,$b,$tmp0 + and $a,$b,$tmp1 + and $c,$tmp0,$tmp0 + or $tmp0,$tmp1,$tmp1 ! Maj(a,b,c) + add $tmp2,$T1,$T1 ! +=K[$i] + add $tmp1,$h,$h + + add $T1,$d,$d + add $T1,$h,$h +___ +} + +########### SHA256 +$BODY_16_XX = sub { +my $i=@_[0]; +my $xi; + + if ($i&1) { + $xi=$tmp32; + $code.="\tsrlx @X[(($i+1)/2)%8],32,$xi\n"; + } else { + $xi=@X[(($i+1)/2)%8]; + } +$code.=<<___; + srl $xi,@sigma0[0],$T1 !! Xupdate($i) + sll $xi,`32-@sigma0[2]`,$tmp1 + srl $xi,@sigma0[1],$tmp0 + xor $tmp1,$T1,$T1 + sll $tmp1,`@sigma0[2]-@sigma0[1]`,$tmp1 + xor $tmp0,$T1,$T1 + srl $xi,@sigma0[2],$tmp0 + xor $tmp1,$T1,$T1 +___ + if ($i&1) { + $xi=@X[(($i+14)/2)%8]; + } else { + $xi=$tmp32; + $code.="\tsrlx @X[(($i+14)/2)%8],32,$xi\n"; + } +$code.=<<___; + srl $xi,@sigma1[0],$tmp2 + xor $tmp0,$T1,$T1 ! T1=sigma0(X[i+1]) + sll $xi,`32-@sigma1[2]`,$tmp1 + srl $xi,@sigma1[1],$tmp0 + xor $tmp1,$tmp2,$tmp2 + sll $tmp1,`@sigma1[2]-@sigma1[1]`,$tmp1 + xor $tmp0,$tmp2,$tmp2 + srl $xi,@sigma1[2],$tmp0 + xor $tmp1,$tmp2,$tmp2 +___ + if ($i&1) { + $xi=@X[($i/2)%8]; +$code.=<<___; + srlx @X[(($i+9)/2)%8],32,$tmp1 ! X[i+9] + xor $tmp0,$tmp2,$tmp2 ! sigma1(X[i+14]) + srl @X[($i/2)%8],0,$tmp0 + add $tmp2,$tmp1,$tmp1 + add $xi,$T1,$T1 ! +=X[i] + xor $tmp0,@X[($i/2)%8],@X[($i/2)%8] + add $tmp1,$T1,$T1 + + srl $T1,0,$T1 + or $T1,@X[($i/2)%8],@X[($i/2)%8] +___ + } else { + $xi=@X[(($i+9)/2)%8]; +$code.=<<___; + srlx @X[($i/2)%8],32,$tmp1 ! X[i] + xor $tmp0,$tmp2,$tmp2 ! sigma1(X[i+14]) + add $xi,$T1,$T1 ! +=X[i+9] + add $tmp2,$tmp1,$tmp1 + srl @X[($i/2)%8],0,@X[($i/2)%8] + add $tmp1,$T1,$T1 + + sllx $T1,32,$tmp0 + or $tmp0,@X[($i/2)%8],@X[($i/2)%8] +___ + } + &BODY_00_15(@_); +} if ($SZ==4); + +########### SHA512 +$BODY_16_XX = sub { +my $i=@_[0]; +my @pair=("%l".eval(($i*2)%8),"%l".eval(($i*2)%8+1)); + +$code.=<<___; + sllx %l2,32,$tmp0 !! Xupdate($i) + or %l3,$tmp0,$tmp0 + + srlx $tmp0,@sigma0[0],$T1 + ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+0`],%l2 + sllx $tmp0,`64-@sigma0[2]`,$tmp1 + ld [%sp+`$bias+$frame+(($i+1+1)%16)*$SZ+4`],%l3 + srlx $tmp0,@sigma0[1],$tmp0 + xor $tmp1,$T1,$T1 + sllx $tmp1,`@sigma0[2]-@sigma0[1]`,$tmp1 + xor $tmp0,$T1,$T1 + srlx $tmp0,`@sigma0[2]-@sigma0[1]`,$tmp0 + xor $tmp1,$T1,$T1 + sllx %l6,32,$tmp2 + xor $tmp0,$T1,$T1 ! sigma0(X[$i+1]) + or %l7,$tmp2,$tmp2 + + srlx $tmp2,@sigma1[0],$tmp1 + ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+0`],%l6 + sllx $tmp2,`64-@sigma1[2]`,$tmp0 + ld [%sp+`$bias+$frame+(($i+1+14)%16)*$SZ+4`],%l7 + srlx $tmp2,@sigma1[1],$tmp2 + xor $tmp0,$tmp1,$tmp1 + sllx $tmp0,`@sigma1[2]-@sigma1[1]`,$tmp0 + xor $tmp2,$tmp1,$tmp1 + srlx $tmp2,`@sigma1[2]-@sigma1[1]`,$tmp2 + xor $tmp0,$tmp1,$tmp1 + sllx %l4,32,$tmp0 + xor $tmp2,$tmp1,$tmp1 ! sigma1(X[$i+14]) + ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+0`],%l4 + or %l5,$tmp0,$tmp0 + ld [%sp+`$bias+$frame+(($i+1+9)%16)*$SZ+4`],%l5 + + sllx %l0,32,$tmp2 + add $tmp1,$T1,$T1 + ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+0`],%l0 + or %l1,$tmp2,$tmp2 + add $tmp0,$T1,$T1 ! +=X[$i+9] + ld [%sp+`$bias+$frame+(($i+1+0)%16)*$SZ+4`],%l1 + add $tmp2,$T1,$T1 ! +=X[$i] + $ST $T1,[%sp+`$bias+$frame+($i%16)*$SZ`] +___ + &BODY_00_15(@_); +} if ($SZ==8); + +$code.=<<___ if ($bits==64); +.register %g2,#scratch +.register %g3,#scratch +___ +$code.=<<___; +.section ".text",#alloc,#execinstr + +.align 64 +K${label}: +.type K${label},#object +___ +if ($SZ==4) { +$code.=<<___; + .long 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 + .long 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 + .long 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 + .long 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 + .long 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc + .long 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da + .long 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 + .long 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 + .long 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 + .long 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 + .long 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 + .long 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 + .long 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 + .long 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 + .long 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 + .long 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +___ +} else { +$code.=<<___; + .long 0x428a2f98,0xd728ae22, 0x71374491,0x23ef65cd + .long 0xb5c0fbcf,0xec4d3b2f, 0xe9b5dba5,0x8189dbbc + .long 0x3956c25b,0xf348b538, 0x59f111f1,0xb605d019 + .long 0x923f82a4,0xaf194f9b, 0xab1c5ed5,0xda6d8118 + .long 0xd807aa98,0xa3030242, 0x12835b01,0x45706fbe + .long 0x243185be,0x4ee4b28c, 0x550c7dc3,0xd5ffb4e2 + .long 0x72be5d74,0xf27b896f, 0x80deb1fe,0x3b1696b1 + .long 0x9bdc06a7,0x25c71235, 0xc19bf174,0xcf692694 + .long 0xe49b69c1,0x9ef14ad2, 0xefbe4786,0x384f25e3 + .long 0x0fc19dc6,0x8b8cd5b5, 0x240ca1cc,0x77ac9c65 + .long 0x2de92c6f,0x592b0275, 0x4a7484aa,0x6ea6e483 + .long 0x5cb0a9dc,0xbd41fbd4, 0x76f988da,0x831153b5 + .long 0x983e5152,0xee66dfab, 0xa831c66d,0x2db43210 + .long 0xb00327c8,0x98fb213f, 0xbf597fc7,0xbeef0ee4 + .long 0xc6e00bf3,0x3da88fc2, 0xd5a79147,0x930aa725 + .long 0x06ca6351,0xe003826f, 0x14292967,0x0a0e6e70 + .long 0x27b70a85,0x46d22ffc, 0x2e1b2138,0x5c26c926 + .long 0x4d2c6dfc,0x5ac42aed, 0x53380d13,0x9d95b3df + .long 0x650a7354,0x8baf63de, 0x766a0abb,0x3c77b2a8 + .long 0x81c2c92e,0x47edaee6, 0x92722c85,0x1482353b + .long 0xa2bfe8a1,0x4cf10364, 0xa81a664b,0xbc423001 + .long 0xc24b8b70,0xd0f89791, 0xc76c51a3,0x0654be30 + .long 0xd192e819,0xd6ef5218, 0xd6990624,0x5565a910 + .long 0xf40e3585,0x5771202a, 0x106aa070,0x32bbd1b8 + .long 0x19a4c116,0xb8d2d0c8, 0x1e376c08,0x5141ab53 + .long 0x2748774c,0xdf8eeb99, 0x34b0bcb5,0xe19b48a8 + .long 0x391c0cb3,0xc5c95a63, 0x4ed8aa4a,0xe3418acb + .long 0x5b9cca4f,0x7763e373, 0x682e6ff3,0xd6b2b8a3 + .long 0x748f82ee,0x5defb2fc, 0x78a5636f,0x43172f60 + .long 0x84c87814,0xa1f0ab72, 0x8cc70208,0x1a6439ec + .long 0x90befffa,0x23631e28, 0xa4506ceb,0xde82bde9 + .long 0xbef9a3f7,0xb2c67915, 0xc67178f2,0xe372532b + .long 0xca273ece,0xea26619c, 0xd186b8c7,0x21c0c207 + .long 0xeada7dd6,0xcde0eb1e, 0xf57d4f7f,0xee6ed178 + .long 0x06f067aa,0x72176fba, 0x0a637dc5,0xa2c898a6 + .long 0x113f9804,0xbef90dae, 0x1b710b35,0x131c471b + .long 0x28db77f5,0x23047d84, 0x32caab7b,0x40c72493 + .long 0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c + .long 0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a + .long 0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817 +___ +} +$code.=<<___; +.size K${label},.-K${label} +.globl sha${label}_block_data_order +sha${label}_block_data_order: + save %sp,`-$frame-$locals`,%sp + and $inp,`$align-1`,$tmp31 + sllx $len,`log(16*$SZ)/log(2)`,$len + andn $inp,`$align-1`,$inp + sll $tmp31,3,$tmp31 + add $inp,$len,$len +___ +$code.=<<___ if ($SZ==8); # SHA512 + mov 32,$tmp32 + sub $tmp32,$tmp31,$tmp32 +___ +$code.=<<___; +.Lpic: call .+8 + add %o7,K${label}-.Lpic,$Ktbl + + $LD [$ctx+`0*$SZ`],$A + $LD [$ctx+`1*$SZ`],$B + $LD [$ctx+`2*$SZ`],$C + $LD [$ctx+`3*$SZ`],$D + $LD [$ctx+`4*$SZ`],$E + $LD [$ctx+`5*$SZ`],$F + $LD [$ctx+`6*$SZ`],$G + $LD [$ctx+`7*$SZ`],$H + +.Lloop: +___ +for ($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } +$code.=".L16_xx:\n"; +for (;$i<32;$i++) { &$BODY_16_XX($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; + and $tmp2,0xfff,$tmp2 + cmp $tmp2,$lastK + bne .L16_xx + add $Ktbl,`16*$SZ`,$Ktbl ! Ktbl+=16 + +___ +$code.=<<___ if ($SZ==4); # SHA256 + $LD [$ctx+`0*$SZ`],@X[0] + $LD [$ctx+`1*$SZ`],@X[1] + $LD [$ctx+`2*$SZ`],@X[2] + $LD [$ctx+`3*$SZ`],@X[3] + $LD [$ctx+`4*$SZ`],@X[4] + $LD [$ctx+`5*$SZ`],@X[5] + $LD [$ctx+`6*$SZ`],@X[6] + $LD [$ctx+`7*$SZ`],@X[7] + + add $A,@X[0],$A + $ST $A,[$ctx+`0*$SZ`] + add $B,@X[1],$B + $ST $B,[$ctx+`1*$SZ`] + add $C,@X[2],$C + $ST $C,[$ctx+`2*$SZ`] + add $D,@X[3],$D + $ST $D,[$ctx+`3*$SZ`] + add $E,@X[4],$E + $ST $E,[$ctx+`4*$SZ`] + add $F,@X[5],$F + $ST $F,[$ctx+`5*$SZ`] + add $G,@X[6],$G + $ST $G,[$ctx+`6*$SZ`] + add $H,@X[7],$H + $ST $H,[$ctx+`7*$SZ`] +___ +$code.=<<___ if ($SZ==8); # SHA512 + ld [$ctx+`0*$SZ+0`],%l0 + ld [$ctx+`0*$SZ+4`],%l1 + ld [$ctx+`1*$SZ+0`],%l2 + ld [$ctx+`1*$SZ+4`],%l3 + ld [$ctx+`2*$SZ+0`],%l4 + ld [$ctx+`2*$SZ+4`],%l5 + ld [$ctx+`3*$SZ+0`],%l6 + + sllx %l0,32,$tmp0 + ld [$ctx+`3*$SZ+4`],%l7 + sllx %l2,32,$tmp1 + or %l1,$tmp0,$tmp0 + or %l3,$tmp1,$tmp1 + add $tmp0,$A,$A + add $tmp1,$B,$B + $ST $A,[$ctx+`0*$SZ`] + sllx %l4,32,$tmp2 + $ST $B,[$ctx+`1*$SZ`] + sllx %l6,32,$T1 + or %l5,$tmp2,$tmp2 + or %l7,$T1,$T1 + add $tmp2,$C,$C + $ST $C,[$ctx+`2*$SZ`] + add $T1,$D,$D + $ST $D,[$ctx+`3*$SZ`] + + ld [$ctx+`4*$SZ+0`],%l0 + ld [$ctx+`4*$SZ+4`],%l1 + ld [$ctx+`5*$SZ+0`],%l2 + ld [$ctx+`5*$SZ+4`],%l3 + ld [$ctx+`6*$SZ+0`],%l4 + ld [$ctx+`6*$SZ+4`],%l5 + ld [$ctx+`7*$SZ+0`],%l6 + + sllx %l0,32,$tmp0 + ld [$ctx+`7*$SZ+4`],%l7 + sllx %l2,32,$tmp1 + or %l1,$tmp0,$tmp0 + or %l3,$tmp1,$tmp1 + add $tmp0,$E,$E + add $tmp1,$F,$F + $ST $E,[$ctx+`4*$SZ`] + sllx %l4,32,$tmp2 + $ST $F,[$ctx+`5*$SZ`] + sllx %l6,32,$T1 + or %l5,$tmp2,$tmp2 + or %l7,$T1,$T1 + add $tmp2,$G,$G + $ST $G,[$ctx+`6*$SZ`] + add $T1,$H,$H + $ST $H,[$ctx+`7*$SZ`] +___ +$code.=<<___; + add $inp,`16*$SZ`,$inp ! advance inp + cmp $inp,$len + bne `$bits==64?"%xcc":"%icc"`,.Lloop + sub $Ktbl,`($rounds-16)*$SZ`,$Ktbl ! rewind Ktbl + + ret + restore +.type sha${label}_block_data_order,#function +.size sha${label}_block_data_order,(.-sha${label}_block_data_order) +.asciz "SHA${label} block transform for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>" +.align 4 +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT; diff --git a/crypto/openssl/crypto/sha/asm/sha512-sse2.pl b/crypto/openssl/crypto/sha/asm/sha512-sse2.pl deleted file mode 100644 index 10902bf673d5..000000000000 --- a/crypto/openssl/crypto/sha/asm/sha512-sse2.pl +++ /dev/null @@ -1,404 +0,0 @@ -#!/usr/bin/env perl -# -# ==================================================================== -# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL -# project. Rights for redistribution and usage in source and binary -# forms are granted according to the OpenSSL license. -# ==================================================================== -# -# SHA512_Transform_SSE2. -# -# As the name suggests, this is an IA-32 SSE2 implementation of -# SHA512_Transform. Motivating factor for the undertaken effort was that -# SHA512 was observed to *consistently* perform *significantly* poorer -# than SHA256 [2x and slower is common] on 32-bit platforms. On 64-bit -# platforms on the other hand SHA512 tend to outperform SHA256 [~50% -# seem to be common improvement factor]. All this is perfectly natural, -# as SHA512 is a 64-bit algorithm. But isn't IA-32 SSE2 essentially -# a 64-bit instruction set? Is it rich enough to implement SHA512? -# If answer was "no," then you wouldn't have been reading this... -# -# Throughput performance in MBps (larger is better): -# -# 2.4GHz P4 1.4GHz AMD32 1.4GHz AMD64(*) -# SHA256/gcc(*) 54 43 59 -# SHA512/gcc 17 23 92 -# SHA512/sse2 61(**) 57(**) -# SHA512/icc 26 28 -# SHA256/icc(*) 65 54 -# -# (*) AMD64 and SHA256 numbers are presented mostly for amusement or -# reference purposes. -# (**) I.e. it gives ~2-3x speed-up if compared with compiler generated -# code. One can argue that hand-coded *non*-SSE2 implementation -# would perform better than compiler generated one as well, and -# that comparison is therefore not exactly fair. Well, as SHA512 -# puts enormous pressure on IA-32 GP register bank, I reckon that -# hand-coded version wouldn't perform significantly better than -# one compiled with icc, ~20% perhaps... So that this code would -# still outperform it with distinguishing marginal. But feel free -# to prove me wrong:-) -# <appro@fy.chalmers.se> -push(@INC,"perlasm","../../perlasm"); -require "x86asm.pl"; - -&asm_init($ARGV[0],"sha512-sse2.pl",$ARGV[$#ARGV] eq "386"); - -$K512="esi"; # K512[80] table, found at the end... -#$W512="esp"; # $W512 is not just W512[16]: it comprises *two* copies - # of W512[16] and a copy of A-H variables... -$W512_SZ=8*(16+16+8); # see above... -#$Kidx="ebx"; # index in K512 table, advances from 0 to 80... -$Widx="edx"; # index in W512, wraps around at 16... -$data="edi"; # 16 qwords of input data... -$A="mm0"; # B-D and -$E="mm1"; # F-H are allocated dynamically... -$Aoff=256+0; # A-H offsets relative to $W512... -$Boff=256+8; -$Coff=256+16; -$Doff=256+24; -$Eoff=256+32; -$Foff=256+40; -$Goff=256+48; -$Hoff=256+56; - -sub SHA2_ROUND() -{ local ($kidx,$widx)=@_; - - # One can argue that one could reorder instructions for better - # performance. Well, I tried and it doesn't seem to make any - # noticeable difference. Modern out-of-order execution cores - # reorder instructions to their liking in either case and they - # apparently do decent job. So we can keep the code more - # readable/regular/comprehensible:-) - - # I adhere to 64-bit %mmX registers in order to avoid/not care - # about #GP exceptions on misaligned 128-bit access, most - # notably in paddq with memory operand. Not to mention that - # SSE2 intructions operating on %mmX can be scheduled every - # cycle [and not every second one if operating on %xmmN]. - - &movq ("mm4",&QWP($Foff,$W512)); # load f - &movq ("mm5",&QWP($Goff,$W512)); # load g - &movq ("mm6",&QWP($Hoff,$W512)); # load h - - &movq ("mm2",$E); # %mm2 is sliding right - &movq ("mm3",$E); # %mm3 is sliding left - &psrlq ("mm2",14); - &psllq ("mm3",23); - &movq ("mm7","mm2"); # %mm7 is T1 - &pxor ("mm7","mm3"); - &psrlq ("mm2",4); - &psllq ("mm3",23); - &pxor ("mm7","mm2"); - &pxor ("mm7","mm3"); - &psrlq ("mm2",23); - &psllq ("mm3",4); - &pxor ("mm7","mm2"); - &pxor ("mm7","mm3"); # T1=Sigma1_512(e) - - &movq (&QWP($Foff,$W512),$E); # f = e - &movq (&QWP($Goff,$W512),"mm4"); # g = f - &movq (&QWP($Hoff,$W512),"mm5"); # h = g - - &pxor ("mm4","mm5"); # f^=g - &pand ("mm4",$E); # f&=e - &pxor ("mm4","mm5"); # f^=g - &paddq ("mm7","mm4"); # T1+=Ch(e,f,g) - - &movq ("mm2",&QWP($Boff,$W512)); # load b - &movq ("mm3",&QWP($Coff,$W512)); # load c - &movq ($E,&QWP($Doff,$W512)); # e = d - - &paddq ("mm7","mm6"); # T1+=h - &paddq ("mm7",&QWP(0,$K512,$kidx,8)); # T1+=K512[i] - &paddq ("mm7",&QWP(0,$W512,$widx,8)); # T1+=W512[i] - &paddq ($E,"mm7"); # e += T1 - - &movq ("mm4",$A); # %mm4 is sliding right - &movq ("mm5",$A); # %mm5 is sliding left - &psrlq ("mm4",28); - &psllq ("mm5",25); - &movq ("mm6","mm4"); # %mm6 is T2 - &pxor ("mm6","mm5"); - &psrlq ("mm4",6); - &psllq ("mm5",5); - &pxor ("mm6","mm4"); - &pxor ("mm6","mm5"); - &psrlq ("mm4",5); - &psllq ("mm5",6); - &pxor ("mm6","mm4"); - &pxor ("mm6","mm5"); # T2=Sigma0_512(a) - - &movq (&QWP($Boff,$W512),$A); # b = a - &movq (&QWP($Coff,$W512),"mm2"); # c = b - &movq (&QWP($Doff,$W512),"mm3"); # d = c - - &movq ("mm4",$A); # %mm4=a - &por ($A,"mm3"); # a=a|c - &pand ("mm4","mm3"); # %mm4=a&c - &pand ($A,"mm2"); # a=(a|c)&b - &por ("mm4",$A); # %mm4=(a&c)|((a|c)&b) - &paddq ("mm6","mm4"); # T2+=Maj(a,b,c) - - &movq ($A,"mm7"); # a=T1 - &paddq ($A,"mm6"); # a+=T2 -} - -$func="sha512_block_sse2"; - -&function_begin_B($func); - if (0) {# Caller is expected to check if it's appropriate to - # call this routine. Below 3 lines are retained for - # debugging purposes... - &picmeup("eax","OPENSSL_ia32cap"); - &bt (&DWP(0,"eax"),26); - &jnc ("SHA512_Transform"); - } - - &push ("ebp"); - &mov ("ebp","esp"); - &push ("ebx"); - &push ("esi"); - &push ("edi"); - - &mov ($Widx,&DWP(8,"ebp")); # A-H state, 1st arg - &mov ($data,&DWP(12,"ebp")); # input data, 2nd arg - &call (&label("pic_point")); # make it PIC! -&set_label("pic_point"); - &blindpop($K512); - &lea ($K512,&DWP(&label("K512")."-".&label("pic_point"),$K512)); - - $W512 = "esp"; # start using %esp as W512 - &sub ($W512,$W512_SZ); - &and ($W512,-16); # ensure 128-bit alignment - - # make private copy of A-H - # v assume the worst and stick to unaligned load - &movdqu ("xmm0",&QWP(0,$Widx)); - &movdqu ("xmm1",&QWP(16,$Widx)); - &movdqu ("xmm2",&QWP(32,$Widx)); - &movdqu ("xmm3",&QWP(48,$Widx)); - -&align(8); -&set_label("_chunk_loop"); - - &movdqa (&QWP($Aoff,$W512),"xmm0"); # a,b - &movdqa (&QWP($Coff,$W512),"xmm1"); # c,d - &movdqa (&QWP($Eoff,$W512),"xmm2"); # e,f - &movdqa (&QWP($Goff,$W512),"xmm3"); # g,h - - &xor ($Widx,$Widx); - - &movdq2q($A,"xmm0"); # load a - &movdq2q($E,"xmm2"); # load e - - # Why aren't loops unrolled? It makes sense to unroll if - # execution time for loop body is comparable with branch - # penalties and/or if whole data-set resides in register bank. - # Neither is case here... Well, it would be possible to - # eliminate few store operations, but it would hardly affect - # so to say stop-watch performance, as there is a lot of - # available memory slots to fill. It will only relieve some - # pressure off memory bus... - - # flip input stream byte order... - &mov ("eax",&DWP(0,$data,$Widx,8)); - &mov ("ebx",&DWP(4,$data,$Widx,8)); - &bswap ("eax"); - &bswap ("ebx"); - &mov (&DWP(0,$W512,$Widx,8),"ebx"); # W512[i] - &mov (&DWP(4,$W512,$Widx,8),"eax"); - &mov (&DWP(128+0,$W512,$Widx,8),"ebx"); # copy of W512[i] - &mov (&DWP(128+4,$W512,$Widx,8),"eax"); - -&align(8); -&set_label("_1st_loop"); # 0-15 - # flip input stream byte order... - &mov ("eax",&DWP(0+8,$data,$Widx,8)); - &mov ("ebx",&DWP(4+8,$data,$Widx,8)); - &bswap ("eax"); - &bswap ("ebx"); - &mov (&DWP(0+8,$W512,$Widx,8),"ebx"); # W512[i] - &mov (&DWP(4+8,$W512,$Widx,8),"eax"); - &mov (&DWP(128+0+8,$W512,$Widx,8),"ebx"); # copy of W512[i] - &mov (&DWP(128+4+8,$W512,$Widx,8),"eax"); -&set_label("_1st_looplet"); - &SHA2_ROUND($Widx,$Widx); &inc($Widx); - -&cmp ($Widx,15) -&jl (&label("_1st_loop")); -&je (&label("_1st_looplet")); # playing similar trick on 2nd loop - # does not improve performance... - - $Kidx = "ebx"; # start using %ebx as Kidx - &mov ($Kidx,$Widx); - -&align(8); -&set_label("_2nd_loop"); # 16-79 - &and($Widx,0xf); - - # 128-bit fragment! I update W512[i] and W512[i+1] in - # parallel:-) Note that I refer to W512[(i&0xf)+N] and not to - # W512[(i+N)&0xf]! This is exactly what I maintain the second - # copy of W512[16] for... - &movdqu ("xmm0",&QWP(8*1,$W512,$Widx,8)); # s0=W512[i+1] - &movdqa ("xmm2","xmm0"); # %xmm2 is sliding right - &movdqa ("xmm3","xmm0"); # %xmm3 is sliding left - &psrlq ("xmm2",1); - &psllq ("xmm3",56); - &movdqa ("xmm0","xmm2"); - &pxor ("xmm0","xmm3"); - &psrlq ("xmm2",6); - &psllq ("xmm3",7); - &pxor ("xmm0","xmm2"); - &pxor ("xmm0","xmm3"); - &psrlq ("xmm2",1); - &pxor ("xmm0","xmm2"); # s0 = sigma0_512(s0); - - &movdqa ("xmm1",&QWP(8*14,$W512,$Widx,8)); # s1=W512[i+14] - &movdqa ("xmm4","xmm1"); # %xmm4 is sliding right - &movdqa ("xmm5","xmm1"); # %xmm5 is sliding left - &psrlq ("xmm4",6); - &psllq ("xmm5",3); - &movdqa ("xmm1","xmm4"); - &pxor ("xmm1","xmm5"); - &psrlq ("xmm4",13); - &psllq ("xmm5",42); - &pxor ("xmm1","xmm4"); - &pxor ("xmm1","xmm5"); - &psrlq ("xmm4",42); - &pxor ("xmm1","xmm4"); # s1 = sigma1_512(s1); - - # + have to explictly load W512[i+9] as it's not 128-bit - # v aligned and paddq would throw an exception... - &movdqu ("xmm6",&QWP(8*9,$W512,$Widx,8)); - &paddq ("xmm0","xmm1"); # s0 += s1 - &paddq ("xmm0","xmm6"); # s0 += W512[i+9] - &paddq ("xmm0",&QWP(0,$W512,$Widx,8)); # s0 += W512[i] - - &movdqa (&QWP(0,$W512,$Widx,8),"xmm0"); # W512[i] = s0 - &movdqa (&QWP(16*8,$W512,$Widx,8),"xmm0"); # copy of W512[i] - - # as the above fragment was 128-bit, we "owe" 2 rounds... - &SHA2_ROUND($Kidx,$Widx); &inc($Kidx); &inc($Widx); - &SHA2_ROUND($Kidx,$Widx); &inc($Kidx); &inc($Widx); - -&cmp ($Kidx,80); -&jl (&label("_2nd_loop")); - - # update A-H state - &mov ($Widx,&DWP(8,"ebp")); # A-H state, 1st arg - &movq (&QWP($Aoff,$W512),$A); # write out a - &movq (&QWP($Eoff,$W512),$E); # write out e - &movdqu ("xmm0",&QWP(0,$Widx)); - &movdqu ("xmm1",&QWP(16,$Widx)); - &movdqu ("xmm2",&QWP(32,$Widx)); - &movdqu ("xmm3",&QWP(48,$Widx)); - &paddq ("xmm0",&QWP($Aoff,$W512)); # 128-bit additions... - &paddq ("xmm1",&QWP($Coff,$W512)); - &paddq ("xmm2",&QWP($Eoff,$W512)); - &paddq ("xmm3",&QWP($Goff,$W512)); - &movdqu (&QWP(0,$Widx),"xmm0"); - &movdqu (&QWP(16,$Widx),"xmm1"); - &movdqu (&QWP(32,$Widx),"xmm2"); - &movdqu (&QWP(48,$Widx),"xmm3"); - -&add ($data,16*8); # advance input data pointer -&dec (&DWP(16,"ebp")); # decrement 3rd arg -&jnz (&label("_chunk_loop")); - - # epilogue - &emms (); # required for at least ELF and Win32 ABIs - &mov ("edi",&DWP(-12,"ebp")); - &mov ("esi",&DWP(-8,"ebp")); - &mov ("ebx",&DWP(-4,"ebp")); - &leave (); -&ret (); - -&align(64); -&set_label("K512"); # Yes! I keep it in the code segment! - &data_word(0xd728ae22,0x428a2f98); # u64 - &data_word(0x23ef65cd,0x71374491); # u64 - &data_word(0xec4d3b2f,0xb5c0fbcf); # u64 - &data_word(0x8189dbbc,0xe9b5dba5); # u64 - &data_word(0xf348b538,0x3956c25b); # u64 - &data_word(0xb605d019,0x59f111f1); # u64 - &data_word(0xaf194f9b,0x923f82a4); # u64 - &data_word(0xda6d8118,0xab1c5ed5); # u64 - &data_word(0xa3030242,0xd807aa98); # u64 - &data_word(0x45706fbe,0x12835b01); # u64 - &data_word(0x4ee4b28c,0x243185be); # u64 - &data_word(0xd5ffb4e2,0x550c7dc3); # u64 - &data_word(0xf27b896f,0x72be5d74); # u64 - &data_word(0x3b1696b1,0x80deb1fe); # u64 - &data_word(0x25c71235,0x9bdc06a7); # u64 - &data_word(0xcf692694,0xc19bf174); # u64 - &data_word(0x9ef14ad2,0xe49b69c1); # u64 - &data_word(0x384f25e3,0xefbe4786); # u64 - &data_word(0x8b8cd5b5,0x0fc19dc6); # u64 - &data_word(0x77ac9c65,0x240ca1cc); # u64 - &data_word(0x592b0275,0x2de92c6f); # u64 - &data_word(0x6ea6e483,0x4a7484aa); # u64 - &data_word(0xbd41fbd4,0x5cb0a9dc); # u64 - &data_word(0x831153b5,0x76f988da); # u64 - &data_word(0xee66dfab,0x983e5152); # u64 - &data_word(0x2db43210,0xa831c66d); # u64 - &data_word(0x98fb213f,0xb00327c8); # u64 - &data_word(0xbeef0ee4,0xbf597fc7); # u64 - &data_word(0x3da88fc2,0xc6e00bf3); # u64 - &data_word(0x930aa725,0xd5a79147); # u64 - &data_word(0xe003826f,0x06ca6351); # u64 - &data_word(0x0a0e6e70,0x14292967); # u64 - &data_word(0x46d22ffc,0x27b70a85); # u64 - &data_word(0x5c26c926,0x2e1b2138); # u64 - &data_word(0x5ac42aed,0x4d2c6dfc); # u64 - &data_word(0x9d95b3df,0x53380d13); # u64 - &data_word(0x8baf63de,0x650a7354); # u64 - &data_word(0x3c77b2a8,0x766a0abb); # u64 - &data_word(0x47edaee6,0x81c2c92e); # u64 - &data_word(0x1482353b,0x92722c85); # u64 - &data_word(0x4cf10364,0xa2bfe8a1); # u64 - &data_word(0xbc423001,0xa81a664b); # u64 - &data_word(0xd0f89791,0xc24b8b70); # u64 - &data_word(0x0654be30,0xc76c51a3); # u64 - &data_word(0xd6ef5218,0xd192e819); # u64 - &data_word(0x5565a910,0xd6990624); # u64 - &data_word(0x5771202a,0xf40e3585); # u64 - &data_word(0x32bbd1b8,0x106aa070); # u64 - &data_word(0xb8d2d0c8,0x19a4c116); # u64 - &data_word(0x5141ab53,0x1e376c08); # u64 - &data_word(0xdf8eeb99,0x2748774c); # u64 - &data_word(0xe19b48a8,0x34b0bcb5); # u64 - &data_word(0xc5c95a63,0x391c0cb3); # u64 - &data_word(0xe3418acb,0x4ed8aa4a); # u64 - &data_word(0x7763e373,0x5b9cca4f); # u64 - &data_word(0xd6b2b8a3,0x682e6ff3); # u64 - &data_word(0x5defb2fc,0x748f82ee); # u64 - &data_word(0x43172f60,0x78a5636f); # u64 - &data_word(0xa1f0ab72,0x84c87814); # u64 - &data_word(0x1a6439ec,0x8cc70208); # u64 - &data_word(0x23631e28,0x90befffa); # u64 - &data_word(0xde82bde9,0xa4506ceb); # u64 - &data_word(0xb2c67915,0xbef9a3f7); # u64 - &data_word(0xe372532b,0xc67178f2); # u64 - &data_word(0xea26619c,0xca273ece); # u64 - &data_word(0x21c0c207,0xd186b8c7); # u64 - &data_word(0xcde0eb1e,0xeada7dd6); # u64 - &data_word(0xee6ed178,0xf57d4f7f); # u64 - &data_word(0x72176fba,0x06f067aa); # u64 - &data_word(0xa2c898a6,0x0a637dc5); # u64 - &data_word(0xbef90dae,0x113f9804); # u64 - &data_word(0x131c471b,0x1b710b35); # u64 - &data_word(0x23047d84,0x28db77f5); # u64 - &data_word(0x40c72493,0x32caab7b); # u64 - &data_word(0x15c9bebc,0x3c9ebe0a); # u64 - &data_word(0x9c100d4c,0x431d67c4); # u64 - &data_word(0xcb3e42b6,0x4cc5d4be); # u64 - &data_word(0xfc657e2a,0x597f299c); # u64 - &data_word(0x3ad6faec,0x5fcb6fab); # u64 - &data_word(0x4a475817,0x6c44198c); # u64 - -&function_end_B($func); - -&asm_finish(); diff --git a/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl index b6252d31eca2..f611a2d898e2 100755 --- a/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl @@ -40,14 +40,18 @@ # sha256_block:-( This is presumably because 64-bit shifts/rotates # apparently are not atomic instructions, but implemented in microcode. -$output=shift; +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or die "can't locate x86_64-xlate.pl"; -open STDOUT,"| $^X $xlate $output"; +open STDOUT,"| $^X $xlate $flavour $output"; if ($output =~ /512/) { $func="sha512_block_data_order"; @@ -91,50 +95,44 @@ sub ROUND_00_15() { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; $code.=<<___; - mov $e,$a0 - mov $e,$a1 + ror \$`$Sigma1[2]-$Sigma1[1]`,$a0 mov $f,$a2 + mov $T1,`$SZ*($i&0xf)`(%rsp) - ror \$$Sigma1[0],$a0 - ror \$$Sigma1[1],$a1 + ror \$`$Sigma0[2]-$Sigma0[1]`,$a1 + xor $e,$a0 xor $g,$a2 # f^g - xor $a1,$a0 - ror \$`$Sigma1[2]-$Sigma1[1]`,$a1 + ror \$`$Sigma1[1]-$Sigma1[0]`,$a0 + add $h,$T1 # T1+=h + xor $a,$a1 + + add ($Tbl,$round,$SZ),$T1 # T1+=K[round] and $e,$a2 # (f^g)&e - mov $T1,`$SZ*($i&0xf)`(%rsp) + mov $b,$h - xor $a1,$a0 # Sigma1(e) + ror \$`$Sigma0[1]-$Sigma0[0]`,$a1 + xor $e,$a0 xor $g,$a2 # Ch(e,f,g)=((f^g)&e)^g - add $h,$T1 # T1+=h - - mov $a,$h - add $a0,$T1 # T1+=Sigma1(e) + xor $c,$h # b^c + xor $a,$a1 add $a2,$T1 # T1+=Ch(e,f,g) - mov $a,$a0 - mov $a,$a1 + mov $b,$a2 - ror \$$Sigma0[0],$h - ror \$$Sigma0[1],$a0 - mov $a,$a2 - add ($Tbl,$round,$SZ),$T1 # T1+=K[round] + ror \$$Sigma1[0],$a0 # Sigma1(e) + and $a,$h # h=(b^c)&a + and $c,$a2 # b&c - xor $a0,$h - ror \$`$Sigma0[2]-$Sigma0[1]`,$a0 - or $c,$a1 # a|c + ror \$$Sigma0[0],$a1 # Sigma0(a) + add $a0,$T1 # T1+=Sigma1(e) + add $a2,$h # h+=b&c (completes +=Maj(a,b,c) - xor $a0,$h # h=Sigma0(a) - and $c,$a2 # a&c add $T1,$d # d+=T1 - - and $b,$a1 # (a|c)&b add $T1,$h # h+=T1 - - or $a2,$a1 # Maj(a,b,c)=((a|c)&b)|(a&c) lea 1($round),$round # round++ + add $a1,$h # h+=Sigma0(a) - add $a1,$h # h+=Maj(a,b,c) ___ } @@ -143,32 +141,30 @@ sub ROUND_16_XX() $code.=<<___; mov `$SZ*(($i+1)&0xf)`(%rsp),$a0 - mov `$SZ*(($i+14)&0xf)`(%rsp),$T1 - - mov $a0,$a2 + mov `$SZ*(($i+14)&0xf)`(%rsp),$a1 + mov $a0,$T1 + mov $a1,$a2 + ror \$`$sigma0[1]-$sigma0[0]`,$T1 + xor $a0,$T1 shr \$$sigma0[2],$a0 - ror \$$sigma0[0],$a2 - - xor $a2,$a0 - ror \$`$sigma0[1]-$sigma0[0]`,$a2 - - xor $a2,$a0 # sigma0(X[(i+1)&0xf]) - mov $T1,$a1 - shr \$$sigma1[2],$T1 - ror \$$sigma1[0],$a1 + ror \$$sigma0[0],$T1 + xor $T1,$a0 # sigma0(X[(i+1)&0xf]) + mov `$SZ*(($i+9)&0xf)`(%rsp),$T1 - xor $a1,$T1 - ror \$`$sigma1[1]-$sigma1[0]`,$a1 - - xor $a1,$T1 # sigma1(X[(i+14)&0xf]) + ror \$`$sigma1[1]-$sigma1[0]`,$a2 + xor $a1,$a2 + shr \$$sigma1[2],$a1 + ror \$$sigma1[0],$a2 add $a0,$T1 - - add `$SZ*(($i+9)&0xf)`(%rsp),$T1 + xor $a2,$a1 # sigma1(X[(i+14)&0xf]) add `$SZ*($i&0xf)`(%rsp),$T1 + mov $e,$a0 + add $a1,$T1 + mov $a,$a1 ___ &ROUND_00_15(@_); } @@ -186,7 +182,7 @@ $func: push %r13 push %r14 push %r15 - mov %rsp,%rbp # copy %rsp + mov %rsp,%r11 # copy %rsp shl \$4,%rdx # num*16 sub \$$framesz,%rsp lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ @@ -194,10 +190,10 @@ $func: mov $ctx,$_ctx # save ctx, 1st arg mov $inp,$_inp # save inp, 2nd arh mov %rdx,$_end # save end pointer, "3rd" arg - mov %rbp,$_rsp # save copy of %rsp + mov %r11,$_rsp # save copy of %rsp +.Lprologue: - .picmeup $Tbl - lea $TABLE-.($Tbl),$Tbl + lea $TABLE(%rip),$Tbl mov $SZ*0($ctx),$A mov $SZ*1($ctx),$B @@ -215,6 +211,8 @@ $func: ___ for($i=0;$i<16;$i++) { $code.=" mov $SZ*$i($inp),$T1\n"; + $code.=" mov @ROT[4],$a0\n"; + $code.=" mov @ROT[0],$a1\n"; $code.=" bswap $T1\n"; &ROUND_00_15($i,@ROT); unshift(@ROT,pop(@ROT)); @@ -257,14 +255,15 @@ $code.=<<___; mov $H,$SZ*7($ctx) jb .Lloop - mov $_rsp,%rsp - pop %r15 - pop %r14 - pop %r13 - pop %r12 - pop %rbp - pop %rbx - + mov $_rsp,%rsi + mov (%rsi),%r15 + mov 8(%rsi),%r14 + mov 16(%rsi),%r13 + mov 24(%rsi),%r12 + mov 32(%rsi),%rbp + mov 40(%rsi),%rbx + lea 48(%rsi),%rsp +.Lepilogue: ret .size $func,.-$func ___ @@ -339,6 +338,113 @@ $TABLE: ___ } +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + lea .Lprologue(%rip),%r10 + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lin_prologue + + mov 152($context),%rax # pull context->Rsp + + lea .Lepilogue(%rip),%r10 + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lin_prologue + + mov 16*$SZ+3*8(%rax),%rax # pull $_rsp + lea 48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R15 + +.Lin_prologue: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_$func + .rva .LSEH_end_$func + .rva .LSEH_info_$func + +.section .xdata +.align 8 +.LSEH_info_$func: + .byte 9,0,0,0 + .rva se_handler +___ +} + $code =~ s/\`([^\`]*)\`/eval $1/gem; print $code; close STDOUT; diff --git a/crypto/openssl/crypto/sha/sha.h b/crypto/openssl/crypto/sha/sha.h index 47a2c29f6627..8a6bf4bbbb1d 100644 --- a/crypto/openssl/crypto/sha/sha.h +++ b/crypto/openssl/crypto/sha/sha.h @@ -81,7 +81,7 @@ extern "C" { * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -#if defined(OPENSSL_SYS_WIN16) || defined(__LP32__) +#if defined(__LP32__) #define SHA_LONG unsigned long #elif defined(OPENSSL_SYS_CRAY) || defined(__ILP64__) #define SHA_LONG unsigned long @@ -116,6 +116,9 @@ unsigned char *SHA(const unsigned char *d, size_t n, unsigned char *md); void SHA_Transform(SHA_CTX *c, const unsigned char *data); #endif #ifndef OPENSSL_NO_SHA1 +#ifdef OPENSSL_FIPS +int private_SHA1_Init(SHA_CTX *c); +#endif int SHA1_Init(SHA_CTX *c); int SHA1_Update(SHA_CTX *c, const void *data, size_t len); int SHA1_Final(unsigned char *md, SHA_CTX *c); @@ -138,6 +141,10 @@ typedef struct SHA256state_st } SHA256_CTX; #ifndef OPENSSL_NO_SHA256 +#ifdef OPENSSL_FIPS +int private_SHA224_Init(SHA256_CTX *c); +int private_SHA256_Init(SHA256_CTX *c); +#endif int SHA224_Init(SHA256_CTX *c); int SHA224_Update(SHA256_CTX *c, const void *data, size_t len); int SHA224_Final(unsigned char *md, SHA256_CTX *c); @@ -185,6 +192,10 @@ typedef struct SHA512state_st #endif #ifndef OPENSSL_NO_SHA512 +#ifdef OPENSSL_FIPS +int private_SHA384_Init(SHA512_CTX *c); +int private_SHA512_Init(SHA512_CTX *c); +#endif int SHA384_Init(SHA512_CTX *c); int SHA384_Update(SHA512_CTX *c, const void *data, size_t len); int SHA384_Final(unsigned char *md, SHA512_CTX *c); diff --git a/crypto/openssl/crypto/sha/sha1_one.c b/crypto/openssl/crypto/sha/sha1_one.c index 4831174198e7..7c65b60276cd 100644 --- a/crypto/openssl/crypto/sha/sha1_one.c +++ b/crypto/openssl/crypto/sha/sha1_one.c @@ -61,7 +61,7 @@ #include <openssl/sha.h> #include <openssl/crypto.h> -#if !defined(OPENSSL_NO_SHA1) +#ifndef OPENSSL_NO_SHA1 unsigned char *SHA1(const unsigned char *d, size_t n, unsigned char *md) { SHA_CTX c; diff --git a/crypto/openssl/crypto/sha/sha1dgst.c b/crypto/openssl/crypto/sha/sha1dgst.c index d31f0781a06a..81219af08834 100644 --- a/crypto/openssl/crypto/sha/sha1dgst.c +++ b/crypto/openssl/crypto/sha/sha1dgst.c @@ -57,16 +57,13 @@ */ #include <openssl/opensslconf.h> +#include <openssl/crypto.h> #if !defined(OPENSSL_NO_SHA1) && !defined(OPENSSL_NO_SHA) #undef SHA_0 #define SHA_1 #include <openssl/opensslv.h> -#ifdef OPENSSL_FIPS -#include <openssl/fips.h> -#endif - const char SHA1_version[]="SHA1" OPENSSL_VERSION_PTEXT; diff --git a/crypto/openssl/crypto/sha/sha1s.cpp b/crypto/openssl/crypto/sha/sha1s.cpp deleted file mode 100644 index af23d1e0f218..000000000000 --- a/crypto/openssl/crypto/sha/sha1s.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// -// gettsc.inl -// -// gives access to the Pentium's (secret) cycle counter -// -// This software was written by Leonard Janke (janke@unixg.ubc.ca) -// in 1996-7 and is entered, by him, into the public domain. - -#if defined(__WATCOMC__) -void GetTSC(unsigned long&); -#pragma aux GetTSC = 0x0f 0x31 "mov [edi], eax" parm [edi] modify [edx eax]; -#elif defined(__GNUC__) -inline -void GetTSC(unsigned long& tsc) -{ - asm volatile(".byte 15, 49\n\t" - : "=eax" (tsc) - : - : "%edx", "%eax"); -} -#elif defined(_MSC_VER) -inline -void GetTSC(unsigned long& tsc) -{ - unsigned long a; - __asm _emit 0fh - __asm _emit 31h - __asm mov a, eax; - tsc=a; -} -#endif - -#include <stdio.h> -#include <stdlib.h> -#include <openssl/sha.h> - -#define sha1_block_x86 sha1_block_asm_data_order -extern "C" { -void sha1_block_x86(SHA_CTX *ctx, unsigned char *buffer,int num); -} - -void main(int argc,char *argv[]) - { - unsigned char buffer[64*256]; - SHA_CTX ctx; - unsigned long s1,s2,e1,e2; - unsigned char k[16]; - unsigned long data[2]; - unsigned char iv[8]; - int i,num=0,numm; - int j=0; - - if (argc >= 2) - num=atoi(argv[1]); - - if (num == 0) num=16; - if (num > 250) num=16; - numm=num+2; -#if 0 - num*=64; - numm*=64; -#endif - - for (j=0; j<6; j++) - { - for (i=0; i<10; i++) /**/ - { - sha1_block_x86(&ctx,buffer,numm); - GetTSC(s1); - sha1_block_x86(&ctx,buffer,numm); - GetTSC(e1); - GetTSC(s2); - sha1_block_x86(&ctx,buffer,num); - GetTSC(e2); - sha1_block_x86(&ctx,buffer,num); - } - - printf("sha1 (%d bytes) %d %d (%.2f)\n",num*64, - e1-s1,e2-s2,(double)((e1-s1)-(e2-s2))/2); - } - } - diff --git a/crypto/openssl/crypto/sha/sha256.c b/crypto/openssl/crypto/sha/sha256.c index 3256a83e98e1..f88d3d6dadba 100644 --- a/crypto/openssl/crypto/sha/sha256.c +++ b/crypto/openssl/crypto/sha/sha256.c @@ -12,39 +12,29 @@ #include <openssl/crypto.h> #include <openssl/sha.h> -#ifdef OPENSSL_FIPS -#include <openssl/fips.h> -#endif - #include <openssl/opensslv.h> const char SHA256_version[]="SHA-256" OPENSSL_VERSION_PTEXT; -int SHA224_Init (SHA256_CTX *c) +fips_md_init_ctx(SHA224, SHA256) { -#ifdef OPENSSL_FIPS - FIPS_selftest_check(); -#endif + memset (c,0,sizeof(*c)); c->h[0]=0xc1059ed8UL; c->h[1]=0x367cd507UL; c->h[2]=0x3070dd17UL; c->h[3]=0xf70e5939UL; c->h[4]=0xffc00b31UL; c->h[5]=0x68581511UL; c->h[6]=0x64f98fa7UL; c->h[7]=0xbefa4fa4UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA224_DIGEST_LENGTH; + c->md_len=SHA224_DIGEST_LENGTH; return 1; } -int SHA256_Init (SHA256_CTX *c) +fips_md_init(SHA256) { -#ifdef OPENSSL_FIPS - FIPS_selftest_check(); -#endif + memset (c,0,sizeof(*c)); c->h[0]=0x6a09e667UL; c->h[1]=0xbb67ae85UL; c->h[2]=0x3c6ef372UL; c->h[3]=0xa54ff53aUL; c->h[4]=0x510e527fUL; c->h[5]=0x9b05688cUL; c->h[6]=0x1f83d9abUL; c->h[7]=0x5be0cd19UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA256_DIGEST_LENGTH; + c->md_len=SHA256_DIGEST_LENGTH; return 1; } @@ -94,21 +84,21 @@ int SHA224_Final (unsigned char *md, SHA256_CTX *c) */ #define HASH_MAKE_STRING(c,s) do { \ unsigned long ll; \ - unsigned int xn; \ + unsigned int nn; \ switch ((c)->md_len) \ { case SHA224_DIGEST_LENGTH: \ - for (xn=0;xn<SHA224_DIGEST_LENGTH/4;xn++) \ - { ll=(c)->h[xn]; HOST_l2c(ll,(s)); } \ + for (nn=0;nn<SHA224_DIGEST_LENGTH/4;nn++) \ + { ll=(c)->h[nn]; HOST_l2c(ll,(s)); } \ break; \ case SHA256_DIGEST_LENGTH: \ - for (xn=0;xn<SHA256_DIGEST_LENGTH/4;xn++) \ - { ll=(c)->h[xn]; HOST_l2c(ll,(s)); } \ + for (nn=0;nn<SHA256_DIGEST_LENGTH/4;nn++) \ + { ll=(c)->h[nn]; HOST_l2c(ll,(s)); } \ break; \ default: \ if ((c)->md_len > SHA256_DIGEST_LENGTH) \ return 0; \ - for (xn=0;xn<(c)->md_len/4;xn++) \ - { ll=(c)->h[xn]; HOST_l2c(ll,(s)); } \ + for (nn=0;nn<(c)->md_len/4;nn++) \ + { ll=(c)->h[nn]; HOST_l2c(ll,(s)); } \ break; \ } \ } while (0) diff --git a/crypto/openssl/crypto/sha/sha512.c b/crypto/openssl/crypto/sha/sha512.c index 9e91bcad0425..50dd7dc74437 100644 --- a/crypto/openssl/crypto/sha/sha512.c +++ b/crypto/openssl/crypto/sha/sha512.c @@ -5,10 +5,6 @@ * ==================================================================== */ #include <openssl/opensslconf.h> -#ifdef OPENSSL_FIPS -#include <openssl/fips.h> -#endif - #if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA512) /* * IMPLEMENTATION NOTES. @@ -63,11 +59,8 @@ const char SHA512_version[]="SHA-512" OPENSSL_VERSION_PTEXT; #define SHA512_BLOCK_CAN_MANAGE_UNALIGNED_DATA #endif -int SHA384_Init (SHA512_CTX *c) +fips_md_init_ctx(SHA384, SHA512) { -#ifdef OPENSSL_FIPS - FIPS_selftest_check(); -#endif c->h[0]=U64(0xcbbb9d5dc1059ed8); c->h[1]=U64(0x629a292a367cd507); c->h[2]=U64(0x9159015a3070dd17); @@ -76,16 +69,14 @@ int SHA384_Init (SHA512_CTX *c) c->h[5]=U64(0x8eb44a8768581511); c->h[6]=U64(0xdb0c2e0d64f98fa7); c->h[7]=U64(0x47b5481dbefa4fa4); + c->Nl=0; c->Nh=0; c->num=0; c->md_len=SHA384_DIGEST_LENGTH; return 1; } -int SHA512_Init (SHA512_CTX *c) +fips_md_init(SHA512) { -#ifdef OPENSSL_FIPS - FIPS_selftest_check(); -#endif c->h[0]=U64(0x6a09e667f3bcc908); c->h[1]=U64(0xbb67ae8584caa73b); c->h[2]=U64(0x3c6ef372fe94f82b); @@ -94,6 +85,7 @@ int SHA512_Init (SHA512_CTX *c) c->h[5]=U64(0x9b05688c2b3e6c1f); c->h[6]=U64(0x1f83d9abfb41bd6b); c->h[7]=U64(0x5be0cd19137e2179); + c->Nl=0; c->Nh=0; c->num=0; c->md_len=SHA512_DIGEST_LENGTH; return 1; @@ -204,7 +196,7 @@ int SHA512_Update (SHA512_CTX *c, const void *_data, size_t len) if (len < n) { - memcpy (p+c->num,data,len), c->num += len; + memcpy (p+c->num,data,len), c->num += (unsigned int)len; return 1; } else { @@ -314,7 +306,7 @@ static const SHA_LONG64 K512[80] = { #ifndef PEDANTIC # if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) # if defined(__x86_64) || defined(__x86_64__) -# define ROTR(a,n) ({ unsigned long ret; \ +# define ROTR(a,n) ({ SHA_LONG64 ret; \ asm ("rorq %1,%0" \ : "=r"(ret) \ : "J"(n),"0"(a) \ @@ -337,20 +329,21 @@ static const SHA_LONG64 K512[80] = { ((SHA_LONG64)hi)<<32|lo; }) # else # define PULL64(x) ({ const unsigned int *p=(const unsigned int *)(&(x));\ - unsigned int hi=p[0],lo=p[1]; \ + unsigned int hi=p[0],lo=p[1]; \ asm ("bswapl %0; bswapl %1;" \ : "=r"(lo),"=r"(hi) \ : "0"(lo),"1"(hi)); \ ((SHA_LONG64)hi)<<32|lo; }) # endif # elif (defined(_ARCH_PPC) && defined(__64BIT__)) || defined(_ARCH_PPC64) -# define ROTR(a,n) ({ unsigned long ret; \ +# define ROTR(a,n) ({ SHA_LONG64 ret; \ asm ("rotrdi %0,%1,%2" \ : "=r"(ret) \ : "r"(a),"K"(n)); ret; }) # endif # elif defined(_MSC_VER) # if defined(_WIN64) /* applies to both IA-64 and AMD64 */ +# pragma intrinsic(_rotr64) # define ROTR(a,n) _rotr64((a),n) # endif # if defined(_M_IX86) && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) @@ -398,15 +391,66 @@ static const SHA_LONG64 K512[80] = { #define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) #define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) -#if defined(OPENSSL_IA32_SSE2) && !defined(OPENSSL_NO_ASM) && !defined(I386_ONLY) -#define GO_FOR_SSE2(ctx,in,num) do { \ - void sha512_block_sse2(void *,const void *,size_t); \ - if (!(OPENSSL_ia32cap_P & (1<<26))) break; \ - sha512_block_sse2(ctx->h,in,num); return; \ - } while (0) + +#if defined(__i386) || defined(__i386__) || defined(_M_IX86) +/* + * This code should give better results on 32-bit CPU with less than + * ~24 registers, both size and performance wise... + */ +static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num) + { + const SHA_LONG64 *W=in; + SHA_LONG64 A,E,T; + SHA_LONG64 X[9+80],*F; + int i; + + while (num--) { + + F = X+80; + A = ctx->h[0]; F[1] = ctx->h[1]; + F[2] = ctx->h[2]; F[3] = ctx->h[3]; + E = ctx->h[4]; F[5] = ctx->h[5]; + F[6] = ctx->h[6]; F[7] = ctx->h[7]; + + for (i=0;i<16;i++,F--) + { +#ifdef B_ENDIAN + T = W[i]; +#else + T = PULL64(W[i]); #endif + F[0] = A; + F[4] = E; + F[8] = T; + T += F[7] + Sigma1(E) + Ch(E,F[5],F[6]) + K512[i]; + E = F[3] + T; + A = T + Sigma0(A) + Maj(A,F[1],F[2]); + } + + for (;i<80;i++,F--) + { + T = sigma0(F[8+16-1]); + T += sigma1(F[8+16-14]); + T += F[8+16] + F[8+16-9]; + + F[0] = A; + F[4] = E; + F[8] = T; + T += F[7] + Sigma1(E) + Ch(E,F[5],F[6]) + K512[i]; + E = F[3] + T; + A = T + Sigma0(A) + Maj(A,F[1],F[2]); + } + + ctx->h[0] += A; ctx->h[1] += F[1]; + ctx->h[2] += F[2]; ctx->h[3] += F[3]; + ctx->h[4] += E; ctx->h[5] += F[5]; + ctx->h[6] += F[6]; ctx->h[7] += F[7]; + + W+=SHA_LBLOCK; + } + } -#ifdef OPENSSL_SMALL_FOOTPRINT +#elif defined(OPENSSL_SMALL_FOOTPRINT) static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num) { @@ -415,10 +459,6 @@ static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num SHA_LONG64 X[16]; int i; -#ifdef GO_FOR_SSE2 - GO_FOR_SSE2(ctx,in,num); -#endif - while (num--) { a = ctx->h[0]; b = ctx->h[1]; c = ctx->h[2]; d = ctx->h[3]; @@ -463,11 +503,11 @@ static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num h = Sigma0(a) + Maj(a,b,c); \ d += T1; h += T1; } while (0) -#define ROUND_16_80(i,a,b,c,d,e,f,g,h,X) do { \ - s0 = X[(i+1)&0x0f]; s0 = sigma0(s0); \ - s1 = X[(i+14)&0x0f]; s1 = sigma1(s1); \ - T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f]; \ - ROUND_00_15(i,a,b,c,d,e,f,g,h); } while (0) +#define ROUND_16_80(i,j,a,b,c,d,e,f,g,h,X) do { \ + s0 = X[(j+1)&0x0f]; s0 = sigma0(s0); \ + s1 = X[(j+14)&0x0f]; s1 = sigma1(s1); \ + T1 = X[(j)&0x0f] += s0 + s1 + X[(j+9)&0x0f]; \ + ROUND_00_15(i+j,a,b,c,d,e,f,g,h); } while (0) static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num) { @@ -476,10 +516,6 @@ static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num SHA_LONG64 X[16]; int i; -#ifdef GO_FOR_SSE2 - GO_FOR_SSE2(ctx,in,num); -#endif - while (num--) { a = ctx->h[0]; b = ctx->h[1]; c = ctx->h[2]; d = ctx->h[3]; @@ -521,16 +557,24 @@ static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num T1 = X[15] = PULL64(W[15]); ROUND_00_15(15,b,c,d,e,f,g,h,a); #endif - for (i=16;i<80;i+=8) + for (i=16;i<80;i+=16) { - ROUND_16_80(i+0,a,b,c,d,e,f,g,h,X); - ROUND_16_80(i+1,h,a,b,c,d,e,f,g,X); - ROUND_16_80(i+2,g,h,a,b,c,d,e,f,X); - ROUND_16_80(i+3,f,g,h,a,b,c,d,e,X); - ROUND_16_80(i+4,e,f,g,h,a,b,c,d,X); - ROUND_16_80(i+5,d,e,f,g,h,a,b,c,X); - ROUND_16_80(i+6,c,d,e,f,g,h,a,b,X); - ROUND_16_80(i+7,b,c,d,e,f,g,h,a,X); + ROUND_16_80(i, 0,a,b,c,d,e,f,g,h,X); + ROUND_16_80(i, 1,h,a,b,c,d,e,f,g,X); + ROUND_16_80(i, 2,g,h,a,b,c,d,e,f,X); + ROUND_16_80(i, 3,f,g,h,a,b,c,d,e,X); + ROUND_16_80(i, 4,e,f,g,h,a,b,c,d,X); + ROUND_16_80(i, 5,d,e,f,g,h,a,b,c,X); + ROUND_16_80(i, 6,c,d,e,f,g,h,a,b,X); + ROUND_16_80(i, 7,b,c,d,e,f,g,h,a,X); + ROUND_16_80(i, 8,a,b,c,d,e,f,g,h,X); + ROUND_16_80(i, 9,h,a,b,c,d,e,f,g,X); + ROUND_16_80(i,10,g,h,a,b,c,d,e,f,X); + ROUND_16_80(i,11,f,g,h,a,b,c,d,e,X); + ROUND_16_80(i,12,e,f,g,h,a,b,c,d,X); + ROUND_16_80(i,13,d,e,f,g,h,a,b,c,X); + ROUND_16_80(i,14,c,d,e,f,g,h,a,b,X); + ROUND_16_80(i,15,b,c,d,e,f,g,h,a,X); } ctx->h[0] += a; ctx->h[1] += b; ctx->h[2] += c; ctx->h[3] += d; @@ -544,13 +588,10 @@ static void sha512_block_data_order (SHA512_CTX *ctx, const void *in, size_t num #endif /* SHA512_ASM */ -#else /* OPENSSL_NO_SHA512 */ - -/* Sensitive compilers ("Compaq C V6.4-005 on OpenVMS VAX V7.3", for - * example) dislike a statement-free file, complaining: - * "%CC-W-EMPTYFILE, Source file does not contain any declarations." - */ +#else /* !OPENSSL_NO_SHA512 */ -int sha512_dummy(); +#if defined(PEDANTIC) || defined(__DECC) || defined(OPENSSL_SYS_MACOSX) +static void *dummy=&dummy; +#endif -#endif /* OPENSSL_NO_SHA512 */ +#endif /* !OPENSSL_NO_SHA512 */ diff --git a/crypto/openssl/crypto/sha/sha_dgst.c b/crypto/openssl/crypto/sha/sha_dgst.c index 598f4d721af9..c946ad827d58 100644 --- a/crypto/openssl/crypto/sha/sha_dgst.c +++ b/crypto/openssl/crypto/sha/sha_dgst.c @@ -58,11 +58,6 @@ #include <openssl/opensslconf.h> #include <openssl/crypto.h> -#ifdef OPENSSL_FIPS -#include <openssl/fips.h> -#endif - -#include <openssl/err.h> #if !defined(OPENSSL_NO_SHA0) && !defined(OPENSSL_NO_SHA) #undef SHA_1 diff --git a/crypto/openssl/crypto/sha/sha_locl.h b/crypto/openssl/crypto/sha/sha_locl.h index da46ddfe7943..7a0c3ca8d8ce 100644 --- a/crypto/openssl/crypto/sha/sha_locl.h +++ b/crypto/openssl/crypto/sha/sha_locl.h @@ -122,23 +122,18 @@ void sha1_block_data_order (SHA_CTX *c, const void *p,size_t num); #define INIT_DATA_h3 0x10325476UL #define INIT_DATA_h4 0xc3d2e1f0UL -#if defined(SHA_0) && defined(OPENSSL_FIPS) -FIPS_NON_FIPS_MD_Init(SHA) +#ifdef SHA_0 +fips_md_init(SHA) #else -int HASH_INIT (SHA_CTX *c) +fips_md_init_ctx(SHA1, SHA) #endif { -#if defined(SHA_1) && defined(OPENSSL_FIPS) - FIPS_selftest_check(); -#endif + memset (c,0,sizeof(*c)); c->h0=INIT_DATA_h0; c->h1=INIT_DATA_h1; c->h2=INIT_DATA_h2; c->h3=INIT_DATA_h3; c->h4=INIT_DATA_h4; - c->Nl=0; - c->Nh=0; - c->num=0; return 1; } diff --git a/crypto/openssl/crypto/sha/shatest.c b/crypto/openssl/crypto/sha/shatest.c index ed0fe06a7bee..27614646d154 100644 --- a/crypto/openssl/crypto/sha/shatest.c +++ b/crypto/openssl/crypto/sha/shatest.c @@ -123,9 +123,9 @@ int main(int argc, char *argv[]) i=1; while (*P != NULL) { - EVP_Digest(*P,strlen((char *)*P),md,NULL,EVP_sha(), NULL); + EVP_Digest(*P,strlen(*P),md,NULL,EVP_sha(), NULL); p=pt(md); - if (strcmp(p,(char *)*R) != 0) + if (strcmp(p,*R) != 0) { printf("error calculating SHA on '%s'\n",*P); printf("got %s instead of %s\n",p,*R); |