AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-12-28 11:18:45 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity
665e7c73a7
linux_dsm_epyc7002/arch/arm/crypto/poly1305-armv4.pl

1237 lines
29 KiB
Perl
Raw Normal View History

crypto: arm/poly1305 - incorporate OpenSSL/CRYPTOGAMS NEON implementation This is a straight import of the OpenSSL/CRYPTOGAMS Poly1305 implementation for NEON authored by Andy Polyakov, and contributed by him to the OpenSSL project. The file 'poly1305-armv4.pl' is taken straight from this upstream GitHub repository [0] at commit ec55a08dc0244ce570c4fc7cade330c60798952f, and already contains all the changes required to build it as part of a Linux kernel module. [0] https://github.com/dot-asm/cryptogams Co-developed-by: Andy Polyakov <appro@cryptogams.org> Signed-off-by: Andy Polyakov <appro@cryptogams.org> Signed-off-by: Ard Biesheuvel <ardb@kernel.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2019-11-08 19:22:25 +07:00
#!/usr/bin/env perl
# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause
#
# ====================================================================
# Written by Andy Polyakov, @dot-asm, initially for the OpenSSL
# project.
# ====================================================================
#
# IALU(*)/gcc-4.4 NEON
#
# ARM11xx(ARMv6) 7.78/+100% -
# Cortex-A5 6.35/+130% 3.00
# Cortex-A8 6.25/+115% 2.36
# Cortex-A9 5.10/+95% 2.55
# Cortex-A15 3.85/+85% 1.25(**)
# Snapdragon S4 5.70/+100% 1.48(**)
#
# (*) this is for -march=armv6, i.e. with bunch of ldrb loading data;
# (**) these are trade-off results, they can be improved by ~8% but at
# the cost of 15/12% regression on Cortex-A5/A7, it's even possible
# to improve Cortex-A9 result, but then A5/A7 loose more than 20%;
$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
if ($flavour && $flavour ne "void") {
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
open STDOUT,">$output";
}
($ctx,$inp,$len,$padbit)=map("r$_",(0..3));
$code.=<<___;
#ifndef __KERNEL__
# include "arm_arch.h"
#else
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
# define poly1305_init poly1305_init_arm
# define poly1305_blocks poly1305_blocks_arm
# define poly1305_emit poly1305_emit_arm
.globl poly1305_blocks_neon
#endif
#if defined(__thumb2__)
.syntax unified
.thumb
#else
.code 32
#endif
.text
.globl poly1305_emit
.globl poly1305_blocks
.globl poly1305_init
.type poly1305_init,%function
.align 5
poly1305_init:
.Lpoly1305_init:
stmdb sp!,{r4-r11}
eor r3,r3,r3
cmp $inp,#0
str r3,[$ctx,#0] @ zero hash value
str r3,[$ctx,#4]
str r3,[$ctx,#8]
str r3,[$ctx,#12]
str r3,[$ctx,#16]
str r3,[$ctx,#36] @ clear is_base2_26
add $ctx,$ctx,#20
#ifdef __thumb2__
it eq
#endif
moveq r0,#0
beq .Lno_key
#if __ARM_MAX_ARCH__>=7
mov r3,#-1
str r3,[$ctx,#28] @ impossible key power value
# ifndef __KERNEL__
adr r11,.Lpoly1305_init
ldr r12,.LOPENSSL_armcap
# endif
#endif
ldrb r4,[$inp,#0]
mov r10,#0x0fffffff
ldrb r5,[$inp,#1]
and r3,r10,#-4 @ 0x0ffffffc
ldrb r6,[$inp,#2]
ldrb r7,[$inp,#3]
orr r4,r4,r5,lsl#8
ldrb r5,[$inp,#4]
orr r4,r4,r6,lsl#16
ldrb r6,[$inp,#5]
orr r4,r4,r7,lsl#24
ldrb r7,[$inp,#6]
and r4,r4,r10
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
# if !defined(_WIN32)
ldr r12,[r11,r12] @ OPENSSL_armcap_P
# endif
# if defined(__APPLE__) || defined(_WIN32)
ldr r12,[r12]
# endif
#endif
ldrb r8,[$inp,#7]
orr r5,r5,r6,lsl#8
ldrb r6,[$inp,#8]
orr r5,r5,r7,lsl#16
ldrb r7,[$inp,#9]
orr r5,r5,r8,lsl#24
ldrb r8,[$inp,#10]
and r5,r5,r3
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
tst r12,#ARMV7_NEON @ check for NEON
# ifdef __thumb2__
adr r9,.Lpoly1305_blocks_neon
adr r11,.Lpoly1305_blocks
it ne
movne r11,r9
adr r12,.Lpoly1305_emit
orr r11,r11,#1 @ thumb-ify addresses
orr r12,r12,#1
# else
add r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init)
ite eq
addeq r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init)
addne r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init)
# endif
#endif
ldrb r9,[$inp,#11]
orr r6,r6,r7,lsl#8
ldrb r7,[$inp,#12]
orr r6,r6,r8,lsl#16
ldrb r8,[$inp,#13]
orr r6,r6,r9,lsl#24
ldrb r9,[$inp,#14]
and r6,r6,r3
ldrb r10,[$inp,#15]
orr r7,r7,r8,lsl#8
str r4,[$ctx,#0]
orr r7,r7,r9,lsl#16
str r5,[$ctx,#4]
orr r7,r7,r10,lsl#24
str r6,[$ctx,#8]
and r7,r7,r3
str r7,[$ctx,#12]
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
stmia r2,{r11,r12} @ fill functions table
mov r0,#1
#else
mov r0,#0
#endif
.Lno_key:
ldmia sp!,{r4-r11}
#if __ARM_ARCH__>=5
ret @ bx lr
#else
tst lr,#1
moveq pc,lr @ be binary compatible with V4, yet
bx lr @ interoperable with Thumb ISA:-)
#endif
.size poly1305_init,.-poly1305_init
___
{
my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
my ($s1,$s2,$s3)=($r1,$r2,$r3);
$code.=<<___;
.type poly1305_blocks,%function
.align 5
poly1305_blocks:
.Lpoly1305_blocks:
stmdb sp!,{r3-r11,lr}
ands $len,$len,#-16
beq .Lno_data
add $len,$len,$inp @ end pointer
sub sp,sp,#32
#if __ARM_ARCH__<7
ldmia $ctx,{$h0-$r3} @ load context
add $ctx,$ctx,#20
str $len,[sp,#16] @ offload stuff
str $ctx,[sp,#12]
#else
ldr lr,[$ctx,#36] @ is_base2_26
ldmia $ctx!,{$h0-$h4} @ load hash value
str $len,[sp,#16] @ offload stuff
str $ctx,[sp,#12]
adds $r0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32
mov $r1,$h1,lsr#6
adcs $r1,$r1,$h2,lsl#20
mov $r2,$h2,lsr#12
adcs $r2,$r2,$h3,lsl#14
mov $r3,$h3,lsr#18
adcs $r3,$r3,$h4,lsl#8
mov $len,#0
teq lr,#0
str $len,[$ctx,#16] @ clear is_base2_26
adc $len,$len,$h4,lsr#24
itttt ne
movne $h0,$r0 @ choose between radixes
movne $h1,$r1
movne $h2,$r2
movne $h3,$r3
ldmia $ctx,{$r0-$r3} @ load key
it ne
movne $h4,$len
#endif
mov lr,$inp
cmp $padbit,#0
str $r1,[sp,#20]
str $r2,[sp,#24]
str $r3,[sp,#28]
b .Loop
.align 4
.Loop:
#if __ARM_ARCH__<7
ldrb r0,[lr],#16 @ load input
# ifdef __thumb2__
it hi
# endif
addhi $h4,$h4,#1 @ 1<<128
ldrb r1,[lr,#-15]
ldrb r2,[lr,#-14]
ldrb r3,[lr,#-13]
orr r1,r0,r1,lsl#8
ldrb r0,[lr,#-12]
orr r2,r1,r2,lsl#16
ldrb r1,[lr,#-11]
orr r3,r2,r3,lsl#24
ldrb r2,[lr,#-10]
adds $h0,$h0,r3 @ accumulate input
ldrb r3,[lr,#-9]
orr r1,r0,r1,lsl#8
ldrb r0,[lr,#-8]
orr r2,r1,r2,lsl#16
ldrb r1,[lr,#-7]
orr r3,r2,r3,lsl#24
ldrb r2,[lr,#-6]
adcs $h1,$h1,r3
ldrb r3,[lr,#-5]
orr r1,r0,r1,lsl#8
ldrb r0,[lr,#-4]
orr r2,r1,r2,lsl#16
ldrb r1,[lr,#-3]
orr r3,r2,r3,lsl#24
ldrb r2,[lr,#-2]
adcs $h2,$h2,r3
ldrb r3,[lr,#-1]
orr r1,r0,r1,lsl#8
str lr,[sp,#8] @ offload input pointer
orr r2,r1,r2,lsl#16
add $s1,$r1,$r1,lsr#2
orr r3,r2,r3,lsl#24
#else
ldr r0,[lr],#16 @ load input
it hi
addhi $h4,$h4,#1 @ padbit
ldr r1,[lr,#-12]
ldr r2,[lr,#-8]
ldr r3,[lr,#-4]
# ifdef __ARMEB__
rev r0,r0
rev r1,r1
rev r2,r2
rev r3,r3
# endif
adds $h0,$h0,r0 @ accumulate input
str lr,[sp,#8] @ offload input pointer
adcs $h1,$h1,r1
add $s1,$r1,$r1,lsr#2
adcs $h2,$h2,r2
#endif
add $s2,$r2,$r2,lsr#2
adcs $h3,$h3,r3
add $s3,$r3,$r3,lsr#2
umull r2,r3,$h1,$r0
adc $h4,$h4,#0
umull r0,r1,$h0,$r0
umlal r2,r3,$h4,$s1
umlal r0,r1,$h3,$s1
ldr $r1,[sp,#20] @ reload $r1
umlal r2,r3,$h2,$s3
umlal r0,r1,$h1,$s3
umlal r2,r3,$h3,$s2
umlal r0,r1,$h2,$s2
umlal r2,r3,$h0,$r1
str r0,[sp,#0] @ future $h0
mul r0,$s2,$h4
ldr $r2,[sp,#24] @ reload $r2
adds r2,r2,r1 @ d1+=d0>>32
eor r1,r1,r1
adc lr,r3,#0 @ future $h2
str r2,[sp,#4] @ future $h1
mul r2,$s3,$h4
eor r3,r3,r3
umlal r0,r1,$h3,$s3
ldr $r3,[sp,#28] @ reload $r3
umlal r2,r3,$h3,$r0
umlal r0,r1,$h2,$r0
umlal r2,r3,$h2,$r1
umlal r0,r1,$h1,$r1
umlal r2,r3,$h1,$r2
umlal r0,r1,$h0,$r2
umlal r2,r3,$h0,$r3
ldr $h0,[sp,#0]
mul $h4,$r0,$h4
ldr $h1,[sp,#4]
adds $h2,lr,r0 @ d2+=d1>>32
ldr lr,[sp,#8] @ reload input pointer
adc r1,r1,#0
adds $h3,r2,r1 @ d3+=d2>>32
ldr r0,[sp,#16] @ reload end pointer
adc r3,r3,#0
add $h4,$h4,r3 @ h4+=d3>>32
and r1,$h4,#-4
and $h4,$h4,#3
add r1,r1,r1,lsr#2 @ *=5
adds $h0,$h0,r1
adcs $h1,$h1,#0
adcs $h2,$h2,#0
adcs $h3,$h3,#0
adc $h4,$h4,#0
cmp r0,lr @ done yet?
bhi .Loop
ldr $ctx,[sp,#12]
add sp,sp,#32
stmdb $ctx,{$h0-$h4} @ store the result
.Lno_data:
#if __ARM_ARCH__>=5
ldmia sp!,{r3-r11,pc}
#else
ldmia sp!,{r3-r11,lr}
tst lr,#1
moveq pc,lr @ be binary compatible with V4, yet
bx lr @ interoperable with Thumb ISA:-)
#endif
.size poly1305_blocks,.-poly1305_blocks
___
}
{
my ($ctx,$mac,$nonce)=map("r$_",(0..2));
my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
my $g4=$ctx;
$code.=<<___;
.type poly1305_emit,%function
.align 5
poly1305_emit:
.Lpoly1305_emit:
stmdb sp!,{r4-r11}
ldmia $ctx,{$h0-$h4}
#if __ARM_ARCH__>=7
ldr ip,[$ctx,#36] @ is_base2_26
adds $g0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32
mov $g1,$h1,lsr#6
adcs $g1,$g1,$h2,lsl#20
mov $g2,$h2,lsr#12
adcs $g2,$g2,$h3,lsl#14
mov $g3,$h3,lsr#18
adcs $g3,$g3,$h4,lsl#8
mov $g4,#0
adc $g4,$g4,$h4,lsr#24
tst ip,ip
itttt ne
movne $h0,$g0
movne $h1,$g1
movne $h2,$g2
movne $h3,$g3
it ne
movne $h4,$g4
#endif
adds $g0,$h0,#5 @ compare to modulus
adcs $g1,$h1,#0
adcs $g2,$h2,#0
adcs $g3,$h3,#0
adc $g4,$h4,#0
tst $g4,#4 @ did it carry/borrow?
#ifdef __thumb2__
it ne
#endif
movne $h0,$g0
ldr $g0,[$nonce,#0]
#ifdef __thumb2__
it ne
#endif
movne $h1,$g1
ldr $g1,[$nonce,#4]
#ifdef __thumb2__
it ne
#endif
movne $h2,$g2
ldr $g2,[$nonce,#8]
#ifdef __thumb2__
it ne
#endif
movne $h3,$g3
ldr $g3,[$nonce,#12]
adds $h0,$h0,$g0
adcs $h1,$h1,$g1
adcs $h2,$h2,$g2
adc $h3,$h3,$g3
#if __ARM_ARCH__>=7
# ifdef __ARMEB__
rev $h0,$h0
rev $h1,$h1
rev $h2,$h2
rev $h3,$h3
# endif
str $h0,[$mac,#0]
str $h1,[$mac,#4]
str $h2,[$mac,#8]
str $h3,[$mac,#12]
#else
strb $h0,[$mac,#0]
mov $h0,$h0,lsr#8
strb $h1,[$mac,#4]
mov $h1,$h1,lsr#8
strb $h2,[$mac,#8]
mov $h2,$h2,lsr#8
strb $h3,[$mac,#12]
mov $h3,$h3,lsr#8
strb $h0,[$mac,#1]
mov $h0,$h0,lsr#8
strb $h1,[$mac,#5]
mov $h1,$h1,lsr#8
strb $h2,[$mac,#9]
mov $h2,$h2,lsr#8
strb $h3,[$mac,#13]
mov $h3,$h3,lsr#8
strb $h0,[$mac,#2]
mov $h0,$h0,lsr#8
strb $h1,[$mac,#6]
mov $h1,$h1,lsr#8
strb $h2,[$mac,#10]
mov $h2,$h2,lsr#8
strb $h3,[$mac,#14]
mov $h3,$h3,lsr#8
strb $h0,[$mac,#3]
strb $h1,[$mac,#7]
strb $h2,[$mac,#11]
strb $h3,[$mac,#15]
#endif
ldmia sp!,{r4-r11}
#if __ARM_ARCH__>=5
ret @ bx lr
#else
tst lr,#1
moveq pc,lr @ be binary compatible with V4, yet
bx lr @ interoperable with Thumb ISA:-)
#endif
.size poly1305_emit,.-poly1305_emit
___
{
my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
my ($T0,$T1,$MASK) = map("q$_",(15,4,0));
my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));
$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.fpu neon
.type poly1305_init_neon,%function
.align 5
poly1305_init_neon:
.Lpoly1305_init_neon:
ldr r3,[$ctx,#48] @ first table element
cmp r3,#-1 @ is value impossible?
bne .Lno_init_neon
ldr r4,[$ctx,#20] @ load key base 2^32
ldr r5,[$ctx,#24]
ldr r6,[$ctx,#28]
ldr r7,[$ctx,#32]
and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26
mov r3,r4,lsr#26
mov r4,r5,lsr#20
orr r3,r3,r5,lsl#6
mov r5,r6,lsr#14
orr r4,r4,r6,lsl#12
mov r6,r7,lsr#8
orr r5,r5,r7,lsl#18
and r3,r3,#0x03ffffff
and r4,r4,#0x03ffffff
and r5,r5,#0x03ffffff
vdup.32 $R0,r2 @ r^1 in both lanes
add r2,r3,r3,lsl#2 @ *5
vdup.32 $R1,r3
add r3,r4,r4,lsl#2
vdup.32 $S1,r2
vdup.32 $R2,r4
add r4,r5,r5,lsl#2
vdup.32 $S2,r3
vdup.32 $R3,r5
add r5,r6,r6,lsl#2
vdup.32 $S3,r4
vdup.32 $R4,r6
vdup.32 $S4,r5
mov $zeros,#2 @ counter
.Lsquare_neon:
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
@ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
@ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
@ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
@ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
vmull.u32 $D0,$R0,${R0}[1]
vmull.u32 $D1,$R1,${R0}[1]
vmull.u32 $D2,$R2,${R0}[1]
vmull.u32 $D3,$R3,${R0}[1]
vmull.u32 $D4,$R4,${R0}[1]
vmlal.u32 $D0,$R4,${S1}[1]
vmlal.u32 $D1,$R0,${R1}[1]
vmlal.u32 $D2,$R1,${R1}[1]
vmlal.u32 $D3,$R2,${R1}[1]
vmlal.u32 $D4,$R3,${R1}[1]
vmlal.u32 $D0,$R3,${S2}[1]
vmlal.u32 $D1,$R4,${S2}[1]
vmlal.u32 $D3,$R1,${R2}[1]
vmlal.u32 $D2,$R0,${R2}[1]
vmlal.u32 $D4,$R2,${R2}[1]
vmlal.u32 $D0,$R2,${S3}[1]
vmlal.u32 $D3,$R0,${R3}[1]
vmlal.u32 $D1,$R3,${S3}[1]
vmlal.u32 $D2,$R4,${S3}[1]
vmlal.u32 $D4,$R1,${R3}[1]
vmlal.u32 $D3,$R4,${S4}[1]
vmlal.u32 $D0,$R1,${S4}[1]
vmlal.u32 $D1,$R2,${S4}[1]
vmlal.u32 $D2,$R3,${S4}[1]
vmlal.u32 $D4,$R0,${R4}[1]
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
@ and P. Schwabe
@
@ H0>>+H1>>+H2>>+H3>>+H4
@ H3>>+H4>>*5+H0>>+H1
@
@ Trivia.
@
@ Result of multiplication of n-bit number by m-bit number is
@ n+m bits wide. However! Even though 2^n is a n+1-bit number,
@ m-bit number multiplied by 2^n is still n+m bits wide.
@
@ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
@ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
@ one is n+1 bits wide.
@
@ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
@ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
@ can be 27. However! In cases when their width exceeds 26 bits
@ they are limited by 2^26+2^6. This in turn means that *sum*
@ of the products with these values can still be viewed as sum
@ of 52-bit numbers as long as the amount of addends is not a
@ power of 2. For example,
@
@ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
@
@ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
@ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
@ 8 * (2^52) or 2^55. However, the value is then multiplied by
@ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
@ which is less than 32 * (2^52) or 2^57. And when processing
@ data we are looking at triple as many addends...
@
@ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
@ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
@ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
@ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
@ instruction accepts 2x32-bit input and writes 2x64-bit result.
@ This means that result of reduction have to be compressed upon
@ loop wrap-around. This can be done in the process of reduction
@ to minimize amount of instructions [as well as amount of
@ 128-bit instructions, which benefits low-end processors], but
@ one has to watch for H2 (which is narrower than H0) and 5*H4
@ not being wider than 58 bits, so that result of right shift
@ by 26 bits fits in 32 bits. This is also useful on x86,
@ because it allows to use paddd in place for paddq, which
@ benefits Atom, where paddq is ridiculously slow.
vshr.u64 $T0,$D3,#26
vmovn.i64 $D3#lo,$D3
vshr.u64 $T1,$D0,#26
vmovn.i64 $D0#lo,$D0
vadd.i64 $D4,$D4,$T0 @ h3 -> h4
vbic.i32 $D3#lo,#0xfc000000 @ &=0x03ffffff
vadd.i64 $D1,$D1,$T1 @ h0 -> h1
vbic.i32 $D0#lo,#0xfc000000
vshrn.u64 $T0#lo,$D4,#26
vmovn.i64 $D4#lo,$D4
vshr.u64 $T1,$D1,#26
vmovn.i64 $D1#lo,$D1
vadd.i64 $D2,$D2,$T1 @ h1 -> h2
vbic.i32 $D4#lo,#0xfc000000
vbic.i32 $D1#lo,#0xfc000000
vadd.i32 $D0#lo,$D0#lo,$T0#lo
vshl.u32 $T0#lo,$T0#lo,#2
vshrn.u64 $T1#lo,$D2,#26
vmovn.i64 $D2#lo,$D2
vadd.i32 $D0#lo,$D0#lo,$T0#lo @ h4 -> h0
vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3
vbic.i32 $D2#lo,#0xfc000000
vshr.u32 $T0#lo,$D0#lo,#26
vbic.i32 $D0#lo,#0xfc000000
vshr.u32 $T1#lo,$D3#lo,#26
vbic.i32 $D3#lo,#0xfc000000
vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1
vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4
subs $zeros,$zeros,#1
beq .Lsquare_break_neon
add $tbl0,$ctx,#(48+0*9*4)
add $tbl1,$ctx,#(48+1*9*4)
vtrn.32 $R0,$D0#lo @ r^2:r^1
vtrn.32 $R2,$D2#lo
vtrn.32 $R3,$D3#lo
vtrn.32 $R1,$D1#lo
vtrn.32 $R4,$D4#lo
vshl.u32 $S2,$R2,#2 @ *5
vshl.u32 $S3,$R3,#2
vshl.u32 $S1,$R1,#2
vshl.u32 $S4,$R4,#2
vadd.i32 $S2,$S2,$R2
vadd.i32 $S1,$S1,$R1
vadd.i32 $S3,$S3,$R3
vadd.i32 $S4,$S4,$R4
vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
vst1.32 {${S4}[0]},[$tbl0,:32]
vst1.32 {${S4}[1]},[$tbl1,:32]
b .Lsquare_neon
.align 4
.Lsquare_break_neon:
add $tbl0,$ctx,#(48+2*4*9)
add $tbl1,$ctx,#(48+3*4*9)
vmov $R0,$D0#lo @ r^4:r^3
vshl.u32 $S1,$D1#lo,#2 @ *5
vmov $R1,$D1#lo
vshl.u32 $S2,$D2#lo,#2
vmov $R2,$D2#lo
vshl.u32 $S3,$D3#lo,#2
vmov $R3,$D3#lo
vshl.u32 $S4,$D4#lo,#2
vmov $R4,$D4#lo
vadd.i32 $S1,$S1,$D1#lo
vadd.i32 $S2,$S2,$D2#lo
vadd.i32 $S3,$S3,$D3#lo
vadd.i32 $S4,$S4,$D4#lo
vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
vst1.32 {${S4}[0]},[$tbl0]
vst1.32 {${S4}[1]},[$tbl1]
.Lno_init_neon:
ret @ bx lr
.size poly1305_init_neon,.-poly1305_init_neon
.type poly1305_blocks_neon,%function
.align 5
poly1305_blocks_neon:
.Lpoly1305_blocks_neon:
ldr ip,[$ctx,#36] @ is_base2_26
cmp $len,#64
blo .Lpoly1305_blocks
stmdb sp!,{r4-r7}
vstmdb sp!,{d8-d15} @ ABI specification says so
tst ip,ip @ is_base2_26?
bne .Lbase2_26_neon
stmdb sp!,{r1-r3,lr}
bl .Lpoly1305_init_neon
ldr r4,[$ctx,#0] @ load hash value base 2^32
ldr r5,[$ctx,#4]
ldr r6,[$ctx,#8]
ldr r7,[$ctx,#12]
ldr ip,[$ctx,#16]
and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26
mov r3,r4,lsr#26
veor $D0#lo,$D0#lo,$D0#lo
mov r4,r5,lsr#20
orr r3,r3,r5,lsl#6
veor $D1#lo,$D1#lo,$D1#lo
mov r5,r6,lsr#14
orr r4,r4,r6,lsl#12
veor $D2#lo,$D2#lo,$D2#lo
mov r6,r7,lsr#8
orr r5,r5,r7,lsl#18
veor $D3#lo,$D3#lo,$D3#lo
and r3,r3,#0x03ffffff
orr r6,r6,ip,lsl#24
veor $D4#lo,$D4#lo,$D4#lo
and r4,r4,#0x03ffffff
mov r1,#1
and r5,r5,#0x03ffffff
str r1,[$ctx,#36] @ set is_base2_26
vmov.32 $D0#lo[0],r2
vmov.32 $D1#lo[0],r3
vmov.32 $D2#lo[0],r4
vmov.32 $D3#lo[0],r5
vmov.32 $D4#lo[0],r6
adr $zeros,.Lzeros
ldmia sp!,{r1-r3,lr}
b .Lhash_loaded
.align 4
.Lbase2_26_neon:
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ load hash value
veor $D0#lo,$D0#lo,$D0#lo
veor $D1#lo,$D1#lo,$D1#lo
veor $D2#lo,$D2#lo,$D2#lo
veor $D3#lo,$D3#lo,$D3#lo
veor $D4#lo,$D4#lo,$D4#lo
vld4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
adr $zeros,.Lzeros
vld1.32 {$D4#lo[0]},[$ctx]
sub $ctx,$ctx,#16 @ rewind
.Lhash_loaded:
add $in2,$inp,#32
mov $padbit,$padbit,lsl#24
tst $len,#31
beq .Leven
vld4.32 {$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
vmov.32 $H4#lo[0],$padbit
sub $len,$len,#16
add $in2,$inp,#32
# ifdef __ARMEB__
vrev32.8 $H0,$H0
vrev32.8 $H3,$H3
vrev32.8 $H1,$H1
vrev32.8 $H2,$H2
# endif
vsri.u32 $H4#lo,$H3#lo,#8 @ base 2^32 -> base 2^26
vshl.u32 $H3#lo,$H3#lo,#18
vsri.u32 $H3#lo,$H2#lo,#14
vshl.u32 $H2#lo,$H2#lo,#12
vadd.i32 $H4#hi,$H4#lo,$D4#lo @ add hash value and move to #hi
vbic.i32 $H3#lo,#0xfc000000
vsri.u32 $H2#lo,$H1#lo,#20
vshl.u32 $H1#lo,$H1#lo,#6
vbic.i32 $H2#lo,#0xfc000000
vsri.u32 $H1#lo,$H0#lo,#26
vadd.i32 $H3#hi,$H3#lo,$D3#lo
vbic.i32 $H0#lo,#0xfc000000
vbic.i32 $H1#lo,#0xfc000000
vadd.i32 $H2#hi,$H2#lo,$D2#lo
vadd.i32 $H0#hi,$H0#lo,$D0#lo
vadd.i32 $H1#hi,$H1#lo,$D1#lo
mov $tbl1,$zeros
add $tbl0,$ctx,#48
cmp $len,$len
b .Long_tail
.align 4
.Leven:
subs $len,$len,#64
it lo
movlo $in2,$zeros
vmov.i32 $H4,#1<<24 @ padbit, yes, always
vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
add $inp,$inp,#64
vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
add $in2,$in2,#64
itt hi
addhi $tbl1,$ctx,#(48+1*9*4)
addhi $tbl0,$ctx,#(48+3*9*4)
# ifdef __ARMEB__
vrev32.8 $H0,$H0
vrev32.8 $H3,$H3
vrev32.8 $H1,$H1
vrev32.8 $H2,$H2
# endif
vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26
vshl.u32 $H3,$H3,#18
vsri.u32 $H3,$H2,#14
vshl.u32 $H2,$H2,#12
vbic.i32 $H3,#0xfc000000
vsri.u32 $H2,$H1,#20
vshl.u32 $H1,$H1,#6
vbic.i32 $H2,#0xfc000000
vsri.u32 $H1,$H0,#26
vbic.i32 $H0,#0xfc000000
vbic.i32 $H1,#0xfc000000
bls .Lskip_loop
vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^2
vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4
vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
b .Loop_neon
.align 5
.Loop_neon:
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
@ \___________________/
@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
@ \___________________/ \____________________/
@
@ Note that we start with inp[2:3]*r^2. This is because it
@ doesn't depend on reduction in previous iteration.
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
@ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
@ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
@ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ inp[2:3]*r^2
vadd.i32 $H2#lo,$H2#lo,$D2#lo @ accumulate inp[0:1]
vmull.u32 $D2,$H2#hi,${R0}[1]
vadd.i32 $H0#lo,$H0#lo,$D0#lo
vmull.u32 $D0,$H0#hi,${R0}[1]
vadd.i32 $H3#lo,$H3#lo,$D3#lo
vmull.u32 $D3,$H3#hi,${R0}[1]
vmlal.u32 $D2,$H1#hi,${R1}[1]
vadd.i32 $H1#lo,$H1#lo,$D1#lo
vmull.u32 $D1,$H1#hi,${R0}[1]
vadd.i32 $H4#lo,$H4#lo,$D4#lo
vmull.u32 $D4,$H4#hi,${R0}[1]
subs $len,$len,#64
vmlal.u32 $D0,$H4#hi,${S1}[1]
it lo
movlo $in2,$zeros
vmlal.u32 $D3,$H2#hi,${R1}[1]
vld1.32 ${S4}[1],[$tbl1,:32]
vmlal.u32 $D1,$H0#hi,${R1}[1]
vmlal.u32 $D4,$H3#hi,${R1}[1]
vmlal.u32 $D0,$H3#hi,${S2}[1]
vmlal.u32 $D3,$H1#hi,${R2}[1]
vmlal.u32 $D4,$H2#hi,${R2}[1]
vmlal.u32 $D1,$H4#hi,${S2}[1]
vmlal.u32 $D2,$H0#hi,${R2}[1]
vmlal.u32 $D3,$H0#hi,${R3}[1]
vmlal.u32 $D0,$H2#hi,${S3}[1]
vmlal.u32 $D4,$H1#hi,${R3}[1]
vmlal.u32 $D1,$H3#hi,${S3}[1]
vmlal.u32 $D2,$H4#hi,${S3}[1]
vmlal.u32 $D3,$H4#hi,${S4}[1]
vmlal.u32 $D0,$H1#hi,${S4}[1]
vmlal.u32 $D4,$H0#hi,${R4}[1]
vmlal.u32 $D1,$H2#hi,${S4}[1]
vmlal.u32 $D2,$H3#hi,${S4}[1]
vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
add $in2,$in2,#64
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ (hash+inp[0:1])*r^4 and accumulate
vmlal.u32 $D3,$H3#lo,${R0}[0]
vmlal.u32 $D0,$H0#lo,${R0}[0]
vmlal.u32 $D4,$H4#lo,${R0}[0]
vmlal.u32 $D1,$H1#lo,${R0}[0]
vmlal.u32 $D2,$H2#lo,${R0}[0]
vld1.32 ${S4}[0],[$tbl0,:32]
vmlal.u32 $D3,$H2#lo,${R1}[0]
vmlal.u32 $D0,$H4#lo,${S1}[0]
vmlal.u32 $D4,$H3#lo,${R1}[0]
vmlal.u32 $D1,$H0#lo,${R1}[0]
vmlal.u32 $D2,$H1#lo,${R1}[0]
vmlal.u32 $D3,$H1#lo,${R2}[0]
vmlal.u32 $D0,$H3#lo,${S2}[0]
vmlal.u32 $D4,$H2#lo,${R2}[0]
vmlal.u32 $D1,$H4#lo,${S2}[0]
vmlal.u32 $D2,$H0#lo,${R2}[0]
vmlal.u32 $D3,$H0#lo,${R3}[0]
vmlal.u32 $D0,$H2#lo,${S3}[0]
vmlal.u32 $D4,$H1#lo,${R3}[0]
vmlal.u32 $D1,$H3#lo,${S3}[0]
vmlal.u32 $D3,$H4#lo,${S4}[0]
vmlal.u32 $D2,$H4#lo,${S3}[0]
vmlal.u32 $D0,$H1#lo,${S4}[0]
vmlal.u32 $D4,$H0#lo,${R4}[0]
vmov.i32 $H4,#1<<24 @ padbit, yes, always
vmlal.u32 $D1,$H2#lo,${S4}[0]
vmlal.u32 $D2,$H3#lo,${S4}[0]
vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
add $inp,$inp,#64
# ifdef __ARMEB__
vrev32.8 $H0,$H0
vrev32.8 $H1,$H1
vrev32.8 $H2,$H2
vrev32.8 $H3,$H3
# endif
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ lazy reduction interleaved with base 2^32 -> base 2^26 of
@ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.
vshr.u64 $T0,$D3,#26
vmovn.i64 $D3#lo,$D3
vshr.u64 $T1,$D0,#26
vmovn.i64 $D0#lo,$D0
vadd.i64 $D4,$D4,$T0 @ h3 -> h4
vbic.i32 $D3#lo,#0xfc000000
vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26
vadd.i64 $D1,$D1,$T1 @ h0 -> h1
vshl.u32 $H3,$H3,#18
vbic.i32 $D0#lo,#0xfc000000
vshrn.u64 $T0#lo,$D4,#26
vmovn.i64 $D4#lo,$D4
vshr.u64 $T1,$D1,#26
vmovn.i64 $D1#lo,$D1
vadd.i64 $D2,$D2,$T1 @ h1 -> h2
vsri.u32 $H3,$H2,#14
vbic.i32 $D4#lo,#0xfc000000
vshl.u32 $H2,$H2,#12
vbic.i32 $D1#lo,#0xfc000000
vadd.i32 $D0#lo,$D0#lo,$T0#lo
vshl.u32 $T0#lo,$T0#lo,#2
vbic.i32 $H3,#0xfc000000
vshrn.u64 $T1#lo,$D2,#26
vmovn.i64 $D2#lo,$D2
vaddl.u32 $D0,$D0#lo,$T0#lo @ h4 -> h0 [widen for a sec]
vsri.u32 $H2,$H1,#20
vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3
vshl.u32 $H1,$H1,#6
vbic.i32 $D2#lo,#0xfc000000
vbic.i32 $H2,#0xfc000000
vshrn.u64 $T0#lo,$D0,#26 @ re-narrow
vmovn.i64 $D0#lo,$D0
vsri.u32 $H1,$H0,#26
vbic.i32 $H0,#0xfc000000
vshr.u32 $T1#lo,$D3#lo,#26
vbic.i32 $D3#lo,#0xfc000000
vbic.i32 $D0#lo,#0xfc000000
vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1
vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4
vbic.i32 $H1,#0xfc000000
bhi .Loop_neon
.Lskip_loop:
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
add $tbl1,$ctx,#(48+0*9*4)
add $tbl0,$ctx,#(48+1*9*4)
adds $len,$len,#32
it ne
movne $len,#0
bne .Long_tail
vadd.i32 $H2#hi,$H2#lo,$D2#lo @ add hash value and move to #hi
vadd.i32 $H0#hi,$H0#lo,$D0#lo
vadd.i32 $H3#hi,$H3#lo,$D3#lo
vadd.i32 $H1#hi,$H1#lo,$D1#lo
vadd.i32 $H4#hi,$H4#lo,$D4#lo
.Long_tail:
vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^1
vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^2
vadd.i32 $H2#lo,$H2#lo,$D2#lo @ can be redundant
vmull.u32 $D2,$H2#hi,$R0
vadd.i32 $H0#lo,$H0#lo,$D0#lo
vmull.u32 $D0,$H0#hi,$R0
vadd.i32 $H3#lo,$H3#lo,$D3#lo
vmull.u32 $D3,$H3#hi,$R0
vadd.i32 $H1#lo,$H1#lo,$D1#lo
vmull.u32 $D1,$H1#hi,$R0
vadd.i32 $H4#lo,$H4#lo,$D4#lo
vmull.u32 $D4,$H4#hi,$R0
vmlal.u32 $D0,$H4#hi,$S1
vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
vmlal.u32 $D3,$H2#hi,$R1
vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
vmlal.u32 $D1,$H0#hi,$R1
vmlal.u32 $D4,$H3#hi,$R1
vmlal.u32 $D2,$H1#hi,$R1
vmlal.u32 $D3,$H1#hi,$R2
vld1.32 ${S4}[1],[$tbl1,:32]
vmlal.u32 $D0,$H3#hi,$S2
vld1.32 ${S4}[0],[$tbl0,:32]
vmlal.u32 $D4,$H2#hi,$R2
vmlal.u32 $D1,$H4#hi,$S2
vmlal.u32 $D2,$H0#hi,$R2
vmlal.u32 $D3,$H0#hi,$R3
it ne
addne $tbl1,$ctx,#(48+2*9*4)
vmlal.u32 $D0,$H2#hi,$S3
it ne
addne $tbl0,$ctx,#(48+3*9*4)
vmlal.u32 $D4,$H1#hi,$R3
vmlal.u32 $D1,$H3#hi,$S3
vmlal.u32 $D2,$H4#hi,$S3
vmlal.u32 $D3,$H4#hi,$S4
vorn $MASK,$MASK,$MASK @ all-ones, can be redundant
vmlal.u32 $D0,$H1#hi,$S4
vshr.u64 $MASK,$MASK,#38
vmlal.u32 $D4,$H0#hi,$R4
vmlal.u32 $D1,$H2#hi,$S4
vmlal.u32 $D2,$H3#hi,$S4
beq .Lshort_tail
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ (hash+inp[0:1])*r^4:r^3 and accumulate
vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^3
vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4
vmlal.u32 $D2,$H2#lo,$R0
vmlal.u32 $D0,$H0#lo,$R0
vmlal.u32 $D3,$H3#lo,$R0
vmlal.u32 $D1,$H1#lo,$R0
vmlal.u32 $D4,$H4#lo,$R0
vmlal.u32 $D0,$H4#lo,$S1
vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
vmlal.u32 $D3,$H2#lo,$R1
vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
vmlal.u32 $D1,$H0#lo,$R1
vmlal.u32 $D4,$H3#lo,$R1
vmlal.u32 $D2,$H1#lo,$R1
vmlal.u32 $D3,$H1#lo,$R2
vld1.32 ${S4}[1],[$tbl1,:32]
vmlal.u32 $D0,$H3#lo,$S2
vld1.32 ${S4}[0],[$tbl0,:32]
vmlal.u32 $D4,$H2#lo,$R2
vmlal.u32 $D1,$H4#lo,$S2
vmlal.u32 $D2,$H0#lo,$R2
vmlal.u32 $D3,$H0#lo,$R3
vmlal.u32 $D0,$H2#lo,$S3
vmlal.u32 $D4,$H1#lo,$R3
vmlal.u32 $D1,$H3#lo,$S3
vmlal.u32 $D2,$H4#lo,$S3
vmlal.u32 $D3,$H4#lo,$S4
vorn $MASK,$MASK,$MASK @ all-ones
vmlal.u32 $D0,$H1#lo,$S4
vshr.u64 $MASK,$MASK,#38
vmlal.u32 $D4,$H0#lo,$R4
vmlal.u32 $D1,$H2#lo,$S4
vmlal.u32 $D2,$H3#lo,$S4
.Lshort_tail:
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ horizontal addition
vadd.i64 $D3#lo,$D3#lo,$D3#hi
vadd.i64 $D0#lo,$D0#lo,$D0#hi
vadd.i64 $D4#lo,$D4#lo,$D4#hi
vadd.i64 $D1#lo,$D1#lo,$D1#hi
vadd.i64 $D2#lo,$D2#lo,$D2#hi
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ lazy reduction, but without narrowing
vshr.u64 $T0,$D3,#26
vand.i64 $D3,$D3,$MASK
vshr.u64 $T1,$D0,#26
vand.i64 $D0,$D0,$MASK
vadd.i64 $D4,$D4,$T0 @ h3 -> h4
vadd.i64 $D1,$D1,$T1 @ h0 -> h1
vshr.u64 $T0,$D4,#26
vand.i64 $D4,$D4,$MASK
vshr.u64 $T1,$D1,#26
vand.i64 $D1,$D1,$MASK
vadd.i64 $D2,$D2,$T1 @ h1 -> h2
vadd.i64 $D0,$D0,$T0
vshl.u64 $T0,$T0,#2
vshr.u64 $T1,$D2,#26
vand.i64 $D2,$D2,$MASK
vadd.i64 $D0,$D0,$T0 @ h4 -> h0
vadd.i64 $D3,$D3,$T1 @ h2 -> h3
vshr.u64 $T0,$D0,#26
vand.i64 $D0,$D0,$MASK
vshr.u64 $T1,$D3,#26
vand.i64 $D3,$D3,$MASK
vadd.i64 $D1,$D1,$T0 @ h0 -> h1
vadd.i64 $D4,$D4,$T1 @ h3 -> h4
cmp $len,#0
bne .Leven
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ store hash value
vst4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
vst1.32 {$D4#lo[0]},[$ctx]
vldmia sp!,{d8-d15} @ epilogue
ldmia sp!,{r4-r7}
ret @ bx lr
.size poly1305_blocks_neon,.-poly1305_blocks_neon
.align 5
.Lzeros:
.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
#ifndef __KERNEL__
.LOPENSSL_armcap:
# ifdef _WIN32
.word OPENSSL_armcap_P
# else
.word OPENSSL_armcap_P-.Lpoly1305_init
# endif
.comm OPENSSL_armcap_P,4,4
.hidden OPENSSL_armcap_P
#endif
#endif
___
} }
$code.=<<___;
.asciz "Poly1305 for ARMv4/NEON, CRYPTOGAMS by \@dot-asm"
.align 2
___
foreach (split("\n",$code)) {
s/\`([^\`]*)\`/eval $1/geo;
s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or
s/\bret\b/bx lr/go or
s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
print $_,"\n";
}
close STDOUT; # enforce flush
Reference in New Issue Copy Permalink