Commit 44d21c3f authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto update from Herbert Xu:
 "Here is the crypto update for 4.2:

  API:

   - Convert RNG interface to new style.

   - New AEAD interface with one SG list for AD and plain/cipher text.
     All external AEAD users have been converted.

   - New asymmetric key interface (akcipher).

  Algorithms:

   - Chacha20, Poly1305 and RFC7539 support.

   - New RSA implementation.

   - Jitter RNG.

   - DRBG is now seeded with both /dev/random and Jitter RNG.  If kernel
     pool isn't ready then DRBG will be reseeded when it is.

   - DRBG is now the default crypto API RNG, replacing krng.

   - 842 compression (previously part of powerpc nx driver).

  Drivers:

   - Accelerated SHA-512 for arm64.

   - New Marvell CESA driver that supports DMA and more algorithms.

   - Updated powerpc nx 842 support.

   - Added support for SEC1 hardware to talitos"

* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (292 commits)
  crypto: marvell/cesa - remove COMPILE_TEST dependency
  crypto: algif_aead - Temporarily disable all AEAD algorithms
  crypto: af_alg - Forbid the use internal algorithms
  crypto: echainiv - Only hold RNG during initialisation
  crypto: seqiv - Add compatibility support without RNG
  crypto: eseqiv - Offer normal cipher functionality without RNG
  crypto: chainiv - Offer normal cipher functionality without RNG
  crypto: user - Add CRYPTO_MSG_DELRNG
  crypto: user - Move cryptouser.h to uapi
  crypto: rng - Do not free default RNG when it becomes unused
  crypto: skcipher - Allow givencrypt to be NULL
  crypto: sahara - propagate the error on clk_disable_unprepare() failure
  crypto: rsa - fix invalid select for AKCIPHER
  crypto: picoxcell - Update to the current clk API
  crypto: nx - Check for bogus firmware properties
  crypto: marvell/cesa - add DT bindings documentation
  crypto: marvell/cesa - add support for Kirkwood and Dove SoCs
  crypto: marvell/cesa - add support for Orion SoCs
  crypto: marvell/cesa - add allhwsupport module parameter
  crypto: marvell/cesa - add support for all armada SoCs
  ...
parents efdfce2b fe55dfdc
......@@ -119,7 +119,7 @@
<para>
Note: The terms "transformation" and cipher algorithm are used
interchangably.
interchangeably.
</para>
</sect1>
......@@ -536,8 +536,8 @@
<para>
For other use cases of AEAD ciphers, the ASCII art applies as
well, but the caller may not use the GIVCIPHER interface. In
this case, the caller must generate the IV.
well, but the caller may not use the AEAD cipher with a separate
IV generator. In this case, the caller must generate the IV.
</para>
<para>
......@@ -584,8 +584,8 @@ kernel crypto API | IPSEC Layer
|
+-----------+ |
| | (1)
| givcipher | <----------------------------------- esp_output
| (seqiv) | ---+
| aead | <----------------------------------- esp_output
| (seqniv) | ---+
+-----------+ |
| (2)
+-----------+ |
......@@ -620,8 +620,8 @@ kernel crypto API | IPSEC Layer
<orderedlist>
<listitem>
<para>
esp_output() invokes crypto_aead_givencrypt() to trigger an encryption
operation of the GIVCIPHER implementation.
esp_output() invokes crypto_aead_encrypt() to trigger an encryption
operation of the AEAD cipher with IV generator.
</para>
<para>
......@@ -1563,7 +1563,7 @@ struct sockaddr_alg sa = {
<sect1><title>Zero-Copy Interface</title>
<para>
In addition to the send/write/read/recv system call familty, the AF_ALG
In addition to the send/write/read/recv system call family, the AF_ALG
interface can be accessed with the zero-copy interface of splice/vmsplice.
As the name indicates, the kernel tries to avoid a copy operation into
kernel space.
......@@ -1669,9 +1669,19 @@ read(opfd, out, outlen);
</chapter>
<chapter id="API"><title>Programming Interface</title>
<para>
Please note that the kernel crypto API contains the AEAD givcrypt
API (crypto_aead_giv* and aead_givcrypt_* function calls in
include/crypto/aead.h). This API is obsolete and will be removed
in the future. To obtain the functionality of an AEAD cipher with
internal IV generation, use the IV generator as a regular cipher.
For example, rfc4106(gcm(aes)) is the AEAD cipher with external
IV generation and seqniv(rfc4106(gcm(aes))) implies that the kernel
crypto API generates the IV. Different IV generators are available.
</para>
<sect1><title>Block Cipher Context Data Structures</title>
!Pinclude/linux/crypto.h Block Cipher Context Data Structures
!Finclude/linux/crypto.h aead_request
!Finclude/crypto/aead.h aead_request
</sect1>
<sect1><title>Block Cipher Algorithm Definitions</title>
!Pinclude/linux/crypto.h Block Cipher Algorithm Definitions
......@@ -1680,7 +1690,7 @@ read(opfd, out, outlen);
!Finclude/linux/crypto.h aead_alg
!Finclude/linux/crypto.h blkcipher_alg
!Finclude/linux/crypto.h cipher_alg
!Finclude/linux/crypto.h rng_alg
!Finclude/crypto/rng.h rng_alg
</sect1>
<sect1><title>Asynchronous Block Cipher API</title>
!Pinclude/linux/crypto.h Asynchronous Block Cipher API
......@@ -1704,26 +1714,27 @@ read(opfd, out, outlen);
!Finclude/linux/crypto.h ablkcipher_request_set_crypt
</sect1>
<sect1><title>Authenticated Encryption With Associated Data (AEAD) Cipher API</title>
!Pinclude/linux/crypto.h Authenticated Encryption With Associated Data (AEAD) Cipher API
!Finclude/linux/crypto.h crypto_alloc_aead
!Finclude/linux/crypto.h crypto_free_aead
!Finclude/linux/crypto.h crypto_aead_ivsize
!Finclude/linux/crypto.h crypto_aead_authsize
!Finclude/linux/crypto.h crypto_aead_blocksize
!Finclude/linux/crypto.h crypto_aead_setkey
!Finclude/linux/crypto.h crypto_aead_setauthsize
!Finclude/linux/crypto.h crypto_aead_encrypt
!Finclude/linux/crypto.h crypto_aead_decrypt
!Pinclude/crypto/aead.h Authenticated Encryption With Associated Data (AEAD) Cipher API
!Finclude/crypto/aead.h crypto_alloc_aead
!Finclude/crypto/aead.h crypto_free_aead
!Finclude/crypto/aead.h crypto_aead_ivsize
!Finclude/crypto/aead.h crypto_aead_authsize
!Finclude/crypto/aead.h crypto_aead_blocksize
!Finclude/crypto/aead.h crypto_aead_setkey
!Finclude/crypto/aead.h crypto_aead_setauthsize
!Finclude/crypto/aead.h crypto_aead_encrypt
!Finclude/crypto/aead.h crypto_aead_decrypt
</sect1>
<sect1><title>Asynchronous AEAD Request Handle</title>
!Pinclude/linux/crypto.h Asynchronous AEAD Request Handle
!Finclude/linux/crypto.h crypto_aead_reqsize
!Finclude/linux/crypto.h aead_request_set_tfm
!Finclude/linux/crypto.h aead_request_alloc
!Finclude/linux/crypto.h aead_request_free
!Finclude/linux/crypto.h aead_request_set_callback
!Finclude/linux/crypto.h aead_request_set_crypt
!Finclude/linux/crypto.h aead_request_set_assoc
!Pinclude/crypto/aead.h Asynchronous AEAD Request Handle
!Finclude/crypto/aead.h crypto_aead_reqsize
!Finclude/crypto/aead.h aead_request_set_tfm
!Finclude/crypto/aead.h aead_request_alloc
!Finclude/crypto/aead.h aead_request_free
!Finclude/crypto/aead.h aead_request_set_callback
!Finclude/crypto/aead.h aead_request_set_crypt
!Finclude/crypto/aead.h aead_request_set_assoc
!Finclude/crypto/aead.h aead_request_set_ad
</sect1>
<sect1><title>Synchronous Block Cipher API</title>
!Pinclude/linux/crypto.h Synchronous Block Cipher API
......
Freescale SoC SEC Security Engines versions 2.x-3.x
Freescale SoC SEC Security Engines versions 1.x-2.x-3.x
Required properties:
- compatible : Should contain entries for this and backward compatible
SEC versions, high to low, e.g., "fsl,sec2.1", "fsl,sec2.0"
SEC versions, high to low, e.g., "fsl,sec2.1", "fsl,sec2.0" (SEC2/3)
e.g., "fsl,sec1.2", "fsl,sec1.0" (SEC1)
warning: SEC1 and SEC2 are mutually exclusive
- reg : Offset and length of the register set for the device
- interrupts : the SEC's interrupt number
- fsl,num-channels : An integer representing the number of channels
......
Marvell Cryptographic Engines And Security Accelerator
Required properties:
- compatible: should be one of the following string
"marvell,orion-crypto"
"marvell,kirkwood-crypto"
"marvell,dove-crypto"
"marvell,armada-370-crypto"
"marvell,armada-xp-crypto"
"marvell,armada-375-crypto"
"marvell,armada-38x-crypto"
- reg: base physical address of the engine and length of memory mapped
region. Can also contain an entry for the SRAM attached to the CESA,
but this representation is deprecated and marvell,crypto-srams should
be used instead
- reg-names: "regs". Can contain an "sram" entry, but this representation
is deprecated and marvell,crypto-srams should be used instead
- interrupts: interrupt number
- clocks: reference to the crypto engines clocks. This property is not
required for orion and kirkwood platforms
- clock-names: "cesaX" and "cesazX", X should be replaced by the crypto engine
id.
This property is not required for the orion and kirkwoord
platforms.
"cesazX" clocks are not required on armada-370 platforms
- marvell,crypto-srams: phandle to crypto SRAM definitions
Optional properties:
- marvell,crypto-sram-size: SRAM size reserved for crypto operations, if not
specified the whole SRAM is used (2KB)
Examples:
crypto@90000 {
compatible = "marvell,armada-xp-crypto";
reg = <0x90000 0x10000>;
reg-names = "regs";
interrupts = <48>, <49>;
clocks = <&gateclk 23>, <&gateclk 23>;
clock-names = "cesa0", "cesa1";
marvell,crypto-srams = <&crypto_sram0>, <&crypto_sram1>;
marvell,crypto-sram-size = <0x600>;
status = "okay";
};
Marvell Cryptographic Engines And Security Accelerator
Required properties:
- compatible : should be "marvell,orion-crypto"
- reg : base physical address of the engine and length of memory mapped
region, followed by base physical address of sram and its memory
length
- reg-names : "regs" , "sram";
- interrupts : interrupt number
- compatible: should be one of the following string
"marvell,orion-crypto"
"marvell,kirkwood-crypto"
"marvell,dove-crypto"
- reg: base physical address of the engine and length of memory mapped
region. Can also contain an entry for the SRAM attached to the CESA,
but this representation is deprecated and marvell,crypto-srams should
be used instead
- reg-names: "regs". Can contain an "sram" entry, but this representation
is deprecated and marvell,crypto-srams should be used instead
- interrupts: interrupt number
- clocks: reference to the crypto engines clocks. This property is only
required for Dove platforms
- marvell,crypto-srams: phandle to crypto SRAM definitions
Optional properties:
- marvell,crypto-sram-size: SRAM size reserved for crypto operations, if not
specified the whole SRAM is used (2KB)
Examples:
crypto@30000 {
compatible = "marvell,orion-crypto";
reg = <0x30000 0x10000>,
<0x4000000 0x800>;
reg-names = "regs" , "sram";
reg = <0x30000 0x10000>;
reg-names = "regs";
interrupts = <22>;
marvell,crypto-srams = <&crypto_sram>;
marvell,crypto-sram-size = <0x600>;
status = "okay";
};
......@@ -4879,13 +4879,23 @@ M: Marcelo Henrique Cerri <mhcerri@linux.vnet.ibm.com>
M: Fionnuala Gunter <fin@linux.vnet.ibm.com>
L: linux-crypto@vger.kernel.org
S: Supported
F: drivers/crypto/nx/
F: drivers/crypto/nx/Makefile
F: drivers/crypto/nx/Kconfig
F: drivers/crypto/nx/nx-aes*
F: drivers/crypto/nx/nx-sha*
F: drivers/crypto/nx/nx.*
F: drivers/crypto/nx/nx_csbcpb.h
F: drivers/crypto/nx/nx_debugfs.h
IBM Power 842 compression accelerator
M: Dan Streetman <ddstreet@us.ibm.com>
S: Supported
F: drivers/crypto/nx/nx-842.c
F: include/linux/nx842.h
F: drivers/crypto/nx/Makefile
F: drivers/crypto/nx/Kconfig
F: drivers/crypto/nx/nx-842*
F: include/linux/sw842.h
F: crypto/842.c
F: lib/842/
IBM Power Linux RAID adapter
M: Brian King <brking@us.ibm.com>
......
......@@ -53,20 +53,13 @@ config CRYPTO_SHA256_ARM
SHA-256 secure hash standard (DFIPS 180-2) implemented
using optimized ARM assembler and NEON, when available.
config CRYPTO_SHA512_ARM_NEON
tristate "SHA384 and SHA512 digest algorithm (ARM NEON)"
depends on KERNEL_MODE_NEON
select CRYPTO_SHA512
config CRYPTO_SHA512_ARM
tristate "SHA-384/512 digest algorithm (ARM-asm and NEON)"
select CRYPTO_HASH
depends on !CPU_V7M
help
SHA-512 secure hash standard (DFIPS 180-2) implemented
using ARM NEON instructions, when available.
This version of SHA implements a 512 bit hash with 256 bits of
security against collision attacks.
This code also includes SHA-384, a 384 bit hash with 192 bits
of security against collision attacks.
using optimized ARM assembler and NEON, when available.
config CRYPTO_AES_ARM
tristate "AES cipher algorithms (ARM-asm)"
......
......@@ -7,7 +7,7 @@ obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o
obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o
ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
......@@ -30,7 +30,8 @@ sha1-arm-y := sha1-armv4-large.o sha1_glue.o
sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o
sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o
sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y)
sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o
sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o
sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y)
sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o
sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o
aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
......@@ -45,4 +46,7 @@ $(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl
$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl
$(call cmd,perl)
.PRECIOUS: $(obj)/aesbs-core.S $(obj)/sha256-core.S
$(src)/sha512-core.S_shipped: $(src)/sha512-armv4.pl
$(call cmd,perl)
.PRECIOUS: $(obj)/aesbs-core.S $(obj)/sha256-core.S $(obj)/sha512-core.S
......@@ -101,15 +101,14 @@
\dround q10, q11
blo 0f @ AES-128: 10 rounds
vld1.8 {q10-q11}, [ip]!
beq 1f @ AES-192: 12 rounds
\dround q12, q13
beq 1f @ AES-192: 12 rounds
vld1.8 {q12-q13}, [ip]
\dround q10, q11
0: \fround q12, q13, q14
bx lr
1: \dround q12, q13
\fround q10, q11, q14
1: \fround q10, q11, q14
bx lr
.endm
......@@ -122,8 +121,8 @@
* q2 : third in/output block (_3x version only)
* q8 : first round key
* q9 : secound round key
* ip : address of 3rd round key
* q14 : final round key
* r2 : address of round key array
* r3 : number of rounds
*/
.align 6
......
#!/usr/bin/env perl
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
#
# Permission to use under GPL terms is granted.
# ====================================================================
# SHA512 block procedure for ARMv4. September 2007.
# This code is ~4.5 (four and a half) times faster than code generated
# by gcc 3.4 and it spends ~72 clock cycles per byte [on single-issue
# Xscale PXA250 core].
#
# July 2010.
#
# Rescheduling for dual-issue pipeline resulted in 6% improvement on
# Cortex A8 core and ~40 cycles per processed byte.
# February 2011.
#
# Profiler-assisted and platform-specific optimization resulted in 7%
# improvement on Coxtex A8 core and ~38 cycles per byte.
# March 2011.
#
# Add NEON implementation. On Cortex A8 it was measured to process
# one byte in 23.3 cycles or ~60% faster than integer-only code.
# August 2012.
#
# Improve NEON performance by 12% on Snapdragon S4. In absolute
# terms it's 22.6 cycles per byte, which is disappointing result.
# Technical writers asserted that 3-way S4 pipeline can sustain
# multiple NEON instructions per cycle, but dual NEON issue could
# not be observed, see http://www.openssl.org/~appro/Snapdragon-S4.html
# for further details. On side note Cortex-A15 processes one byte in
# 16 cycles.
# Byte order [in]dependence. =========================================
#
# Originally caller was expected to maintain specific *dword* order in
# h[0-7], namely with most significant dword at *lower* address, which
# was reflected in below two parameters as 0 and 4. Now caller is
# expected to maintain native byte order for whole 64-bit values.
$hi="HI";
$lo="LO";
# ====================================================================
while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";
$ctx="r0"; # parameter block
$inp="r1";
$len="r2";
$Tlo="r3";
$Thi="r4";
$Alo="r5";
$Ahi="r6";
$Elo="r7";
$Ehi="r8";
$t0="r9";
$t1="r10";
$t2="r11";
$t3="r12";
############ r13 is stack pointer
$Ktbl="r14";
############ r15 is program counter
$Aoff=8*0;
$Boff=8*1;
$Coff=8*2;
$Doff=8*3;
$Eoff=8*4;
$Foff=8*5;
$Goff=8*6;
$Hoff=8*7;
$Xoff=8*8;
sub BODY_00_15() {
my $magic = shift;
$code.=<<___;
@ Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41))
@ LO lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23
@ HI hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23
mov $t0,$Elo,lsr#14
str $Tlo,[sp,#$Xoff+0]
mov $t1,$Ehi,lsr#14
str $Thi,[sp,#$Xoff+4]
eor $t0,$t0,$Ehi,lsl#18
ldr $t2,[sp,#$Hoff+0] @ h.lo
eor $t1,$t1,$Elo,lsl#18
ldr $t3,[sp,#$Hoff+4] @ h.hi
eor $t0,$t0,$Elo,lsr#18
eor $t1,$t1,$Ehi,lsr#18
eor $t0,$t0,$Ehi,lsl#14
eor $t1,$t1,$Elo,lsl#14
eor $t0,$t0,$Ehi,lsr#9
eor $t1,$t1,$Elo,lsr#9
eor $t0,$t0,$Elo,lsl#23
eor $t1,$t1,$Ehi,lsl#23 @ Sigma1(e)
adds $Tlo,$Tlo,$t0
ldr $t0,[sp,#$Foff+0] @ f.lo
adc $Thi,$Thi,$t1 @ T += Sigma1(e)
ldr $t1,[sp,#$Foff+4] @ f.hi
adds $Tlo,$Tlo,$t2
ldr $t2,[sp,#$Goff+0] @ g.lo
adc $Thi,$Thi,$t3 @ T += h
ldr $t3,[sp,#$Goff+4] @ g.hi
eor $t0,$t0,$t2
str $Elo,[sp,#$Eoff+0]
eor $t1,$t1,$t3
str $Ehi,[sp,#$Eoff+4]
and $t0,$t0,$Elo
str $Alo,[sp,#$Aoff+0]
and $t1,$t1,$Ehi
str $Ahi,[sp,#$Aoff+4]
eor $t0,$t0,$t2
ldr $t2,[$Ktbl,#$lo] @ K[i].lo
eor $t1,$t1,$t3 @ Ch(e,f,g)
ldr $t3,[$Ktbl,#$hi] @ K[i].hi
adds $Tlo,$Tlo,$t0
ldr $Elo,[sp,#$Doff+0] @ d.lo
adc $Thi,$Thi,$t1 @ T += Ch(e,f,g)
ldr $Ehi,[sp,#$Doff+4] @ d.hi
adds $Tlo,$Tlo,$t2
and $t0,$t2,#0xff
adc $Thi,$Thi,$t3 @ T += K[i]
adds $Elo,$Elo,$Tlo
ldr $t2,[sp,#$Boff+0] @ b.lo
adc $Ehi,$Ehi,$Thi @ d += T
teq $t0,#$magic
ldr $t3,[sp,#$Coff+0] @ c.lo
#if __ARM_ARCH__>=7
it eq @ Thumb2 thing, sanity check in ARM
#endif
orreq $Ktbl,$Ktbl,#1
@ Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39))
@ LO lo>>28^hi<<4 ^ hi>>2^lo<<30 ^ hi>>7^lo<<25
@ HI hi>>28^lo<<4 ^ lo>>2^hi<<30 ^ lo>>7^hi<<25
mov $t0,$Alo,lsr#28
mov $t1,$Ahi,lsr#28
eor $t0,$t0,$Ahi,lsl#4
eor $t1,$t1,$Alo,lsl#4
eor $t0,$t0,$Ahi,lsr#2
eor $t1,$t1,$Alo,lsr#2
eor $t0,$t0,$Alo,lsl#30
eor $t1,$t1,$Ahi,lsl#30
eor $t0,$t0,$Ahi,lsr#7
eor $t1,$t1,$Alo,lsr#7
eor $t0,$t0,$Alo,lsl#25
eor $t1,$t1,$Ahi,lsl#25 @ Sigma0(a)
adds $Tlo,$Tlo,$t0
and $t0,$Alo,$t2
adc $Thi,$Thi,$t1 @ T += Sigma0(a)
ldr $t1,[sp,#$Boff+4] @ b.hi
orr $Alo,$Alo,$t2
ldr $t2,[sp,#$Coff+4] @ c.hi
and $Alo,$Alo,$t3
and $t3,$Ahi,$t1
orr $Ahi,$Ahi,$t1
orr $Alo,$Alo,$t0 @ Maj(a,b,c).lo
and $Ahi,$Ahi,$t2
adds $Alo,$Alo,$Tlo
orr $Ahi,$Ahi,$t3 @ Maj(a,b,c).hi
sub sp,sp,#8
adc $Ahi,$Ahi,$Thi @ h += T
tst $Ktbl,#1
add $Ktbl,$Ktbl,#8
___
}
$code=<<___;
#ifndef __KERNEL__
# include "arm_arch.h"
# define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
# define VFP_ABI_POP vldmia sp!,{d8-d15}
#else
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
# define __ARM_MAX_ARCH__ 7
# define VFP_ABI_PUSH
# define VFP_ABI_POP
#endif
#ifdef __ARMEL__
# define LO 0
# define HI 4
# define WORD64(hi0,lo0,hi1,lo1) .word lo0,hi0, lo1,hi1
#else
# define HI 0
# define LO 4
# define WORD64(hi0,lo0,hi1,lo1) .word hi0,lo0, hi1,lo1
#endif
.text
#if __ARM_ARCH__<7
.code 32
#else
.syntax unified
# ifdef __thumb2__
# define adrl adr
.thumb
# else
.code 32
# endif
#endif
.type K512,%object
.align 5
K512:
WORD64(0x428a2f98,0xd728ae22, 0x71374491,0x23ef65cd)
WORD64(0xb5c0fbcf,0xec4d3b2f, 0xe9b5dba5,0x8189dbbc)
WORD64(0x3956c25b,0xf348b538, 0x59f111f1,0xb605d019)
WORD64(0x923f82a4,0xaf194f9b, 0xab1c5ed5,0xda6d8118)
WORD64(0xd807aa98,0xa3030242, 0x12835b01,0x45706fbe)
WORD64(0x243185be,0x4ee4b28c, 0x550c7dc3,0xd5ffb4e2)
WORD64(0x72be5d74,0xf27b896f, 0x80deb1fe,0x3b1696b1)
WORD64(0x9bdc06a7,0x25c71235, 0xc19bf174,0xcf692694)
WORD64(0xe49b69c1,0x9ef14ad2, 0xefbe4786,0x384f25e3)
WORD64(0x0fc19dc6,0x8b8cd5b5, 0x240ca1cc,0x77ac9c65)
WORD64(0x2de92c6f,0x592b0275, 0x4a7484aa,0x6ea6e483)
WORD64(0x5cb0a9dc,0xbd41fbd4, 0x76f988da,0x831153b5)
WORD64(0x983e5152,0xee66dfab, 0xa831c66d,0x2db43210)
WORD64(0xb00327c8,0x98fb213f, 0xbf597fc7,0xbeef0ee4)
WORD64(0xc6e00bf3,0x3da88fc2, 0xd5a79147,0x930aa725)
WORD64(0x06ca6351,0xe003826f, 0x14292967,0x0a0e6e70)
WORD64(0x27b70a85,0x46d22ffc, 0x2e1b2138,0x5c26c926)
WORD64(0x4d2c6dfc,0x5ac42aed, 0x53380d13,0x9d95b3df)
WORD64(0x650a7354,0x8baf63de, 0x766a0abb,0x3c77b2a8)
WORD64(0x81c2c92e,0x47edaee6, 0x92722c85,0x1482353b)
WORD64(0xa2bfe8a1,0x4cf10364, 0xa81a664b,0xbc423001)
WORD64(0xc24b8b70,0xd0f89791, 0xc76c51a3,0x0654be30)
WORD64(0xd192e819,0xd6ef5218, 0xd6990624,0x5565a910)
WORD64(0xf40e3585,0x5771202a, 0x106aa070,0x32bbd1b8)
WORD64(0x19a4c116,0xb8d2d0c8, 0x1e376c08,0x5141ab53)
WORD64(0x2748774c,0xdf8eeb99, 0x34b0bcb5,0xe19b48a8)
WORD64(0x391c0cb3,0xc5c95a63, 0x4ed8aa4a,0xe3418acb)
WORD64(0x5b9cca4f,0x7763e373, 0x682e6ff3,0xd6b2b8a3)
WORD64(0x748f82ee,0x5defb2fc, 0x78a5636f,0x43172f60)
WORD64(0x84c87814,0xa1f0ab72, 0x8cc70208,0x1a6439ec)
WORD64(0x90befffa,0x23631e28, 0xa4506ceb,0xde82bde9)
WORD64(0xbef9a3f7,0xb2c67915, 0xc67178f2,0xe372532b)
WORD64(0xca273ece,0xea26619c, 0xd186b8c7,0x21c0c207)
WORD64(0xeada7dd6,0xcde0eb1e, 0xf57d4f7f,0xee6ed178)
WORD64(0x06f067aa,0x72176fba, 0x0a637dc5,0xa2c898a6)
WORD64(0x113f9804,0xbef90dae, 0x1b710b35,0x131c471b)
WORD64(0x28db77f5,0x23047d84, 0x32caab7b,0x40c72493)
WORD64(0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c)
WORD64(0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a)
WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817)
.size K512,.-K512
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
.LOPENSSL_armcap:
.word OPENSSL_armcap_P-sha512_block_data_order
.skip 32-4
#else
.skip 32
#endif
.global sha512_block_data_order
.type sha512_block_data_order,%function
sha512_block_data_order:
#if __ARM_ARCH__<7
sub r3,pc,#8 @ sha512_block_data_order
#else
adr r3,sha512_block_data_order
#endif
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
ldr r12,.LOPENSSL_armcap
ldr r12,[r3,r12] @ OPENSSL_armcap_P
tst r12,#1
bne .LNEON
#endif
add $len,$inp,$len,lsl#7 @ len to point at the end of inp
stmdb sp!,{r4-r12,lr}
sub $Ktbl,r3,#672 @ K512
sub sp,sp,#9*8
ldr $Elo,[$ctx,#$Eoff+$lo]
ldr $Ehi,[$ctx,#$Eoff+$hi]
ldr $t0, [$ctx,#$Goff+$lo]
ldr $t1, [$ctx,#$Goff+$hi]
ldr $t2, [$ctx,#$Hoff+$lo]