Commit 87d7bcee authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto update from Herbert Xu:
 - add multibuffer infrastructure (single_task_running scheduler helper,
   OKed by Peter on lkml.
 - add SHA1 multibuffer implementation for AVX2.
 - reenable "by8" AVX CTR optimisation after fixing counter overflow.
 - add APM X-Gene SoC RNG support.
 - SHA256/SHA512 now handles unaligned input correctly.
 - set lz4 decompressed length correctly.
 - fix algif socket buffer allocation failure for 64K page machines.
 - misc fixes

* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (47 commits)
  crypto: sha - Handle unaligned input data in generic sha256 and sha512.
  Revert "crypto: aesni - disable "by8" AVX CTR optimization"
  crypto: aesni - remove unused defines in "by8" variant
  crypto: aesni - fix counter overflow handling in "by8" variant
  hwrng: printk replacement
  crypto: qat - Removed unneeded partial state
  crypto: qat - Fix typo in name of tasklet_struct
  crypto: caam - Dynamic allocation of addresses for various memory blocks in CAAM.
  crypto: mcryptd - Fix typos in CRYPTO_MCRYPTD description
  crypto: algif - avoid excessive use of socket buffer in skcipher
  arm64: dts: add random number generator dts node to APM X-Gene platform.
  Documentation: rng: Add X-Gene SoC RNG driver documentation
  hwrng: xgene - add support for APM X-Gene SoC RNG support
  crypto: mv_cesa - Add missing #define
  crypto: testmgr - add test for lz4 and lz4hc
  crypto: lz4,lz4hc - fix decompression
  crypto: qat - Use pci_enable_msix_exact() instead of pci_enable_msix()
  crypto: drbg - fix maximum value checks on 32 bit systems
  crypto: drbg - fix sparse warning for cpu_to_be[32|64]
  crypto: sha-mb - sha1_mb_alg_state can be static
  ...
parents 0223f9aa be34c4ef
APM X-Gene SoC random number generator.
Required properties:
- compatible : should be "apm,xgene-rng"
- reg : specifies base physical address and size of the registers map
- clocks : phandle to clock-controller plus clock-specifier pair
- interrupts : specify the fault interrupt for the RNG device
Example:
rng: rng@10520000 {
compatible = "apm,xgene-rng";
reg = <0x0 0x10520000 0x0 0x100>;
interrupts = <0x0 0x41 0x4>;
clocks = <&rngpkaclk 0>;
};
......@@ -269,6 +269,19 @@
enable-mask = <0x2>;
clock-output-names = "rtcclk";
};
rngpkaclk: rngpkaclk@17000000 {
compatible = "apm,xgene-device-clock";
#clock-cells = <1>;
clocks = <&socplldiv2 0>;
reg = <0x0 0x17000000 0x0 0x2000>;
reg-names = "csr-reg";
csr-offset = <0xc>;
csr-mask = <0x10>;
enable-offset = <0x10>;
enable-mask = <0x10>;
clock-output-names = "rngpkaclk";
};
};
serial0: serial@1c020000 {
......@@ -421,5 +434,13 @@
};
};
rng: rng@10520000 {
compatible = "apm,xgene-rng";
reg = <0x0 0x10520000 0x0 0x100>;
interrupts = <0x0 0x41 0x4>;
clocks = <&rngpkaclk 0>;
};
};
};
......@@ -26,6 +26,7 @@ obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
obj-$(CONFIG_CRYPTO_SHA1_MB) += sha-mb/
obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o
obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
......
......@@ -79,9 +79,6 @@
#define xcounter %xmm8
#define xbyteswap %xmm9
#define xkey0 %xmm10
#define xkey3 %xmm11
#define xkey6 %xmm12
#define xkey9 %xmm13
#define xkey4 %xmm11
#define xkey8 %xmm12
#define xkey12 %xmm13
......@@ -108,6 +105,10 @@
byteswap_const:
.octa 0x000102030405060708090A0B0C0D0E0F
ddq_low_msk:
.octa 0x0000000000000000FFFFFFFFFFFFFFFF
ddq_high_add_1:
.octa 0x00000000000000010000000000000000
ddq_add_1:
.octa 0x00000000000000000000000000000001
ddq_add_2:
......@@ -169,7 +170,12 @@ ddq_add_8:
.rept (by - 1)
club DDQ_DATA, i
club XDATA, i
vpaddd var_ddq_add(%rip), xcounter, var_xdata
vpaddq var_ddq_add(%rip), xcounter, var_xdata
vptest ddq_low_msk(%rip), var_xdata
jnz 1f
vpaddq ddq_high_add_1(%rip), var_xdata, var_xdata
vpaddq ddq_high_add_1(%rip), xcounter, xcounter
1:
vpshufb xbyteswap, var_xdata, var_xdata
.set i, (i +1)
.endr
......@@ -178,7 +184,11 @@ ddq_add_8:
vpxor xkey0, xdata0, xdata0
club DDQ_DATA, by
vpaddd var_ddq_add(%rip), xcounter, xcounter
vpaddq var_ddq_add(%rip), xcounter, xcounter
vptest ddq_low_msk(%rip), xcounter
jnz 1f
vpaddq ddq_high_add_1(%rip), xcounter, xcounter
1:
.set i, 1
.rept (by - 1)
......
......@@ -481,7 +481,7 @@ static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
crypto_inc(ctrblk, AES_BLOCK_SIZE);
}
#if 0 /* temporary disabled due to failing crypto tests */
#ifdef CONFIG_AS_AVX
static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
const u8 *in, unsigned int len, u8 *iv)
{
......@@ -1522,7 +1522,7 @@ static int __init aesni_init(void)
aesni_gcm_dec_tfm = aesni_gcm_dec;
}
aesni_ctr_enc_tfm = aesni_ctr_enc;
#if 0 /* temporary disabled due to failing crypto tests */
#ifdef CONFIG_AS_AVX
if (cpu_has_avx) {
/* optimize performance of ctr mode encryption transform */
aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
......
#
# Arch-specific CryptoAPI modules.
#
avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
$(comma)4)$(comma)%ymm2,yes,no)
ifeq ($(avx2_supported),yes)
obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb.o
sha1-mb-y := sha1_mb.o sha1_mb_mgr_flush_avx2.o \
sha1_mb_mgr_init_avx2.o sha1_mb_mgr_submit_avx2.o sha1_x8_avx2.o
endif
This diff is collapsed.
/*
* Header file for multi buffer SHA1 algorithm data structure
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright(c) 2014 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* Contact Information:
* James Guilford <james.guilford@intel.com>
* Tim Chen <tim.c.chen@linux.intel.com>
*
* BSD LICENSE
*
* Copyright(c) 2014 Intel Corporation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
# Macros for defining data structures
# Usage example
#START_FIELDS # JOB_AES
### name size align
#FIELD _plaintext, 8, 8 # pointer to plaintext
#FIELD _ciphertext, 8, 8 # pointer to ciphertext
#FIELD _IV, 16, 8 # IV
#FIELD _keys, 8, 8 # pointer to keys
#FIELD _len, 4, 4 # length in bytes
#FIELD _status, 4, 4 # status enumeration
#FIELD _user_data, 8, 8 # pointer to user data
#UNION _union, size1, align1, \
# size2, align2, \
# size3, align3, \
# ...
#END_FIELDS
#%assign _JOB_AES_size _FIELD_OFFSET
#%assign _JOB_AES_align _STRUCT_ALIGN
#########################################################################
# Alternate "struc-like" syntax:
# STRUCT job_aes2
# RES_Q .plaintext, 1
# RES_Q .ciphertext, 1
# RES_DQ .IV, 1
# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN
# RES_U .union, size1, align1, \
# size2, align2, \
# ...
# ENDSTRUCT
# # Following only needed if nesting
# %assign job_aes2_size _FIELD_OFFSET
# %assign job_aes2_align _STRUCT_ALIGN
#
# RES_* macros take a name, a count and an optional alignment.
# The count in in terms of the base size of the macro, and the
# default alignment is the base size.
# The macros are:
# Macro Base size
# RES_B 1
# RES_W 2
# RES_D 4
# RES_Q 8
# RES_DQ 16
# RES_Y 32
# RES_Z 64
#
# RES_U defines a union. It's arguments are a name and two or more
# pairs of "size, alignment"
#
# The two assigns are only needed if this structure is being nested
# within another. Even if the assigns are not done, one can still use
# STRUCT_NAME_size as the size of the structure.
#
# Note that for nesting, you still need to assign to STRUCT_NAME_size.
#
# The differences between this and using "struc" directly are that each
# type is implicitly aligned to its natural length (although this can be
# over-ridden with an explicit third parameter), and that the structure
# is padded at the end to its overall alignment.
#
#########################################################################
#ifndef _SHA1_MB_MGR_DATASTRUCT_ASM_
#define _SHA1_MB_MGR_DATASTRUCT_ASM_
## START_FIELDS
.macro START_FIELDS
_FIELD_OFFSET = 0
_STRUCT_ALIGN = 0
.endm
## FIELD name size align
.macro FIELD name size align
_FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1))
\name = _FIELD_OFFSET
_FIELD_OFFSET = _FIELD_OFFSET + (\size)
.if (\align > _STRUCT_ALIGN)
_STRUCT_ALIGN = \align
.endif
.endm
## END_FIELDS
.macro END_FIELDS
_FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
.endm
########################################################################
.macro STRUCT p1
START_FIELDS
.struc \p1
.endm
.macro ENDSTRUCT
tmp = _FIELD_OFFSET
END_FIELDS
tmp = (_FIELD_OFFSET - %%tmp)
.if (tmp > 0)
.lcomm tmp
.endif
.endstruc
.endm
## RES_int name size align
.macro RES_int p1 p2 p3
name = \p1
size = \p2
align = .\p3
_FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1))
.align align
.lcomm name size
_FIELD_OFFSET = _FIELD_OFFSET + (size)
.if (align > _STRUCT_ALIGN)
_STRUCT_ALIGN = align
.endif
.endm
# macro RES_B name, size [, align]
.macro RES_B _name, _size, _align=1
RES_int _name _size _align
.endm
# macro RES_W name, size [, align]
.macro RES_W _name, _size, _align=2
RES_int _name 2*(_size) _align
.endm
# macro RES_D name, size [, align]
.macro RES_D _name, _size, _align=4
RES_int _name 4*(_size) _align
.endm
# macro RES_Q name, size [, align]
.macro RES_Q _name, _size, _align=8
RES_int _name 8*(_size) _align
.endm
# macro RES_DQ name, size [, align]
.macro RES_DQ _name, _size, _align=16
RES_int _name 16*(_size) _align
.endm
# macro RES_Y name, size [, align]
.macro RES_Y _name, _size, _align=32
RES_int _name 32*(_size) _align
.endm
# macro RES_Z name, size [, align]
.macro RES_Z _name, _size, _align=64
RES_int _name 64*(_size) _align
.endm
#endif
########################################################################
#### Define constants
########################################################################
########################################################################
#### Define SHA1 Out Of Order Data Structures
########################################################################
START_FIELDS # LANE_DATA
### name size align
FIELD _job_in_lane, 8, 8 # pointer to job object
END_FIELDS
_LANE_DATA_size = _FIELD_OFFSET
_LANE_DATA_align = _STRUCT_ALIGN
########################################################################
START_FIELDS # SHA1_ARGS_X8
### name size align
FIELD _digest, 4*5*8, 16 # transposed digest
FIELD _data_ptr, 8*8, 8 # array of pointers to data
END_FIELDS
_SHA1_ARGS_X4_size = _FIELD_OFFSET
_SHA1_ARGS_X4_align = _STRUCT_ALIGN
_SHA1_ARGS_X8_size = _FIELD_OFFSET
_SHA1_ARGS_X8_align = _STRUCT_ALIGN
########################################################################
START_FIELDS # MB_MGR
### name size align
FIELD _args, _SHA1_ARGS_X4_size, _SHA1_ARGS_X4_align
FIELD _lens, 4*8, 8
FIELD _unused_lanes, 8, 8
FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align
END_FIELDS
_MB_MGR_size = _FIELD_OFFSET
_MB_MGR_align = _STRUCT_ALIGN
_args_digest = _args + _digest
_args_data_ptr = _args + _data_ptr
########################################################################
#### Define constants
########################################################################
#define STS_UNKNOWN 0
#define STS_BEING_PROCESSED 1
#define STS_COMPLETED 2
########################################################################
#### Define JOB_SHA1 structure
########################################################################
START_FIELDS # JOB_SHA1
### name size align
FIELD _buffer, 8, 8 # pointer to buffer
FIELD _len, 4, 4 # length in bytes
FIELD _result_digest, 5*4, 32 # Digest (output)
FIELD _status, 4, 4
FIELD _user_data, 8, 8
END_FIELDS
_JOB_SHA1_size = _FIELD_OFFSET
_JOB_SHA1_align = _STRUCT_ALIGN
/*
* Flush routine for SHA1 multibuffer
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright(c) 2014 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* Contact Information:
* James Guilford <james.guilford@intel.com>
* Tim Chen <tim.c.chen@linux.intel.com>
*
* BSD LICENSE
*
* Copyright(c) 2014 Intel Corporation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <linux/linkage.h>
#include "sha1_mb_mgr_datastruct.S"
.extern sha1_x8_avx2
# LINUX register definitions
#define arg1 %rdi
#define arg2 %rsi
# Common definitions
#define state arg1
#define job arg2
#define len2 arg2
# idx must be a register not clobbered by sha1_x8_avx2
#define idx %r8
#define DWORD_idx %r8d
#define unused_lanes %rbx
#define lane_data %rbx
#define tmp2 %rbx
#define tmp2_w %ebx
#define job_rax %rax
#define tmp1 %rax
#define size_offset %rax
#define tmp %rax
#define start_offset %rax
#define tmp3 %arg1
#define extra_blocks %arg2
#define p %arg2
# STACK_SPACE needs to be an odd multiple of 8
_XMM_SAVE_SIZE = 10*16
_GPR_SAVE_SIZE = 8*8
_ALIGN_SIZE = 8
_XMM_SAVE = 0
_GPR_SAVE = _XMM_SAVE + _XMM_SAVE_SIZE
STACK_SPACE = _GPR_SAVE + _GPR_SAVE_SIZE + _ALIGN_SIZE
.macro LABEL prefix n
\prefix\n\():
.endm
.macro JNE_SKIP i
jne skip_\i
.endm
.altmacro
.macro SET_OFFSET _offset
offset = \_offset
.endm
.noaltmacro
# JOB* sha1_mb_mgr_flush_avx2(MB_MGR *state)
# arg 1 : rcx : state
ENTRY(sha1_mb_mgr_flush_avx2)
mov %rsp, %r10
sub $STACK_SPACE, %rsp
and $~31, %rsp
mov %rbx, _GPR_SAVE(%rsp)
mov %r10, _GPR_SAVE+8*1(%rsp) #save rsp
mov %rbp, _GPR_SAVE+8*3(%rsp)
mov %r12, _GPR_SAVE+8*4(%rsp)
mov %r13, _GPR_SAVE+8*5(%rsp)
mov %r14, _GPR_SAVE+8*6(%rsp)
mov %r15, _GPR_SAVE+8*7(%rsp)
# If bit (32+3) is set, then all lanes are empty
mov _unused_lanes(state), unused_lanes
bt $32+3, unused_lanes
jc return_null
# find a lane with a non-null job
xor idx, idx
offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
cmovne one(%rip), idx
offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
cmovne two(%rip), idx
offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
cmovne three(%rip), idx
offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
cmovne four(%rip), idx
offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
cmovne five(%rip), idx
offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
cmovne six(%rip), idx
offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
cmovne seven(%rip), idx
# copy idx to empty lanes
copy_lane_data:
offset = (_args + _data_ptr)
mov offset(state,idx,8), tmp
I = 0
.rep 8
offset = (_ldata + I * _LANE_DATA_size + _job_in_lane)
cmpq $0, offset(state)
.altmacro