From 8db41da676ac8368ef7c2549d56239a5ff5eedde Mon Sep 17 00:00:00 2001 From: Rutger Broekhoff Date: Tue, 2 Jan 2024 18:56:31 +0100 Subject: Delete vendor directory --- .../minio/sha256-simd/sha256blockAvx512_amd64.asm | 686 --------------------- 1 file changed, 686 deletions(-) delete mode 100644 vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.asm (limited to 'vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.asm') diff --git a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.asm b/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.asm deleted file mode 100644 index c959b1a..0000000 --- a/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.asm +++ /dev/null @@ -1,686 +0,0 @@ - -// 16x Parallel implementation of SHA256 for AVX512 - -// -// Minio Cloud Storage, (C) 2017 Minio, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// -// This code is based on the Intel Multi-Buffer Crypto for IPSec library -// and more specifically the following implementation: -// https://github.com/intel/intel-ipsec-mb/blob/master/avx512/sha256_x16_avx512.asm -// -// For Golang it has been converted into Plan 9 assembly with the help of -// github.com/minio/asm2plan9s to assemble the AVX512 instructions -// - -// Copyright (c) 2017, Intel Corporation -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// * Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of Intel Corporation nor the names of its contributors -// may be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#define SHA256_DIGEST_ROW_SIZE 64 - -// arg1 -#define STATE rdi -#define STATE_P9 DI -// arg2 -#define INP_SIZE rsi -#define INP_SIZE_P9 SI - -#define IDX rcx -#define TBL rdx -#define TBL_P9 DX - -#define INPUT rax -#define INPUT_P9 AX - -#define inp0 r9 -#define SCRATCH_P9 R12 -#define SCRATCH r12 -#define maskp r13 -#define MASKP_P9 R13 -#define mask r14 -#define MASK_P9 R14 - -#define A zmm0 -#define B zmm1 -#define C zmm2 -#define D zmm3 -#define E zmm4 -#define F zmm5 -#define G zmm6 -#define H zmm7 -#define T1 zmm8 -#define TMP0 zmm9 -#define TMP1 zmm10 -#define TMP2 zmm11 -#define TMP3 zmm12 -#define TMP4 zmm13 -#define TMP5 zmm14 -#define TMP6 zmm15 - -#define W0 zmm16 -#define W1 zmm17 -#define W2 zmm18 -#define W3 zmm19 -#define W4 zmm20 -#define W5 zmm21 -#define W6 zmm22 -#define W7 zmm23 -#define W8 zmm24 -#define W9 zmm25 -#define W10 zmm26 -#define W11 zmm27 -#define W12 zmm28 -#define W13 zmm29 -#define W14 zmm30 -#define W15 zmm31 - - -#define TRANSPOSE16(_r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7, _r8, _r9, _r10, _r11, _r12, _r13, _r14, _r15, _t0, _t1) \ - \ - \ // input r0 = {a15 a14 a13 a12 a11 a10 a9 a8 a7 a6 a5 a4 a3 a2 a1 a0} - \ // r1 = {b15 b14 b13 b12 b11 b10 b9 b8 b7 b6 b5 b4 b3 b2 b1 b0} - \ // r2 = {c15 c14 c13 c12 c11 c10 c9 c8 c7 c6 c5 c4 c3 c2 c1 c0} - \ // r3 = {d15 d14 d13 d12 d11 d10 d9 d8 d7 d6 d5 d4 d3 d2 d1 d0} - \ // r4 = {e15 e14 e13 e12 e11 e10 e9 e8 e7 e6 e5 e4 e3 e2 e1 e0} - \ // r5 = {f15 f14 f13 f12 f11 f10 f9 f8 f7 f6 f5 f4 f3 f2 f1 f0} - \ // r6 = {g15 g14 g13 g12 g11 g10 g9 g8 g7 g6 g5 g4 g3 g2 g1 g0} - \ // r7 = {h15 h14 h13 h12 h11 h10 h9 h8 h7 h6 h5 h4 h3 h2 h1 h0} - \ // r8 = {i15 i14 i13 i12 i11 i10 i9 i8 i7 i6 i5 i4 i3 i2 i1 i0} - \ // r9 = {j15 j14 j13 j12 j11 j10 j9 j8 j7 j6 j5 j4 j3 j2 j1 j0} - \ // r10 = {k15 k14 k13 k12 k11 k10 k9 k8 k7 k6 k5 k4 k3 k2 k1 k0} - \ // r11 = {l15 l14 l13 l12 l11 l10 l9 l8 l7 l6 l5 l4 l3 l2 l1 l0} - \ // r12 = {m15 m14 m13 m12 m11 m10 m9 m8 m7 m6 m5 m4 m3 m2 m1 m0} - \ // r13 = {n15 n14 n13 n12 n11 n10 n9 n8 n7 n6 n5 n4 n3 n2 n1 n0} - \ // r14 = {o15 o14 o13 o12 o11 o10 o9 o8 o7 o6 o5 o4 o3 o2 o1 o0} - \ // r15 = {p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0} - \ - \ // output r0 = { p0 o0 n0 m0 l0 k0 j0 i0 h0 g0 f0 e0 d0 c0 b0 a0} - \ // r1 = { p1 o1 n1 m1 l1 k1 j1 i1 h1 g1 f1 e1 d1 c1 b1 a1} - \ // r2 = { p2 o2 n2 m2 l2 k2 j2 i2 h2 g2 f2 e2 d2 c2 b2 a2} - \ // r3 = { p3 o3 n3 m3 l3 k3 j3 i3 h3 g3 f3 e3 d3 c3 b3 a3} - \ // r4 = { p4 o4 n4 m4 l4 k4 j4 i4 h4 g4 f4 e4 d4 c4 b4 a4} - \ // r5 = { p5 o5 n5 m5 l5 k5 j5 i5 h5 g5 f5 e5 d5 c5 b5 a5} - \ // r6 = { p6 o6 n6 m6 l6 k6 j6 i6 h6 g6 f6 e6 d6 c6 b6 a6} - \ // r7 = { p7 o7 n7 m7 l7 k7 j7 i7 h7 g7 f7 e7 d7 c7 b7 a7} - \ // r8 = { p8 o8 n8 m8 l8 k8 j8 i8 h8 g8 f8 e8 d8 c8 b8 a8} - \ // r9 = { p9 o9 n9 m9 l9 k9 j9 i9 h9 g9 f9 e9 d9 c9 b9 a9} - \ // r10 = {p10 o10 n10 m10 l10 k10 j10 i10 h10 g10 f10 e10 d10 c10 b10 a10} - \ // r11 = {p11 o11 n11 m11 l11 k11 j11 i11 h11 g11 f11 e11 d11 c11 b11 a11} - \ // r12 = {p12 o12 n12 m12 l12 k12 j12 i12 h12 g12 f12 e12 d12 c12 b12 a12} - \ // r13 = {p13 o13 n13 m13 l13 k13 j13 i13 h13 g13 f13 e13 d13 c13 b13 a13} - \ // r14 = {p14 o14 n14 m14 l14 k14 j14 i14 h14 g14 f14 e14 d14 c14 b14 a14} - \ // r15 = {p15 o15 n15 m15 l15 k15 j15 i15 h15 g15 f15 e15 d15 c15 b15 a15} - \ - \ // process top half - vshufps _t0, _r0, _r1, 0x44 \ // t0 = {b13 b12 a13 a12 b9 b8 a9 a8 b5 b4 a5 a4 b1 b0 a1 a0} - vshufps _r0, _r0, _r1, 0xEE \ // r0 = {b15 b14 a15 a14 b11 b10 a11 a10 b7 b6 a7 a6 b3 b2 a3 a2} - vshufps _t1, _r2, _r3, 0x44 \ // t1 = {d13 d12 c13 c12 d9 d8 c9 c8 d5 d4 c5 c4 d1 d0 c1 c0} - vshufps _r2, _r2, _r3, 0xEE \ // r2 = {d15 d14 c15 c14 d11 d10 c11 c10 d7 d6 c7 c6 d3 d2 c3 c2} - \ - vshufps _r3, _t0, _t1, 0xDD \ // r3 = {d13 c13 b13 a13 d9 c9 b9 a9 d5 c5 b5 a5 d1 c1 b1 a1} - vshufps _r1, _r0, _r2, 0x88 \ // r1 = {d14 c14 b14 a14 d10 c10 b10 a10 d6 c6 b6 a6 d2 c2 b2 a2} - vshufps _r0, _r0, _r2, 0xDD \ // r0 = {d15 c15 b15 a15 d11 c11 b11 a11 d7 c7 b7 a7 d3 c3 b3 a3} - vshufps _t0, _t0, _t1, 0x88 \ // t0 = {d12 c12 b12 a12 d8 c8 b8 a8 d4 c4 b4 a4 d0 c0 b0 a0} - \ - \ // use r2 in place of t0 - vshufps _r2, _r4, _r5, 0x44 \ // r2 = {f13 f12 e13 e12 f9 f8 e9 e8 f5 f4 e5 e4 f1 f0 e1 e0} - vshufps _r4, _r4, _r5, 0xEE \ // r4 = {f15 f14 e15 e14 f11 f10 e11 e10 f7 f6 e7 e6 f3 f2 e3 e2} - vshufps _t1, _r6, _r7, 0x44 \ // t1 = {h13 h12 g13 g12 h9 h8 g9 g8 h5 h4 g5 g4 h1 h0 g1 g0} - vshufps _r6, _r6, _r7, 0xEE \ // r6 = {h15 h14 g15 g14 h11 h10 g11 g10 h7 h6 g7 g6 h3 h2 g3 g2} - \ - vshufps _r7, _r2, _t1, 0xDD \ // r7 = {h13 g13 f13 e13 h9 g9 f9 e9 h5 g5 f5 e5 h1 g1 f1 e1} - vshufps _r5, _r4, _r6, 0x88 \ // r5 = {h14 g14 f14 e14 h10 g10 f10 e10 h6 g6 f6 e6 h2 g2 f2 e2} - vshufps _r4, _r4, _r6, 0xDD \ // r4 = {h15 g15 f15 e15 h11 g11 f11 e11 h7 g7 f7 e7 h3 g3 f3 e3} - vshufps _r2, _r2, _t1, 0x88 \ // r2 = {h12 g12 f12 e12 h8 g8 f8 e8 h4 g4 f4 e4 h0 g0 f0 e0} - \ - \ // use r6 in place of t0 - vshufps _r6, _r8, _r9, 0x44 \ // r6 = {j13 j12 i13 i12 j9 j8 i9 i8 j5 j4 i5 i4 j1 j0 i1 i0} - vshufps _r8, _r8, _r9, 0xEE \ // r8 = {j15 j14 i15 i14 j11 j10 i11 i10 j7 j6 i7 i6 j3 j2 i3 i2} - vshufps _t1, _r10, _r11, 0x44 \ // t1 = {l13 l12 k13 k12 l9 l8 k9 k8 l5 l4 k5 k4 l1 l0 k1 k0} - vshufps _r10, _r10, _r11, 0xEE \ // r10 = {l15 l14 k15 k14 l11 l10 k11 k10 l7 l6 k7 k6 l3 l2 k3 k2} - \ - vshufps _r11, _r6, _t1, 0xDD \ // r11 = {l13 k13 j13 113 l9 k9 j9 i9 l5 k5 j5 i5 l1 k1 j1 i1} - vshufps _r9, _r8, _r10, 0x88 \ // r9 = {l14 k14 j14 114 l10 k10 j10 i10 l6 k6 j6 i6 l2 k2 j2 i2} - vshufps _r8, _r8, _r10, 0xDD \ // r8 = {l15 k15 j15 115 l11 k11 j11 i11 l7 k7 j7 i7 l3 k3 j3 i3} - vshufps _r6, _r6, _t1, 0x88 \ // r6 = {l12 k12 j12 112 l8 k8 j8 i8 l4 k4 j4 i4 l0 k0 j0 i0} - \ - \ // use r10 in place of t0 - vshufps _r10, _r12, _r13, 0x44 \ // r10 = {n13 n12 m13 m12 n9 n8 m9 m8 n5 n4 m5 m4 n1 n0 a1 m0} - vshufps _r12, _r12, _r13, 0xEE \ // r12 = {n15 n14 m15 m14 n11 n10 m11 m10 n7 n6 m7 m6 n3 n2 a3 m2} - vshufps _t1, _r14, _r15, 0x44 \ // t1 = {p13 p12 013 012 p9 p8 09 08 p5 p4 05 04 p1 p0 01 00} - vshufps _r14, _r14, _r15, 0xEE \ // r14 = {p15 p14 015 014 p11 p10 011 010 p7 p6 07 06 p3 p2 03 02} - \ - vshufps _r15, _r10, _t1, 0xDD \ // r15 = {p13 013 n13 m13 p9 09 n9 m9 p5 05 n5 m5 p1 01 n1 m1} - vshufps _r13, _r12, _r14, 0x88 \ // r13 = {p14 014 n14 m14 p10 010 n10 m10 p6 06 n6 m6 p2 02 n2 m2} - vshufps _r12, _r12, _r14, 0xDD \ // r12 = {p15 015 n15 m15 p11 011 n11 m11 p7 07 n7 m7 p3 03 n3 m3} - vshufps _r10, _r10, _t1, 0x88 \ // r10 = {p12 012 n12 m12 p8 08 n8 m8 p4 04 n4 m4 p0 00 n0 m0} - \ - \ // At this point, the registers that contain interesting data are: - \ // t0, r3, r1, r0, r2, r7, r5, r4, r6, r11, r9, r8, r10, r15, r13, r12 - \ // Can use t1 and r14 as scratch registers - LEAQ PSHUFFLE_TRANSPOSE16_MASK1<>(SB), BX \ - LEAQ PSHUFFLE_TRANSPOSE16_MASK2<>(SB), R8 \ - \ - vmovdqu32 _r14, [rbx] \ - vpermi2q _r14, _t0, _r2 \ // r14 = {h8 g8 f8 e8 d8 c8 b8 a8 h0 g0 f0 e0 d0 c0 b0 a0} - vmovdqu32 _t1, [r8] \ - vpermi2q _t1, _t0, _r2 \ // t1 = {h12 g12 f12 e12 d12 c12 b12 a12 h4 g4 f4 e4 d4 c4 b4 a4} - \ - vmovdqu32 _r2, [rbx] \ - vpermi2q _r2, _r3, _r7 \ // r2 = {h9 g9 f9 e9 d9 c9 b9 a9 h1 g1 f1 e1 d1 c1 b1 a1} - vmovdqu32 _t0, [r8] \ - vpermi2q _t0, _r3, _r7 \ // t0 = {h13 g13 f13 e13 d13 c13 b13 a13 h5 g5 f5 e5 d5 c5 b5 a5} - \ - vmovdqu32 _r3, [rbx] \ - vpermi2q _r3, _r1, _r5 \ // r3 = {h10 g10 f10 e10 d10 c10 b10 a10 h2 g2 f2 e2 d2 c2 b2 a2} - vmovdqu32 _r7, [r8] \ - vpermi2q _r7, _r1, _r5 \ // r7 = {h14 g14 f14 e14 d14 c14 b14 a14 h6 g6 f6 e6 d6 c6 b6 a6} - \ - vmovdqu32 _r1, [rbx] \ - vpermi2q _r1, _r0, _r4 \ // r1 = {h11 g11 f11 e11 d11 c11 b11 a11 h3 g3 f3 e3 d3 c3 b3 a3} - vmovdqu32 _r5, [r8] \ - vpermi2q _r5, _r0, _r4 \ // r5 = {h15 g15 f15 e15 d15 c15 b15 a15 h7 g7 f7 e7 d7 c7 b7 a7} - \ - vmovdqu32 _r0, [rbx] \ - vpermi2q _r0, _r6, _r10 \ // r0 = {p8 o8 n8 m8 l8 k8 j8 i8 p0 o0 n0 m0 l0 k0 j0 i0} - vmovdqu32 _r4, [r8] \ - vpermi2q _r4, _r6, _r10 \ // r4 = {p12 o12 n12 m12 l12 k12 j12 i12 p4 o4 n4 m4 l4 k4 j4 i4} - \ - vmovdqu32 _r6, [rbx] \ - vpermi2q _r6, _r11, _r15 \ // r6 = {p9 o9 n9 m9 l9 k9 j9 i9 p1 o1 n1 m1 l1 k1 j1 i1} - vmovdqu32 _r10, [r8] \ - vpermi2q _r10, _r11, _r15 \ // r10 = {p13 o13 n13 m13 l13 k13 j13 i13 p5 o5 n5 m5 l5 k5 j5 i5} - \ - vmovdqu32 _r11, [rbx] \ - vpermi2q _r11, _r9, _r13 \ // r11 = {p10 o10 n10 m10 l10 k10 j10 i10 p2 o2 n2 m2 l2 k2 j2 i2} - vmovdqu32 _r15, [r8] \ - vpermi2q _r15, _r9, _r13 \ // r15 = {p14 o14 n14 m14 l14 k14 j14 i14 p6 o6 n6 m6 l6 k6 j6 i6} - \ - vmovdqu32 _r9, [rbx] \ - vpermi2q _r9, _r8, _r12 \ // r9 = {p11 o11 n11 m11 l11 k11 j11 i11 p3 o3 n3 m3 l3 k3 j3 i3} - vmovdqu32 _r13, [r8] \ - vpermi2q _r13, _r8, _r12 \ // r13 = {p15 o15 n15 m15 l15 k15 j15 i15 p7 o7 n7 m7 l7 k7 j7 i7} - \ - \ // At this point r8 and r12 can be used as scratch registers - vshuff64x2 _r8, _r14, _r0, 0xEE \ // r8 = {p8 o8 n8 m8 l8 k8 j8 i8 h8 g8 f8 e8 d8 c8 b8 a8} - vshuff64x2 _r0, _r14, _r0, 0x44 \ // r0 = {p0 o0 n0 m0 l0 k0 j0 i0 h0 g0 f0 e0 d0 c0 b0 a0} - \ - vshuff64x2 _r12, _t1, _r4, 0xEE \ // r12 = {p12 o12 n12 m12 l12 k12 j12 i12 h12 g12 f12 e12 d12 c12 b12 a12} - vshuff64x2 _r4, _t1, _r4, 0x44 \ // r4 = {p4 o4 n4 m4 l4 k4 j4 i4 h4 g4 f4 e4 d4 c4 b4 a4} - \ - vshuff64x2 _r14, _r7, _r15, 0xEE \ // r14 = {p14 o14 n14 m14 l14 k14 j14 i14 h14 g14 f14 e14 d14 c14 b14 a14} - vshuff64x2 _t1, _r7, _r15, 0x44 \ // t1 = {p6 o6 n6 m6 l6 k6 j6 i6 h6 g6 f6 e6 d6 c6 b6 a6} - \ - vshuff64x2 _r15, _r5, _r13, 0xEE \ // r15 = {p15 o15 n15 m15 l15 k15 j15 i15 h15 g15 f15 e15 d15 c15 b15 a15} - vshuff64x2 _r7, _r5, _r13, 0x44 \ // r7 = {p7 o7 n7 m7 l7 k7 j7 i7 h7 g7 f7 e7 d7 c7 b7 a7} - \ - vshuff64x2 _r13, _t0, _r10, 0xEE \ // r13 = {p13 o13 n13 m13 l13 k13 j13 i13 h13 g13 f13 e13 d13 c13 b13 a13} - vshuff64x2 _r5, _t0, _r10, 0x44 \ // r5 = {p5 o5 n5 m5 l5 k5 j5 i5 h5 g5 f5 e5 d5 c5 b5 a5} - \ - vshuff64x2 _r10, _r3, _r11, 0xEE \ // r10 = {p10 o10 n10 m10 l10 k10 j10 i10 h10 g10 f10 e10 d10 c10 b10 a10} - vshuff64x2 _t0, _r3, _r11, 0x44 \ // t0 = {p2 o2 n2 m2 l2 k2 j2 i2 h2 g2 f2 e2 d2 c2 b2 a2} - \ - vshuff64x2 _r11, _r1, _r9, 0xEE \ // r11 = {p11 o11 n11 m11 l11 k11 j11 i11 h11 g11 f11 e11 d11 c11 b11 a11} - vshuff64x2 _r3, _r1, _r9, 0x44 \ // r3 = {p3 o3 n3 m3 l3 k3 j3 i3 h3 g3 f3 e3 d3 c3 b3 a3} - \ - vshuff64x2 _r9, _r2, _r6, 0xEE \ // r9 = {p9 o9 n9 m9 l9 k9 j9 i9 h9 g9 f9 e9 d9 c9 b9 a9} - vshuff64x2 _r1, _r2, _r6, 0x44 \ // r1 = {p1 o1 n1 m1 l1 k1 j1 i1 h1 g1 f1 e1 d1 c1 b1 a1} - \ - vmovdqu32 _r2, _t0 \ // r2 = {p2 o2 n2 m2 l2 k2 j2 i2 h2 g2 f2 e2 d2 c2 b2 a2} - vmovdqu32 _r6, _t1 \ // r6 = {p6 o6 n6 m6 l6 k6 j6 i6 h6 g6 f6 e6 d6 c6 b6 a6} - - -// CH(A, B, C) = (A&B) ^ (~A&C) -// MAJ(E, F, G) = (E&F) ^ (E&G) ^ (F&G) -// SIGMA0 = ROR_2 ^ ROR_13 ^ ROR_22 -// SIGMA1 = ROR_6 ^ ROR_11 ^ ROR_25 -// sigma0 = ROR_7 ^ ROR_18 ^ SHR_3 -// sigma1 = ROR_17 ^ ROR_19 ^ SHR_10 - -// Main processing loop per round -#define PROCESS_LOOP(_WT, _ROUND, _A, _B, _C, _D, _E, _F, _G, _H) \ - \ // T1 = H + SIGMA1(E) + CH(E, F, G) + Kt + Wt - \ // T2 = SIGMA0(A) + MAJ(A, B, C) - \ // H=G, G=F, F=E, E=D+T1, D=C, C=B, B=A, A=T1+T2 - \ - \ // H becomes T2, then add T1 for A - \ // D becomes D + T1 for E - \ - vpaddd T1, _H, TMP3 \ // T1 = H + Kt - vmovdqu32 TMP0, _E \ - vprord TMP1, _E, 6 \ // ROR_6(E) - vprord TMP2, _E, 11 \ // ROR_11(E) - vprord TMP3, _E, 25 \ // ROR_25(E) - vpternlogd TMP0, _F, _G, 0xCA \ // TMP0 = CH(E,F,G) - vpaddd T1, T1, _WT \ // T1 = T1 + Wt - vpternlogd TMP1, TMP2, TMP3, 0x96 \ // TMP1 = SIGMA1(E) - vpaddd T1, T1, TMP0 \ // T1 = T1 + CH(E,F,G) - vpaddd T1, T1, TMP1 \ // T1 = T1 + SIGMA1(E) - vpaddd _D, _D, T1 \ // D = D + T1 - \ - vprord _H, _A, 2 \ // ROR_2(A) - vprord TMP2, _A, 13 \ // ROR_13(A) - vprord TMP3, _A, 22 \ // ROR_22(A) - vmovdqu32 TMP0, _A \ - vpternlogd TMP0, _B, _C, 0xE8 \ // TMP0 = MAJ(A,B,C) - vpternlogd _H, TMP2, TMP3, 0x96 \ // H(T2) = SIGMA0(A) - vpaddd _H, _H, TMP0 \ // H(T2) = SIGMA0(A) + MAJ(A,B,C) - vpaddd _H, _H, T1 \ // H(A) = H(T2) + T1 - \ - vmovdqu32 TMP3, [TBL + ((_ROUND+1)*64)] \ // Next Kt - - -#define MSG_SCHED_ROUND_16_63(_WT, _WTp1, _WTp9, _WTp14) \ - vprord TMP4, _WTp14, 17 \ // ROR_17(Wt-2) - vprord TMP5, _WTp14, 19 \ // ROR_19(Wt-2) - vpsrld TMP6, _WTp14, 10 \ // SHR_10(Wt-2) - vpternlogd TMP4, TMP5, TMP6, 0x96 \ // TMP4 = sigma1(Wt-2) - \ - vpaddd _WT, _WT, TMP4 \ // Wt = Wt-16 + sigma1(Wt-2) - vpaddd _WT, _WT, _WTp9 \ // Wt = Wt-16 + sigma1(Wt-2) + Wt-7 - \ - vprord TMP4, _WTp1, 7 \ // ROR_7(Wt-15) - vprord TMP5, _WTp1, 18 \ // ROR_18(Wt-15) - vpsrld TMP6, _WTp1, 3 \ // SHR_3(Wt-15) - vpternlogd TMP4, TMP5, TMP6, 0x96 \ // TMP4 = sigma0(Wt-15) - \ - vpaddd _WT, _WT, TMP4 \ // Wt = Wt-16 + sigma1(Wt-2) + - \ // Wt-7 + sigma0(Wt-15) + - - -// Note this is reading in a block of data for one lane -// When all 16 are read, the data must be transposed to build msg schedule -#define MSG_SCHED_ROUND_00_15(_WT, OFFSET, LABEL) \ - TESTQ $(1<(SB), TBL_P9 - vmovdqu32 TMP2, [TBL] - - // Get first K from table - MOVQ table+16(FP), TBL_P9 - vmovdqu32 TMP3, [TBL] - - // Save digests for later addition - vmovdqu32 [SCRATCH + 64*0], A - vmovdqu32 [SCRATCH + 64*1], B - vmovdqu32 [SCRATCH + 64*2], C - vmovdqu32 [SCRATCH + 64*3], D - vmovdqu32 [SCRATCH + 64*4], E - vmovdqu32 [SCRATCH + 64*5], F - vmovdqu32 [SCRATCH + 64*6], G - vmovdqu32 [SCRATCH + 64*7], H - - add IDX, 64 - - // Transpose input data - TRANSPOSE16(W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1) - - vpshufb W0, W0, TMP2 - vpshufb W1, W1, TMP2 - vpshufb W2, W2, TMP2 - vpshufb W3, W3, TMP2 - vpshufb W4, W4, TMP2 - vpshufb W5, W5, TMP2 - vpshufb W6, W6, TMP2 - vpshufb W7, W7, TMP2 - vpshufb W8, W8, TMP2 - vpshufb W9, W9, TMP2 - vpshufb W10, W10, TMP2 - vpshufb W11, W11, TMP2 - vpshufb W12, W12, TMP2 - vpshufb W13, W13, TMP2 - vpshufb W14, W14, TMP2 - vpshufb W15, W15, TMP2 - - // MSG Schedule for W0-W15 is now complete in registers - // Process first 48 rounds - // Calculate next Wt+16 after processing is complete and Wt is unneeded - - PROCESS_LOOP( W0, 0, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_16_63( W0, W1, W9, W14) - PROCESS_LOOP( W1, 1, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_16_63( W1, W2, W10, W15) - PROCESS_LOOP( W2, 2, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_16_63( W2, W3, W11, W0) - PROCESS_LOOP( W3, 3, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_16_63( W3, W4, W12, W1) - PROCESS_LOOP( W4, 4, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_16_63( W4, W5, W13, W2) - PROCESS_LOOP( W5, 5, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_16_63( W5, W6, W14, W3) - PROCESS_LOOP( W6, 6, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_16_63( W6, W7, W15, W4) - PROCESS_LOOP( W7, 7, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_16_63( W7, W8, W0, W5) - PROCESS_LOOP( W8, 8, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_16_63( W8, W9, W1, W6) - PROCESS_LOOP( W9, 9, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_16_63( W9, W10, W2, W7) - PROCESS_LOOP(W10, 10, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_16_63(W10, W11, W3, W8) - PROCESS_LOOP(W11, 11, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_16_63(W11, W12, W4, W9) - PROCESS_LOOP(W12, 12, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_16_63(W12, W13, W5, W10) - PROCESS_LOOP(W13, 13, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_16_63(W13, W14, W6, W11) - PROCESS_LOOP(W14, 14, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_16_63(W14, W15, W7, W12) - PROCESS_LOOP(W15, 15, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_16_63(W15, W0, W8, W13) - PROCESS_LOOP( W0, 16, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_16_63( W0, W1, W9, W14) - PROCESS_LOOP( W1, 17, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_16_63( W1, W2, W10, W15) - PROCESS_LOOP( W2, 18, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_16_63( W2, W3, W11, W0) - PROCESS_LOOP( W3, 19, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_16_63( W3, W4, W12, W1) - PROCESS_LOOP( W4, 20, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_16_63( W4, W5, W13, W2) - PROCESS_LOOP( W5, 21, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_16_63( W5, W6, W14, W3) - PROCESS_LOOP( W6, 22, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_16_63( W6, W7, W15, W4) - PROCESS_LOOP( W7, 23, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_16_63( W7, W8, W0, W5) - PROCESS_LOOP( W8, 24, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_16_63( W8, W9, W1, W6) - PROCESS_LOOP( W9, 25, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_16_63( W9, W10, W2, W7) - PROCESS_LOOP(W10, 26, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_16_63(W10, W11, W3, W8) - PROCESS_LOOP(W11, 27, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_16_63(W11, W12, W4, W9) - PROCESS_LOOP(W12, 28, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_16_63(W12, W13, W5, W10) - PROCESS_LOOP(W13, 29, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_16_63(W13, W14, W6, W11) - PROCESS_LOOP(W14, 30, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_16_63(W14, W15, W7, W12) - PROCESS_LOOP(W15, 31, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_16_63(W15, W0, W8, W13) - PROCESS_LOOP( W0, 32, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_16_63( W0, W1, W9, W14) - PROCESS_LOOP( W1, 33, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_16_63( W1, W2, W10, W15) - PROCESS_LOOP( W2, 34, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_16_63( W2, W3, W11, W0) - PROCESS_LOOP( W3, 35, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_16_63( W3, W4, W12, W1) - PROCESS_LOOP( W4, 36, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_16_63( W4, W5, W13, W2) - PROCESS_LOOP( W5, 37, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_16_63( W5, W6, W14, W3) - PROCESS_LOOP( W6, 38, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_16_63( W6, W7, W15, W4) - PROCESS_LOOP( W7, 39, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_16_63( W7, W8, W0, W5) - PROCESS_LOOP( W8, 40, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_16_63( W8, W9, W1, W6) - PROCESS_LOOP( W9, 41, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_16_63( W9, W10, W2, W7) - PROCESS_LOOP(W10, 42, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_16_63(W10, W11, W3, W8) - PROCESS_LOOP(W11, 43, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_16_63(W11, W12, W4, W9) - PROCESS_LOOP(W12, 44, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_16_63(W12, W13, W5, W10) - PROCESS_LOOP(W13, 45, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_16_63(W13, W14, W6, W11) - PROCESS_LOOP(W14, 46, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_16_63(W14, W15, W7, W12) - PROCESS_LOOP(W15, 47, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_16_63(W15, W0, W8, W13) - - // Check if this is the last block - sub INP_SIZE, 1 - JE lastLoop - - // Load next mask for inputs - ADDQ $8, MASKP_P9 - MOVQ (MASKP_P9), MASK_P9 - - // Process last 16 rounds - // Read in next block msg data for use in first 16 words of msg sched - - PROCESS_LOOP( W0, 48, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_00_15( W0, 0, skipNext0) - PROCESS_LOOP( W1, 49, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_00_15( W1, 1, skipNext1) - PROCESS_LOOP( W2, 50, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_00_15( W2, 2, skipNext2) - PROCESS_LOOP( W3, 51, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_00_15( W3, 3, skipNext3) - PROCESS_LOOP( W4, 52, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_00_15( W4, 4, skipNext4) - PROCESS_LOOP( W5, 53, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_00_15( W5, 5, skipNext5) - PROCESS_LOOP( W6, 54, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_00_15( W6, 6, skipNext6) - PROCESS_LOOP( W7, 55, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_00_15( W7, 7, skipNext7) - PROCESS_LOOP( W8, 56, A, B, C, D, E, F, G, H) - MSG_SCHED_ROUND_00_15( W8, 8, skipNext8) - PROCESS_LOOP( W9, 57, H, A, B, C, D, E, F, G) - MSG_SCHED_ROUND_00_15( W9, 9, skipNext9) - PROCESS_LOOP(W10, 58, G, H, A, B, C, D, E, F) - MSG_SCHED_ROUND_00_15(W10, 10, skipNext10) - PROCESS_LOOP(W11, 59, F, G, H, A, B, C, D, E) - MSG_SCHED_ROUND_00_15(W11, 11, skipNext11) - PROCESS_LOOP(W12, 60, E, F, G, H, A, B, C, D) - MSG_SCHED_ROUND_00_15(W12, 12, skipNext12) - PROCESS_LOOP(W13, 61, D, E, F, G, H, A, B, C) - MSG_SCHED_ROUND_00_15(W13, 13, skipNext13) - PROCESS_LOOP(W14, 62, C, D, E, F, G, H, A, B) - MSG_SCHED_ROUND_00_15(W14, 14, skipNext14) - PROCESS_LOOP(W15, 63, B, C, D, E, F, G, H, A) - MSG_SCHED_ROUND_00_15(W15, 15, skipNext15) - - // Add old digest - vmovdqu32 TMP2, A - vmovdqu32 A, [SCRATCH + 64*0] - vpaddd A{k1}, A, TMP2 - vmovdqu32 TMP2, B - vmovdqu32 B, [SCRATCH + 64*1] - vpaddd B{k1}, B, TMP2 - vmovdqu32 TMP2, C - vmovdqu32 C, [SCRATCH + 64*2] - vpaddd C{k1}, C, TMP2 - vmovdqu32 TMP2, D - vmovdqu32 D, [SCRATCH + 64*3] - vpaddd D{k1}, D, TMP2 - vmovdqu32 TMP2, E - vmovdqu32 E, [SCRATCH + 64*4] - vpaddd E{k1}, E, TMP2 - vmovdqu32 TMP2, F - vmovdqu32 F, [SCRATCH + 64*5] - vpaddd F{k1}, F, TMP2 - vmovdqu32 TMP2, G - vmovdqu32 G, [SCRATCH + 64*6] - vpaddd G{k1}, G, TMP2 - vmovdqu32 TMP2, H - vmovdqu32 H, [SCRATCH + 64*7] - vpaddd H{k1}, H, TMP2 - - kmovq k1, mask - JMP lloop - -lastLoop: - // Process last 16 rounds - PROCESS_LOOP( W0, 48, A, B, C, D, E, F, G, H) - PROCESS_LOOP( W1, 49, H, A, B, C, D, E, F, G) - PROCESS_LOOP( W2, 50, G, H, A, B, C, D, E, F) - PROCESS_LOOP( W3, 51, F, G, H, A, B, C, D, E) - PROCESS_LOOP( W4, 52, E, F, G, H, A, B, C, D) - PROCESS_LOOP( W5, 53, D, E, F, G, H, A, B, C) - PROCESS_LOOP( W6, 54, C, D, E, F, G, H, A, B) - PROCESS_LOOP( W7, 55, B, C, D, E, F, G, H, A) - PROCESS_LOOP( W8, 56, A, B, C, D, E, F, G, H) - PROCESS_LOOP( W9, 57, H, A, B, C, D, E, F, G) - PROCESS_LOOP(W10, 58, G, H, A, B, C, D, E, F) - PROCESS_LOOP(W11, 59, F, G, H, A, B, C, D, E) - PROCESS_LOOP(W12, 60, E, F, G, H, A, B, C, D) - PROCESS_LOOP(W13, 61, D, E, F, G, H, A, B, C) - PROCESS_LOOP(W14, 62, C, D, E, F, G, H, A, B) - PROCESS_LOOP(W15, 63, B, C, D, E, F, G, H, A) - - // Add old digest - vmovdqu32 TMP2, A - vmovdqu32 A, [SCRATCH + 64*0] - vpaddd A{k1}, A, TMP2 - vmovdqu32 TMP2, B - vmovdqu32 B, [SCRATCH + 64*1] - vpaddd B{k1}, B, TMP2 - vmovdqu32 TMP2, C - vmovdqu32 C, [SCRATCH + 64*2] - vpaddd C{k1}, C, TMP2 - vmovdqu32 TMP2, D - vmovdqu32 D, [SCRATCH + 64*3] - vpaddd D{k1}, D, TMP2 - vmovdqu32 TMP2, E - vmovdqu32 E, [SCRATCH + 64*4] - vpaddd E{k1}, E, TMP2 - vmovdqu32 TMP2, F - vmovdqu32 F, [SCRATCH + 64*5] - vpaddd F{k1}, F, TMP2 - vmovdqu32 TMP2, G - vmovdqu32 G, [SCRATCH + 64*6] - vpaddd G{k1}, G, TMP2 - vmovdqu32 TMP2, H - vmovdqu32 H, [SCRATCH + 64*7] - vpaddd H{k1}, H, TMP2 - - // Write out digest - vmovdqu32 [STATE + 0*SHA256_DIGEST_ROW_SIZE], A - vmovdqu32 [STATE + 1*SHA256_DIGEST_ROW_SIZE], B - vmovdqu32 [STATE + 2*SHA256_DIGEST_ROW_SIZE], C - vmovdqu32 [STATE + 3*SHA256_DIGEST_ROW_SIZE], D - vmovdqu32 [STATE + 4*SHA256_DIGEST_ROW_SIZE], E - vmovdqu32 [STATE + 5*SHA256_DIGEST_ROW_SIZE], F - vmovdqu32 [STATE + 6*SHA256_DIGEST_ROW_SIZE], G - vmovdqu32 [STATE + 7*SHA256_DIGEST_ROW_SIZE], H - - VZEROUPPER - RET - -// -// Tables -// - -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x000(SB)/8, $0x0405060700010203 -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x008(SB)/8, $0x0c0d0e0f08090a0b -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x010(SB)/8, $0x0405060700010203 -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x018(SB)/8, $0x0c0d0e0f08090a0b -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x020(SB)/8, $0x0405060700010203 -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x028(SB)/8, $0x0c0d0e0f08090a0b -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x030(SB)/8, $0x0405060700010203 -DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x038(SB)/8, $0x0c0d0e0f08090a0b -GLOBL PSHUFFLE_BYTE_FLIP_MASK<>(SB), 8, $64 - -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x000(SB)/8, $0x0000000000000000 -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x008(SB)/8, $0x0000000000000001 -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x010(SB)/8, $0x0000000000000008 -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x018(SB)/8, $0x0000000000000009 -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x020(SB)/8, $0x0000000000000004 -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x028(SB)/8, $0x0000000000000005 -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x030(SB)/8, $0x000000000000000C -DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x038(SB)/8, $0x000000000000000D -GLOBL PSHUFFLE_TRANSPOSE16_MASK1<>(SB), 8, $64 - -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x000(SB)/8, $0x0000000000000002 -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x008(SB)/8, $0x0000000000000003 -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x010(SB)/8, $0x000000000000000A -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x018(SB)/8, $0x000000000000000B -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x020(SB)/8, $0x0000000000000006 -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x028(SB)/8, $0x0000000000000007 -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x030(SB)/8, $0x000000000000000E -DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x038(SB)/8, $0x000000000000000F -GLOBL PSHUFFLE_TRANSPOSE16_MASK2<>(SB), 8, $64 -- cgit v1.2.3