// Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Based on CRYPTOGAMS code with the following comment: // # ==================================================================== // # Written by Andy Polyakov for the OpenSSL // # project. The module is, however, dual licensed under OpenSSL and // # CRYPTOGAMS licenses depending on where you obtain it. For further // # details see http://www.openssl.org/~appro/cryptogams/. // # ==================================================================== // Original code can be found at the link below: // https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91e5c39ca79126a4a876d5d8ff // There are some differences between CRYPTOGAMS code and this one. The round // loop for "_int" isn't the same as the original. Some adjustments were // necessary because there are less vector registers available. For example, some // X variables (r12, r13, r14, and r15) share the same register used by the // counter. The original code uses ctr to name the counter. Here we use CNT // because golang uses CTR as the counter register name. // +build ppc64le,!gccgo,!appengine #include "textflag.h" #define OUT R3 #define INP R4 #define LEN R5 #define KEY R6 #define CNT R7 #define TEMP R8 #define X0 R11 #define X1 R12 #define X2 R14 #define X3 R15 #define X4 R16 #define X5 R17 #define X6 R18 #define X7 R19 #define X8 R20 #define X9 R21 #define X10 R22 #define X11 R23 #define X12 R24 #define X13 R25 #define X14 R26 #define X15 R27 #define CON0 X0 #define CON1 X1 #define CON2 X2 #define CON3 X3 #define KEY0 X4 #define KEY1 X5 #define KEY2 X6 #define KEY3 X7 #define KEY4 X8 #define KEY5 X9 #define KEY6 X10 #define KEY7 X11 #define CNT0 X12 #define CNT1 X13 #define CNT2 X14 #define CNT3 X15 #define TMP0 R9 #define TMP1 R10 #define TMP2 R28 #define TMP3 R29 #define CONSTS R8 #define A0 V0 #define B0 V1 #define C0 V2 #define D0 V3 #define A1 V4 #define B1 V5 #define C1 V6 #define D1 V7 #define A2 V8 #define B2 V9 #define C2 V10 #define D2 V11 #define T0 V12 #define T1 V13 #define T2 V14 #define K0 V15 #define K1 V16 #define K2 V17 #define K3 V18 #define K4 V19 #define K5 V20 #define FOUR V21 #define SIXTEEN V22 #define TWENTY4 V23 #define TWENTY V24 #define TWELVE V25 #define TWENTY5 V26 #define SEVEN V27 #define INPPERM V28 #define OUTPERM V29 #define OUTMASK V30 #define DD0 V31 #define DD1 SEVEN #define DD2 T0 #define DD3 T1 #define DD4 T2 DATA ·consts+0x00(SB)/8, $0x3320646e61707865 DATA ·consts+0x08(SB)/8, $0x6b20657479622d32 DATA ·consts+0x10(SB)/8, $0x0000000000000001 DATA ·consts+0x18(SB)/8, $0x0000000000000000 DATA ·consts+0x20(SB)/8, $0x0000000000000004 DATA ·consts+0x28(SB)/8, $0x0000000000000000 DATA ·consts+0x30(SB)/8, $0x0a0b08090e0f0c0d DATA ·consts+0x38(SB)/8, $0x0203000106070405 DATA ·consts+0x40(SB)/8, $0x090a0b080d0e0f0c DATA ·consts+0x48(SB)/8, $0x0102030005060704 GLOBL ·consts(SB), RODATA, $80 //func chaCha20_ctr32_vmx(out, inp *byte, len int, key *[32]byte, counter *[16]byte) TEXT ·chaCha20_ctr32_vmx(SB),NOSPLIT|NOFRAME,$0 // Load the arguments inside the registers MOVD out+0(FP), OUT MOVD inp+8(FP), INP MOVD len+16(FP), LEN MOVD key+24(FP), KEY MOVD cnt+32(FP), CNT MOVD $·consts(SB), CONSTS // point to consts addr MOVD $16, X0 MOVD $32, X1 MOVD $48, X2 MOVD $64, X3 MOVD $31, X4 MOVD $15, X5 // Load key LVX (KEY)(R0), K1 LVSR (KEY)(R0), T0 LVX (KEY)(X0), K2 LVX (KEY)(X4), DD0 // Load counter LVX (CNT)(R0), K3 LVSR (CNT)(R0), T1 LVX (CNT)(X5), DD1 // Load constants LVX (CONSTS)(R0), K0 LVX (CONSTS)(X0), K5 LVX (CONSTS)(X1), FOUR LVX (CONSTS)(X2), SIXTEEN LVX (CONSTS)(X3), TWENTY4 // Align key and counter VPERM K2, K1, T0, K1 VPERM DD0, K2, T0, K2 VPERM DD1, K3, T1, K3 // Load counter to GPR MOVWZ 0(CNT), CNT0 MOVWZ 4(CNT), CNT1 MOVWZ 8(CNT), CNT2 MOVWZ 12(CNT), CNT3 // Adjust vectors for the initial state VADDUWM K3, K5, K3 VADDUWM K3, K5, K4 VADDUWM K4, K5, K5 // Synthesized constants VSPLTISW $-12, TWENTY VSPLTISW $12, TWELVE VSPLTISW $-7, TWENTY5 VXOR T0, T0, T0 VSPLTISW $-1, OUTMASK LVSR (INP)(R0), INPPERM LVSL (OUT)(R0), OUTPERM VPERM OUTMASK, T0, OUTPERM, OUTMASK loop_outer_vmx: // Load constant MOVD $0x61707865, CON0 MOVD $0x3320646e, CON1 MOVD $0x79622d32, CON2 MOVD $0x6b206574, CON3 VOR K0, K0, A0 VOR K0, K0, A1 VOR K0, K0, A2 VOR K1, K1, B0 MOVD $10, TEMP // Load key to GPR MOVWZ 0(KEY), X4 MOVWZ 4(KEY), X5 MOVWZ 8(KEY), X6 MOVWZ 12(KEY), X7 VOR K1, K1, B1 VOR K1, K1, B2 MOVWZ 16(KEY), X8 MOVWZ 0(CNT), X12 MOVWZ 20(KEY), X9 MOVWZ 4(CNT), X13 VOR K2, K2, C0 VOR K2, K2, C1 MOVWZ 24(KEY), X10 MOVWZ 8(CNT), X14 VOR K2, K2, C2 VOR K3, K3, D0 MOVWZ 28(KEY), X11 MOVWZ 12(CNT), X15 VOR K4, K4, D1 VOR K5, K5, D2 MOVD X4, TMP0 MOVD X5, TMP1 MOVD X6, TMP2 MOVD X7, TMP3 VSPLTISW $7, SEVEN MOVD TEMP, CTR loop_vmx: // CRYPTOGAMS uses a macro to create a loop using perl. This isn't possible // using assembly macros. Therefore, the macro expansion result was used // in order to maintain the algorithm efficiency. // This loop generates three keystream blocks using VMX instructions and, // in parallel, one keystream block using scalar instructions. ADD X4, X0, X0 ADD X5, X1, X1 VADDUWM A0, B0, A0 VADDUWM A1, B1, A1 ADD X6, X2, X2 ADD X7, X3, X3 VADDUWM A2, B2, A2 VXOR D0, A0, D0 XOR X0, X12, X12 XOR X1, X13, X13 VXOR D1, A1, D1 VXOR D2, A2, D2 XOR X2, X14, X14 XOR X3, X15, X15 VPERM D0, D0, SIXTEEN, D0 VPERM D1, D1, SIXTEEN, D1 ROTLW $16, X12, X12 ROTLW $16, X13, X13 VPERM D2, D2, SIXTEEN, D2 VADDUWM C0, D0, C0 ROTLW $16, X14, X14 ROTLW $16, X15, X15 VADDUWM C1, D1, C1 VADDUWM C2, D2, C2 ADD X12, X8, X8 ADD X13, X9, X9 VXOR B0, C0, T0 VXOR B1, C1, T1 ADD X14, X10, X10 ADD X15, X11, X11 VXOR B2, C2, T2 VRLW T0, TWELVE, B0 XOR X8, X4, X4 XOR X9, X5, X5 VRLW T1, TWELVE, B1 VRLW T2, TWELVE, B2 XOR X10, X6, X6 XOR X11, X7, X7 VADDUWM A0, B0, A0 VADDUWM A1, B1, A1 ROTLW $12, X4, X4 ROTLW $12, X5, X5 VADDUWM A2, B2, A2 VXOR D0, A0, D0 ROTLW $12, X6, X6 ROTLW $12, X7, X7 VXOR D1, A1, D1 VXOR D2, A2, D2 ADD X4, X0, X0 ADD X5, X1, X1 VPERM D0, D0, TWENTY4, D0 VPERM D1, D1, TWENTY4, D1 ADD X6, X2, X2 ADD X7, X3, X3 VPERM D2, D2, TWENTY4, D2 VADDUWM C0, D0, C0 XOR X0, X12, X12 XOR X1, X13, X13 VADDUWM C1, D1, C1 VADDUWM C2, D2, C2 XOR X2, X14, X14 XOR X3, X15, X15 VXOR B0, C0, T0 VXOR B1, C1, T1 ROTLW $8, X12, X12 ROTLW $8, X13, X13 VXOR B2, C2, T2 VRLW T0, SEVEN, B0 ROTLW $8, X14, X14 ROTLW $8, X15, X15 VRLW T1, SEVEN, B1 VRLW T2, SEVEN, B2 ADD X12, X8, X8 ADD X13, X9, X9 VSLDOI $8, C0, C0, C0 VSLDOI $8, C1, C1, C1 ADD X14, X10, X10 ADD X15, X11, X11 VSLDOI $8, C2, C2, C2 VSLDOI $12, B0, B0, B0 XOR X8, X4, X4 XOR X9, X5, X5 VSLDOI $12, B1, B1, B1 VSLDOI $12, B2, B2, B2 XOR X10, X6, X6 XOR X11, X7, X7 VSLDOI $4, D0, D0, D0 VSLDOI $4, D1, D1, D1 ROTLW $7, X4, X4 ROTLW $7, X5, X5 VSLDOI $4, D2, D2, D2 VADDUWM A0, B0, A0 ROTLW $7, X6, X6 ROTLW $7, X7, X7 VADDUWM A1, B1, A1 VADDUWM A2, B2, A2 ADD X5, X0, X0 ADD X6, X1, X1 VXOR D0, A0, D0 VXOR D1, A1, D1 ADD X7, X2, X2 ADD X4, X3, X3 VXOR D2, A2, D2 VPERM D0, D0, SIXTEEN, D0 XOR X0, X15, X15 XOR X1, X12, X12 VPERM D1, D1, SIXTEEN, D1 VPERM D2, D2, SIXTEEN, D2 XOR X2, X13, X13 XOR X3, X14, X14 VADDUWM C0, D0, C0 VADDUWM C1, D1, C1 ROTLW $16, X15, X15 ROTLW $16, X12, X12 VADDUWM C2, D2, C2 VXOR B0, C0, T0 ROTLW $16, X13, X13 ROTLW $16, X14, X14 VXOR B1, C1, T1 VXOR B2, C2, T2 ADD X15, X10, X10 ADD X12, X11, X11 VRLW T0, TWELVE, B0 VRLW T1, TWELVE, B1 ADD X13, X8, X8 ADD X14, X9, X9 VRLW T2, TWELVE, B2 VADDUWM A0, B0, A0 XOR X10, X5, X5 XOR X11, X6, X6 VADDUWM A1, B1, A1 VADDUWM A2, B2, A2 XOR X8, X7, X7 XOR X9, X4, X4 VXOR D0, A0, D0 VXOR D1, A1, D1 ROTLW $12, X5, X5 ROTLW $12, X6, X6 VXOR D2, A2, D2 VPERM D0, D0, TWENTY4, D0 ROTLW $12, X7, X7 ROTLW $12, X4, X4 VPERM D1, D1, TWENTY4, D1 VPERM D2, D2, TWENTY4, D2 ADD X5, X0, X0 ADD X6, X1, X1 VADDUWM C0, D0, C0 VADDUWM C1, D1, C1 ADD X7, X2, X2 ADD X4, X3, X3 VADDUWM C2, D2, C2 VXOR B0, C0, T0 XOR X0, X15, X15 XOR X1, X12, X12 VXOR B1, C1, T1 VXOR B2, C2, T2 XOR X2, X13, X13 XOR X3, X14, X14 VRLW T0, SEVEN, B0 VRLW T1, SEVEN, B1 ROTLW $8, X15, X15 ROTLW $8, X12, X12 VRLW T2, SEVEN, B2 VSLDOI $8, C0, C0, C0 ROTLW $8, X13, X13 ROTLW $8, X14, X14 VSLDOI $8, C1, C1, C1 VSLDOI $8, C2, C2, C2 ADD X15, X10, X10 ADD X12, X11, X11 VSLDOI $4, B0, B0, B0 VSLDOI $4, B1, B1, B1 ADD X13, X8, X8 ADD X14, X9, X9 VSLDOI $4, B2, B2, B2 VSLDOI $12, D0, D0, D0 XOR X10, X5, X5 XOR X11, X6, X6 VSLDOI $12, D1, D1, D1 VSLDOI $12, D2, D2, D2 XOR X8, X7, X7 XOR X9, X4, X4 ROTLW $7, X5, X5 ROTLW $7, X6, X6 ROTLW $7, X7, X7 ROTLW $7, X4, X4 BC 0x10, 0, loop_vmx SUB $256, LEN, LEN // Accumulate key block ADD $0x61707865, X0, X0 ADD $0x3320646e, X1, X1 ADD $0x79622d32, X2, X2 ADD $0x6b206574, X3, X3 ADD TMP0, X4, X4 ADD TMP1, X5, X5 ADD TMP2, X6, X6 ADD TMP3, X7, X7 MOVWZ 16(KEY), TMP0 MOVWZ 20(KEY), TMP1 MOVWZ 24(KEY), TMP2 MOVWZ 28(KEY), TMP3 ADD TMP0, X8, X8 ADD TMP1, X9, X9 ADD TMP2, X10, X10 ADD TMP3, X11, X11 MOVWZ 12(CNT), TMP0 MOVWZ 8(CNT), TMP1 MOVWZ 4(CNT), TMP2 MOVWZ 0(CNT), TEMP ADD TMP0, X15, X15 ADD TMP1, X14, X14 ADD TMP2, X13, X13 ADD TEMP, X12, X12 // Accumulate key block VADDUWM A0, K0, A0 VADDUWM A1, K0, A1 VADDUWM A2, K0, A2 VADDUWM B0, K1, B0 VADDUWM B1, K1, B1 VADDUWM B2, K1, B2 VADDUWM C0, K2, C0 VADDUWM C1, K2, C1 VADDUWM C2, K2, C2 VADDUWM D0, K3, D0 VADDUWM D1, K4, D1 VADDUWM D2, K5, D2 // Increment counter ADD $4, TEMP, TEMP MOVW TEMP, 0(CNT) VADDUWM K3, FOUR, K3 VADDUWM K4, FOUR, K4 VADDUWM K5, FOUR, K5 // XOR the input slice (INP) with the keystream, which is stored in GPRs (X0-X3). // Load input (aligned or not) MOVWZ 0(INP), TMP0 MOVWZ 4(INP), TMP1 MOVWZ 8(INP), TMP2 MOVWZ 12(INP), TMP3 // XOR with input XOR TMP0, X0, X0 XOR TMP1, X1, X1 XOR TMP2, X2, X2 XOR TMP3, X3, X3 MOVWZ 16(INP), TMP0 MOVWZ 20(INP), TMP1 MOVWZ 24(INP), TMP2 MOVWZ 28(INP), TMP3 XOR TMP0, X4, X4 XOR TMP1, X5, X5 XOR TMP2, X6, X6 XOR TMP3, X7, X7 MOVWZ 32(INP), TMP0 MOVWZ 36(INP), TMP1 MOVWZ 40(INP), TMP2 MOVWZ 44(INP), TMP3 XOR TMP0, X8, X8 XOR TMP1, X9, X9 XOR TMP2, X10, X10 XOR TMP3, X11, X11 MOVWZ 48(INP), TMP0 MOVWZ 52(INP), TMP1 MOVWZ 56(INP), TMP2 MOVWZ 60(INP), TMP3 XOR TMP0, X12, X12 XOR TMP1, X13, X13 XOR TMP2, X14, X14 XOR TMP3, X15, X15 // Store output (aligned or not) MOVW X0, 0(OUT) MOVW X1, 4(OUT) MOVW X2, 8(OUT) MOVW X3, 12(OUT) ADD $64, INP, INP // INP points to the end of the slice for the alignment code below MOVW X4, 16(OUT) MOVD $16, TMP0 MOVW X5, 20(OUT) MOVD $32, TMP1 MOVW X6, 24(OUT) MOVD $48, TMP2 MOVW X7, 28(OUT) MOVD $64, TMP3 MOVW X8, 32(OUT) MOVW X9, 36(OUT) MOVW X10, 40(OUT) MOVW X11, 44(OUT) MOVW X12, 48(OUT) MOVW X13, 52(OUT) MOVW X14, 56(OUT) MOVW X15, 60(OUT) ADD $64, OUT, OUT // Load input LVX (INP)(R0), DD0 LVX (INP)(TMP0), DD1 LVX (INP)(TMP1), DD2 LVX (INP)(TMP2), DD3 LVX (INP)(TMP3), DD4 ADD $64, INP, INP VPERM DD1, DD0, INPPERM, DD0 // Align input VPERM DD2, DD1, INPPERM, DD1 VPERM DD3, DD2, INPPERM, DD2 VPERM DD4, DD3, INPPERM, DD3 VXOR A0, DD0, A0 // XOR with input VXOR B0, DD1, B0 LVX (INP)(TMP0), DD1 // Keep loading input VXOR C0, DD2, C0 LVX (INP)(TMP1), DD2 VXOR D0, DD3, D0 LVX (INP)(TMP2), DD3 LVX (INP)(TMP3), DD0 ADD $64, INP, INP MOVD $63, TMP3 // 63 is not a typo VPERM A0, A0, OUTPERM, A0 VPERM B0, B0, OUTPERM, B0 VPERM C0, C0, OUTPERM, C0 VPERM D0, D0, OUTPERM, D0 VPERM DD1, DD4, INPPERM, DD4 // Align input VPERM DD2, DD1, INPPERM, DD1 VPERM DD3, DD2, INPPERM, DD2 VPERM DD0, DD3, INPPERM, DD3 VXOR A1, DD4, A1 VXOR B1, DD1, B1 LVX (INP)(TMP0), DD1 // Keep loading VXOR C1, DD2, C1 LVX (INP)(TMP1), DD2 VXOR D1, DD3, D1 LVX (INP)(TMP2), DD3 // Note that the LVX address is always rounded down to the nearest 16-byte // boundary, and that it always points to at most 15 bytes beyond the end of // the slice, so we cannot cross a page boundary. LVX (INP)(TMP3), DD4 // Redundant in aligned case. ADD $64, INP, INP VPERM A1, A1, OUTPERM, A1 // Pre-misalign output VPERM B1, B1, OUTPERM, B1 VPERM C1, C1, OUTPERM, C1 VPERM D1, D1, OUTPERM, D1 VPERM DD1, DD0, INPPERM, DD0 // Align Input VPERM DD2, DD1, INPPERM, DD1 VPERM DD3, DD2, INPPERM, DD2 VPERM DD4, DD3, INPPERM, DD3 VXOR A2, DD0, A2 VXOR B2, DD1, B2 VXOR C2, DD2, C2 VXOR D2, DD3, D2 VPERM A2, A2, OUTPERM, A2 VPERM B2, B2, OUTPERM, B2 VPERM C2, C2, OUTPERM, C2 VPERM D2, D2, OUTPERM, D2 ANDCC $15, OUT, X1 // Is out aligned? MOVD OUT, X0 VSEL A0, B0, OUTMASK, DD0 // Collect pre-misaligned output VSEL B0, C0, OUTMASK, DD1 VSEL C0, D0, OUTMASK, DD2 VSEL D0, A1, OUTMASK, DD3 VSEL A1, B1, OUTMASK, B0 VSEL B1, C1, OUTMASK, C0 VSEL C1, D1, OUTMASK, D0 VSEL D1, A2, OUTMASK, A1 VSEL A2, B2, OUTMASK, B1 VSEL B2, C2, OUTMASK, C1 VSEL C2, D2, OUTMASK, D1 STVX DD0, (OUT+TMP0) STVX DD1, (OUT+TMP1) STVX DD2, (OUT+TMP2) ADD $64, OUT, OUT STVX DD3, (OUT+R0) STVX B0, (OUT+TMP0) STVX C0, (OUT+TMP1) STVX D0, (OUT+TMP2) ADD $64, OUT, OUT STVX A1, (OUT+R0) STVX B1, (OUT+TMP0) STVX C1, (OUT+TMP1) STVX D1, (OUT+TMP2) ADD $64, OUT, OUT BEQ aligned_vmx SUB X1, OUT, X2 // in misaligned case edges MOVD $0, X3 // are written byte-by-byte unaligned_tail_vmx: STVEBX D2, (X2+X3) ADD $1, X3, X3 CMPW X3, X1 BNE unaligned_tail_vmx SUB X1, X0, X2 unaligned_head_vmx: STVEBX A0, (X2+X1) CMPW X1, $15 ADD $1, X1, X1 BNE unaligned_head_vmx CMPU LEN, $255 // done with 256-byte block yet? BGT loop_outer_vmx JMP done_vmx aligned_vmx: STVX A0, (X0+R0) CMPU LEN, $255 // done with 256-byte block yet? BGT loop_outer_vmx done_vmx: RET