path: root/src/crypto/aes/gcm_ppc64x.s

// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build (ppc64 || ppc64le) && !purego

// Portions based on CRYPTOGAMS code with the following comment:
// # ====================================================================
// # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
// # project. The module is, however, dual licensed under OpenSSL and
// # CRYPTOGAMS licenses depending on where you obtain it. For further
// # details see http://www.openssl.org/~appro/cryptogams/.
// # ====================================================================

// The implementations for gcmHash, gcmInit and gcmMul are based on the generated asm
// from the script https://github.com/dot-asm/cryptogams/blob/master/ppc/ghashp8-ppc.pl
// from commit d47afb3c.

// Changes were made due to differences in the ABI and some register usage.
// Some arguments were changed due to the way the Go code passes them.

// Portions that use the stitched AES-GCM approach in counterCryptASM
// are based on code found in
// https://github.com/IBM/ipcri/blob/main/aes/p10_aes_gcm.s

#include "textflag.h"

#define XIP    R3
#define HTBL   R4
#define INP    R5
#define LEN    R6

#define XL     V0
#define XM     V1
#define XH     V2
#define IN     V3
#define ZERO   V4
#define T0     V5
#define T1     V6
#define T2     V7
#define XC2    V8
#define H      V9
#define HH     V10
#define HL     V11
#define LEMASK V12
#define XL1    V13
#define XM1    V14
#define XH1    V15
#define IN1    V16
#define H2     V17
#define H2H    V18
#define H2L    V19
#define XL3    V20
#define XM2    V21
#define IN2    V22
#define H3L    V23
#define H3     V24
#define H3H    V25
#define XH3    V26
#define XM3    V27
#define IN3    V28
#define H4L    V29
#define H4     V30
#define H4H    V31

#define IN0    IN
#define H21L   HL
#define H21H   HH
#define LOPERM H2L
#define HIPERM H2H

#define VXL    VS32
#define VIN    VS35
#define VXC2   VS40
#define VH     VS41
#define VHH    VS42
#define VHL    VS43
#define VIN1   VS48
#define VH2    VS49
#define VH2H   VS50
#define VH2L   VS51

#define VIN2   VS54
#define VH3L   VS55
#define VH3    VS56
#define VH3H   VS57
#define VIN3   VS60
#define VH4L   VS61
#define VH4    VS62
#define VH4H   VS63

#define VIN0   VIN

#define ESPERM V10
#define TMP2 V11

// The following macros provide appropriate
// implementations for endianness as well as
// ISA specific for power8 and power9.
#ifdef GOARCH_ppc64le
#  ifdef GOPPC64_power9
#define P8_LXVB16X(RA,RB,VT)   LXVB16X (RA)(RB), VT
#define P8_STXVB16X(VS,RA,RB)  STXVB16X VS, (RA)(RB)
#  else
#define NEEDS_ESPERM
#define P8_LXVB16X(RA,RB,VT) \
	LXVD2X  (RA+RB), VT \
	VPERM	VT, VT, ESPERM, VT

#define P8_STXVB16X(VS,RA,RB) \
	VPERM	VS, VS, ESPERM, TMP2; \
	STXVD2X TMP2, (RA+RB)

#  endif
#else
#define P8_LXVB16X(RA,RB,VT) \
	LXVD2X  (RA+RB), VT

#define P8_STXVB16X(VS,RA,RB) \
	STXVD2X VS, (RA+RB)

#endif

#define MASK_PTR   R8

#define MASKV   V0
#define INV     V1

// The following macros are used for
// the stitched implementation within
// counterCryptASM.

// Load the initial GCM counter value
// in V30 and set up the counter increment
// in V31
#define SETUP_COUNTER \
	P8_LXVB16X(COUNTER, R0, V30); \
	VSPLTISB $1, V28; \
	VXOR V31, V31, V31; \
	VSLDOI $1, V31, V28, V31

// These macros set up the initial value
// for a single encryption, or 4 or 8
// stitched encryptions implemented
// with interleaving vciphers.
//
// The input value for each encryption
// is generated by XORing the counter
// from V30 with the first key in VS0
// and incrementing the counter.
//
// Single encryption in V15
#define GEN_VCIPHER_INPUT \
	XXLOR VS0, VS0, V29 \
	VXOR V30, V29, V15; \
	VADDUWM V30, V31, V30

// 4 encryptions in V15 - V18
#define GEN_VCIPHER_4_INPUTS \
	XXLOR VS0, VS0, V29; \
	VXOR V30, V29, V15; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V16; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V17; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V18; \
	VADDUWM V30, V31, V30

// 8 encryptions in V15 - V22
#define GEN_VCIPHER_8_INPUTS \
	XXLOR VS0, VS0, V29; \
	VXOR V30, V29, V15; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V16; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V17; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V18; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V19; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V20; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V21; \
	VADDUWM V30, V31, V30; \
	VXOR V30, V29, V22; \
	VADDUWM V30, V31, V30

// Load the keys to be used for
// encryption based on key_len.
// Keys are in VS0 - VS14
// depending on key_len.
// Valid keys sizes are verified
// here. CR2 is set and used
// throughout to check key_len.
#define LOAD_KEYS(blk_key, key_len) \
	MOVD	$16, R16; \
	MOVD	$32, R17; \
	MOVD	$48, R18; \
	MOVD	$64, R19; \
	LXVD2X (blk_key)(R0), VS0; \
	LXVD2X (blk_key)(R16), VS1; \
	LXVD2X (blk_key)(R17), VS2; \
	LXVD2X (blk_key)(R18), VS3; \
	LXVD2X (blk_key)(R19), VS4; \
	ADD $64, R16; \
	ADD $64, R17; \
	ADD $64, R18; \
	ADD $64, R19; \
	LXVD2X (blk_key)(R16), VS5; \
	LXVD2X (blk_key)(R17), VS6; \
	LXVD2X (blk_key)(R18), VS7; \
	LXVD2X (blk_key)(R19), VS8; \
	ADD $64, R16; \
	ADD $64, R17; \
	ADD $64, R18; \
	ADD $64, R19; \
	LXVD2X (blk_key)(R16), VS9; \
	LXVD2X (blk_key)(R17), VS10; \
	CMP key_len, $12, CR2; \
	CMP key_len, $10; \
	BEQ keysLoaded; \
	LXVD2X (blk_key)(R18), VS11; \
	LXVD2X (blk_key)(R19), VS12; \
	BEQ CR2, keysLoaded; \
	ADD $64, R16; \
	ADD $64, R17; \
	LXVD2X (blk_key)(R16), VS13; \
	LXVD2X (blk_key)(R17), VS14; \
	CMP key_len, $14; \
	BEQ keysLoaded; \
	MOVD R0,0(R0); \
keysLoaded:

// Encrypt 1 (vin) with first 9
// keys from VS1 - VS9.
#define VCIPHER_1X9_KEYS(vin) \
	XXLOR VS1, VS1, V23; \
	XXLOR VS2, VS2, V24; \
	XXLOR VS3, VS3, V25; \
	XXLOR VS4, VS4, V26; \
	XXLOR VS5, VS5, V27; \
	VCIPHER vin, V23, vin; \
	VCIPHER vin, V24, vin; \
	VCIPHER vin, V25, vin; \
	VCIPHER vin, V26, vin; \
	VCIPHER vin, V27, vin; \
	XXLOR VS6, VS6, V23; \
	XXLOR VS7, VS7, V24; \
	XXLOR VS8, VS8, V25; \
	XXLOR VS9, VS9, V26; \
	VCIPHER vin, V23, vin; \
	VCIPHER vin, V24, vin; \
	VCIPHER vin, V25, vin; \
	VCIPHER	vin, V26, vin

// Encrypt 1 value (vin) with
// 2 specified keys
#define VCIPHER_1X2_KEYS(vin, key1, key2) \
	XXLOR key1, key1, V25; \
	XXLOR key2, key2, V26; \
	VCIPHER vin, V25, vin; \
	VCIPHER vin, V26, vin

// Encrypt 4 values in V15 - V18
// with the specified key from
// VS1 - VS9.
#define VCIPHER_4X1_KEY(key) \
	XXLOR key, key, V23; \
	VCIPHER V15, V23, V15; \
	VCIPHER V16, V23, V16; \
	VCIPHER V17, V23, V17; \
	VCIPHER V18, V23, V18

// Encrypt 8 values in V15 - V22
// with the specified key,
// assuming it is a VSreg
#define VCIPHER_8X1_KEY(key) \
	XXLOR key, key, V23; \
	VCIPHER V15, V23, V15; \
	VCIPHER V16, V23, V16; \
	VCIPHER V17, V23, V17; \
	VCIPHER V18, V23, V18; \
	VCIPHER V19, V23, V19; \
	VCIPHER V20, V23, V20; \
	VCIPHER V21, V23, V21; \
	VCIPHER V22, V23, V22

// Load input block into V1-V4
// in big endian order and
// update blk_inp by 64.
#define LOAD_INPUT_BLOCK64(blk_inp) \
	MOVD $16, R16; \
	MOVD $32, R17; \
	MOVD $48, R18; \
	P8_LXVB16X(blk_inp,R0,V1); \
	P8_LXVB16X(blk_inp,R16,V2); \
	P8_LXVB16X(blk_inp,R17,V3); \
	P8_LXVB16X(blk_inp,R18,V4); \
	ADD $64, blk_inp

// Load input block into V1-V8
// in big endian order and
// Update blk_inp by 128
#define LOAD_INPUT_BLOCK128(blk_inp) \
	MOVD $16, R16; \
	MOVD $32, R17; \
	MOVD $48, R18; \
	MOVD $64, R19; \
	MOVD $80, R20; \
	MOVD $96, R21; \
	MOVD $112, R22; \
	P8_LXVB16X(blk_inp,R0,V1); \
	P8_LXVB16X(blk_inp,R16,V2); \
	P8_LXVB16X(blk_inp,R17,V3); \
	P8_LXVB16X(blk_inp,R18,V4); \
	P8_LXVB16X(blk_inp,R19,V5); \
	P8_LXVB16X(blk_inp,R20,V6); \
	P8_LXVB16X(blk_inp,R21,V7); \
	P8_LXVB16X(blk_inp,R22,V8); \
	ADD $128, blk_inp

// Finish encryption on 8 streams and
// XOR with input block
#define VCIPHERLAST8_XOR_INPUT \
	VCIPHERLAST     V15, V23, V15; \
	VCIPHERLAST     V16, V23, V16; \
	VCIPHERLAST     V17, V23, V17; \
	VCIPHERLAST     V18, V23, V18; \
	VCIPHERLAST     V19, V23, V19; \
	VCIPHERLAST     V20, V23, V20; \
	VCIPHERLAST     V21, V23, V21; \
	VCIPHERLAST     V22, V23, V22; \
	XXLXOR          V1, V15, V1; \
	XXLXOR          V2, V16, V2; \
	XXLXOR          V3, V17, V3; \
	XXLXOR          V4, V18, V4; \
	XXLXOR          V5, V19, V5; \
	XXLXOR          V6, V20, V6; \
	XXLXOR          V7, V21, V7; \
	XXLXOR          V8, V22, V8

// Finish encryption on 4 streams and
// XOR with input block
#define VCIPHERLAST4_XOR_INPUT \
	VCIPHERLAST     V15, V23, V15; \
	VCIPHERLAST     V16, V23, V16; \
	VCIPHERLAST     V17, V23, V17; \
	VCIPHERLAST     V18, V23, V18; \
	XXLXOR          V1, V15, V1; \
	XXLXOR          V2, V16, V2; \
	XXLXOR          V3, V17, V3; \
	XXLXOR          V4, V18, V4

// Store output block from V1-V8
// in big endian order and
// Update blk_out by 128
#define STORE_OUTPUT_BLOCK128(blk_out) \
	P8_STXVB16X(V1,blk_out,R0); \
	P8_STXVB16X(V2,blk_out,R16); \
	P8_STXVB16X(V3,blk_out,R17); \
	P8_STXVB16X(V4,blk_out,R18); \
	P8_STXVB16X(V5,blk_out,R19); \
	P8_STXVB16X(V6,blk_out,R20); \
	P8_STXVB16X(V7,blk_out,R21); \
	P8_STXVB16X(V8,blk_out,R22); \
	ADD $128, blk_out

// Store output block from V1-V4
// in big endian order and
// Update blk_out by 64
#define STORE_OUTPUT_BLOCK64(blk_out) \
	P8_STXVB16X(V1,blk_out,R0); \
	P8_STXVB16X(V2,blk_out,R16); \
	P8_STXVB16X(V3,blk_out,R17); \
	P8_STXVB16X(V4,blk_out,R18); \
	ADD $64, blk_out

// func gcmInit(productTable *[256]byte, h []byte)
TEXT ·gcmInit(SB), NOSPLIT, $0-32
	MOVD productTable+0(FP), XIP
	MOVD h+8(FP), HTBL

	MOVD   $0x10, R8
	MOVD   $0x20, R9
	MOVD   $0x30, R10
	LXVD2X (HTBL)(R0), VH // Load H

	VSPLTISB $-16, XC2           // 0xf0
	VSPLTISB $1, T0              // one
	VADDUBM  XC2, XC2, XC2       // 0xe0
	VXOR     ZERO, ZERO, ZERO
	VOR      XC2, T0, XC2        // 0xe1
	VSLDOI   $15, XC2, ZERO, XC2 // 0xe1...
	VSLDOI   $1, ZERO, T0, T1    // ...1
	VADDUBM  XC2, XC2, XC2       // 0xc2...
	VSPLTISB $7, T2
	VOR      XC2, T1, XC2        // 0xc2....01
	VSPLTB   $0, H, T1           // most significant byte
	VSL      H, T0, H            // H<<=1
	VSRAB    T1, T2, T1          // broadcast carry bit
	VAND     T1, XC2, T1
	VXOR     H, T1, IN           // twisted H

	VSLDOI $8, IN, IN, H      // twist even more ...
	VSLDOI $8, ZERO, XC2, XC2 // 0xc2.0
	VSLDOI $8, ZERO, H, HL    // ... and split
	VSLDOI $8, H, ZERO, HH

	STXVD2X VXC2, (XIP+R0) // save pre-computed table
	STXVD2X VHL, (XIP+R8)
	MOVD    $0x40, R8
	STXVD2X VH, (XIP+R9)
	MOVD    $0x50, R9
	STXVD2X VHH, (XIP+R10)
	MOVD    $0x60, R10

	VPMSUMD IN, HL, XL // H.lo·H.lo
	VPMSUMD IN, H, XM  // H.hi·H.lo+H.lo·H.hi
	VPMSUMD IN, HH, XH // H.hi·H.hi

	VPMSUMD XL, XC2, T2 // 1st reduction phase

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH

	VSLDOI $8, XL, XL, XL
	VXOR   XL, T2, XL

	VSLDOI  $8, XL, XL, T1 // 2nd reduction phase
	VPMSUMD XL, XC2, XL
	VXOR    T1, XH, T1
	VXOR    XL, T1, IN1

	VSLDOI $8, IN1, IN1, H2
	VSLDOI $8, ZERO, H2, H2L
	VSLDOI $8, H2, ZERO, H2H

	STXVD2X VH2L, (XIP+R8)  // save H^2
	MOVD    $0x70, R8
	STXVD2X VH2, (XIP+R9)
	MOVD    $0x80, R9
	STXVD2X VH2H, (XIP+R10)
	MOVD    $0x90, R10

	VPMSUMD IN, H2L, XL   // H.lo·H^2.lo
	VPMSUMD IN1, H2L, XL1 // H^2.lo·H^2.lo
	VPMSUMD IN, H2, XM    // H.hi·H^2.lo+H.lo·H^2.hi
	VPMSUMD IN1, H2, XM1  // H^2.hi·H^2.lo+H^2.lo·H^2.hi
	VPMSUMD IN, H2H, XH   // H.hi·H^2.hi
	VPMSUMD IN1, H2H, XH1 // H^2.hi·H^2.hi

	VPMSUMD XL, XC2, T2  // 1st reduction phase
	VPMSUMD XL1, XC2, HH // 1st reduction phase

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VSLDOI $8, XM1, ZERO, HL
	VSLDOI $8, ZERO, XM1, H
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH
	VXOR   XL1, HL, XL1
	VXOR   XH1, H, XH1

	VSLDOI $8, XL, XL, XL
	VSLDOI $8, XL1, XL1, XL1
	VXOR   XL, T2, XL
	VXOR   XL1, HH, XL1

	VSLDOI  $8, XL, XL, T1  // 2nd reduction phase
	VSLDOI  $8, XL1, XL1, H // 2nd reduction phase
	VPMSUMD XL, XC2, XL
	VPMSUMD XL1, XC2, XL1
	VXOR    T1, XH, T1
	VXOR    H, XH1, H
	VXOR    XL, T1, XL
	VXOR    XL1, H, XL1

	VSLDOI $8, XL, XL, H
	VSLDOI $8, XL1, XL1, H2
	VSLDOI $8, ZERO, H, HL
	VSLDOI $8, H, ZERO, HH
	VSLDOI $8, ZERO, H2, H2L
	VSLDOI $8, H2, ZERO, H2H

	STXVD2X VHL, (XIP+R8)   // save H^3
	MOVD    $0xa0, R8
	STXVD2X VH, (XIP+R9)
	MOVD    $0xb0, R9
	STXVD2X VHH, (XIP+R10)
	MOVD    $0xc0, R10
	STXVD2X VH2L, (XIP+R8)  // save H^4
	STXVD2X VH2, (XIP+R9)
	STXVD2X VH2H, (XIP+R10)

	RET

// func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int)
TEXT ·gcmHash(SB), NOSPLIT, $0-64
	MOVD output+0(FP), XIP
	MOVD productTable+24(FP), HTBL
	MOVD inp+32(FP), INP
	MOVD len+56(FP), LEN

	MOVD   $0x10, R8
	MOVD   $0x20, R9
	MOVD   $0x30, R10
	LXVD2X (XIP)(R0), VXL // load Xi

	LXVD2X   (HTBL)(R8), VHL    // load pre-computed table
	MOVD     $0x40, R8
	LXVD2X   (HTBL)(R9), VH
	MOVD     $0x50, R9
	LXVD2X   (HTBL)(R10), VHH
	MOVD     $0x60, R10
	LXVD2X   (HTBL)(R0), VXC2
#ifdef GOARCH_ppc64le
	LVSL     (R0)(R0), LEMASK
	VSPLTISB $0x07, T0
	VXOR     LEMASK, T0, LEMASK
	VPERM    XL, XL, LEMASK, XL
#endif
	VXOR     ZERO, ZERO, ZERO

	CMPU LEN, $64
	BGE  gcm_ghash_p8_4x

	LXVD2X (INP)(R0), VIN
	ADD    $16, INP, INP
	SUBCCC $16, LEN, LEN
#ifdef GOARCH_ppc64le
	VPERM  IN, IN, LEMASK, IN
#endif
	VXOR   IN, XL, IN
	BEQ    short

	LXVD2X (HTBL)(R8), VH2L  // load H^2
	MOVD   $16, R8
	LXVD2X (HTBL)(R9), VH2
	ADD    LEN, INP, R9      // end of input
	LXVD2X (HTBL)(R10), VH2H

loop_2x:
	LXVD2X (INP)(R0), VIN1
#ifdef GOARCH_ppc64le
	VPERM  IN1, IN1, LEMASK, IN1
#endif

	SUBC    $32, LEN, LEN
	VPMSUMD IN, H2L, XL   // H^2.lo·Xi.lo
	VPMSUMD IN1, HL, XL1  // H.lo·Xi+1.lo
	SUBE    R11, R11, R11 // borrow?-1:0
	VPMSUMD IN, H2, XM    // H^2.hi·Xi.lo+H^2.lo·Xi.hi
	VPMSUMD IN1, H, XM1   // H.hi·Xi+1.lo+H.lo·Xi+1.hi
	AND     LEN, R11, R11
	VPMSUMD IN, H2H, XH   // H^2.hi·Xi.hi
	VPMSUMD IN1, HH, XH1  // H.hi·Xi+1.hi
	ADD     R11, INP, INP

	VXOR XL, XL1, XL
	VXOR XM, XM1, XM

	VPMSUMD XL, XC2, T2 // 1st reduction phase

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VXOR   XH, XH1, XH
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH

	VSLDOI $8, XL, XL, XL
	VXOR   XL, T2, XL
	LXVD2X (INP)(R8), VIN
	ADD    $32, INP, INP

	VSLDOI  $8, XL, XL, T1     // 2nd reduction phase
	VPMSUMD XL, XC2, XL
#ifdef GOARCH_ppc64le
	VPERM   IN, IN, LEMASK, IN
#endif
	VXOR    T1, XH, T1
	VXOR    IN, T1, IN
	VXOR    IN, XL, IN
	CMP     R9, INP
	BGT     loop_2x            // done yet?

	CMPWU LEN, $0
	BNE   even

short:
	VPMSUMD IN, HL, XL // H.lo·Xi.lo
	VPMSUMD IN, H, XM  // H.hi·Xi.lo+H.lo·Xi.hi
	VPMSUMD IN, HH, XH // H.hi·Xi.hi

	VPMSUMD XL, XC2, T2 // 1st reduction phase

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH

	VSLDOI $8, XL, XL, XL
	VXOR   XL, T2, XL

	VSLDOI  $8, XL, XL, T1 // 2nd reduction phase
	VPMSUMD XL, XC2, XL
	VXOR    T1, XH, T1

even:
	VXOR    XL, T1, XL
#ifdef GOARCH_ppc64le
	VPERM   XL, XL, LEMASK, XL
#endif
	STXVD2X VXL, (XIP+R0)

	OR R12, R12, R12 // write out Xi
	RET

gcm_ghash_p8_4x:
	LVSL     (R8)(R0), T0      // 0x0001..0e0f
	MOVD     $0x70, R8
	LXVD2X   (HTBL)(R9), VH2
	MOVD     $0x80, R9
	VSPLTISB $8, T1            // 0x0808..0808
	MOVD     $0x90, R10
	LXVD2X   (HTBL)(R8), VH3L  // load H^3
	MOVD     $0xa0, R8
	LXVD2X   (HTBL)(R9), VH3
	MOVD     $0xb0, R9
	LXVD2X   (HTBL)(R10), VH3H
	MOVD     $0xc0, R10
	LXVD2X   (HTBL)(R8), VH4L  // load H^4
	MOVD     $0x10, R8
	LXVD2X   (HTBL)(R9), VH4
	MOVD     $0x20, R9
	LXVD2X   (HTBL)(R10), VH4H
	MOVD     $0x30, R10

	VSLDOI  $8, ZERO, T1, T2   // 0x0000..0808
	VADDUBM T0, T2, HIPERM     // 0x0001..1617
	VADDUBM T1, HIPERM, LOPERM // 0x0809..1e1f

	SRD $4, LEN, LEN // this allows to use sign bit as carry

	LXVD2X (INP)(R0), VIN0       // load input
	LXVD2X (INP)(R8), VIN1
	SUBCCC $8, LEN, LEN
	LXVD2X (INP)(R9), VIN2
	LXVD2X (INP)(R10), VIN3
	ADD    $0x40, INP, INP
#ifdef GOARCH_ppc64le
	VPERM  IN0, IN0, LEMASK, IN0
	VPERM  IN1, IN1, LEMASK, IN1
	VPERM  IN2, IN2, LEMASK, IN2
	VPERM  IN3, IN3, LEMASK, IN3
#endif

	VXOR IN0, XL, XH

	VPMSUMD IN1, H3L, XL1
	VPMSUMD IN1, H3, XM1
	VPMSUMD IN1, H3H, XH1

	VPERM   H2, H, HIPERM, H21L
	VPERM   IN2, IN3, LOPERM, T0
	VPERM   H2, H, LOPERM, H21H
	VPERM   IN2, IN3, HIPERM, T1
	VPMSUMD IN2, H2, XM2         // H^2.lo·Xi+2.hi+H^2.hi·Xi+2.lo
	VPMSUMD T0, H21L, XL3        // H^2.lo·Xi+2.lo+H.lo·Xi+3.lo
	VPMSUMD IN3, H, XM3          // H.hi·Xi+3.lo  +H.lo·Xi+3.hi
	VPMSUMD T1, H21H, XH3        // H^2.hi·Xi+2.hi+H.hi·Xi+3.hi

	VXOR XM2, XM1, XM2
	VXOR XL3, XL1, XL3
	VXOR XM3, XM2, XM3
	VXOR XH3, XH1, XH3

	BLT tail_4x

loop_4x:
	LXVD2X (INP)(R0), VIN0
	LXVD2X (INP)(R8), VIN1
	SUBCCC $4, LEN, LEN
	LXVD2X (INP)(R9), VIN2
	LXVD2X (INP)(R10), VIN3
	ADD    $0x40, INP, INP
#ifdef GOARCH_ppc64le
	VPERM  IN1, IN1, LEMASK, IN1
	VPERM  IN2, IN2, LEMASK, IN2
	VPERM  IN3, IN3, LEMASK, IN3
	VPERM  IN0, IN0, LEMASK, IN0
#endif

	VPMSUMD XH, H4L, XL   // H^4.lo·Xi.lo
	VPMSUMD XH, H4, XM    // H^4.hi·Xi.lo+H^4.lo·Xi.hi
	VPMSUMD XH, H4H, XH   // H^4.hi·Xi.hi
	VPMSUMD IN1, H3L, XL1
	VPMSUMD IN1, H3, XM1
	VPMSUMD IN1, H3H, XH1

	VXOR  XL, XL3, XL
	VXOR  XM, XM3, XM
	VXOR  XH, XH3, XH
	VPERM IN2, IN3, LOPERM, T0
	VPERM IN2, IN3, HIPERM, T1

	VPMSUMD XL, XC2, T2   // 1st reduction phase
	VPMSUMD T0, H21L, XL3 // H.lo·Xi+3.lo  +H^2.lo·Xi+2.lo
	VPMSUMD T1, H21H, XH3 // H.hi·Xi+3.hi  +H^2.hi·Xi+2.hi

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH

	VSLDOI $8, XL, XL, XL
	VXOR   XL, T2, XL

	VSLDOI  $8, XL, XL, T1 // 2nd reduction phase
	VPMSUMD IN2, H2, XM2   // H^2.hi·Xi+2.lo+H^2.lo·Xi+2.hi
	VPMSUMD IN3, H, XM3    // H.hi·Xi+3.lo  +H.lo·Xi+3.hi
	VPMSUMD XL, XC2, XL

	VXOR XL3, XL1, XL3
	VXOR XH3, XH1, XH3
	VXOR XH, IN0, XH
	VXOR XM2, XM1, XM2
	VXOR XH, T1, XH
	VXOR XM3, XM2, XM3
	VXOR XH, XL, XH
	BGE  loop_4x

tail_4x:
	VPMSUMD XH, H4L, XL // H^4.lo·Xi.lo
	VPMSUMD XH, H4, XM  // H^4.hi·Xi.lo+H^4.lo·Xi.hi
	VPMSUMD XH, H4H, XH // H^4.hi·Xi.hi

	VXOR XL, XL3, XL
	VXOR XM, XM3, XM

	VPMSUMD XL, XC2, T2 // 1st reduction phase

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VXOR   XH, XH3, XH
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH

	VSLDOI $8, XL, XL, XL
	VXOR   XL, T2, XL

	VSLDOI  $8, XL, XL, T1 // 2nd reduction phase
	VPMSUMD XL, XC2, XL
	VXOR    T1, XH, T1
	VXOR    XL, T1, XL

	ADDCCC $4, LEN, LEN
	BEQ    done_4x

	LXVD2X (INP)(R0), VIN0
	CMPU   LEN, $2
	MOVD   $-4, LEN
	BLT    one
	LXVD2X (INP)(R8), VIN1
	BEQ    two

three:
	LXVD2X (INP)(R9), VIN2
#ifdef GOARCH_ppc64le
	VPERM  IN0, IN0, LEMASK, IN0
	VPERM  IN1, IN1, LEMASK, IN1
	VPERM  IN2, IN2, LEMASK, IN2
#endif

	VXOR IN0, XL, XH
	VOR  H3L, H3L, H4L
	VOR  H3, H3, H4
	VOR  H3H, H3H, H4H

	VPERM   IN1, IN2, LOPERM, T0
	VPERM   IN1, IN2, HIPERM, T1
	VPMSUMD IN1, H2, XM2         // H^2.lo·Xi+1.hi+H^2.hi·Xi+1.lo
	VPMSUMD IN2, H, XM3          // H.hi·Xi+2.lo  +H.lo·Xi+2.hi
	VPMSUMD T0, H21L, XL3        // H^2.lo·Xi+1.lo+H.lo·Xi+2.lo
	VPMSUMD T1, H21H, XH3        // H^2.hi·Xi+1.hi+H.hi·Xi+2.hi

	VXOR XM3, XM2, XM3
	JMP  tail_4x

two:
#ifdef GOARCH_ppc64le
	VPERM IN0, IN0, LEMASK, IN0
	VPERM IN1, IN1, LEMASK, IN1
#endif

	VXOR  IN, XL, XH
	VPERM ZERO, IN1, LOPERM, T0
	VPERM ZERO, IN1, HIPERM, T1

	VSLDOI $8, ZERO, H2, H4L
	VOR    H2, H2, H4
	VSLDOI $8, H2, ZERO, H4H

	VPMSUMD T0, H21L, XL3 // H.lo·Xi+1.lo
	VPMSUMD IN1, H, XM3   // H.hi·Xi+1.lo+H.lo·Xi+2.hi
	VPMSUMD T1, H21H, XH3 // H.hi·Xi+1.hi

	JMP tail_4x

one:
#ifdef GOARCH_ppc64le
	VPERM IN0, IN0, LEMASK, IN0
#endif

	VSLDOI $8, ZERO, H, H4L
	VOR    H, H, H4
	VSLDOI $8, H, ZERO, H4H

	VXOR IN0, XL, XH
	VXOR XL3, XL3, XL3
	VXOR XM3, XM3, XM3
	VXOR XH3, XH3, XH3

	JMP tail_4x

done_4x:
#ifdef GOARCH_ppc64le
	VPERM   XL, XL, LEMASK, XL
#endif
	STXVD2X VXL, (XIP+R0)      // write out Xi
	RET

// func gcmMul(output []byte, productTable *[256]byte)
TEXT ·gcmMul(SB), NOSPLIT, $0-32
	MOVD output+0(FP), XIP
	MOVD productTable+24(FP), HTBL

	MOVD   $0x10, R8
	MOVD   $0x20, R9
	MOVD   $0x30, R10
	LXVD2X (XIP)(R0), VIN // load Xi

	LXVD2X   (HTBL)(R8), VHL    // Load pre-computed table
	LXVD2X   (HTBL)(R9), VH
	LXVD2X   (HTBL)(R10), VHH
	LXVD2X   (HTBL)(R0), VXC2
#ifdef GOARCH_ppc64le
	VSPLTISB $0x07, T0
	VXOR     LEMASK, T0, LEMASK
	VPERM    IN, IN, LEMASK, IN
#endif
	VXOR     ZERO, ZERO, ZERO

	VPMSUMD IN, HL, XL // H.lo·Xi.lo
	VPMSUMD IN, H, XM  // H.hi·Xi.lo+H.lo·Xi.hi
	VPMSUMD IN, HH, XH // H.hi·Xi.hi

	VPMSUMD XL, XC2, T2 // 1st reduction phase

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH

	VSLDOI $8, XL, XL, XL
	VXOR   XL, T2, XL

	VSLDOI  $8, XL, XL, T1 // 2nd reduction phase
	VPMSUMD XL, XC2, XL
	VXOR    T1, XH, T1
	VXOR    XL, T1, XL

#ifdef GOARCH_ppc64le
	VPERM   XL, XL, LEMASK, XL
#endif
	STXVD2X VXL, (XIP+R0)      // write out Xi
	RET

#define BLK_INP    R3
#define BLK_OUT    R4
#define BLK_KEY    R5
#define KEY_LEN    R6
#define BLK_IDX    R7
#define IDX        R8
#define IN_LEN     R9
#define COUNTER    R10
#define CONPTR     R14
#define MASK       V5

// Implementation of the counterCrypt function in assembler.
// Original loop is unrolled to allow for multiple encryption
// streams to be done in parallel, which is achieved by interleaving
// vcipher instructions from each stream. This is also referred to as
// stitching, and provides significant performance improvements.
// Some macros are defined which enable execution for big or little
// endian as well as different ISA targets.
//func (g *gcmAsm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte, key[gcmBlockSize]uint32)
//func counterCryptASM(xr, out, in, counter, key)
TEXT ·counterCryptASM(SB), NOSPLIT, $16-72
	MOVD	xr(FP), KEY_LEN
	MOVD    out+8(FP), BLK_OUT
	MOVD    out_len+16(FP), R8
	MOVD    in+32(FP), BLK_INP
	MOVD    in_len+40(FP), IN_LEN
	MOVD    counter+56(FP), COUNTER
	MOVD    key+64(FP), BLK_KEY

// Set up permute string when needed.
#ifdef NEEDS_ESPERM
	MOVD    $·rcon(SB), R14
	LVX     (R14), ESPERM   // Permute value for P8_ macros.
#endif
	SETUP_COUNTER		// V30 Counter V31 BE {0, 0, 0, 1}
	LOAD_KEYS(BLK_KEY, KEY_LEN)	// VS1 - VS10/12/14 based on keysize
	CMP     IN_LEN, $128
	BLT	block64
block128_loop:
	// Do 8 encryptions in parallel by setting
	// input values in V15-V22 and executing
	// vcipher on the updated value and the keys.
	GEN_VCIPHER_8_INPUTS
	VCIPHER_8X1_KEY(VS1)
	VCIPHER_8X1_KEY(VS2)
	VCIPHER_8X1_KEY(VS3)
	VCIPHER_8X1_KEY(VS4)
	VCIPHER_8X1_KEY(VS5)
	VCIPHER_8X1_KEY(VS6)
	VCIPHER_8X1_KEY(VS7)
	VCIPHER_8X1_KEY(VS8)
	VCIPHER_8X1_KEY(VS9)
	// Additional encryptions are done based on
	// the key length, with the last key moved
	// to V23 for use with VCIPHERLAST.
	// CR2 = CMP key_len, $12
	XXLOR VS10, VS10, V23
	BLT	CR2, block128_last // key_len = 10
	VCIPHER_8X1_KEY(VS10)
	VCIPHER_8X1_KEY(VS11)
	XXLOR VS12,VS12,V23
	BEQ	CR2, block128_last // ken_len = 12
	VCIPHER_8X1_KEY(VS12)
	VCIPHER_8X1_KEY(VS13)
	XXLOR VS14,VS14,V23	// key_len = 14
block128_last:
	// vcipher encryptions are in V15-V22 at this
	// point with vcipherlast remaining to be done.
	// Load input block into V1-V8, setting index offsets
	// in R16-R22 to use with the STORE.
	LOAD_INPUT_BLOCK128(BLK_INP)
	// Do VCIPHERLAST on the last key for each encryption
	// stream and XOR the result with the corresponding
	// value from the input block.
	VCIPHERLAST8_XOR_INPUT
	// Store the results (8*16) and update BLK_OUT by 128.
	STORE_OUTPUT_BLOCK128(BLK_OUT)
	ADD	$-128, IN_LEN	// input size
	CMP     IN_LEN, $128	// check if >= blocksize
	BGE	block128_loop	// next input block
	CMP	IN_LEN, $0
	BEQ	done
block64:
	CMP	IN_LEN, $64	// Check if >= 64
	BLT	block16_loop
	// Do 4 encryptions in parallel by setting
	// input values in V15-V18 and executing
	// vcipher on the updated value and the keys.
	GEN_VCIPHER_4_INPUTS
	VCIPHER_4X1_KEY(VS1)
	VCIPHER_4X1_KEY(VS2)
	VCIPHER_4X1_KEY(VS3)
	VCIPHER_4X1_KEY(VS4)
	VCIPHER_4X1_KEY(VS5)
	VCIPHER_4X1_KEY(VS6)
	VCIPHER_4X1_KEY(VS7)
	VCIPHER_4X1_KEY(VS8)
	VCIPHER_4X1_KEY(VS9)
	// Check key length based on CR2
	// Move last key to V23 for use with later vcipherlast
	XXLOR	VS10, VS10, V23
	BLT	CR2, block64_last	// size = 10
	VCIPHER_4X1_KEY(VS10)		// Encrypt next 2 keys
	VCIPHER_4X1_KEY(VS11)
	XXLOR	VS12, VS12, V23
	BEQ	CR2, block64_last	// size = 12
	VCIPHER_4X1_KEY(VS12)		// Encrypt last 2 keys
	VCIPHER_4X1_KEY(VS13)
	XXLOR	VS14, VS14, V23		// size = 14
block64_last:
	LOAD_INPUT_BLOCK64(BLK_INP)	// Load 64 bytes of input
	// Do VCIPHERLAST on the last for each encryption
	// stream and XOR the result with the corresponding
	// value from the input block.
	VCIPHERLAST4_XOR_INPUT
	// Store the results (4*16) and update BLK_OUT by 64.
	STORE_OUTPUT_BLOCK64(BLK_OUT)
	ADD	$-64, IN_LEN		// decrement input block length
	CMP	IN_LEN, $0		// check for remaining length
	BEQ	done
block16_loop:
	CMP	IN_LEN, $16		// More input
	BLT	final_block		// If not, then handle partial block
	// Single encryption, no stitching
	GEN_VCIPHER_INPUT		// Generate input value for single encryption
	VCIPHER_1X9_KEYS(V15)		// Encrypt V15 value with 9 keys
	XXLOR	VS10, VS10, V23		// Last key -> V23 for later vcipiherlast
	// Key length based on CR2. (LT=10, EQ=12, GT=14)
	BLT	CR2, block16_last	// Finish for key size 10
	VCIPHER_1X2_KEYS(V15, VS10, VS11) // Encrypt V15 with 2 more keys
	XXLOR	VS12, VS12, V23		// Last key -> V23 for later vcipherlast
	BEQ	CR2, block16_last	// Finish for key size 12
	VCIPHER_1X2_KEYS(V15, VS12, VS13) // Encrypt V15 with last 2 keys
	XXLOR	VS14, VS14, V23		// Last key -> V23 for vcipherlast with key size 14
block16_last:
	P8_LXVB16X(BLK_INP, R0, V1)	// Load input
	VCIPHERLAST V15, V23, V15	// Encrypt last value in V23
	XXLXOR	V15, V1, V1		// XOR with input
	P8_STXVB16X(V1,R0,BLK_OUT)	// Store final encryption value to output
	ADD	$16, BLK_INP		// Increment input pointer
	ADD	$16, BLK_OUT		// Increment output pointer
	ADD	$-16, IN_LEN		// Decrement input length
	BR	block16_loop		// Check for next
final_block:
	CMP	IN_LEN, $0
	BEQ	done
	GEN_VCIPHER_INPUT		// Generate input value for partial encryption
	VCIPHER_1X9_KEYS(V15)		// Encrypt V15 with 9 keys
	XXLOR	VS10, VS10, V23		// Save possible last key
	BLT	CR2, final_block_last
	VCIPHER_1X2_KEYS(V15, VS10, VS11)	// Encrypt V15 with next 2 keys
	XXLOR	VS12, VS12, V23		// Save possible last key
	BEQ	CR2, final_block_last
	VCIPHER_1X2_KEYS(V15, VS12, VS13) // Encrypt V15 with last 2 keys
	XXLOR	VS14, VS14, V23		// Save last key
final_block_last:
	VCIPHERLAST V15, V23, V15	// Finish encryption
#ifdef GOPPC64_power10
	// set up length
	SLD	$56, IN_LEN, R17
	LXVLL	BLK_INP, R17, V25
	VXOR	V25, V15, V25
	STXVLL	V25, BLK_OUT, R17
#else
	ADD	$32, R1, MASK_PTR
	MOVD	$0, R16
	P8_STXVB16X(V15, MASK_PTR, R0)
	CMP	IN_LEN, $8
	BLT	next4
	MOVD	0(MASK_PTR), R14
	MOVD	0(BLK_INP), R15
	XOR	R14, R15, R14
	MOVD	R14, 0(BLK_OUT)
	ADD	$8, R16
	ADD	$-8, IN_LEN
next4:
	CMP	IN_LEN, $4
	BLT	next2
	MOVWZ	(BLK_INP)(R16), R15
	MOVWZ	(MASK_PTR)(R16), R14
	XOR	R14, R15, R14
	MOVW	R14, (R16)(BLK_OUT)
	ADD	$4, R16
	ADD	$-4, IN_LEN
next2:
	CMP	IN_LEN, $2
	BLT	next1
	MOVHZ	(BLK_INP)(R16), R15
	MOVHZ	(MASK_PTR)(R16), R14
	XOR	R14, R15, R14
	MOVH	R14, (R16)(BLK_OUT)
	ADD	$2, R16
	ADD	$-2, IN_LEN
next1:
	CMP	IN_LEN, $1
	BLT	done
	MOVBZ	(MASK_PTR)(R16), R14
	MOVBZ	(BLK_INP)(R16), R15
	XOR	R14, R15, R14
	MOVB	R14, (R16)(BLK_OUT)
#endif
done:
	// Save the updated counter value
	P8_STXVB16X(V30, COUNTER, R0)
	// Clear the keys
	XXLXOR	VS0, VS0, VS0
	XXLXOR	VS1, VS1, VS1
	XXLXOR	VS2, VS2, VS2
	XXLXOR	VS3, VS3, VS3
	XXLXOR	VS4, VS4, VS4
	XXLXOR	VS5, VS5, VS5
	XXLXOR	VS6, VS6, VS6
	XXLXOR	VS7, VS7, VS7
	XXLXOR	VS8, VS8, VS8
	XXLXOR	VS9, VS9, VS9
	XXLXOR	VS10, VS10, VS10
	XXLXOR	VS11, VS11, VS11
	XXLXOR	VS12, VS12, VS12
	XXLXOR	VS13, VS13, VS13
	XXLXOR	VS14, VS14, VS14
	RET