#include "arm_arch.h"

.text
.arch	armv8-a+crypto
.global	gcm_init_v8
.type	gcm_init_v8,%function
.align	4
gcm_init_v8:
	ld1		{v17.2d},[x1]		//load input H
	movi		v19.16b,#0xe1
	shl	v19.2d,v19.2d,#57		//0xc2.0
	ext		v3.16b,v17.16b,v17.16b,#8
	ushr	v18.2d,v19.2d,#63
	dup		v17.4s,v17.s[1]
	ext		v16.16b,v18.16b,v19.16b,#8		//t0=0xc2....01
	ushr	v18.2d,v3.2d,#63
	sshr	v17.4s,v17.4s,#31		//broadcast carry bit
	and		v18.16b,v18.16b,v16.16b
	shl	v3.2d,v3.2d,#1
	ext		v18.16b,v18.16b,v18.16b,#8
	and		v16.16b,v16.16b,v17.16b
	orr		v3.16b,v3.16b,v18.16b		//H<<<=1
	eor		v20.16b,v3.16b,v16.16b		//twisted H
	st1		{v20.2d},[x0],#16		//store Htable[0]

	//calculate H^2
	ext		v16.16b,v20.16b,v20.16b,#8		//Karatsuba pre-processing
	pmull	v0.1q,v20.1d,v20.1d
	eor		v16.16b,v16.16b,v20.16b
	pmull2	v2.1q,v20.2d,v20.2d
	pmull	v1.1q,v16.1d,v16.1d

	ext		v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
	eor		v18.16b,v0.16b,v2.16b
	eor		v1.16b,v1.16b,v17.16b
	eor		v1.16b,v1.16b,v18.16b
	pmull	v18.1q,v0.1d,v19.1d		//1st phase

	ins	v2.d[0],v1.d[1]
	ins	v1.d[1],v0.d[0]
	eor		v0.16b,v1.16b,v18.16b

	ext		v18.16b,v0.16b,v0.16b,#8		//2nd phase
	pmull	v0.1q,v0.1d,v19.1d
	eor		v18.16b,v18.16b,v2.16b
	eor		v22.16b,v0.16b,v18.16b

	ext		v17.16b,v22.16b,v22.16b,#8		//Karatsuba pre-processing
	eor		v17.16b,v17.16b,v22.16b
	ext		v21.16b,v16.16b,v17.16b,#8		//pack Karatsuba pre-processed
	st1		{v21.2d-v22.2d},[x0]		//store Htable[1..2]

	ret
.size	gcm_init_v8,.-gcm_init_v8
.global	gcm_gmult_v8
.type	gcm_gmult_v8,%function
.align	4
gcm_gmult_v8:
	ld1		{v17.2d},[x0]		//load Xi
	movi		v19.16b,#0xe1
	ld1		{v20.2d-v21.2d},[x1]	//load twisted H, ...
	shl	v19.2d,v19.2d,#57
#ifndef __ARMEB__
	rev64	v17.16b,v17.16b
#endif
	ext		v3.16b,v17.16b,v17.16b,#8

	pmull	v0.1q,v20.1d,v3.1d		//H.lo·Xi.lo
	eor		v17.16b,v17.16b,v3.16b		//Karatsuba pre-processing
	pmull2	v2.1q,v20.2d,v3.2d		//H.hi·Xi.hi
	pmull	v1.1q,v21.1d,v17.1d		//(H.lo+H.hi)·(Xi.lo+Xi.hi)

	ext		v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
	eor		v18.16b,v0.16b,v2.16b
	eor		v1.16b,v1.16b,v17.16b
	eor		v1.16b,v1.16b,v18.16b
	pmull	v18.1q,v0.1d,v19.1d		//1st phase of reduction

	ins	v2.d[0],v1.d[1]
	ins	v1.d[1],v0.d[0]
	eor		v0.16b,v1.16b,v18.16b

	ext		v18.16b,v0.16b,v0.16b,#8		//2nd phase of reduction
	pmull	v0.1q,v0.1d,v19.1d
	eor		v18.16b,v18.16b,v2.16b
	eor		v0.16b,v0.16b,v18.16b

#ifndef __ARMEB__
	rev64	v0.16b,v0.16b
#endif
	ext		v0.16b,v0.16b,v0.16b,#8
	st1		{v0.2d},[x0]		//write out Xi

	ret
.size	gcm_gmult_v8,.-gcm_gmult_v8
.global	gcm_ghash_v8
.type	gcm_ghash_v8,%function
.align	4
gcm_ghash_v8:
	ld1		{v0.2d},[x0]		//load [rotated] Xi
						//"[rotated]" means that
						//loaded value would have
						//to be rotated in order to
						//make it appear as in
						//alorithm specification
	subs		x3,x3,#32		//see if x3 is 32 or larger
	mov		x12,#16		//x12 is used as post-
						//increment for input pointer;
						//as loop is modulo-scheduled
						//x12 is zeroed just in time
						//to preclude oversteping
						//inp[len], which means that
						//last block[s] are actually
						//loaded twice, but last
						//copy is not processed
	ld1		{v20.2d-v21.2d},[x1],#32	//load twisted H, ..., H^2
	movi		v19.16b,#0xe1
	ld1		{v22.2d},[x1]
	csel	x12,xzr,x12,eq			//is it time to zero x12?
	ext		v0.16b,v0.16b,v0.16b,#8		//rotate Xi
	ld1		{v16.2d},[x2],#16	//load [rotated] I[0]
	shl	v19.2d,v19.2d,#57		//compose 0xc2.0 constant
#ifndef __ARMEB__
	rev64	v16.16b,v16.16b
	rev64	v0.16b,v0.16b
#endif
	ext		v3.16b,v16.16b,v16.16b,#8		//rotate I[0]
	b.lo		.Lodd_tail_v8		//x3 was less than 32
	ld1		{v17.2d},[x2],x12	//load [rotated] I[1]
#ifndef __ARMEB__
	rev64	v17.16b,v17.16b
#endif
	ext		v7.16b,v17.16b,v17.16b,#8
	eor		v3.16b,v3.16b,v0.16b		//I[i]^=Xi
	pmull	v4.1q,v20.1d,v7.1d		//H·Ii+1
	eor		v17.16b,v17.16b,v7.16b		//Karatsuba pre-processing
	pmull2	v6.1q,v20.2d,v7.2d
	b		.Loop_mod2x_v8

.align	4
.Loop_mod2x_v8:
	ext		v18.16b,v3.16b,v3.16b,#8
	subs		x3,x3,#32		//is there more data?
	pmull	v0.1q,v22.1d,v3.1d		//H^2.lo·Xi.lo
	csel	x12,xzr,x12,lo			//is it time to zero x12?

	 pmull	v5.1q,v21.1d,v17.1d
	eor		v18.16b,v18.16b,v3.16b		//Karatsuba pre-processing
	pmull2	v2.1q,v22.2d,v3.2d		//H^2.hi·Xi.hi
	eor		v0.16b,v0.16b,v4.16b		//accumulate
	pmull2	v1.1q,v21.2d,v18.2d		//(H^2.lo+H^2.hi)·(Xi.lo+Xi.hi)
	 ld1	{v16.2d},[x2],x12	//load [rotated] I[i+2]

	eor		v2.16b,v2.16b,v6.16b
	 csel	x12,xzr,x12,eq			//is it time to zero x12?
	eor		v1.16b,v1.16b,v5.16b

	ext		v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
	eor		v18.16b,v0.16b,v2.16b
	eor		v1.16b,v1.16b,v17.16b
	 ld1	{v17.2d},[x2],x12	//load [rotated] I[i+3]
#ifndef __ARMEB__
	 rev64	v16.16b,v16.16b
#endif
	eor		v1.16b,v1.16b,v18.16b
	pmull	v18.1q,v0.1d,v19.1d		//1st phase of reduction

#ifndef __ARMEB__
	 rev64	v17.16b,v17.16b
#endif
	ins	v2.d[0],v1.d[1]
	ins	v1.d[1],v0.d[0]
	 ext		v7.16b,v17.16b,v17.16b,#8
	 ext		v3.16b,v16.16b,v16.16b,#8
	eor		v0.16b,v1.16b,v18.16b
	 pmull	v4.1q,v20.1d,v7.1d		//H·Ii+1
	eor		v3.16b,v3.16b,v2.16b		//accumulate v3.16b early

	ext		v18.16b,v0.16b,v0.16b,#8		//2nd phase of reduction
	pmull	v0.1q,v0.1d,v19.1d
	eor		v3.16b,v3.16b,v18.16b
	 eor		v17.16b,v17.16b,v7.16b		//Karatsuba pre-processing
	eor		v3.16b,v3.16b,v0.16b
	 pmull2	v6.1q,v20.2d,v7.2d
	b.hs		.Loop_mod2x_v8		//there was at least 32 more bytes

	eor		v2.16b,v2.16b,v18.16b
	ext		v3.16b,v16.16b,v16.16b,#8		//re-construct v3.16b
	adds		x3,x3,#32		//re-construct x3
	eor		v0.16b,v0.16b,v2.16b		//re-construct v0.16b
	b.eq		.Ldone_v8		//is x3 zero?
.Lodd_tail_v8:
	ext		v18.16b,v0.16b,v0.16b,#8
	eor		v3.16b,v3.16b,v0.16b		//inp^=Xi
	eor		v17.16b,v16.16b,v18.16b		//v17.16b is rotated inp^Xi

	pmull	v0.1q,v20.1d,v3.1d		//H.lo·Xi.lo
	eor		v17.16b,v17.16b,v3.16b		//Karatsuba pre-processing
	pmull2	v2.1q,v20.2d,v3.2d		//H.hi·Xi.hi
	pmull	v1.1q,v21.1d,v17.1d		//(H.lo+H.hi)·(Xi.lo+Xi.hi)

	ext		v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
	eor		v18.16b,v0.16b,v2.16b
	eor		v1.16b,v1.16b,v17.16b
	eor		v1.16b,v1.16b,v18.16b
	pmull	v18.1q,v0.1d,v19.1d		//1st phase of reduction

	ins	v2.d[0],v1.d[1]
	ins	v1.d[1],v0.d[0]
	eor		v0.16b,v1.16b,v18.16b

	ext		v18.16b,v0.16b,v0.16b,#8		//2nd phase of reduction
	pmull	v0.1q,v0.1d,v19.1d
	eor		v18.16b,v18.16b,v2.16b
	eor		v0.16b,v0.16b,v18.16b

.Ldone_v8:
#ifndef __ARMEB__
	rev64	v0.16b,v0.16b
#endif
	ext		v0.16b,v0.16b,v0.16b,#8
	st1		{v0.2d},[x0]		//write out Xi

	ret
.size	gcm_ghash_v8,.-gcm_ghash_v8
.asciz  "GHASH for ARMv8, CRYPTOGAMS by <appro@openssl.org>"
.align  2
