#include "arm_arch.h" #if __ARM_MAX_ARCH__>=8 .arch armv8-a+crypto .text .globl unroll8_eor3_aes_gcm_enc_128_kernel .type unroll8_eor3_aes_gcm_enc_128_kernel,%function .align 4 unroll8_eor3_aes_gcm_enc_128_kernel: AARCH64_VALID_CALL_TARGET cbz x1, .L128_enc_ret stp d8, d9, [sp, #-80]! lsr x9, x1, #3 mov x16, x4 mov x8, x5 stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x5, #0xc200000000000000 stp x5, xzr, [sp, #64] add x10, sp, #64 mov x15, #0x100000000 //set up counter increment movi v31.16b, #0x0 mov v31.d[1], x15 mov x5, x9 ld1 { v0.16b}, [x16] //CTR block 0 sub x5, x5, #1 //byte_len - 1 and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) rev32 v30.16b, v0.16b //set up reversed counter add v30.4s, v30.4s, v31.4s //CTR block 0 rev32 v1.16b, v30.16b //CTR block 1 add v30.4s, v30.4s, v31.4s //CTR block 1 rev32 v2.16b, v30.16b //CTR block 2 add v30.4s, v30.4s, v31.4s //CTR block 2 rev32 v3.16b, v30.16b //CTR block 3 add v30.4s, v30.4s, v31.4s //CTR block 3 rev32 v4.16b, v30.16b //CTR block 4 add v30.4s, v30.4s, v31.4s //CTR block 4 rev32 v5.16b, v30.16b //CTR block 5 add v30.4s, v30.4s, v31.4s //CTR block 5 ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v6.16b, v30.16b //CTR block 6 add v30.4s, v30.4s, v31.4s //CTR block 6 rev32 v7.16b, v30.16b //CTR block 7 add v30.4s, v30.4s, v31.4s //CTR block 7 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 2 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 2 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 3 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 4 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 4 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 5 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 5 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 7 ld1 { v19.16b}, [x3] ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 7 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 ldr q27, [x8, #160] //load rk10 aese v3.16b, v26.16b //AES block 8k+11 - round 9 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v2.16b, v26.16b //AES block 8k+10 - round 9 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v6.16b, v26.16b //AES block 8k+14 - round 9 aese v4.16b, v26.16b //AES block 8k+12 - round 9 add x5, x5, x0 aese v0.16b, v26.16b //AES block 8k+8 - round 9 aese v7.16b, v26.16b //AES block 8k+15 - round 9 aese v5.16b, v26.16b //AES block 8k+13 - round 9 aese v1.16b, v26.16b //AES block 8k+9 - round 9 add x4, x0, x1, lsr #3 //end_input_ptr cmp x0, x5 //check if we have <= 8 blocks b.ge .L128_enc_tail //handle tail ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext cmp x0, x5 //check if we have <= 8 blocks .inst 0xce006d08 //eor3 v8.16b, v8.16b, v0.16b, v27.16b //AES block 0 - result rev32 v0.16b, v30.16b //CTR block 8 add v30.4s, v30.4s, v31.4s //CTR block 8 .inst 0xce016d29 //eor3 v9.16b, v9.16b, v1.16b, v27.16b //AES block 1 - result stp q8, q9, [x2], #32 //AES block 0, 1 - store result rev32 v1.16b, v30.16b //CTR block 9 .inst 0xce056dad //eor3 v13.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result add v30.4s, v30.4s, v31.4s //CTR block 9 .inst 0xce026d4a //eor3 v10.16b, v10.16b, v2.16b, v27.16b //AES block 2 - result .inst 0xce066dce //eor3 v14.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result .inst 0xce046d8c //eor3 v12.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result rev32 v2.16b, v30.16b //CTR block 10 add v30.4s, v30.4s, v31.4s //CTR block 10 .inst 0xce036d6b //eor3 v11.16b, v11.16b, v3.16b, v27.16b //AES block 3 - result .inst 0xce076def //eor3 v15.16b, v15.16b, v7.16b,v27.16b //AES block 7 - result stp q10, q11, [x2], #32 //AES block 2, 3 - store result rev32 v3.16b, v30.16b //CTR block 11 add v30.4s, v30.4s, v31.4s //CTR block 11 stp q12, q13, [x2], #32 //AES block 4, 5 - store result stp q14, q15, [x2], #32 //AES block 6, 7 - store result rev32 v4.16b, v30.16b //CTR block 12 add v30.4s, v30.4s, v31.4s //CTR block 12 b.ge .L128_enc_prepretail //do prepretail .L128_enc_main_loop: //main loop start rev32 v5.16b, v30.16b //CTR block 8k+13 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 rev64 v9.16b, v9.16b //GHASH block 8k+1 rev64 v8.16b, v8.16b //GHASH block 8k ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) rev64 v11.16b, v11.16b //GHASH block 8k+3 ldp q26, q27, [x8, #0] //load rk0, rk1 eor v8.16b, v8.16b, v19.16b //PRE 1 rev32 v7.16b, v30.16b //CTR block 8k+15 rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high rev64 v10.16b, v10.16b //GHASH block 8k+2 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h3l | h3h ext v25.16b, v25.16b, v25.16b, #8 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b,v9.16b //GHASH block 8k+2, 8k+3 - high trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid ldp q28, q26, [x8, #32] //load rk2, rk3 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) ldp q27, q28, [x8, #64] //load rk4, rk5 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h1l | h1h ext v22.16b, v22.16b, v22.16b, #8 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 ldp q26, q27, [x8, #96] //load rk6, rk7 trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid ldr d16, [x10] //MODULO - load modulo constant pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 rev32 v20.16b, v30.16b //CTR block 8k+16 add v30.4s, v30.4s, v31.4s //CTR block 8k+16 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid ldp q28, q26, [x8, #128] //load rk8, rk9 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 rev32 v22.16b, v30.16b //CTR block 8k+17 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load plaintext add v30.4s, v30.4s, v31.4s //CTR block 8k+17 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up ldr q27, [x8, #160] //load rk10 ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment rev32 v23.16b, v30.16b //CTR block 8k+18 add v30.4s, v30.4s, v31.4s //CTR block 8k+18 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 aese v2.16b, v26.16b //AES block 8k+10 - round 9 aese v4.16b, v26.16b //AES block 8k+12 - round 9 aese v1.16b, v26.16b //AES block 8k+9 - round 9 ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load plaintext rev32 v25.16b, v30.16b //CTR block 8k+19 add v30.4s, v30.4s, v31.4s //CTR block 8k+19 cmp x0, x5 //.LOOP CONTROL .inst 0xce046d8c //eor3 v12.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result aese v7.16b, v26.16b //AES block 8k+15 - round 9 aese v6.16b, v26.16b //AES block 8k+14 - round 9 aese v3.16b, v26.16b //AES block 8k+11 - round 9 .inst 0xce026d4a //eor3 v10.16b, v10.16b, v2.16b, v27.16b //AES block 8k+10 - result mov v2.16b, v23.16b //CTR block 8k+18 aese v0.16b, v26.16b //AES block 8k+8 - round 9 rev32 v4.16b, v30.16b //CTR block 8k+20 add v30.4s, v30.4s, v31.4s //CTR block 8k+20 .inst 0xce076def //eor3 v15.16b, v15.16b, v7.16b, v27.16b //AES block 7 - result aese v5.16b, v26.16b //AES block 8k+13 - round 9 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low .inst 0xce016d29 //eor3 v9.16b, v9.16b, v1.16b, v27.16b //AES block 8k+9 - result .inst 0xce036d6b //eor3 v11.16b, v11.16b, v3.16b, v27.16b //AES block 8k+11 - result mov v3.16b, v25.16b //CTR block 8k+19 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment .inst 0xce056dad //eor3 v13.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result mov v1.16b, v22.16b //CTR block 8k+17 .inst 0xce006d08 //eor3 v8.16b, v8.16b, v0.16b, v27.16b //AES block 8k+8 - result mov v0.16b, v20.16b //CTR block 8k+16 stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result .inst 0xce066dce //eor3 v14.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result stp q12, q13, [x2], #32 //AES block 8k+12, 8k+13 - store result .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low stp q14, q15, [x2], #32 //AES block 8k+14, 8k+15 - store result b.lt .L128_enc_main_loop .L128_enc_prepretail: //PREPRETAIL rev32 v5.16b, v30.16b //CTR block 8k+13 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 rev64 v8.16b, v8.16b //GHASH block 8k rev64 v9.16b, v9.16b //GHASH block 8k+1 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h6k | h5k add v30.4s, v30.4s, v31.4s //CTR block 8k+13 rev64 v11.16b, v11.16b //GHASH block 8k+3 rev64 v10.16b, v10.16b //GHASH block 8k+2 eor v8.16b, v8.16b, v19.16b //PRE 1 rev32 v6.16b, v30.16b //CTR block 8k+14 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid ldp q26, q27, [x8, #0] //load rk0, rk1 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid rev32 v7.16b, v30.16b //CTR block 8k+15 rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 ldp q27, q28, [x8, #64] //load rk4, rk5 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h1l | h1h ext v22.16b, v22.16b, v22.16b, #8 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high ldp q26, q27, [x8, #96] //load rk6, rk7 pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 ldr d16, [x10] //MODULO - load modulo constant aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up ldp q28, q26, [x8, #128] //load rk8, rk9 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 .inst 0xce114a73 //eor3 v19.16b, v19.16b, v17.16b, v18.16b //MODULO - fold into low aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 ldr q27, [x8, #160] //load rk10 aese v6.16b, v26.16b //AES block 8k+14 - round 9 aese v2.16b, v26.16b //AES block 8k+10 - round 9 aese v0.16b, v26.16b //AES block 8k+8 - round 9 aese v1.16b, v26.16b //AES block 8k+9 - round 9 aese v3.16b, v26.16b //AES block 8k+11 - round 9 aese v5.16b, v26.16b //AES block 8k+13 - round 9 aese v4.16b, v26.16b //AES block 8k+12 - round 9 aese v7.16b, v26.16b //AES block 8k+15 - round 9 .L128_enc_tail: //TAIL sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldr q8, [x0], #16 //AES block 8k+8 - load plaintext mov v29.16b, v27.16b ldp q20, q21, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 .inst 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag ldp q22, q23, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 ext v23.16b, v23.16b, v23.16b, #8 ldp q24, q25, [x3, #192] //load h8k | h7k ext v25.16b, v25.16b, v25.16b, #8 cmp x5, #112 b.gt .L128_enc_blocks_more_than_7 mov v7.16b, v6.16b mov v6.16b, v5.16b movi v17.8b, #0 cmp x5, #96 sub v30.4s, v30.4s, v31.4s mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v2.16b mov v2.16b, v1.16b movi v19.8b, #0 movi v18.8b, #0 b.gt .L128_enc_blocks_more_than_6 mov v7.16b, v6.16b cmp x5, #80 sub v30.4s, v30.4s, v31.4s mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v1.16b b.gt .L128_enc_blocks_more_than_5 cmp x5, #64 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v1.16b b.gt .L128_enc_blocks_more_than_4 mov v7.16b, v6.16b sub v30.4s, v30.4s, v31.4s mov v6.16b, v5.16b mov v5.16b, v1.16b cmp x5, #48 b.gt .L128_enc_blocks_more_than_3 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b mov v6.16b, v1.16b cmp x5, #32 ldr q24, [x3, #96] //load h4k | h3k b.gt .L128_enc_blocks_more_than_2 cmp x5, #16 sub v30.4s, v30.4s, v31.4s mov v7.16b, v1.16b b.gt .L128_enc_blocks_more_than_1 ldr q21, [x3, #48] //load h2k | h1k sub v30.4s, v30.4s, v31.4s b .L128_enc_blocks_less_than_1 .L128_enc_blocks_more_than_7: //blocks left > 7 st1 { v9.16b}, [x2], #16 //AES final-7 block - store result rev64 v8.16b, v9.16b //GHASH final-7 block ldr q9, [x0], #16 //AES final-6 block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-7 block - mid pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high ins v18.d[0], v24.d[1] //GHASH final-7 block - mid eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid movi v16.8b, #0 //suppress further partial tag feed in .inst 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low .L128_enc_blocks_more_than_6: //blocks left > 6 st1 { v9.16b}, [x2], #16 //AES final-6 block - store result rev64 v8.16b, v9.16b //GHASH final-6 block ldr q9, [x0], #16 //AES final-5 block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-6 block - mid .inst 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid movi v16.8b, #0 //suppress further partial tag feed in pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high .L128_enc_blocks_more_than_5: //blocks left > 5 st1 { v9.16b}, [x2], #16 //AES final-5 block - store result rev64 v8.16b, v9.16b //GHASH final-5 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-5 block - mid ldr q9, [x0], #16 //AES final-4 block - load plaintext pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid ins v27.d[1], v27.d[0] //GHASH final-5 block - mid .inst 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low movi v16.8b, #0 //suppress further partial tag feed in pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid .L128_enc_blocks_more_than_4: //blocks left > 4 st1 { v9.16b}, [x2], #16 //AES final-4 block - store result rev64 v8.16b, v9.16b //GHASH final-4 block ldr q9, [x0], #16 //AES final-3 block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-4 block - mid movi v16.8b, #0 //suppress further partial tag feed in pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low .inst 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid .L128_enc_blocks_more_than_3: //blocks left > 3 st1 { v9.16b}, [x2], #16 //AES final-3 block - store result ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 rev64 v8.16b, v9.16b //GHASH final-3 block eor v8.16b, v8.16b, v16.16b //feed in partial tag movi v16.8b, #0 //suppress further partial tag feed in ins v27.d[0], v8.d[1] //GHASH final-3 block - mid ldr q24, [x3, #96] //load h4k | h3k pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low ldr q9, [x0], #16 //AES final-2 block - load plaintext eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid ins v27.d[1], v27.d[0] //GHASH final-3 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low .inst 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high .L128_enc_blocks_more_than_2: //blocks left > 2 st1 { v9.16b}, [x2], #16 //AES final-2 block - store result rev64 v8.16b, v9.16b //GHASH final-2 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-1 block - load plaintext ins v27.d[0], v8.d[1] //GHASH final-2 block - mid ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 movi v16.8b, #0 //suppress further partial tag feed in eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid .inst 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low .L128_enc_blocks_more_than_1: //blocks left > 1 st1 { v9.16b}, [x2], #16 //AES final-1 block - store result ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 rev64 v8.16b, v9.16b //GHASH final-1 block ldr q9, [x0], #16 //AES final block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag movi v16.8b, #0 //suppress further partial tag feed in ins v27.d[0], v8.d[1] //GHASH final-1 block - mid .inst 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid ldr q21, [x3, #48] //load h2k | h1k ins v27.d[1], v27.d[0] //GHASH final-1 block - mid pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low .L128_enc_blocks_less_than_1: //blocks left <= 1 rev32 v30.16b, v30.16b str q30, [x16] //store the updated counter and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) mvn x6, xzr //temp0_x = 0xffffffffffffffff ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored and x1, x1, #127 //bit_length %= 128 lsr x6, x6, x1 //temp0_x is mask for top 64b of last block mvn x7, xzr //temp1_x = 0xffffffffffffffff cmp x1, #64 csel x13, x7, x6, lt csel x14, x6, xzr, lt mov v0.d[1], x14 mov v0.d[0], x13 //ctr0b is mask for last block and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits rev64 v8.16b, v9.16b //GHASH final block bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing st1 { v9.16b}, [x2] //store all 16B eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v16.d[0], v8.d[1] //GHASH final block - mid eor v16.8b, v16.8b, v8.8b //GHASH final block - mid ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high eor v18.16b, v18.16b, v16.16b //GHASH final block - mid ldr d16, [x10] //MODULO - load modulo constant pmull v26.1q, v8.1d, v20.1d //GHASH final block - low eor v17.16b, v17.16b, v28.16b //GHASH final block - high eor v19.16b, v19.16b, v26.16b //GHASH final block - low ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b st1 { v19.16b }, [x3] mov x0, x9 ldp d10, d11, [sp, #16] ldp d12, d13, [sp, #32] ldp d14, d15, [sp, #48] ldp d8, d9, [sp], #80 ret .L128_enc_ret: mov w0, #0x0 ret .size unroll8_eor3_aes_gcm_enc_128_kernel,.-unroll8_eor3_aes_gcm_enc_128_kernel .globl unroll8_eor3_aes_gcm_dec_128_kernel .type unroll8_eor3_aes_gcm_dec_128_kernel,%function .align 4 unroll8_eor3_aes_gcm_dec_128_kernel: AARCH64_VALID_CALL_TARGET cbz x1, .L128_dec_ret stp d8, d9, [sp, #-80]! lsr x9, x1, #3 mov x16, x4 mov x8, x5 stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x5, #0xc200000000000000 stp x5, xzr, [sp, #64] add x10, sp, #64 mov x5, x9 ld1 { v0.16b}, [x16] //CTR block 0 ldp q26, q27, [x8, #0] //load rk0, rk1 sub x5, x5, #1 //byte_len - 1 mov x15, #0x100000000 //set up counter increment movi v31.16b, #0x0 mov v31.d[1], x15 ld1 { v19.16b}, [x3] ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b rev32 v30.16b, v0.16b //set up reversed counter aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 add v30.4s, v30.4s, v31.4s //CTR block 0 rev32 v1.16b, v30.16b //CTR block 1 add v30.4s, v30.4s, v31.4s //CTR block 1 and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) rev32 v2.16b, v30.16b //CTR block 2 add v30.4s, v30.4s, v31.4s //CTR block 2 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 rev32 v3.16b, v30.16b //CTR block 3 add v30.4s, v30.4s, v31.4s //CTR block 3 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 rev32 v4.16b, v30.16b //CTR block 4 add v30.4s, v30.4s, v31.4s //CTR block 4 rev32 v5.16b, v30.16b //CTR block 5 add v30.4s, v30.4s, v31.4s //CTR block 5 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 rev32 v6.16b, v30.16b //CTR block 6 add v30.4s, v30.4s, v31.4s //CTR block 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 0 rev32 v7.16b, v30.16b //CTR block 7 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 0 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 1 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 3 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 4 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 3 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 4 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 5 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 6 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 6 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 7 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 7 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 7 add x5, x5, x0 add v30.4s, v30.4s, v31.4s //CTR block 7 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 8 aese v0.16b, v26.16b //AES block 0 - round 9 aese v1.16b, v26.16b //AES block 1 - round 9 aese v6.16b, v26.16b //AES block 6 - round 9 ldr q27, [x8, #160] //load rk10 aese v4.16b, v26.16b //AES block 4 - round 9 aese v3.16b, v26.16b //AES block 3 - round 9 aese v2.16b, v26.16b //AES block 2 - round 9 aese v5.16b, v26.16b //AES block 5 - round 9 aese v7.16b, v26.16b //AES block 7 - round 9 add x4, x0, x1, lsr #3 //end_input_ptr cmp x0, x5 //check if we have <= 8 blocks b.ge .L128_dec_tail //handle tail ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext .inst 0xce006d00 //eor3 v0.16b, v8.16b, v0.16b, v27.16b //AES block 0 - result .inst 0xce016d21 //eor3 v1.16b, v9.16b, v1.16b, v27.16b //AES block 1 - result stp q0, q1, [x2], #32 //AES block 0, 1 - store result rev32 v0.16b, v30.16b //CTR block 8 add v30.4s, v30.4s, v31.4s //CTR block 8 ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext rev32 v1.16b, v30.16b //CTR block 9 add v30.4s, v30.4s, v31.4s //CTR block 9 ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext .inst 0xce036d63 //eor3 v3.16b, v11.16b, v3.16b, v27.16b //AES block 3 - result .inst 0xce026d42 //eor3 v2.16b, v10.16b, v2.16b, v27.16b //AES block 2 - result stp q2, q3, [x2], #32 //AES block 2, 3 - store result rev32 v2.16b, v30.16b //CTR block 10 add v30.4s, v30.4s, v31.4s //CTR block 10 .inst 0xce066dc6 //eor3 v6.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result rev32 v3.16b, v30.16b //CTR block 11 add v30.4s, v30.4s, v31.4s //CTR block 11 .inst 0xce046d84 //eor3 v4.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result .inst 0xce056da5 //eor3 v5.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result stp q4, q5, [x2], #32 //AES block 4, 5 - store result .inst 0xce076de7 //eor3 v7.16b, v15.16b, v7.16b, v27.16b //AES block 7 - result stp q6, q7, [x2], #32 //AES block 6, 7 - store result rev32 v4.16b, v30.16b //CTR block 12 cmp x0, x5 //check if we have <= 8 blocks add v30.4s, v30.4s, v31.4s //CTR block 12 b.ge .L128_dec_prepretail //do prepretail .L128_dec_main_loop: //main loop start ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 rev64 v9.16b, v9.16b //GHASH block 8k+1 rev64 v8.16b, v8.16b //GHASH block 8k ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev64 v14.16b, v14.16b //GHASH block 8k+6 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 eor v8.16b, v8.16b, v19.16b //PRE 1 rev32 v5.16b, v30.16b //CTR block 8k+13 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 rev64 v10.16b, v10.16b //GHASH block 8k+2 rev64 v12.16b, v12.16b //GHASH block 8k+4 ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high rev64 v11.16b, v11.16b //GHASH block 8k+3 rev32 v7.16b, v30.16b //CTR block 8k+15 trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid rev64 v13.16b, v13.16b //GHASH block 8k+5 pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high ldp q28, q26, [x8, #32] //load rk2, rk3 trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 rev64 v15.16b, v15.16b //GHASH block 8k+7 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high ldp q27, q28, [x8, #64] //load rk4, rk5 pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid ldp q26, q27, [x8, #96] //load rk6, rk7 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 add v30.4s, v30.4s, v31.4s //CTR block 8k+15 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 ldr d16, [x10] //MODULO - load modulo constant .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 rev32 v20.16b, v30.16b //CTR block 8k+16 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid add v30.4s, v30.4s, v31.4s //CTR block 8k+16 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 rev32 v22.16b, v30.16b //CTR block 8k+17 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 add v30.4s, v30.4s, v31.4s //CTR block 8k+17 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 rev32 v23.16b, v30.16b //CTR block 8k+18 ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 add v30.4s, v30.4s, v31.4s //CTR block 8k+18 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 aese v0.16b, v26.16b //AES block 8k+8 - round 9 aese v1.16b, v26.16b //AES block 8k+9 - round 9 ldr q27, [x8, #160] //load rk10 aese v6.16b, v26.16b //AES block 8k+14 - round 9 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v2.16b, v26.16b //AES block 8k+10 - round 9 aese v7.16b, v26.16b //AES block 8k+15 - round 9 aese v4.16b, v26.16b //AES block 8k+12 - round 9 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment rev32 v25.16b, v30.16b //CTR block 8k+19 add v30.4s, v30.4s, v31.4s //CTR block 8k+19 aese v3.16b, v26.16b //AES block 8k+11 - round 9 aese v5.16b, v26.16b //AES block 8k+13 - round 9 .inst 0xce016d21 //eor3 v1.16b, v9.16b, v1.16b, v27.16b //AES block 8k+9 - result .inst 0xce006d00 //eor3 v0.16b, v8.16b, v0.16b, v27.16b //AES block 8k+8 - result .inst 0xce076de7 //eor3 v7.16b, v15.16b, v7.16b, v27.16b //AES block 8k+15 - result .inst 0xce066dc6 //eor3 v6.16b, v14.16b, v6.16b, v27.16b //AES block 8k+14 - result .inst 0xce026d42 //eor3 v2.16b, v10.16b, v2.16b, v27.16b //AES block 8k+10 - result stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result mov v1.16b, v22.16b //CTR block 8k+17 .inst 0xce046d84 //eor3 v4.16b, v12.16b, v4.16b, v27.16b //AES block 8k+12 - result .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low mov v0.16b, v20.16b //CTR block 8k+16 .inst 0xce036d63 //eor3 v3.16b, v11.16b, v3.16b, v27.16b //AES block 8k+11 - result cmp x0, x5 //.LOOP CONTROL stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result .inst 0xce056da5 //eor3 v5.16b, v13.16b, v5.16b, v27.16b //AES block 8k+13 - result mov v2.16b, v23.16b //CTR block 8k+18 stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result rev32 v4.16b, v30.16b //CTR block 8k+20 add v30.4s, v30.4s, v31.4s //CTR block 8k+20 stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result mov v3.16b, v25.16b //CTR block 8k+19 b.lt .L128_dec_main_loop .L128_dec_prepretail: //PREPRETAIL rev64 v11.16b, v11.16b //GHASH block 8k+3 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev64 v8.16b, v8.16b //GHASH block 8k rev64 v10.16b, v10.16b //GHASH block 8k+2 rev32 v5.16b, v30.16b //CTR block 8k+13 ldp q26, q27, [x8, #0] //load rk0, rk1 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 eor v8.16b, v8.16b, v19.16b //PRE 1 rev64 v9.16b, v9.16b //GHASH block 8k+1 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 rev64 v13.16b, v13.16b //GHASH block 8k+5 rev64 v12.16b, v12.16b //GHASH block 8k+4 rev64 v14.16b, v14.16b //GHASH block 8k+6 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low rev32 v7.16b, v30.16b //CTR block 8k+15 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 ldp q28, q26, [x8, #32] //load rk2, rk3 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid ldp q27, q28, [x8, #64] //load rk4, rk5 rev64 v15.16b, v15.16b //GHASH block 8k+7 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid ldp q26, q27, [x8, #96] //load rk6, rk7 .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 ldr d16, [x10] //MODULO - load modulo constant pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up ldp q28, q26, [x8, #128] //load rk8, rk9 pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid ldr q27, [x8, #160] //load rk10 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v6.16b, v26.16b //AES block 8k+14 - round 9 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v2.16b, v26.16b //AES block 8k+10 - round 9 aese v3.16b, v26.16b //AES block 8k+11 - round 9 aese v5.16b, v26.16b //AES block 8k+13 - round 9 aese v0.16b, v26.16b //AES block 8k+8 - round 9 aese v4.16b, v26.16b //AES block 8k+12 - round 9 aese v1.16b, v26.16b //AES block 8k+9 - round 9 aese v7.16b, v26.16b //AES block 8k+15 - round 9 .L128_dec_tail: //TAIL mov v29.16b, v27.16b sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process cmp x5, #112 ldp q24, q25, [x3, #192] //load h8k | h7k ext v25.16b, v25.16b, v25.16b, #8 ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext ldp q20, q21, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag ldp q22, q23, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 ext v23.16b, v23.16b, v23.16b, #8 .inst 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result b.gt .L128_dec_blocks_more_than_7 cmp x5, #96 mov v7.16b, v6.16b movi v19.8b, #0 movi v17.8b, #0 mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v2.16b mov v2.16b, v1.16b movi v18.8b, #0 sub v30.4s, v30.4s, v31.4s b.gt .L128_dec_blocks_more_than_6 cmp x5, #80 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v1.16b b.gt .L128_dec_blocks_more_than_5 cmp x5, #64 mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v1.16b sub v30.4s, v30.4s, v31.4s b.gt .L128_dec_blocks_more_than_4 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v1.16b cmp x5, #48 b.gt .L128_dec_blocks_more_than_3 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b cmp x5, #32 ldr q24, [x3, #96] //load h4k | h3k mov v6.16b, v1.16b b.gt .L128_dec_blocks_more_than_2 cmp x5, #16 mov v7.16b, v1.16b sub v30.4s, v30.4s, v31.4s b.gt .L128_dec_blocks_more_than_1 sub v30.4s, v30.4s, v31.4s ldr q21, [x3, #48] //load h2k | h1k b .L128_dec_blocks_less_than_1 .L128_dec_blocks_more_than_7: //blocks left > 7 rev64 v8.16b, v9.16b //GHASH final-7 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v18.d[0], v24.d[1] //GHASH final-7 block - mid pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low ins v27.d[0], v8.d[1] //GHASH final-7 block - mid movi v16.8b, #0 //suppress further partial tag feed in ldr q9, [x0], #16 //AES final-6 block - load ciphertext eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high st1 { v12.16b}, [x2], #16 //AES final-7 block - store result .inst 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid .L128_dec_blocks_more_than_6: //blocks left > 6 rev64 v8.16b, v9.16b //GHASH final-6 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-6 block - mid eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low ldr q9, [x0], #16 //AES final-5 block - load ciphertext movi v16.8b, #0 //suppress further partial tag feed in pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid st1 { v12.16b}, [x2], #16 //AES final-6 block - store result pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid .inst 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result .L128_dec_blocks_more_than_5: //blocks left > 5 rev64 v8.16b, v9.16b //GHASH final-5 block ldr q9, [x0], #16 //AES final-4 block - load ciphertext st1 { v12.16b}, [x2], #16 //AES final-5 block - store result eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-5 block - mid .inst 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid ins v27.d[1], v27.d[0] //GHASH final-5 block - mid pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low movi v16.8b, #0 //suppress further partial tag feed in pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high .L128_dec_blocks_more_than_4: //blocks left > 4 rev64 v8.16b, v9.16b //GHASH final-4 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-3 block - load ciphertext ins v27.d[0], v8.d[1] //GHASH final-4 block - mid movi v16.8b, #0 //suppress further partial tag feed in pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high st1 { v12.16b}, [x2], #16 //AES final-4 block - store result eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid .inst 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid .L128_dec_blocks_more_than_3: //blocks left > 3 st1 { v12.16b}, [x2], #16 //AES final-3 block - store result rev64 v8.16b, v9.16b //GHASH final-3 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-3 block - mid ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 ldr q24, [x3, #96] //load h4k | h3k eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid ldr q9, [x0], #16 //AES final-2 block - load ciphertext ins v27.d[1], v27.d[0] //GHASH final-3 block - mid pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high movi v16.8b, #0 //suppress further partial tag feed in .inst 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid .L128_dec_blocks_more_than_2: //blocks left > 2 rev64 v8.16b, v9.16b //GHASH final-2 block st1 { v12.16b}, [x2], #16 //AES final-2 block - store result eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 movi v16.8b, #0 //suppress further partial tag feed in ins v27.d[0], v8.d[1] //GHASH final-2 block - mid eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid ldr q9, [x0], #16 //AES final-1 block - load ciphertext eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low .inst 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high .L128_dec_blocks_more_than_1: //blocks left > 1 st1 { v12.16b}, [x2], #16 //AES final-1 block - store result rev64 v8.16b, v9.16b //GHASH final-1 block ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 eor v8.16b, v8.16b, v16.16b //feed in partial tag movi v16.8b, #0 //suppress further partial tag feed in ins v27.d[0], v8.d[1] //GHASH final-1 block - mid ldr q9, [x0], #16 //AES final block - load ciphertext pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high ldr q21, [x3, #48] //load h2k | h1k ins v27.d[1], v27.d[0] //GHASH final-1 block - mid .inst 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid .L128_dec_blocks_less_than_1: //blocks left <= 1 and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) mvn x6, xzr //temp0_x = 0xffffffffffffffff and x1, x1, #127 //bit_length %= 128 lsr x6, x6, x1 //temp0_x is mask for top 64b of last block cmp x1, #64 mvn x7, xzr //temp1_x = 0xffffffffffffffff csel x13, x7, x6, lt csel x14, x6, xzr, lt mov v0.d[1], x14 mov v0.d[0], x13 //ctr0b is mask for last block ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits rev64 v8.16b, v9.16b //GHASH final block eor v8.16b, v8.16b, v16.16b //feed in partial tag pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high ins v16.d[0], v8.d[1] //GHASH final block - mid eor v17.16b, v17.16b, v28.16b //GHASH final block - high eor v16.8b, v16.8b, v8.8b //GHASH final block - mid bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid st1 { v12.16b}, [x2] //store all 16B pmull v26.1q, v8.1d, v20.1d //GHASH final block - low eor v18.16b, v18.16b, v16.16b //GHASH final block - mid ldr d16, [x10] //MODULO - load modulo constant eor v19.16b, v19.16b, v26.16b //GHASH final block - low eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up .inst 0xce115652 //eor3 v18.16b, v18.16b, v17.16b, v21.16b //MODULO - fold into mid pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment .inst 0xce124673 //eor3 v19.16b, v19.16b, v18.16b, v17.16b //MODULO - fold into low ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b st1 { v19.16b }, [x3] rev32 v30.16b, v30.16b str q30, [x16] //store the updated counter mov x0, x9 ldp d10, d11, [sp, #16] ldp d12, d13, [sp, #32] ldp d14, d15, [sp, #48] ldp d8, d9, [sp], #80 ret .L128_dec_ret: mov w0, #0x0 ret .size unroll8_eor3_aes_gcm_dec_128_kernel,.-unroll8_eor3_aes_gcm_dec_128_kernel .globl unroll8_eor3_aes_gcm_enc_192_kernel .type unroll8_eor3_aes_gcm_enc_192_kernel,%function .align 4 unroll8_eor3_aes_gcm_enc_192_kernel: AARCH64_VALID_CALL_TARGET cbz x1, .L192_enc_ret stp d8, d9, [sp, #-80]! lsr x9, x1, #3 mov x16, x4 mov x8, x5 stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x5, #0xc200000000000000 stp x5, xzr, [sp, #64] add x10, sp, #64 mov x5, x9 ld1 { v0.16b}, [x16] //CTR block 0 mov x15, #0x100000000 //set up counter increment movi v31.16b, #0x0 mov v31.d[1], x15 rev32 v30.16b, v0.16b //set up reversed counter add v30.4s, v30.4s, v31.4s //CTR block 0 rev32 v1.16b, v30.16b //CTR block 1 add v30.4s, v30.4s, v31.4s //CTR block 1 rev32 v2.16b, v30.16b //CTR block 2 add v30.4s, v30.4s, v31.4s //CTR block 2 rev32 v3.16b, v30.16b //CTR block 3 add v30.4s, v30.4s, v31.4s //CTR block 3 rev32 v4.16b, v30.16b //CTR block 4 add v30.4s, v30.4s, v31.4s //CTR block 4 sub x5, x5, #1 //byte_len - 1 and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) rev32 v5.16b, v30.16b //CTR block 5 add v30.4s, v30.4s, v31.4s //CTR block 5 ldp q26, q27, [x8, #0] //load rk0, rk1 add x5, x5, x0 rev32 v6.16b, v30.16b //CTR block 6 add v30.4s, v30.4s, v31.4s //CTR block 6 rev32 v7.16b, v30.16b //CTR block 7 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 1 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 2 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 2 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 3 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 3 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 4 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 5 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 add v30.4s, v30.4s, v31.4s //CTR block 7 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 6 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 7 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 7 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 8 add x4, x0, x1, lsr #3 //end_input_ptr cmp x0, x5 //check if we have <= 8 blocks aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 9 ld1 { v19.16b}, [x3] ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b ldp q27, q28, [x8, #160] //load rk10, rk11 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 9 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 9 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 9 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 14 - round 10 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 9 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 11 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 9 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 13 - round 10 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 12 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 10 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 15 - round 10 aese v6.16b, v28.16b //AES block 14 - round 11 aese v3.16b, v28.16b //AES block 11 - round 11 aese v4.16b, v28.16b //AES block 12 - round 11 aese v7.16b, v28.16b //AES block 15 - round 11 ldr q26, [x8, #192] //load rk12 aese v1.16b, v28.16b //AES block 9 - round 11 aese v5.16b, v28.16b //AES block 13 - round 11 aese v2.16b, v28.16b //AES block 10 - round 11 aese v0.16b, v28.16b //AES block 8 - round 11 b.ge .L192_enc_tail //handle tail ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext .inst 0xce006908 //eor3 v8.16b, v8.16b, v0.16b, v26.16b //AES block 0 - result rev32 v0.16b, v30.16b //CTR block 8 add v30.4s, v30.4s, v31.4s //CTR block 8 .inst 0xce03696b //eor3 v11.16b, v11.16b, v3.16b, v26.16b //AES block 3 - result .inst 0xce016929 //eor3 v9.16b, v9.16b, v1.16b, v26.16b //AES block 1 - result rev32 v1.16b, v30.16b //CTR block 9 add v30.4s, v30.4s, v31.4s //CTR block 9 .inst 0xce04698c //eor3 v12.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result .inst 0xce0569ad //eor3 v13.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result .inst 0xce0769ef //eor3 v15.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result stp q8, q9, [x2], #32 //AES block 0, 1 - store result .inst 0xce02694a //eor3 v10.16b, v10.16b, v2.16b, v26.16b //AES block 2 - result rev32 v2.16b, v30.16b //CTR block 10 add v30.4s, v30.4s, v31.4s //CTR block 10 stp q10, q11, [x2], #32 //AES block 2, 3 - store result cmp x0, x5 //check if we have <= 8 blocks rev32 v3.16b, v30.16b //CTR block 11 add v30.4s, v30.4s, v31.4s //CTR block 11 .inst 0xce0669ce //eor3 v14.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result stp q12, q13, [x2], #32 //AES block 4, 5 - store result rev32 v4.16b, v30.16b //CTR block 12 stp q14, q15, [x2], #32 //AES block 6, 7 - store result add v30.4s, v30.4s, v31.4s //CTR block 12 b.ge .L192_enc_prepretail //do prepretail .L192_enc_main_loop: //main loop start rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) ldp q26, q27, [x8, #0] //load rk0, rk1 rev64 v10.16b, v10.16b //GHASH block 8k+2 rev32 v5.16b, v30.16b //CTR block 8k+13 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev64 v8.16b, v8.16b //GHASH block 8k ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 rev64 v9.16b, v9.16b //GHASH block 8k+1 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 eor v8.16b, v8.16b, v19.16b //PRE 1 rev64 v11.16b, v11.16b //GHASH block 8k+3 rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 rev32 v7.16b, v30.16b //CTR block 8k+15 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid ldp q28, q26, [x8, #128] //load rk8, rk9 pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 add v30.4s, v30.4s, v31.4s //CTR block 8k+15 ldr d16, [x10] //MODULO - load modulo constant .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 ldp q27, q28, [x8, #160] //load rk10, rk11 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low rev32 v20.16b, v30.16b //CTR block 8k+16 add v30.4s, v30.4s, v31.4s //CTR block 8k+16 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid rev32 v22.16b, v30.16b //CTR block 8k+17 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 add v30.4s, v30.4s, v31.4s //CTR block 8k+17 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 ldr q26, [x8, #192] //load rk12 ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext aese v4.16b, v28.16b //AES block 8k+12 - round 11 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load plaintext ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load plaintext aese v2.16b, v28.16b //AES block 8k+10 - round 11 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 rev32 v23.16b, v30.16b //CTR block 8k+18 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 aese v5.16b, v28.16b //AES block 8k+13 - round 11 add v30.4s, v30.4s, v31.4s //CTR block 8k+18 aese v7.16b, v28.16b //AES block 8k+15 - round 11 aese v0.16b, v28.16b //AES block 8k+8 - round 11 .inst 0xce04698c //eor3 v12.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result aese v6.16b, v28.16b //AES block 8k+14 - round 11 aese v3.16b, v28.16b //AES block 8k+11 - round 11 aese v1.16b, v28.16b //AES block 8k+9 - round 11 rev32 v25.16b, v30.16b //CTR block 8k+19 add v30.4s, v30.4s, v31.4s //CTR block 8k+19 .inst 0xce0769ef //eor3 v15.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result .inst 0xce02694a //eor3 v10.16b, v10.16b, v2.16b, v26.16b //AES block 8k+10 - result .inst 0xce006908 //eor3 v8.16b, v8.16b, v0.16b, v26.16b //AES block 8k+8 - result mov v2.16b, v23.16b //CTR block 8k+18 .inst 0xce016929 //eor3 v9.16b, v9.16b, v1.16b, v26.16b //AES block 8k+9 - result mov v1.16b, v22.16b //CTR block 8k+17 stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment .inst 0xce0669ce //eor3 v14.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result mov v0.16b, v20.16b //CTR block 8k+16 rev32 v4.16b, v30.16b //CTR block 8k+20 add v30.4s, v30.4s, v31.4s //CTR block 8k+20 .inst 0xce0569ad //eor3 v13.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low .inst 0xce03696b //eor3 v11.16b, v11.16b, v3.16b, v26.16b //AES block 8k+11 - result mov v3.16b, v25.16b //CTR block 8k+19 stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result stp q12, q13, [x2], #32 //AES block 8k+12, 8k+13 - store result cmp x0, x5 //.LOOP CONTROL stp q14, q15, [x2], #32 //AES block 8k+14, 8k+15 - store result b.lt .L192_enc_main_loop .L192_enc_prepretail: //PREPRETAIL rev32 v5.16b, v30.16b //CTR block 8k+13 ldp q26, q27, [x8, #0] //load rk0, rk1 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 rev64 v8.16b, v8.16b //GHASH block 8k ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k rev64 v11.16b, v11.16b //GHASH block 8k+3 rev64 v10.16b, v10.16b //GHASH block 8k+2 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 eor v8.16b, v8.16b, v19.16b //PRE 1 rev32 v7.16b, v30.16b //CTR block 8k+15 rev64 v9.16b, v9.16b //GHASH block 8k+1 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid ldp q27, q28, [x8, #64] //load rk4, rk5 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 ldr d16, [x10] //MODULO - load modulo constant aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 ldp q27, q28, [x8, #160] //load rk10, rk11 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ldr q26, [x8, #192] //load rk12 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 aese v1.16b, v28.16b //AES block 8k+9 - round 11 aese v7.16b, v28.16b //AES block 8k+15 - round 11 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 aese v3.16b, v28.16b //AES block 8k+11 - round 11 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v2.16b, v28.16b //AES block 8k+10 - round 11 aese v0.16b, v28.16b //AES block 8k+8 - round 11 aese v6.16b, v28.16b //AES block 8k+14 - round 11 aese v4.16b, v28.16b //AES block 8k+12 - round 11 aese v5.16b, v28.16b //AES block 8k+13 - round 11 .L192_enc_tail: //TAIL ldp q20, q21, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldr q8, [x0], #16 //AES block 8k+8 - l3ad plaintext ldp q24, q25, [x3, #192] //load h8k | h7k ext v25.16b, v25.16b, v25.16b, #8 mov v29.16b, v26.16b ldp q22, q23, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 ext v23.16b, v23.16b, v23.16b, #8 cmp x5, #112 .inst 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag b.gt .L192_enc_blocks_more_than_7 cmp x5, #96 mov v7.16b, v6.16b movi v17.8b, #0 mov v6.16b, v5.16b movi v19.8b, #0 sub v30.4s, v30.4s, v31.4s mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v2.16b mov v2.16b, v1.16b movi v18.8b, #0 b.gt .L192_enc_blocks_more_than_6 mov v7.16b, v6.16b cmp x5, #80 mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v1.16b sub v30.4s, v30.4s, v31.4s b.gt .L192_enc_blocks_more_than_5 cmp x5, #64 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v1.16b b.gt .L192_enc_blocks_more_than_4 mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v1.16b sub v30.4s, v30.4s, v31.4s cmp x5, #48 b.gt .L192_enc_blocks_more_than_3 mov v7.16b, v6.16b mov v6.16b, v1.16b sub v30.4s, v30.4s, v31.4s ldr q24, [x3, #96] //load h4k | h3k cmp x5, #32 b.gt .L192_enc_blocks_more_than_2 sub v30.4s, v30.4s, v31.4s cmp x5, #16 mov v7.16b, v1.16b b.gt .L192_enc_blocks_more_than_1 sub v30.4s, v30.4s, v31.4s ldr q21, [x3, #48] //load h2k | h1k b .L192_enc_blocks_less_than_1 .L192_enc_blocks_more_than_7: //blocks left > 7 st1 { v9.16b}, [x2], #16 //AES final-7 block - store result rev64 v8.16b, v9.16b //GHASH final-7 block ins v18.d[0], v24.d[1] //GHASH final-7 block - mid eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-7 block - mid ldr q9, [x0], #16 //AES final-6 block - load plaintext eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid movi v16.8b, #0 //suppress further partial tag feed in pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid .inst 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result .L192_enc_blocks_more_than_6: //blocks left > 6 st1 { v9.16b}, [x2], #16 //AES final-6 block - store result rev64 v8.16b, v9.16b //GHASH final-6 block ldr q9, [x0], #16 //AES final-5 block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-6 block - mid pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low .inst 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result movi v16.8b, #0 //suppress further partial tag feed in pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid .L192_enc_blocks_more_than_5: //blocks left > 5 st1 { v9.16b}, [x2], #16 //AES final-5 block - store result rev64 v8.16b, v9.16b //GHASH final-5 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-5 block - mid ldr q9, [x0], #16 //AES final-4 block - load plaintext pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high ins v27.d[1], v27.d[0] //GHASH final-5 block - mid pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid .inst 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result movi v16.8b, #0 //suppress further partial tag feed in eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid .L192_enc_blocks_more_than_4: //blocks left > 4 st1 { v9.16b}, [x2], #16 //AES final-4 block - store result rev64 v8.16b, v9.16b //GHASH final-4 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-3 block - load plaintext pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high ins v27.d[0], v8.d[1] //GHASH final-4 block - mid pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid movi v16.8b, #0 //suppress further partial tag feed in eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid .inst 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result .L192_enc_blocks_more_than_3: //blocks left > 3 ldr q24, [x3, #96] //load h4k | h3k st1 { v9.16b}, [x2], #16 //AES final-3 block - store result rev64 v8.16b, v9.16b //GHASH final-3 block eor v8.16b, v8.16b, v16.16b //feed in partial tag movi v16.8b, #0 //suppress further partial tag feed in ldr q9, [x0], #16 //AES final-2 block - load plaintext ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 ins v27.d[0], v8.d[1] //GHASH final-3 block - mid .inst 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid ins v27.d[1], v27.d[0] //GHASH final-3 block - mid pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high .L192_enc_blocks_more_than_2: //blocks left > 2 st1 { v9.16b}, [x2], #16 //AES final-2 block - store result rev64 v8.16b, v9.16b //GHASH final-2 block ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-1 block - load plaintext ins v27.d[0], v8.d[1] //GHASH final-2 block - mid eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high movi v16.8b, #0 //suppress further partial tag feed in pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid .inst 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result .L192_enc_blocks_more_than_1: //blocks left > 1 ldr q22, [x3, #64] //load h1l | h1h ext v22.16b, v22.16b, v22.16b, #8 st1 { v9.16b}, [x2], #16 //AES final-1 block - store result rev64 v8.16b, v9.16b //GHASH final-1 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-1 block - mid pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid ldr q9, [x0], #16 //AES final block - load plaintext ldr q21, [x3, #48] //load h2k | h1k ins v27.d[1], v27.d[0] //GHASH final-1 block - mid .inst 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid movi v16.8b, #0 //suppress further partial tag feed in eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high .L192_enc_blocks_less_than_1: //blocks left <= 1 mvn x6, xzr //temp0_x = 0xffffffffffffffff and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) and x1, x1, #127 //bit_length %= 128 lsr x6, x6, x1 //temp0_x is mask for top 64b of last block cmp x1, #64 mvn x7, xzr //temp1_x = 0xffffffffffffffff csel x13, x7, x6, lt csel x14, x6, xzr, lt mov v0.d[1], x14 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored mov v0.d[0], x13 //ctr0b is mask for last block and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits rev64 v8.16b, v9.16b //GHASH final block bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing st1 { v9.16b}, [x2] //store all 16B eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v16.d[0], v8.d[1] //GHASH final block - mid pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high eor v17.16b, v17.16b, v28.16b //GHASH final block - high pmull v26.1q, v8.1d, v20.1d //GHASH final block - low eor v16.8b, v16.8b, v8.8b //GHASH final block - mid pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid eor v18.16b, v18.16b, v16.16b //GHASH final block - mid ldr d16, [x10] //MODULO - load modulo constant eor v19.16b, v19.16b, v26.16b //GHASH final block - low ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment rev32 v30.16b, v30.16b str q30, [x16] //store the updated counter .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b st1 { v19.16b }, [x3] mov x0, x9 //return sizes ldp d10, d11, [sp, #16] ldp d12, d13, [sp, #32] ldp d14, d15, [sp, #48] ldp d8, d9, [sp], #80 ret .L192_enc_ret: mov w0, #0x0 ret .size unroll8_eor3_aes_gcm_enc_192_kernel,.-unroll8_eor3_aes_gcm_enc_192_kernel .globl unroll8_eor3_aes_gcm_dec_192_kernel .type unroll8_eor3_aes_gcm_dec_192_kernel,%function .align 4 unroll8_eor3_aes_gcm_dec_192_kernel: AARCH64_VALID_CALL_TARGET cbz x1, .L192_dec_ret stp d8, d9, [sp, #-80]! lsr x9, x1, #3 mov x16, x4 mov x8, x5 stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x5, #0xc200000000000000 stp x5, xzr, [sp, #64] add x10, sp, #64 mov x5, x9 ld1 { v0.16b}, [x16] //CTR block 0 ld1 { v19.16b}, [x3] mov x15, #0x100000000 //set up counter increment movi v31.16b, #0x0 mov v31.d[1], x15 rev32 v30.16b, v0.16b //set up reversed counter add v30.4s, v30.4s, v31.4s //CTR block 0 rev32 v1.16b, v30.16b //CTR block 1 add v30.4s, v30.4s, v31.4s //CTR block 1 rev32 v2.16b, v30.16b //CTR block 2 add v30.4s, v30.4s, v31.4s //CTR block 2 rev32 v3.16b, v30.16b //CTR block 3 add v30.4s, v30.4s, v31.4s //CTR block 3 rev32 v4.16b, v30.16b //CTR block 4 add v30.4s, v30.4s, v31.4s //CTR block 4 rev32 v5.16b, v30.16b //CTR block 5 add v30.4s, v30.4s, v31.4s //CTR block 5 ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v6.16b, v30.16b //CTR block 6 add v30.4s, v30.4s, v31.4s //CTR block 6 rev32 v7.16b, v30.16b //CTR block 7 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 1 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 1 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 1 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 2 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 3 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 3 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 5 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 4 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 5 sub x5, x5, #1 //byte_len - 1 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 add v30.4s, v30.4s, v31.4s //CTR block 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 7 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 7 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 8 and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 8 add x4, x0, x1, lsr #3 //end_input_ptr aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 9 ld1 { v19.16b}, [x3] ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b ldp q27, q28, [x8, #160] //load rk10, rk11 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 9 add x5, x5, x0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 9 cmp x0, x5 //check if we have <= 8 blocks aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 9 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 9 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 10 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 10 ldr q26, [x8, #192] //load rk12 aese v0.16b, v28.16b //AES block 0 - round 11 aese v1.16b, v28.16b //AES block 1 - round 11 aese v4.16b, v28.16b //AES block 4 - round 11 aese v6.16b, v28.16b //AES block 6 - round 11 aese v5.16b, v28.16b //AES block 5 - round 11 aese v7.16b, v28.16b //AES block 7 - round 11 aese v2.16b, v28.16b //AES block 2 - round 11 aese v3.16b, v28.16b //AES block 3 - round 11 b.ge .L192_dec_tail //handle tail ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext .inst 0xce016921 //eor3 v1.16b, v9.16b, v1.16b, v26.16b //AES block 1 - result .inst 0xce006900 //eor3 v0.16b, v8.16b, v0.16b, v26.16b //AES block 0 - result stp q0, q1, [x2], #32 //AES block 0, 1 - store result rev32 v0.16b, v30.16b //CTR block 8 add v30.4s, v30.4s, v31.4s //CTR block 8 rev32 v1.16b, v30.16b //CTR block 9 add v30.4s, v30.4s, v31.4s //CTR block 9 .inst 0xce036963 //eor3 v3.16b, v11.16b, v3.16b, v26.16b //AES block 3 - result .inst 0xce026942 //eor3 v2.16b, v10.16b, v2.16b, v26.16b //AES block 2 - result stp q2, q3, [x2], #32 //AES block 2, 3 - store result ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext rev32 v2.16b, v30.16b //CTR block 10 add v30.4s, v30.4s, v31.4s //CTR block 10 .inst 0xce046984 //eor3 v4.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result rev32 v3.16b, v30.16b //CTR block 11 add v30.4s, v30.4s, v31.4s //CTR block 11 .inst 0xce0569a5 //eor3 v5.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result stp q4, q5, [x2], #32 //AES block 4, 5 - store result cmp x0, x5 //check if we have <= 8 blocks .inst 0xce0669c6 //eor3 v6.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result .inst 0xce0769e7 //eor3 v7.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result rev32 v4.16b, v30.16b //CTR block 12 add v30.4s, v30.4s, v31.4s //CTR block 12 stp q6, q7, [x2], #32 //AES block 6, 7 - store result b.ge .L192_dec_prepretail //do prepretail .L192_dec_main_loop: //main loop start rev64 v9.16b, v9.16b //GHASH block 8k+1 ldp q26, q27, [x8, #0] //load rk0, rk1 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev64 v8.16b, v8.16b //GHASH block 8k rev32 v5.16b, v30.16b //CTR block 8k+13 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 rev64 v12.16b, v12.16b //GHASH block 8k+4 rev64 v11.16b, v11.16b //GHASH block 8k+3 eor v8.16b, v8.16b, v19.16b //PRE 1 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 rev64 v13.16b, v13.16b //GHASH block 8k+5 rev32 v7.16b, v30.16b //CTR block 8k+15 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high ldp q28, q26, [x8, #32] //load rk2, rk3 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid rev64 v10.16b, v10.16b //GHASH block 8k+2 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 ldp q27, q28, [x8, #64] //load rk4, rk5 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid add v30.4s, v30.4s, v31.4s //CTR block 8k+15 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 rev64 v15.16b, v15.16b //GHASH block 8k+7 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 rev64 v14.16b, v14.16b //GHASH block 8k+6 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid ldr d16, [x10] //MODULO - load modulo constant pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high rev32 v20.16b, v30.16b //CTR block 8k+16 add v30.4s, v30.4s, v31.4s //CTR block 8k+16 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 ldp q27, q28, [x8, #160] //load rk10, rk11 .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext rev32 v22.16b, v30.16b //CTR block 8k+17 pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid add v30.4s, v30.4s, v31.4s //CTR block 8k+17 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext rev32 v23.16b, v30.16b //CTR block 8k+18 add v30.4s, v30.4s, v31.4s //CTR block 8k+18 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 ldr q26, [x8, #192] //load rk12 ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 aese v0.16b, v28.16b //AES block 8k+8 - round 11 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v1.16b, v28.16b //AES block 8k+9 - round 11 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 aese v6.16b, v28.16b //AES block 8k+14 - round 11 aese v3.16b, v28.16b //AES block 8k+11 - round 11 .inst 0xce006900 //eor3 v0.16b, v8.16b, v0.16b, v26.16b //AES block 8k+8 - result rev32 v25.16b, v30.16b //CTR block 8k+19 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v4.16b, v28.16b //AES block 8k+12 - round 11 aese v2.16b, v28.16b //AES block 8k+10 - round 11 add v30.4s, v30.4s, v31.4s //CTR block 8k+19 aese v7.16b, v28.16b //AES block 8k+15 - round 11 aese v5.16b, v28.16b //AES block 8k+13 - round 11 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low .inst 0xce016921 //eor3 v1.16b, v9.16b, v1.16b, v26.16b //AES block 8k+9 - result stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result .inst 0xce036963 //eor3 v3.16b, v11.16b, v3.16b, v26.16b //AES block 8k+11 - result .inst 0xce026942 //eor3 v2.16b, v10.16b, v2.16b, v26.16b //AES block 8k+10 - result .inst 0xce0769e7 //eor3 v7.16b, v15.16b, v7.16b, v26.16b //AES block 8k+15 - result stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result .inst 0xce0569a5 //eor3 v5.16b, v13.16b, v5.16b, v26.16b //AES block 8k+13 - result .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low mov v3.16b, v25.16b //CTR block 8k+19 .inst 0xce046984 //eor3 v4.16b, v12.16b, v4.16b, v26.16b //AES block 8k+12 - result stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result cmp x0, x5 //.LOOP CONTROL .inst 0xce0669c6 //eor3 v6.16b, v14.16b, v6.16b, v26.16b //AES block 8k+14 - result stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result mov v0.16b, v20.16b //CTR block 8k+16 mov v1.16b, v22.16b //CTR block 8k+17 mov v2.16b, v23.16b //CTR block 8k+18 rev32 v4.16b, v30.16b //CTR block 8k+20 add v30.4s, v30.4s, v31.4s //CTR block 8k+20 b.lt .L192_dec_main_loop .L192_dec_prepretail: //PREPRETAIL ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v5.16b, v30.16b //CTR block 8k+13 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 rev64 v8.16b, v8.16b //GHASH block 8k ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev64 v11.16b, v11.16b //GHASH block 8k+3 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 eor v8.16b, v8.16b, v19.16b //PRE 1 rev64 v10.16b, v10.16b //GHASH block 8k+2 rev64 v9.16b, v9.16b //GHASH block 8k+1 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 rev32 v7.16b, v30.16b //CTR block 8k+15 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 rev64 v13.16b, v13.16b //GHASH block 8k+5 pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low ldp q27, q28, [x8, #64] //load rk4, rk5 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 rev64 v15.16b, v15.16b //GHASH block 8k+7 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid rev64 v12.16b, v12.16b //GHASH block 8k+4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 rev64 v14.16b, v14.16b //GHASH block 8k+6 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 ldp q28, q26, [x8, #128] //load rk8, rk9 pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 ldr d16, [x10] //MODULO - load modulo constant .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 ldp q27, q28, [x8, #160] //load rk10, rk11 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ldr q26, [x8, #192] //load rk12 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 aese v0.16b, v28.16b //AES block 8k+8 - round 11 .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low aese v5.16b, v28.16b //AES block 8k+13 - round 11 aese v2.16b, v28.16b //AES block 8k+10 - round 11 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 aese v6.16b, v28.16b //AES block 8k+14 - round 11 aese v4.16b, v28.16b //AES block 8k+12 - round 11 add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v3.16b, v28.16b //AES block 8k+11 - round 11 aese v1.16b, v28.16b //AES block 8k+9 - round 11 aese v7.16b, v28.16b //AES block 8k+15 - round 11 .L192_dec_tail: //TAIL sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldp q20, q21, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext ldp q24, q25, [x3, #192] //load h8k | h7k ext v25.16b, v25.16b, v25.16b, #8 mov v29.16b, v26.16b ldp q22, q23, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 ext v23.16b, v23.16b, v23.16b, #8 ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag .inst 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result cmp x5, #112 b.gt .L192_dec_blocks_more_than_7 mov v7.16b, v6.16b movi v17.8b, #0 sub v30.4s, v30.4s, v31.4s mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b cmp x5, #96 movi v19.8b, #0 mov v3.16b, v2.16b mov v2.16b, v1.16b movi v18.8b, #0 b.gt .L192_dec_blocks_more_than_6 mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v1.16b sub v30.4s, v30.4s, v31.4s cmp x5, #80 b.gt .L192_dec_blocks_more_than_5 mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v1.16b cmp x5, #64 sub v30.4s, v30.4s, v31.4s b.gt .L192_dec_blocks_more_than_4 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v1.16b cmp x5, #48 b.gt .L192_dec_blocks_more_than_3 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b cmp x5, #32 mov v6.16b, v1.16b ldr q24, [x3, #96] //load h4k | h3k b.gt .L192_dec_blocks_more_than_2 sub v30.4s, v30.4s, v31.4s mov v7.16b, v1.16b cmp x5, #16 b.gt .L192_dec_blocks_more_than_1 sub v30.4s, v30.4s, v31.4s ldr q21, [x3, #48] //load h2k | h1k b .L192_dec_blocks_less_than_1 .L192_dec_blocks_more_than_7: //blocks left > 7 rev64 v8.16b, v9.16b //GHASH final-7 block ins v18.d[0], v24.d[1] //GHASH final-7 block - mid eor v8.16b, v8.16b, v16.16b //feed in partial tag pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high ins v27.d[0], v8.d[1] //GHASH final-7 block - mid ldr q9, [x0], #16 //AES final-6 block - load ciphertext pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid st1 { v12.16b}, [x2], #16 //AES final-7 block - store result .inst 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid movi v16.8b, #0 //suppress further partial tag feed in .L192_dec_blocks_more_than_6: //blocks left > 6 rev64 v8.16b, v9.16b //GHASH final-6 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-5 block - load ciphertext ins v27.d[0], v8.d[1] //GHASH final-6 block - mid eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid movi v16.8b, #0 //suppress further partial tag feed in pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high st1 { v12.16b}, [x2], #16 //AES final-6 block - store result .inst 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low .L192_dec_blocks_more_than_5: //blocks left > 5 rev64 v8.16b, v9.16b //GHASH final-5 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-5 block - mid eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid ins v27.d[1], v27.d[0] //GHASH final-5 block - mid pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high ldr q9, [x0], #16 //AES final-4 block - load ciphertext eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low movi v16.8b, #0 //suppress further partial tag feed in st1 { v12.16b}, [x2], #16 //AES final-5 block - store result eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid .inst 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result .L192_dec_blocks_more_than_4: //blocks left > 4 rev64 v8.16b, v9.16b //GHASH final-4 block eor v8.16b, v8.16b, v16.16b //feed in partial tag movi v16.8b, #0 //suppress further partial tag feed in ldr q9, [x0], #16 //AES final-3 block - load ciphertext ins v27.d[0], v8.d[1] //GHASH final-4 block - mid pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid st1 { v12.16b}, [x2], #16 //AES final-4 block - store result pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high .inst 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high .L192_dec_blocks_more_than_3: //blocks left > 3 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 rev64 v8.16b, v9.16b //GHASH final-3 block ldr q9, [x0], #16 //AES final-2 block - load ciphertext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-3 block - mid pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high movi v16.8b, #0 //suppress further partial tag feed in pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low st1 { v12.16b}, [x2], #16 //AES final-3 block - store result eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid .inst 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low ldr q24, [x3, #96] //load h4k | h3k ins v27.d[1], v27.d[0] //GHASH final-3 block - mid pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid .L192_dec_blocks_more_than_2: //blocks left > 2 rev64 v8.16b, v9.16b //GHASH final-2 block ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-2 block - mid ldr q9, [x0], #16 //AES final-1 block - load ciphertext pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid movi v16.8b, #0 //suppress further partial tag feed in eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low st1 { v12.16b}, [x2], #16 //AES final-2 block - store result eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid .inst 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result .L192_dec_blocks_more_than_1: //blocks left > 1 rev64 v8.16b, v9.16b //GHASH final-1 block ldr q9, [x0], #16 //AES final block - load ciphertext ldr q22, [x3, #64] //load h1l | h1h ext v22.16b, v22.16b, v22.16b, #8 eor v8.16b, v8.16b, v16.16b //feed in partial tag movi v16.8b, #0 //suppress further partial tag feed in ldr q21, [x3, #48] //load h2k | h1k pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low ins v27.d[0], v8.d[1] //GHASH final-1 block - mid st1 { v12.16b}, [x2], #16 //AES final-1 block - store result pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high .inst 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid ins v27.d[1], v27.d[0] //GHASH final-1 block - mid pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high .L192_dec_blocks_less_than_1: //blocks left <= 1 rev32 v30.16b, v30.16b and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 str q30, [x16] //store the updated counter neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) mvn x6, xzr //temp0_x = 0xffffffffffffffff and x1, x1, #127 //bit_length %= 128 mvn x7, xzr //temp1_x = 0xffffffffffffffff lsr x6, x6, x1 //temp0_x is mask for top 64b of last block cmp x1, #64 csel x13, x7, x6, lt csel x14, x6, xzr, lt ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 mov v0.d[1], x14 ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored mov v0.d[0], x13 //ctr0b is mask for last block and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing rev64 v8.16b, v9.16b //GHASH final block st1 { v12.16b}, [x2] //store all 16B eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v16.d[0], v8.d[1] //GHASH final block - mid pmull v26.1q, v8.1d, v20.1d //GHASH final block - low eor v16.8b, v16.8b, v8.8b //GHASH final block - mid pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high eor v19.16b, v19.16b, v26.16b //GHASH final block - low pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid eor v17.16b, v17.16b, v28.16b //GHASH final block - high eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up eor v18.16b, v18.16b, v16.16b //GHASH final block - mid ldr d16, [x10] //MODULO - load modulo constant pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up .inst 0xce115652 //eor3 v18.16b, v18.16b, v17.16b, v21.16b //MODULO - fold into mid pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment .inst 0xce124673 //eor3 v19.16b, v19.16b, v18.16b, v17.16b //MODULO - fold into low ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b st1 { v19.16b }, [x3] mov x0, x9 ldp d10, d11, [sp, #16] ldp d12, d13, [sp, #32] ldp d14, d15, [sp, #48] ldp d8, d9, [sp], #80 ret .L192_dec_ret: mov w0, #0x0 ret .size unroll8_eor3_aes_gcm_dec_192_kernel,.-unroll8_eor3_aes_gcm_dec_192_kernel .globl unroll8_eor3_aes_gcm_enc_256_kernel .type unroll8_eor3_aes_gcm_enc_256_kernel,%function .align 4 unroll8_eor3_aes_gcm_enc_256_kernel: AARCH64_VALID_CALL_TARGET cbz x1, .L256_enc_ret stp d8, d9, [sp, #-80]! lsr x9, x1, #3 mov x16, x4 mov x8, x5 stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x5, #0xc200000000000000 stp x5, xzr, [sp, #64] add x10, sp, #64 ld1 { v0.16b}, [x16] //CTR block 0 mov x5, x9 mov x15, #0x100000000 //set up counter increment movi v31.16b, #0x0 mov v31.d[1], x15 sub x5, x5, #1 //byte_len - 1 and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) add x5, x5, x0 rev32 v30.16b, v0.16b //set up reversed counter add v30.4s, v30.4s, v31.4s //CTR block 0 rev32 v1.16b, v30.16b //CTR block 1 add v30.4s, v30.4s, v31.4s //CTR block 1 rev32 v2.16b, v30.16b //CTR block 2 add v30.4s, v30.4s, v31.4s //CTR block 2 rev32 v3.16b, v30.16b //CTR block 3 add v30.4s, v30.4s, v31.4s //CTR block 3 rev32 v4.16b, v30.16b //CTR block 4 add v30.4s, v30.4s, v31.4s //CTR block 4 rev32 v5.16b, v30.16b //CTR block 5 add v30.4s, v30.4s, v31.4s //CTR block 5 ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v6.16b, v30.16b //CTR block 6 add v30.4s, v30.4s, v31.4s //CTR block 6 rev32 v7.16b, v30.16b //CTR block 7 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 1 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 3 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 4 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 5 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 6 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 7 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 8 ld1 { v19.16b}, [x3] ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b ldp q27, q28, [x8, #160] //load rk10, rk11 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 9 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 9 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 9 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 10 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 10 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 9 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 10 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 10 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 11 ldp q26, q27, [x8, #192] //load rk12, rk13 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 11 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 11 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 11 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 11 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 11 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 11 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 11 add v30.4s, v30.4s, v31.4s //CTR block 7 ldr q28, [x8, #224] //load rk14 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 12 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 12 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 12 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 12 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 12 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 12 aese v2.16b, v27.16b //AES block 2 - round 13 aese v1.16b, v27.16b //AES block 1 - round 13 aese v4.16b, v27.16b //AES block 4 - round 13 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 12 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 12 aese v0.16b, v27.16b //AES block 0 - round 13 aese v5.16b, v27.16b //AES block 5 - round 13 aese v6.16b, v27.16b //AES block 6 - round 13 aese v7.16b, v27.16b //AES block 7 - round 13 aese v3.16b, v27.16b //AES block 3 - round 13 add x4, x0, x1, lsr #3 //end_input_ptr cmp x0, x5 //check if we have <= 8 blocks b.ge .L256_enc_tail //handle tail ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext .inst 0xce007108 //eor3 v8.16b, v8.16b, v0.16b, v28.16b //AES block 0 - result rev32 v0.16b, v30.16b //CTR block 8 add v30.4s, v30.4s, v31.4s //CTR block 8 .inst 0xce017129 //eor3 v9.16b, v9.16b, v1.16b, v28.16b //AES block 1 - result .inst 0xce03716b //eor3 v11.16b, v11.16b, v3.16b, v28.16b //AES block 3 - result rev32 v1.16b, v30.16b //CTR block 9 add v30.4s, v30.4s, v31.4s //CTR block 9 ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext .inst 0xce02714a //eor3 v10.16b, v10.16b, v2.16b, v28.16b //AES block 2 - result cmp x0, x5 //check if we have <= 8 blocks rev32 v2.16b, v30.16b //CTR block 10 add v30.4s, v30.4s, v31.4s //CTR block 10 stp q8, q9, [x2], #32 //AES block 0, 1 - store result stp q10, q11, [x2], #32 //AES block 2, 3 - store result rev32 v3.16b, v30.16b //CTR block 11 add v30.4s, v30.4s, v31.4s //CTR block 11 .inst 0xce04718c //eor3 v12.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result .inst 0xce0771ef //eor3 v15.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result .inst 0xce0671ce //eor3 v14.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result .inst 0xce0571ad //eor3 v13.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result stp q12, q13, [x2], #32 //AES block 4, 5 - store result rev32 v4.16b, v30.16b //CTR block 12 stp q14, q15, [x2], #32 //AES block 6, 7 - store result add v30.4s, v30.4s, v31.4s //CTR block 12 b.ge .L256_enc_prepretail //do prepretail .L256_enc_main_loop: //main loop start ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v5.16b, v30.16b //CTR block 8k+13 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k rev64 v11.16b, v11.16b //GHASH block 8k+3 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 rev64 v9.16b, v9.16b //GHASH block 8k+1 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 rev64 v8.16b, v8.16b //GHASH block 8k rev64 v12.16b, v12.16b //GHASH block 8k+4 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 rev32 v7.16b, v30.16b //CTR block 8k+15 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 eor v8.16b, v8.16b, v19.16b //PRE 1 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 rev64 v14.16b, v14.16b //GHASH block 8k+6 pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 ldp q27, q28, [x8, #64] //load rk4, rk5 rev64 v10.16b, v10.16b //GHASH block 8k+2 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high rev64 v13.16b, v13.16b //GHASH block 8k+5 pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid ldp q26, q27, [x8, #96] //load rk6, rk7 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 rev64 v15.16b, v15.16b //GHASH block 8k+7 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 ldp q27, q28, [x8, #160] //load rk10, rk11 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low ldr d16, [x10] //MODULO - load modulo constant pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high ldp q26, q27, [x8, #192] //load rk12, rk13 rev32 v20.16b, v30.16b //CTR block 8k+16 ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 add v30.4s, v30.4s, v31.4s //CTR block 8k+16 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 rev32 v22.16b, v30.16b //CTR block 8k+17 add v30.4s, v30.4s, v31.4s //CTR block 8k+17 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 ldr q28, [x8, #224] //load rk14 aese v7.16b, v27.16b //AES block 8k+15 - round 13 ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext aese v2.16b, v27.16b //AES block 8k+10 - round 13 aese v4.16b, v27.16b //AES block 8k+12 - round 13 rev32 v23.16b, v30.16b //CTR block 8k+18 add v30.4s, v30.4s, v31.4s //CTR block 8k+18 aese v5.16b, v27.16b //AES block 8k+13 - round 13 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 aese v3.16b, v27.16b //AES block 8k+11 - round 13 cmp x0, x5 //.LOOP CONTROL .inst 0xce02714a //eor3 v10.16b, v10.16b, v2.16b, v28.16b //AES block 8k+10 - result rev32 v25.16b, v30.16b //CTR block 8k+19 add v30.4s, v30.4s, v31.4s //CTR block 8k+19 aese v0.16b, v27.16b //AES block 8k+8 - round 13 aese v6.16b, v27.16b //AES block 8k+14 - round 13 .inst 0xce0571ad //eor3 v13.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v1.16b, v27.16b //AES block 8k+9 - round 13 .inst 0xce04718c //eor3 v12.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result rev32 v4.16b, v30.16b //CTR block 8k+20 .inst 0xce03716b //eor3 v11.16b, v11.16b, v3.16b, v28.16b //AES block 8k+11 - result mov v3.16b, v25.16b //CTR block 8k+19 .inst 0xce017129 //eor3 v9.16b, v9.16b, v1.16b, v28.16b //AES block 8k+9 - result .inst 0xce007108 //eor3 v8.16b, v8.16b, v0.16b, v28.16b //AES block 8k+8 - result add v30.4s, v30.4s, v31.4s //CTR block 8k+20 stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result mov v2.16b, v23.16b //CTR block 8k+18 .inst 0xce0771ef //eor3 v15.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result .inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result .inst 0xce0671ce //eor3 v14.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result mov v1.16b, v22.16b //CTR block 8k+17 stp q12, q13, [x2], #32 //AES block 4, 5 - store result stp q14, q15, [x2], #32 //AES block 6, 7 - store result mov v0.16b, v20.16b //CTR block 8k+16 b.lt .L256_enc_main_loop .L256_enc_prepretail: //PREPRETAIL rev32 v5.16b, v30.16b //CTR block 8k+13 ldp q26, q27, [x8, #0] //load rk0, rk1 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 rev64 v10.16b, v10.16b //GHASH block 8k+2 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 rev64 v13.16b, v13.16b //GHASH block 8k+5 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k rev32 v7.16b, v30.16b //CTR block 8k+15 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev64 v8.16b, v8.16b //GHASH block 8k aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 rev64 v9.16b, v9.16b //GHASH block 8k+1 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 eor v8.16b, v8.16b, v19.16b //PRE 1 rev64 v11.16b, v11.16b //GHASH block 8k+3 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 ldp q27, q28, [x8, #64] //load rk4, rk5 trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high rev64 v14.16b, v14.16b //GHASH block 8k+6 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid rev64 v12.16b, v12.16b //GHASH block 8k+4 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 rev64 v15.16b, v15.16b //GHASH block 8k+7 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high ldp q27, q28, [x8, #160] //load rk10, rk11 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid ldr d16, [x10] //MODULO - load modulo constant .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 ldp q26, q27, [x8, #192] //load rk12, rk13 ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 ldr q28, [x8, #224] //load rk14 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 aese v0.16b, v27.16b //AES block 8k+8 - round 13 .inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low aese v5.16b, v27.16b //AES block 8k+13 - round 13 aese v1.16b, v27.16b //AES block 8k+9 - round 13 aese v3.16b, v27.16b //AES block 8k+11 - round 13 aese v4.16b, v27.16b //AES block 8k+12 - round 13 aese v7.16b, v27.16b //AES block 8k+15 - round 13 aese v2.16b, v27.16b //AES block 8k+10 - round 13 aese v6.16b, v27.16b //AES block 8k+14 - round 13 .L256_enc_tail: //TAIL ldp q24, q25, [x3, #192] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldr q8, [x0], #16 //AES block 8k+8 - load plaintext ldp q20, q21, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag ldp q22, q23, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 ext v23.16b, v23.16b, v23.16b, #8 mov v29.16b, v28.16b cmp x5, #112 .inst 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result b.gt .L256_enc_blocks_more_than_7 movi v19.8b, #0 mov v7.16b, v6.16b movi v17.8b, #0 mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v2.16b sub v30.4s, v30.4s, v31.4s mov v2.16b, v1.16b movi v18.8b, #0 cmp x5, #96 b.gt .L256_enc_blocks_more_than_6 mov v7.16b, v6.16b mov v6.16b, v5.16b cmp x5, #80 mov v5.16b, v4.16b mov v4.16b, v3.16b mov v3.16b, v1.16b sub v30.4s, v30.4s, v31.4s b.gt .L256_enc_blocks_more_than_5 mov v7.16b, v6.16b sub v30.4s, v30.4s, v31.4s mov v6.16b, v5.16b mov v5.16b, v4.16b cmp x5, #64 mov v4.16b, v1.16b b.gt .L256_enc_blocks_more_than_4 cmp x5, #48 mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v1.16b sub v30.4s, v30.4s, v31.4s b.gt .L256_enc_blocks_more_than_3 cmp x5, #32 mov v7.16b, v6.16b ldr q24, [x3, #96] //load h4k | h3k mov v6.16b, v1.16b sub v30.4s, v30.4s, v31.4s b.gt .L256_enc_blocks_more_than_2 mov v7.16b, v1.16b sub v30.4s, v30.4s, v31.4s cmp x5, #16 b.gt .L256_enc_blocks_more_than_1 sub v30.4s, v30.4s, v31.4s ldr q21, [x3, #48] //load h2k | h1k b .L256_enc_blocks_less_than_1 .L256_enc_blocks_more_than_7: //blocks left > 7 st1 { v9.16b}, [x2], #16 //AES final-7 block - store result rev64 v8.16b, v9.16b //GHASH final-7 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-6 block - load plaintext pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high ins v27.d[0], v8.d[1] //GHASH final-7 block - mid ins v18.d[0], v24.d[1] //GHASH final-7 block - mid movi v16.8b, #0 //suppress further partial tag feed in eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid .inst 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low .L256_enc_blocks_more_than_6: //blocks left > 6 st1 { v9.16b}, [x2], #16 //AES final-6 block - store result rev64 v8.16b, v9.16b //GHASH final-6 block eor v8.16b, v8.16b, v16.16b //feed in partial tag pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low ins v27.d[0], v8.d[1] //GHASH final-6 block - mid pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high ldr q9, [x0], #16 //AES final-5 block - load plaintext eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid .inst 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result movi v16.8b, #0 //suppress further partial tag feed in eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high .L256_enc_blocks_more_than_5: //blocks left > 5 st1 { v9.16b}, [x2], #16 //AES final-5 block - store result rev64 v8.16b, v9.16b //GHASH final-5 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-5 block - mid pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid ins v27.d[1], v27.d[0] //GHASH final-5 block - mid ldr q9, [x0], #16 //AES final-4 block - load plaintext pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid movi v16.8b, #0 //suppress further partial tag feed in eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid .inst 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result .L256_enc_blocks_more_than_4: //blocks left > 4 st1 { v9.16b}, [x2], #16 //AES final-4 block - store result rev64 v8.16b, v9.16b //GHASH final-4 block ldr q9, [x0], #16 //AES final-3 block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-4 block - mid pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high .inst 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid movi v16.8b, #0 //suppress further partial tag feed in eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high .L256_enc_blocks_more_than_3: //blocks left > 3 st1 { v9.16b}, [x2], #16 //AES final-3 block - store result ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 rev64 v8.16b, v9.16b //GHASH final-3 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-3 block - mid pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid ldr q24, [x3, #96] //load h4k | h3k ins v27.d[1], v27.d[0] //GHASH final-3 block - mid ldr q9, [x0], #16 //AES final-2 block - load plaintext pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low .inst 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result movi v16.8b, #0 //suppress further partial tag feed in eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low .L256_enc_blocks_more_than_2: //blocks left > 2 ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 st1 { v9.16b}, [x2], #16 //AES final-2 block - store result rev64 v8.16b, v9.16b //GHASH final-2 block ldr q9, [x0], #16 //AES final-1 block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-2 block - mid movi v16.8b, #0 //suppress further partial tag feed in pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high .inst 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low .L256_enc_blocks_more_than_1: //blocks left > 1 st1 { v9.16b}, [x2], #16 //AES final-1 block - store result ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 rev64 v8.16b, v9.16b //GHASH final-1 block ldr q9, [x0], #16 //AES final block - load plaintext eor v8.16b, v8.16b, v16.16b //feed in partial tag movi v16.8b, #0 //suppress further partial tag feed in ins v27.d[0], v8.d[1] //GHASH final-1 block - mid pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high .inst 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid ldr q21, [x3, #48] //load h2k | h1k eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low ins v27.d[1], v27.d[0] //GHASH final-1 block - mid pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid .L256_enc_blocks_less_than_1: //blocks left <= 1 and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) mvn x6, xzr //temp0_x = 0xffffffffffffffff and x1, x1, #127 //bit_length %= 128 lsr x6, x6, x1 //temp0_x is mask for top 64b of last block cmp x1, #64 mvn x7, xzr //temp1_x = 0xffffffffffffffff csel x14, x6, xzr, lt csel x13, x7, x6, lt mov v0.d[0], x13 //ctr0b is mask for last block ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored mov v0.d[1], x14 and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits rev64 v8.16b, v9.16b //GHASH final block rev32 v30.16b, v30.16b bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing str q30, [x16] //store the updated counter eor v8.16b, v8.16b, v16.16b //feed in partial tag st1 { v9.16b}, [x2] //store all 16B ins v16.d[0], v8.d[1] //GHASH final block - mid pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high pmull v26.1q, v8.1d, v20.1d //GHASH final block - low eor v17.16b, v17.16b, v28.16b //GHASH final block - high eor v19.16b, v19.16b, v26.16b //GHASH final block - low eor v16.8b, v16.8b, v8.8b //GHASH final block - mid pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid eor v18.16b, v18.16b, v16.16b //GHASH final block - mid ldr d16, [x10] //MODULO - load modulo constant ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment .inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b st1 { v19.16b }, [x3] mov x0, x9 //return sizes ldp d10, d11, [sp, #16] ldp d12, d13, [sp, #32] ldp d14, d15, [sp, #48] ldp d8, d9, [sp], #80 ret .L256_enc_ret: mov w0, #0x0 ret .size unroll8_eor3_aes_gcm_enc_256_kernel,.-unroll8_eor3_aes_gcm_enc_256_kernel .globl unroll8_eor3_aes_gcm_dec_256_kernel .type unroll8_eor3_aes_gcm_dec_256_kernel,%function .align 4 unroll8_eor3_aes_gcm_dec_256_kernel: AARCH64_VALID_CALL_TARGET cbz x1, .L256_dec_ret stp d8, d9, [sp, #-80]! lsr x9, x1, #3 mov x16, x4 mov x8, x5 stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x5, #0xc200000000000000 stp x5, xzr, [sp, #64] add x10, sp, #64 ld1 { v0.16b}, [x16] //CTR block 0 mov x15, #0x100000000 //set up counter increment movi v31.16b, #0x0 mov v31.d[1], x15 mov x5, x9 sub x5, x5, #1 //byte_len - 1 rev32 v30.16b, v0.16b //set up reversed counter add v30.4s, v30.4s, v31.4s //CTR block 0 rev32 v1.16b, v30.16b //CTR block 1 add v30.4s, v30.4s, v31.4s //CTR block 1 rev32 v2.16b, v30.16b //CTR block 2 add v30.4s, v30.4s, v31.4s //CTR block 2 ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v3.16b, v30.16b //CTR block 3 add v30.4s, v30.4s, v31.4s //CTR block 3 rev32 v4.16b, v30.16b //CTR block 4 add v30.4s, v30.4s, v31.4s //CTR block 4 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 rev32 v5.16b, v30.16b //CTR block 5 add v30.4s, v30.4s, v31.4s //CTR block 5 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 rev32 v6.16b, v30.16b //CTR block 6 add v30.4s, v30.4s, v31.4s //CTR block 6 rev32 v7.16b, v30.16b //CTR block 7 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 2 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 2 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 3 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 3 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 3 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 4 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 4 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 7 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 7 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 7 and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 8 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 8 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 9 ld1 { v19.16b}, [x3] ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b ldp q27, q28, [x8, #160] //load rk10, rk11 add x4, x0, x1, lsr #3 //end_input_ptr add x5, x5, x0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 9 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 9 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 9 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 9 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 4 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 7 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 5 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 2 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 0 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 6 - round 10 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 3 - round 10 ldp q26, q27, [x8, #192] //load rk12, rk13 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 0 - round 11 add v30.4s, v30.4s, v31.4s //CTR block 7 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 7 - round 11 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 3 - round 11 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 1 - round 11 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 5 - round 11 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 4 - round 11 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 2 - round 11 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 6 - round 11 ldr q28, [x8, #224] //load rk14 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 1 - round 12 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 4 - round 12 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 5 - round 12 cmp x0, x5 //check if we have <= 8 blocks aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 3 - round 12 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 12 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 6 - round 12 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 0 - round 12 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 7 - round 12 aese v5.16b, v27.16b //AES block 5 - round 13 aese v1.16b, v27.16b //AES block 1 - round 13 aese v2.16b, v27.16b //AES block 2 - round 13 aese v0.16b, v27.16b //AES block 0 - round 13 aese v4.16b, v27.16b //AES block 4 - round 13 aese v6.16b, v27.16b //AES block 6 - round 13 aese v3.16b, v27.16b //AES block 3 - round 13 aese v7.16b, v27.16b //AES block 7 - round 13 b.ge .L256_dec_tail //handle tail ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext cmp x0, x5 //check if we have <= 8 blocks .inst 0xce017121 //eor3 v1.16b, v9.16b, v1.16b, v28.16b //AES block 1 - result .inst 0xce007100 //eor3 v0.16b, v8.16b, v0.16b, v28.16b //AES block 0 - result stp q0, q1, [x2], #32 //AES block 0, 1 - store result rev32 v0.16b, v30.16b //CTR block 8 add v30.4s, v30.4s, v31.4s //CTR block 8 .inst 0xce037163 //eor3 v3.16b, v11.16b, v3.16b, v28.16b //AES block 3 - result .inst 0xce0571a5 //eor3 v5.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result .inst 0xce047184 //eor3 v4.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result rev32 v1.16b, v30.16b //CTR block 9 add v30.4s, v30.4s, v31.4s //CTR block 9 .inst 0xce027142 //eor3 v2.16b, v10.16b, v2.16b, v28.16b //AES block 2 - result stp q2, q3, [x2], #32 //AES block 2, 3 - store result rev32 v2.16b, v30.16b //CTR block 10 add v30.4s, v30.4s, v31.4s //CTR block 10 .inst 0xce0671c6 //eor3 v6.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result rev32 v3.16b, v30.16b //CTR block 11 add v30.4s, v30.4s, v31.4s //CTR block 11 stp q4, q5, [x2], #32 //AES block 4, 5 - store result .inst 0xce0771e7 //eor3 v7.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result stp q6, q7, [x2], #32 //AES block 6, 7 - store result rev32 v4.16b, v30.16b //CTR block 12 add v30.4s, v30.4s, v31.4s //CTR block 12 b.ge .L256_dec_prepretail //do prepretail .L256_dec_main_loop: //main loop start rev32 v5.16b, v30.16b //CTR block 8k+13 ldp q26, q27, [x8, #0] //load rk0, rk1 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 rev64 v9.16b, v9.16b //GHASH block 8k+1 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 rev32 v6.16b, v30.16b //CTR block 8k+14 add v30.4s, v30.4s, v31.4s //CTR block 8k+14 rev64 v8.16b, v8.16b //GHASH block 8k ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 rev64 v12.16b, v12.16b //GHASH block 8k+4 rev64 v11.16b, v11.16b //GHASH block 8k+3 rev32 v7.16b, v30.16b //CTR block 8k+15 rev64 v15.16b, v15.16b //GHASH block 8k+7 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 eor v8.16b, v8.16b, v19.16b //PRE 1 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 rev64 v10.16b, v10.16b //GHASH block 8k+2 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 ldp q27, q28, [x8, #64] //load rk4, rk5 pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low ldp q26, q27, [x8, #96] //load rk6, rk7 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid rev64 v13.16b, v13.16b //GHASH block 8k+5 pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 rev64 v14.16b, v14.16b //GHASH block 8k+6 eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 ldp q28, q26, [x8, #128] //load rk8, rk9 ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 ldp q27, q28, [x8, #160] //load rk10, rk11 pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid add v30.4s, v30.4s, v31.4s //CTR block 8k+15 .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low rev32 v20.16b, v30.16b //CTR block 8k+16 ldr d16, [x10] //MODULO - load modulo constant add v30.4s, v30.4s, v31.4s //CTR block 8k+16 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 ldp q26, q27, [x8, #192] //load rk12, rk13 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid rev32 v22.16b, v30.16b //CTR block 8k+17 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 add v30.4s, v30.4s, v31.4s //CTR block 8k+17 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 rev32 v23.16b, v30.16b //CTR block 8k+18 add v30.4s, v30.4s, v31.4s //CTR block 8k+18 pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 ldr q28, [x8, #224] //load rk14 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext aese v1.16b, v27.16b //AES block 8k+9 - round 13 aese v2.16b, v27.16b //AES block 8k+10 - round 13 ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext aese v0.16b, v27.16b //AES block 8k+8 - round 13 aese v5.16b, v27.16b //AES block 8k+13 - round 13 rev32 v25.16b, v30.16b //CTR block 8k+19 .inst 0xce027142 //eor3 v2.16b, v10.16b, v2.16b, v28.16b //AES block 8k+10 - result .inst 0xce017121 //eor3 v1.16b, v9.16b, v1.16b, v28.16b //AES block 8k+9 - result ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v7.16b, v27.16b //AES block 8k+15 - round 13 add v30.4s, v30.4s, v31.4s //CTR block 8k+19 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v4.16b, v27.16b //AES block 8k+12 - round 13 .inst 0xce0571a5 //eor3 v5.16b, v13.16b, v5.16b, v28.16b //AES block 8k+13 - result .inst 0xce007100 //eor3 v0.16b, v8.16b, v0.16b, v28.16b //AES block 8k+8 - result aese v3.16b, v27.16b //AES block 8k+11 - round 13 stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result mov v0.16b, v20.16b //CTR block 8k+16 .inst 0xce047184 //eor3 v4.16b, v12.16b, v4.16b, v28.16b //AES block 8k+12 - result .inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low .inst 0xce037163 //eor3 v3.16b, v11.16b, v3.16b, v28.16b //AES block 8k+11 - result stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result mov v3.16b, v25.16b //CTR block 8k+19 mov v2.16b, v23.16b //CTR block 8k+18 aese v6.16b, v27.16b //AES block 8k+14 - round 13 mov v1.16b, v22.16b //CTR block 8k+17 stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result .inst 0xce0771e7 //eor3 v7.16b, v15.16b, v7.16b, v28.16b //AES block 8k+15 - result .inst 0xce0671c6 //eor3 v6.16b, v14.16b, v6.16b, v28.16b //AES block 8k+14 - result rev32 v4.16b, v30.16b //CTR block 8k+20 add v30.4s, v30.4s, v31.4s //CTR block 8k+20 cmp x0, x5 //.LOOP CONTROL stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result b.lt .L256_dec_main_loop .L256_dec_prepretail: //PREPRETAIL ldp q26, q27, [x8, #0] //load rk0, rk1 rev32 v5.16b, v30.16b //CTR block 8k+13 add v30.4s, v30.4s, v31.4s //CTR block 8k+13 rev64 v12.16b, v12.16b //GHASH block 8k+4 ldr q21, [x3, #144] //load h6k | h5k ldr q24, [x3, #192] //load h8k | h7k rev32 v6.16b, v30.16b //CTR block 8k+14 rev64 v8.16b, v8.16b //GHASH block 8k add v30.4s, v30.4s, v31.4s //CTR block 8k+14 ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 ldr q23, [x3, #176] //load h7l | h7h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #208] //load h8l | h8h ext v25.16b, v25.16b, v25.16b, #8 rev64 v9.16b, v9.16b //GHASH block 8k+1 rev32 v7.16b, v30.16b //CTR block 8k+15 rev64 v10.16b, v10.16b //GHASH block 8k+2 ldr q20, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 ldp q28, q26, [x8, #32] //load rk2, rk3 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 eor v8.16b, v8.16b, v19.16b //PRE 1 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low rev64 v11.16b, v11.16b //GHASH block 8k+3 pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 rev64 v14.16b, v14.16b //GHASH block 8k+6 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 ldp q27, q28, [x8, #64] //load rk4, rk5 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 .inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 ldp q26, q27, [x8, #96] //load rk6, rk7 ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 rev64 v15.16b, v15.16b //GHASH block 8k+7 rev64 v13.16b, v13.16b //GHASH block 8k+5 .inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 ldr q21, [x3, #48] //load h2k | h1k ldr q24, [x3, #96] //load h4k | h3k aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 ldp q28, q26, [x8, #128] //load rk8, rk9 pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 .inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low ldp q27, q28, [x8, #160] //load rk10, rk11 .inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low .inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 .inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high .inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low ldr d16, [x10] //MODULO - load modulo constant .inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid aese v4.16b, v27.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 aese v6.16b, v27.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 aese v5.16b, v27.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 aese v0.16b, v27.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 .inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up aese v7.16b, v27.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 ldp q26, q27, [x8, #192] //load rk12, rk13 ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment aese v2.16b, v28.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid aese v3.16b, v28.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 aese v7.16b, v28.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 aese v6.16b, v28.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 aese v4.16b, v28.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 aese v5.16b, v28.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 .inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid aese v3.16b, v27.16b //AES block 8k+11 - round 13 aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 aese v6.16b, v26.16b aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low aese v4.16b, v26.16b aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 aese v7.16b, v26.16b aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 aese v0.16b, v26.16b aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 ldr q28, [x8, #224] //load rk14 aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 aese v4.16b, v27.16b //AES block 8k+12 - round 13 ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment aese v5.16b, v26.16b aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 aese v6.16b, v27.16b //AES block 8k+14 - round 13 aese v2.16b, v27.16b //AES block 8k+10 - round 13 aese v1.16b, v27.16b //AES block 8k+9 - round 13 aese v5.16b, v27.16b //AES block 8k+13 - round 13 .inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low add v30.4s, v30.4s, v31.4s //CTR block 8k+15 aese v7.16b, v27.16b //AES block 8k+15 - round 13 aese v0.16b, v27.16b //AES block 8k+8 - round 13 .L256_dec_tail: //TAIL ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process cmp x5, #112 ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext ldp q24, q25, [x3, #192] //load h8k | h7k ext v25.16b, v25.16b, v25.16b, #8 mov v29.16b, v28.16b ldp q20, q21, [x3, #128] //load h5l | h5h ext v20.16b, v20.16b, v20.16b, #8 .inst 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result ldp q22, q23, [x3, #160] //load h6l | h6h ext v22.16b, v22.16b, v22.16b, #8 ext v23.16b, v23.16b, v23.16b, #8 b.gt .L256_dec_blocks_more_than_7 mov v7.16b, v6.16b sub v30.4s, v30.4s, v31.4s mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v3.16b movi v19.8b, #0 movi v17.8b, #0 movi v18.8b, #0 mov v3.16b, v2.16b cmp x5, #96 mov v2.16b, v1.16b b.gt .L256_dec_blocks_more_than_6 mov v7.16b, v6.16b mov v6.16b, v5.16b mov v5.16b, v4.16b cmp x5, #80 sub v30.4s, v30.4s, v31.4s mov v4.16b, v3.16b mov v3.16b, v1.16b b.gt .L256_dec_blocks_more_than_5 cmp x5, #64 mov v7.16b, v6.16b sub v30.4s, v30.4s, v31.4s mov v6.16b, v5.16b mov v5.16b, v4.16b mov v4.16b, v1.16b b.gt .L256_dec_blocks_more_than_4 sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b cmp x5, #48 mov v6.16b, v5.16b mov v5.16b, v1.16b b.gt .L256_dec_blocks_more_than_3 ldr q24, [x3, #96] //load h4k | h3k sub v30.4s, v30.4s, v31.4s mov v7.16b, v6.16b cmp x5, #32 mov v6.16b, v1.16b b.gt .L256_dec_blocks_more_than_2 sub v30.4s, v30.4s, v31.4s mov v7.16b, v1.16b cmp x5, #16 b.gt .L256_dec_blocks_more_than_1 sub v30.4s, v30.4s, v31.4s ldr q21, [x3, #48] //load h2k | h1k b .L256_dec_blocks_less_than_1 .L256_dec_blocks_more_than_7: //blocks left > 7 rev64 v8.16b, v9.16b //GHASH final-7 block ldr q9, [x0], #16 //AES final-6 block - load ciphertext st1 { v12.16b}, [x2], #16 //AES final-7 block - store result ins v18.d[0], v24.d[1] //GHASH final-7 block - mid eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-7 block - mid .inst 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid movi v16.8b, #0 //suppress further partial tag feed in pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid .L256_dec_blocks_more_than_6: //blocks left > 6 rev64 v8.16b, v9.16b //GHASH final-6 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-5 block - load ciphertext movi v16.8b, #0 //suppress further partial tag feed in ins v27.d[0], v8.d[1] //GHASH final-6 block - mid st1 { v12.16b}, [x2], #16 //AES final-6 block - store result pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low .inst 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high .L256_dec_blocks_more_than_5: //blocks left > 5 rev64 v8.16b, v9.16b //GHASH final-5 block eor v8.16b, v8.16b, v16.16b //feed in partial tag pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high ins v27.d[0], v8.d[1] //GHASH final-5 block - mid ldr q9, [x0], #16 //AES final-4 block - load ciphertext eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid st1 { v12.16b}, [x2], #16 //AES final-5 block - store result pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low ins v27.d[1], v27.d[0] //GHASH final-5 block - mid pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high .inst 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid movi v16.8b, #0 //suppress further partial tag feed in .L256_dec_blocks_more_than_4: //blocks left > 4 rev64 v8.16b, v9.16b //GHASH final-4 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-4 block - mid ldr q9, [x0], #16 //AES final-3 block - load ciphertext movi v16.8b, #0 //suppress further partial tag feed in pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low st1 { v12.16b}, [x2], #16 //AES final-4 block - store result eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid .inst 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result .L256_dec_blocks_more_than_3: //blocks left > 3 ldr q25, [x3, #112] //load h4l | h4h ext v25.16b, v25.16b, v25.16b, #8 rev64 v8.16b, v9.16b //GHASH final-3 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ldr q9, [x0], #16 //AES final-2 block - load ciphertext ldr q24, [x3, #96] //load h4k | h3k ins v27.d[0], v8.d[1] //GHASH final-3 block - mid st1 { v12.16b}, [x2], #16 //AES final-3 block - store result .inst 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid ins v27.d[1], v27.d[0] //GHASH final-3 block - mid pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high movi v16.8b, #0 //suppress further partial tag feed in pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid .L256_dec_blocks_more_than_2: //blocks left > 2 rev64 v8.16b, v9.16b //GHASH final-2 block ldr q23, [x3, #80] //load h3l | h3h ext v23.16b, v23.16b, v23.16b, #8 ldr q9, [x0], #16 //AES final-1 block - load ciphertext eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-2 block - mid pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low st1 { v12.16b}, [x2], #16 //AES final-2 block - store result .inst 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low movi v16.8b, #0 //suppress further partial tag feed in pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high .L256_dec_blocks_more_than_1: //blocks left > 1 rev64 v8.16b, v9.16b //GHASH final-1 block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v27.d[0], v8.d[1] //GHASH final-1 block - mid ldr q22, [x3, #64] //load h2l | h2h ext v22.16b, v22.16b, v22.16b, #8 eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid ldr q9, [x0], #16 //AES final block - load ciphertext st1 { v12.16b}, [x2], #16 //AES final-1 block - store result ldr q21, [x3, #48] //load h2k | h1k pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low ins v27.d[1], v27.d[0] //GHASH final-1 block - mid eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low .inst 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid movi v16.8b, #0 //suppress further partial tag feed in eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid .L256_dec_blocks_less_than_1: //blocks left <= 1 ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored mvn x6, xzr //temp0_x = 0xffffffffffffffff and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 rev32 v30.16b, v30.16b str q30, [x16] //store the updated counter neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) and x1, x1, #127 //bit_length %= 128 lsr x6, x6, x1 //temp0_x is mask for top 64b of last block cmp x1, #64 mvn x7, xzr //temp1_x = 0xffffffffffffffff csel x14, x6, xzr, lt csel x13, x7, x6, lt mov v0.d[0], x13 //ctr0b is mask for last block mov v0.d[1], x14 and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits ldr q20, [x3, #32] //load h1l | h1h ext v20.16b, v20.16b, v20.16b, #8 bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing rev64 v8.16b, v9.16b //GHASH final block eor v8.16b, v8.16b, v16.16b //feed in partial tag ins v16.d[0], v8.d[1] //GHASH final block - mid pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high eor v16.8b, v16.8b, v8.8b //GHASH final block - mid pmull v26.1q, v8.1d, v20.1d //GHASH final block - low eor v17.16b, v17.16b, v28.16b //GHASH final block - high pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid eor v18.16b, v18.16b, v16.16b //GHASH final block - mid ldr d16, [x10] //MODULO - load modulo constant eor v19.16b, v19.16b, v26.16b //GHASH final block - low pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment st1 { v12.16b}, [x2] //store all 16B eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up eor v21.16b, v17.16b, v21.16b //MODULO - fold into mid eor v18.16b, v18.16b, v21.16b //MODULO - fold into mid pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment eor v19.16b, v19.16b, v17.16b //MODULO - fold into low eor v19.16b, v19.16b, v18.16b //MODULO - fold into low ext v19.16b, v19.16b, v19.16b, #8 rev64 v19.16b, v19.16b st1 { v19.16b }, [x3] mov x0, x9 ldp d10, d11, [sp, #16] ldp d12, d13, [sp, #32] ldp d14, d15, [sp, #48] ldp d8, d9, [sp], #80 ret .L256_dec_ret: mov w0, #0x0 ret .size unroll8_eor3_aes_gcm_dec_256_kernel,.-unroll8_eor3_aes_gcm_dec_256_kernel .byte 65,69,83,32,71,67,77,32,109,111,100,117,108,101,32,102,111,114,32,65,82,77,118,56,44,32,83,80,68,88,32,66,83,68,45,51,45,67,108,97,117,115,101,32,98,121,32,60,120,105,97,111,107,97,110,103,46,113,105,97,110,64,97,114,109,46,99,111,109,62,0 .align 2 .align 2 #endif