#include "Constants.hpp"

#include "AES.hpp"

#ifdef __GNUC__

#pragma GCC diagnostic ignored "-Wstrict-aliasing"

#endif

#define Te1_r(x) ZT_ROR32(Te0[x], 8U)

#define Te2_r(x) ZT_ROR32(Te0[x], 16U)

#define Te3_r(x) ZT_ROR32(Te0[x], 24U)

#define Td1_r(x) ZT_ROR32(Td0[x], 8U)

#define Td2_r(x) ZT_ROR32(Td0[x], 16U)

#define Td3_r(x) ZT_ROR32(Td0[x], 24U)

namespace ZeroTier {

namespace {

#ifdef ZT_AES_NEON

ZT_INLINE uint8x16_t s_clmul_armneon_crypto(uint8x16_t h, uint8x16_t y, const uint8_t b[16]) noexcept

}

#endif // ZT_AES_NEON

#define s_bmul32(N, x, y, rh, rl) \

uint32_t x0t_##N = (x) & 0x11111111U; \

uint32_t x1t_##N = (x) & 0x22222222U; \

uint32_t x2t_##N = (x) & 0x44444444U; \

uint32_t x3t_##N = (x) & 0x88888888U; \

uint32_t y0t_##N = (y) & 0x11111111U; \

uint32_t y1t_##N = (y) & 0x22222222U; \

uint32_t y2t_##N = (y) & 0x44444444U; \

uint32_t y3t_##N = (y) & 0x88888888U; \

uint64_t z0t_##N = (((uint64_t)x0t_##N * y0t_##N) ^ ((uint64_t)x1t_##N * y3t_##N) ^ ((uint64_t)x2t_##N * y2t_##N) ^ ((uint64_t)x3t_##N * y1t_##N)) & 0x1111111111111111ULL; \

uint64_t z1t_##N = (((uint64_t)x0t_##N * y1t_##N) ^ ((uint64_t)x1t_##N * y0t_##N) ^ ((uint64_t)x2t_##N * y3t_##N) ^ ((uint64_t)x3t_##N * y2t_##N)) & 0x2222222222222222ULL; \

uint64_t z2t_##N = (((uint64_t)x0t_##N * y2t_##N) ^ ((uint64_t)x1t_##N * y1t_##N) ^ ((uint64_t)x2t_##N * y0t_##N) ^ ((uint64_t)x3t_##N * y3t_##N)) & 0x4444444444444444ULL; \

z0t_##N |= z1t_##N; \

z2t_##N |= z0t_##N; \

uint64_t zt_##N = z2t_##N | ((((uint64_t)x0t_##N * y3t_##N) ^ ((uint64_t)x1t_##N * y2t_##N) ^ ((uint64_t)x2t_##N * y1t_##N) ^ ((uint64_t)x3t_##N * y0t_##N)) & 0x8888888888888888ULL); \

(rh) = (uint32_t)(zt_##N >> 32U); \

(rl) = (uint32_t)zt_##N;

void s_gfmul(const uint64_t hh, const uint64_t hl, uint64_t &y0, uint64_t &y1) noexcept

}

} // anonymous namespace

#ifdef ZT_AES_AESNI

static const __m128i s_sseSwapBytes = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);

static __m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept

}

#endif

void AES::GMAC::update(const void *const data, unsigned int len) noexcept

{

const uint8_t *in = reinterpret_cast<const uint8_t *>(data);

_len += len;

#ifdef ZT_AES_AESNI

if (likely(Utils::CPUID.aes)) {

__m128i y = _mm_loadu_si128(reinterpret_cast<const __m128i *>(_y));

// Handle anything left over from a previous run that wasn't a multiple of 16 bytes.

if (_rp) {

for (;;) {

if (!len)

return;

--len;

_r[_rp++] = *(in++);

if (_rp == 16) {

y = p_gmacPCLMUL128(_aes._k.ni.h[0], _mm_xor_si128(y, _mm_loadu_si128(reinterpret_cast<__m128i *>(_r))));

break;

}

}

}

if (likely(len >= 64)) {

const __m128i sb = s_sseSwapBytes;

const __m128i h = _aes._k.ni.h[0];

const __m128i hh = _aes._k.ni.h[1];

const __m128i hhh = _aes._k.ni.h[2];

const __m128i hhhh = _aes._k.ni.h[3];

const __m128i h2 = _aes._k.ni.h2[0];

const __m128i hh2 = _aes._k.ni.h2[1];

const __m128i hhh2 = _aes._k.ni.h2[2];

const __m128i hhhh2 = _aes._k.ni.h2[3];

const uint8_t *const end64 = in + (len & ~((unsigned int)63));

len &= 63;

do {

__m128i d1 = _mm_shuffle_epi8(_mm_xor_si128(y, _mm_loadu_si128(reinterpret_cast<const __m128i *>(in))), sb);

__m128i d2 = _mm_shuffle_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(in + 16)), sb);

__m128i d3 = _mm_shuffle_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(in + 32)), sb);

__m128i d4 = _mm_shuffle_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(in + 48)), sb);

in += 64;

__m128i a = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(hhhh, d1, 0x00), _mm_clmulepi64_si128(hhh, d2, 0x00)), _mm_xor_si128(_mm_clmulepi64_si128(hh, d3, 0x00), _mm_clmulepi64_si128(h, d4, 0x00)));

__m128i b = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(hhhh, d1, 0x11), _mm_clmulepi64_si128(hhh, d2, 0x11)), _mm_xor_si128(_mm_clmulepi64_si128(hh, d3, 0x11), _mm_clmulepi64_si128(h, d4, 0x11)));

__m128i c = _mm_xor_si128(_mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(hhhh2, _mm_xor_si128(_mm_shuffle_epi32(d1, 78), d1), 0x00), _mm_clmulepi64_si128(hhh2, _mm_xor_si128(_mm_shuffle_epi32(d2, 78), d2), 0x00)), _mm_xor_si128(_mm_clmulepi64_si128(hh2, _mm_xor_si128(_mm_shuffle_epi32(d3, 78), d3), 0x00), _mm_clmulepi64_si128(h2, _mm_xor_si128(_mm_shuffle_epi32(d4, 78), d4), 0x00))), _mm_xor_si128(a, b));

a = _mm_xor_si128(_mm_slli_si128(c, 8), a);

b = _mm_xor_si128(_mm_srli_si128(c, 8), b);

c = _mm_srli_epi32(a, 31);

a = _mm_or_si128(_mm_slli_epi32(a, 1), _mm_slli_si128(c, 4));

b = _mm_or_si128(_mm_or_si128(_mm_slli_epi32(b, 1), _mm_slli_si128(_mm_srli_epi32(b, 31), 4)), _mm_srli_si128(c, 12));

c = _mm_xor_si128(_mm_slli_epi32(a, 31), _mm_xor_si128(_mm_slli_epi32(a, 30), _mm_slli_epi32(a, 25)));

a = _mm_xor_si128(a, _mm_slli_si128(c, 12));

b = _mm_xor_si128(b, _mm_xor_si128(a, _mm_xor_si128(_mm_xor_si128(_mm_srli_epi32(a, 1), _mm_srli_si128(c, 4)), _mm_xor_si128(_mm_srli_epi32(a, 2), _mm_srli_epi32(a, 7)))));

y = _mm_shuffle_epi8(b, sb);

} while (likely(in != end64));

}

while (len >= 16) {

y = p_gmacPCLMUL128(_aes._k.ni.h[0], _mm_xor_si128(y, _mm_loadu_si128(reinterpret_cast<const __m128i *>(in))));

in += 16;

len -= 16;

}

_mm_storeu_si128(reinterpret_cast<__m128i *>(_y), y);

// Any overflow is cached for a later run or finish().

for (unsigned int i = 0; i < len; ++i)

_r[i] = in[i];

_rp = len; // len is always less than 16 here

return;

}

#endif // ZT_AES_AESNI

#ifdef ZT_AES_NEON

if (Utils::ARMCAP.pmull) {

uint8x16_t y = vld1q_u8(reinterpret_cast<const uint8_t *>(_y));

const uint8x16_t h = _aes._k.neon.h;

if (_rp) {

for(;;) {

if (!len)

return;

--len;

_r[_rp++] = *(in++);

if (_rp == 16) {

y = s_clmul_armneon_crypto(h, y, _r);

break;

}

}

}

while (len >= 16) {

y = s_clmul_armneon_crypto(h, y, in);

in += 16;

len -= 16;

}

vst1q_u8(reinterpret_cast<uint8_t *>(_y), y);

for (unsigned int i = 0; i < len; ++i)

_r[i] = in[i];

_rp = len; // len is always less than 16 here

return;

}

#endif // ZT_AES_NEON

const uint64_t h0 = _aes._k.sw.h[0];

const uint64_t h1 = _aes._k.sw.h[1];

uint64_t y0 = _y[0];

uint64_t y1 = _y[1];

if (_rp) {

for (;;) {

if (!len)

return;

--len;

_r[_rp++] = *(in++);

if (_rp == 16) {

y0 ^= Utils::loadMachineEndian< uint64_t >(_r);

y1 ^= Utils::loadMachineEndian< uint64_t >(_r + 8);

s_gfmul(h0, h1, y0, y1);

break;

}

}

}

if (likely(((uintptr_t)in & 7U) == 0U)) {

while (len >= 16) {

y0 ^= *reinterpret_cast<const uint64_t *>(in);

y1 ^= *reinterpret_cast<const uint64_t *>(in + 8);

in += 16;

s_gfmul(h0, h1, y0, y1);

len -= 16;

}

} else {

while (len >= 16) {

y0 ^= Utils::loadMachineEndian< uint64_t >(in);

y1 ^= Utils::loadMachineEndian< uint64_t >(in + 8);

in += 16;

s_gfmul(h0, h1, y0, y1);

len -= 16;

}

}

_y[0] = y0;

_y[1] = y1;

for (unsigned int i = 0; i < len; ++i)

_r[i] = in[i];

_rp = len; // len is always less than 16 here

}

void AES::GMAC::finish(uint8_t tag[16]) noexcept

{

#ifdef ZT_AES_AESNI

if (likely(Utils::CPUID.aes)) {

__m128i y = _mm_loadu_si128(reinterpret_cast<const __m128i *>(_y));

// Handle any remaining bytes, padding the last block with zeroes.

if (_rp) {

while (_rp < 16)

_r[_rp++] = 0;

y = p_gmacPCLMUL128(_aes._k.ni.h[0], _mm_xor_si128(y, _mm_loadu_si128(reinterpret_cast<__m128i *>(_r))));

}

// Interleave encryption of IV with the final GHASH of y XOR (length * 8).

// Then XOR these together to get the final tag.

const __m128i *const k = _aes._k.ni.k;

const __m128i h = _aes._k.ni.h[0];

y = _mm_xor_si128(y, _mm_set_epi64x(0LL, (long long)Utils::hton((uint64_t)_len << 3U)));

y = _mm_shuffle_epi8(y, s_sseSwapBytes);

__m128i encIV = _mm_xor_si128(_mm_loadu_si128(reinterpret_cast<const __m128i *>(_iv)), k[0]);

__m128i t1 = _mm_clmulepi64_si128(h, y, 0x00);

__m128i t2 = _mm_clmulepi64_si128(h, y, 0x01);

__m128i t3 = _mm_clmulepi64_si128(h, y, 0x10);

__m128i t4 = _mm_clmulepi64_si128(h, y, 0x11);

encIV = _mm_aesenc_si128(encIV, k[1]);

t2 = _mm_xor_si128(t2, t3);

t3 = _mm_slli_si128(t2, 8);

encIV = _mm_aesenc_si128(encIV, k[2]);

t2 = _mm_srli_si128(t2, 8);

t1 = _mm_xor_si128(t1, t3);

encIV = _mm_aesenc_si128(encIV, k[3]);

t4 = _mm_xor_si128(t4, t2);

__m128i t5 = _mm_srli_epi32(t1, 31);

t1 = _mm_slli_epi32(t1, 1);

__m128i t6 = _mm_srli_epi32(t4, 31);

encIV = _mm_aesenc_si128(encIV, k[4]);

t4 = _mm_slli_epi32(t4, 1);

t3 = _mm_srli_si128(t5, 12);

encIV = _mm_aesenc_si128(encIV, k[5]);

t6 = _mm_slli_si128(t6, 4);

t5 = _mm_slli_si128(t5, 4);

encIV = _mm_aesenc_si128(encIV, k[6]);

t1 = _mm_or_si128(t1, t5);

t4 = _mm_or_si128(t4, t6);

encIV = _mm_aesenc_si128(encIV, k[7]);

t4 = _mm_or_si128(t4, t3);

t5 = _mm_slli_epi32(t1, 31);

encIV = _mm_aesenc_si128(encIV, k[8]);

t6 = _mm_slli_epi32(t1, 30);

t3 = _mm_slli_epi32(t1, 25);

encIV = _mm_aesenc_si128(encIV, k[9]);

t5 = _mm_xor_si128(t5, t6);

t5 = _mm_xor_si128(t5, t3);

encIV = _mm_aesenc_si128(encIV, k[10]);

t6 = _mm_srli_si128(t5, 4);

t4 = _mm_xor_si128(t4, t6);

encIV = _mm_aesenc_si128(encIV, k[11]);

t5 = _mm_slli_si128(t5, 12);

t1 = _mm_xor_si128(t1, t5);

t4 = _mm_xor_si128(t4, t1);

t5 = _mm_srli_epi32(t1, 1);

encIV = _mm_aesenc_si128(encIV, k[12]);

t2 = _mm_srli_epi32(t1, 2);

t3 = _mm_srli_epi32(t1, 7);

encIV = _mm_aesenc_si128(encIV, k[13]);

t4 = _mm_xor_si128(t4, t2);

t4 = _mm_xor_si128(t4, t3);

encIV = _mm_aesenclast_si128(encIV, k[14]);

t4 = _mm_xor_si128(t4, t5);

_mm_storeu_si128(reinterpret_cast<__m128i *>(tag), _mm_xor_si128(_mm_shuffle_epi8(t4, s_sseSwapBytes), encIV));

return;

}

#endif // ZT_AES_AESNI

#ifdef ZT_AES_NEON

if (Utils::ARMCAP.pmull) {

uint64_t tmp[2];

uint8x16_t y = vld1q_u8(reinterpret_cast<const uint8_t *>(_y));

const uint8x16_t h = _aes._k.neon.h;

if (_rp) {

while (_rp < 16)

_r[_rp++] = 0;

y = s_clmul_armneon_crypto(h, y, _r);

}

tmp[0] = Utils::hton((uint64_t)_len << 3U);

tmp[1] = 0;

y = s_clmul_armneon_crypto(h, y, reinterpret_cast<const uint8_t *>(tmp));

Utils::copy< 12 >(tmp, _iv);

#if __BYTE_ORDER == __BIG_ENDIAN

reinterpret_cast<uint32_t *>(tmp)[3] = 0x00000001;

#else

reinterpret_cast<uint32_t *>(tmp)[3] = 0x01000000;

#endif

_aes.encrypt(tmp, tmp);

uint8x16_t yy = y;

Utils::storeMachineEndian< uint64_t >(tag, tmp[0] ^ reinterpret_cast<const uint64_t *>(&yy)[0]);

Utils::storeMachineEndian< uint64_t >(tag + 8, tmp[1] ^ reinterpret_cast<const uint64_t *>(&yy)[1]);

return;

}

#endif // ZT_AES_NEON

const uint64_t h0 = _aes._k.sw.h[0];

const uint64_t h1 = _aes._k.sw.h[1];

uint64_t y0 = _y[0];

uint64_t y1 = _y[1];

if (_rp) {

while (_rp < 16)

_r[_rp++] = 0;

y0 ^= Utils::loadMachineEndian< uint64_t >(_r);

y1 ^= Utils::loadMachineEndian< uint64_t >(_r + 8);

s_gfmul(h0, h1, y0, y1);

}

y0 ^= Utils::hton((uint64_t)_len << 3U);

s_gfmul(h0, h1, y0, y1);

uint64_t iv2[2];

Utils::copy< 12 >(iv2, _iv);

#if __BYTE_ORDER == __BIG_ENDIAN

reinterpret_cast<uint32_t *>(iv2)[3] = 0x00000001;

#else

reinterpret_cast<uint32_t *>(iv2)[3] = 0x01000000;

#endif

_aes.encrypt(iv2, iv2);

Utils::storeMachineEndian< uint64_t >(tag, iv2[0] ^ y0);

Utils::storeMachineEndian< uint64_t >(tag + 8, iv2[1] ^ y1);

}

#ifdef ZT_AES_AESNI

#if !defined(__WINDOWS__) && ((__GNUC__ >= 8) || (__clang_major__ >= 7))

#define ZT_AES_VAES512 1

__attribute__((__target__("sse4,avx,avx2,vaes,avx512f,avx512bw")))

void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept

}

#define ZT_AES_VAES256 1

__attribute__((__target__("sse4,avx,avx2,vaes")))

void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept

}

#endif // does compiler support AVX2 and AVX512 AES intrinsics?

#endif // ZT_AES_AESNI

void AES::CTR::crypt(const void *const input, unsigned int len) noexcept

{

const uint8_t *in = reinterpret_cast<const uint8_t *>(input);

uint8_t *out = _out;

#ifdef ZT_AES_AESNI

if (likely(Utils::CPUID.aes)) {

const __m128i dd = _mm_set_epi64x(0, (long long)_ctr[0]);

uint64_t c1 = Utils::ntoh(_ctr[1]);

const __m128i *const k = _aes._k.ni.k;

const __m128i k0 = k[0];

const __m128i k1 = k[1];

const __m128i k2 = k[2];

const __m128i k3 = k[3];

const __m128i k4 = k[4];

const __m128i k5 = k[5];

const __m128i k6 = k[6];

const __m128i k7 = k[7];

const __m128i k8 = k[8];

const __m128i k9 = k[9];

const __m128i k10 = k[10];

const __m128i k11 = k[11];

const __m128i k12 = k[12];

const __m128i k13 = k[13];

const __m128i k14 = k[14];

// Complete any unfinished blocks from previous calls to crypt().

unsigned int totalLen = _len;

if ((totalLen & 15U)) {

for (;;) {

if (unlikely(!len)) {

_ctr[1] = Utils::hton(c1);

_len = totalLen;

return;

}

--len;

out[totalLen++] = *(in++);

if (!(totalLen & 15U)) {

__m128i d0 = _mm_insert_epi64(dd, (long long)Utils::hton(c1++), 1);

d0 = _mm_xor_si128(d0, k0);

d0 = _mm_aesenc_si128(d0, k1);

d0 = _mm_aesenc_si128(d0, k2);

d0 = _mm_aesenc_si128(d0, k3);

d0 = _mm_aesenc_si128(d0, k4);

d0 = _mm_aesenc_si128(d0, k5);

d0 = _mm_aesenc_si128(d0, k6);

d0 = _mm_aesenc_si128(d0, k7);

d0 = _mm_aesenc_si128(d0, k8);

d0 = _mm_aesenc_si128(d0, k9);

d0 = _mm_aesenc_si128(d0, k10);

__m128i *const outblk = reinterpret_cast<__m128i *>(out + (totalLen - 16));

d0 = _mm_aesenc_si128(d0, k11);

const __m128i p0 = _mm_loadu_si128(outblk);

d0 = _mm_aesenc_si128(d0, k12);

d0 = _mm_aesenc_si128(d0, k13);

d0 = _mm_aesenclast_si128(d0, k14);

_mm_storeu_si128(outblk, _mm_xor_si128(p0, d0));

break;

}

}

}

out += totalLen;

_len = totalLen + len;

if (likely(len >= 64)) {

#if defined(ZT_AES_VAES512) && defined(ZT_AES_VAES256)

if (Utils::CPUID.vaes && (len >= 256)) {

if (Utils::CPUID.avx512f) {

p_aesCtrInnerVAES512(len, _ctr[0], c1, in, out, k);

} else {

p_aesCtrInnerVAES256(len, _ctr[0], c1, in, out, k);

}

goto skip_conventional_aesni_64;

}

#endif

#if !defined(ZT_AES_VAES512) && defined(ZT_AES_VAES256)

if (Utils::CPUID.vaes && (len >= 256)) {

p_aesCtrInnerVAES256(len, _ctr[0], c1, in, out, k);

goto skip_conventional_aesni_64;

}

#endif

const uint8_t *const eof64 = in + (len & ~((unsigned int)63));

len &= 63;

__m128i d0, d1, d2, d3;

do {

const uint64_t c10 = Utils::hton(c1);

const uint64_t c11 = Utils::hton(c1 + 1ULL);

const uint64_t c12 = Utils::hton(c1 + 2ULL);

const uint64_t c13 = Utils::hton(c1 + 3ULL);

d0 = _mm_insert_epi64(dd, (long long)c10, 1);

d1 = _mm_insert_epi64(dd, (long long)c11, 1);

d2 = _mm_insert_epi64(dd, (long long)c12, 1);

d3 = _mm_insert_epi64(dd, (long long)c13, 1);

c1 += 4;

d0 = _mm_xor_si128(d0, k0);

d1 = _mm_xor_si128(d1, k0);

d2 = _mm_xor_si128(d2, k0);

d3 = _mm_xor_si128(d3, k0);

d0 = _mm_aesenc_si128(d0, k1);

d1 = _mm_aesenc_si128(d1, k1);

d2 = _mm_aesenc_si128(d2, k1);

d3 = _mm_aesenc_si128(d3, k1);

d0 = _mm_aesenc_si128(d0, k2);

d1 = _mm_aesenc_si128(d1, k2);

d2 = _mm_aesenc_si128(d2, k2);

d3 = _mm_aesenc_si128(d3, k2);

d0 = _mm_aesenc_si128(d0, k3);

d1 = _mm_aesenc_si128(d1, k3);

d2 = _mm_aesenc_si128(d2, k3);

d3 = _mm_aesenc_si128(d3, k3);

d0 = _mm_aesenc_si128(d0, k4);

d1 = _mm_aesenc_si128(d1, k4);

d2 = _mm_aesenc_si128(d2, k4);

d3 = _mm_aesenc_si128(d3, k4);

d0 = _mm_aesenc_si128(d0, k5);

d1 = _mm_aesenc_si128(d1, k5);

d2 = _mm_aesenc_si128(d2, k5);

d3 = _mm_aesenc_si128(d3, k5);

d0 = _mm_aesenc_si128(d0, k6);

d1 = _mm_aesenc_si128(d1, k6);

d2 = _mm_aesenc_si128(d2, k6);

d3 = _mm_aesenc_si128(d3, k6);

d0 = _mm_aesenc_si128(d0, k7);

d1 = _mm_aesenc_si128(d1, k7);

d2 = _mm_aesenc_si128(d2, k7);

d3 = _mm_aesenc_si128(d3, k7);

d0 = _mm_aesenc_si128(d0, k8);

d1 = _mm_aesenc_si128(d1, k8);

d2 = _mm_aesenc_si128(d2, k8);

d3 = _mm_aesenc_si128(d3, k8);

d0 = _mm_aesenc_si128(d0, k9);

d1 = _mm_aesenc_si128(d1, k9);

d2 = _mm_aesenc_si128(d2, k9);

d3 = _mm_aesenc_si128(d3, k9);

d0 = _mm_aesenc_si128(d0, k10);

d1 = _mm_aesenc_si128(d1, k10);

d2 = _mm_aesenc_si128(d2, k10);

d3 = _mm_aesenc_si128(d3, k10);

d0 = _mm_aesenc_si128(d0, k11);

d1 = _mm_aesenc_si128(d1, k11);

d2 = _mm_aesenc_si128(d2, k11);

d3 = _mm_aesenc_si128(d3, k11);

d0 = _mm_aesenc_si128(d0, k12);

d1 = _mm_aesenc_si128(d1, k12);

d2 = _mm_aesenc_si128(d2, k12);

d3 = _mm_aesenc_si128(d3, k12);

d0 = _mm_aesenc_si128(d0, k13);

d1 = _mm_aesenc_si128(d1, k13);

d2 = _mm_aesenc_si128(d2, k13);

d3 = _mm_aesenc_si128(d3, k13);

d0 = _mm_xor_si128(_mm_aesenclast_si128(d0, k14), _mm_loadu_si128(reinterpret_cast<const __m128i *>(in)));

d1 = _mm_xor_si128(_mm_aesenclast_si128(d1, k14), _mm_loadu_si128(reinterpret_cast<const __m128i *>(in + 16)));

d2 = _mm_xor_si128(_mm_aesenclast_si128(d2, k14), _mm_loadu_si128(reinterpret_cast<const __m128i *>(in + 32)));

d3 = _mm_xor_si128(_mm_aesenclast_si128(d3, k14), _mm_loadu_si128(reinterpret_cast<const __m128i *>(in + 48)));

in += 64;

_mm_storeu_si128(reinterpret_cast<__m128i *>(out), d0);

_mm_storeu_si128(reinterpret_cast<__m128i *>(out + 16), d1);

_mm_storeu_si128(reinterpret_cast<__m128i *>(out + 32), d2);

_mm_storeu_si128(reinterpret_cast<__m128i *>(out + 48), d3);

out += 64;

} while (likely(in != eof64));

}

skip_conventional_aesni_64:

while (len >= 16) {

__m128i d0 = _mm_insert_epi64(dd, (long long)Utils::hton(c1++), 1);

d0 = _mm_xor_si128(d0, k0);

d0 = _mm_aesenc_si128(d0, k1);

d0 = _mm_aesenc_si128(d0, k2);

d0 = _mm_aesenc_si128(d0, k3);

d0 = _mm_aesenc_si128(d0, k4);

d0 = _mm_aesenc_si128(d0, k5);

d0 = _mm_aesenc_si128(d0, k6);

d0 = _mm_aesenc_si128(d0, k7);

d0 = _mm_aesenc_si128(d0, k8);

d0 = _mm_aesenc_si128(d0, k9);

d0 = _mm_aesenc_si128(d0, k10);

d0 = _mm_aesenc_si128(d0, k11);

d0 = _mm_aesenc_si128(d0, k12);

d0 = _mm_aesenc_si128(d0, k13);

_mm_storeu_si128(reinterpret_cast<__m128i *>(out), _mm_xor_si128(_mm_aesenclast_si128(d0, k14), _mm_loadu_si128(reinterpret_cast<const __m128i *>(in))));

in += 16;

len -= 16;

out += 16;

}

// Any remaining input is placed in _out. This will be picked up and crypted

// on subsequent calls to crypt() or finish() as it'll mean _len will not be

// an even multiple of 16.

for (unsigned int i = 0; i < len; ++i)

out[i] = in[i];

_ctr[1] = Utils::hton(c1);

return;

}

#endif // ZT_AES_AESNI

#ifdef ZT_AES_NEON

if (Utils::ARMCAP.aes) {

uint8x16_t dd = vrev32q_u8(vld1q_u8(reinterpret_cast<uint8_t *>(_ctr)));

const uint32x4_t one = {0,0,0,1};

uint8x16_t k0 = _aes._k.neon.ek[0];

uint8x16_t k1 = _aes._k.neon.ek[1];

uint8x16_t k2 = _aes._k.neon.ek[2];

uint8x16_t k3 = _aes._k.neon.ek[3];

uint8x16_t k4 = _aes._k.neon.ek[4];

uint8x16_t k5 = _aes._k.neon.ek[5];

uint8x16_t k6 = _aes._k.neon.ek[6];

uint8x16_t k7 = _aes._k.neon.ek[7];

uint8x16_t k8 = _aes._k.neon.ek[8];

uint8x16_t k9 = _aes._k.neon.ek[9];

uint8x16_t k10 = _aes._k.neon.ek[10];

uint8x16_t k11 = _aes._k.neon.ek[11];

uint8x16_t k12 = _aes._k.neon.ek[12];

uint8x16_t k13 = _aes._k.neon.ek[13];

uint8x16_t k14 = _aes._k.neon.ek[14];

unsigned int totalLen = _len;

if ((totalLen & 15U)) {

for (;;) {

if (unlikely(!len)) {

vst1q_u8(reinterpret_cast<uint8_t *>(_ctr), vrev32q_u8(dd));

_len = totalLen;

return;

}

--len;

out[totalLen++] = *(in++);

if (!(totalLen & 15U)) {

uint8_t *const otmp = out + (totalLen - 16);

uint8x16_t d0 = vrev32q_u8(dd);

uint8x16_t pt = vld1q_u8(otmp);

d0 = vaesmcq_u8(vaeseq_u8(d0, k0));

d0 = vaesmcq_u8(vaeseq_u8(d0, k1));

d0 = vaesmcq_u8(vaeseq_u8(d0, k2));

d0 = vaesmcq_u8(vaeseq_u8(d0, k3));

d0 = vaesmcq_u8(vaeseq_u8(d0, k4));

d0 = vaesmcq_u8(vaeseq_u8(d0, k5));

d0 = vaesmcq_u8(vaeseq_u8(d0, k6));

d0 = vaesmcq_u8(vaeseq_u8(d0, k7));

d0 = vaesmcq_u8(vaeseq_u8(d0, k8));

d0 = vaesmcq_u8(vaeseq_u8(d0, k9));

d0 = vaesmcq_u8(vaeseq_u8(d0, k10));

d0 = vaesmcq_u8(vaeseq_u8(d0, k11));

d0 = vaesmcq_u8(vaeseq_u8(d0, k12));

d0 = veorq_u8(vaeseq_u8(d0, k13), k14);

vst1q_u8(otmp, veorq_u8(pt, d0));

dd = (uint8x16_t)vaddq_u32((uint32x4_t)dd, one);

break;

}

}

}

out += totalLen;

_len = totalLen + len;

if (likely(len >= 64)) {

const uint32x4_t four = vshlq_n_u32(one, 2);

uint8x16_t dd1 = (uint8x16_t)vaddq_u32((uint32x4_t)dd, one);

uint8x16_t dd2 = (uint8x16_t)vaddq_u32((uint32x4_t)dd1, one);

uint8x16_t dd3 = (uint8x16_t)vaddq_u32((uint32x4_t)dd2, one);

for (;;) {

len -= 64;

uint8x16_t d0 = vrev32q_u8(dd);

uint8x16_t d1 = vrev32q_u8(dd1);

uint8x16_t d2 = vrev32q_u8(dd2);

uint8x16_t d3 = vrev32q_u8(dd3);

uint8x16_t pt0 = vld1q_u8(in);

in += 16;

d0 = vaesmcq_u8(vaeseq_u8(d0, k0));

d1 = vaesmcq_u8(vaeseq_u8(d1, k0));

d2 = vaesmcq_u8(vaeseq_u8(d2, k0));

d3 = vaesmcq_u8(vaeseq_u8(d3, k0));

d0 = vaesmcq_u8(vaeseq_u8(d0, k1));

d1 = vaesmcq_u8(vaeseq_u8(d1, k1));

d2 = vaesmcq_u8(vaeseq_u8(d2, k1));

d3 = vaesmcq_u8(vaeseq_u8(d3, k1));

d0 = vaesmcq_u8(vaeseq_u8(d0, k2));

d1 = vaesmcq_u8(vaeseq_u8(d1, k2));

d2 = vaesmcq_u8(vaeseq_u8(d2, k2));

d3 = vaesmcq_u8(vaeseq_u8(d3, k2));

uint8x16_t pt1 = vld1q_u8(in);

in += 16;

d0 = vaesmcq_u8(vaeseq_u8(d0, k3));

d1 = vaesmcq_u8(vaeseq_u8(d1, k3));

d2 = vaesmcq_u8(vaeseq_u8(d2, k3));

d3 = vaesmcq_u8(vaeseq_u8(d3, k3));

d0 = vaesmcq_u8(vaeseq_u8(d0, k4));

d1 = vaesmcq_u8(vaeseq_u8(d1, k4));

d2 = vaesmcq_u8(vaeseq_u8(d2, k4));

d3 = vaesmcq_u8(vaeseq_u8(d3, k4));

d0 = vaesmcq_u8(vaeseq_u8(d0, k5));

d1 = vaesmcq_u8(vaeseq_u8(d1, k5));

d2 = vaesmcq_u8(vaeseq_u8(d2, k5));

d3 = vaesmcq_u8(vaeseq_u8(d3, k5));

uint8x16_t pt2 = vld1q_u8(in);

in += 16;

d0 = vaesmcq_u8(vaeseq_u8(d0, k6));

d1 = vaesmcq_u8(vaeseq_u8(d1, k6));

d2 = vaesmcq_u8(vaeseq_u8(d2, k6));

d3 = vaesmcq_u8(vaeseq_u8(d3, k6));

d0 = vaesmcq_u8(vaeseq_u8(d0, k7));

d1 = vaesmcq_u8(vaeseq_u8(d1, k7));

d2 = vaesmcq_u8(vaeseq_u8(d2, k7));

d3 = vaesmcq_u8(vaeseq_u8(d3, k7));

d0 = vaesmcq_u8(vaeseq_u8(d0, k8));

d1 = vaesmcq_u8(vaeseq_u8(d1, k8));

d2 = vaesmcq_u8(vaeseq_u8(d2, k8));

d3 = vaesmcq_u8(vaeseq_u8(d3, k8));

uint8x16_t pt3 = vld1q_u8(in);

in += 16;

d0 = vaesmcq_u8(vaeseq_u8(d0, k9));

d1 = vaesmcq_u8(vaeseq_u8(d1, k9));

d2 = vaesmcq_u8(vaeseq_u8(d2, k9));

d3 = vaesmcq_u8(vaeseq_u8(d3, k9));

d0 = vaesmcq_u8(vaeseq_u8(d0, k10));

d1 = vaesmcq_u8(vaeseq_u8(d1, k10));

d2 = vaesmcq_u8(vaeseq_u8(d2, k10));

d3 = vaesmcq_u8(vaeseq_u8(d3, k10));

d0 = vaesmcq_u8(vaeseq_u8(d0, k11));

d1 = vaesmcq_u8(vaeseq_u8(d1, k11));

d2 = vaesmcq_u8(vaeseq_u8(d2, k11));

d3 = vaesmcq_u8(vaeseq_u8(d3, k11));

d0 = vaesmcq_u8(vaeseq_u8(d0, k12));

d1 = vaesmcq_u8(vaeseq_u8(d1, k12));

d2 = vaesmcq_u8(vaeseq_u8(d2, k12));

d3 = vaesmcq_u8(vaeseq_u8(d3, k12));

d0 = veorq_u8(vaeseq_u8(d0, k13), k14);

d1 = veorq_u8(vaeseq_u8(d1, k13), k14);

d2 = veorq_u8(vaeseq_u8(d2, k13), k14);

d3 = veorq_u8(vaeseq_u8(d3, k13), k14);

d0 = veorq_u8(pt0, d0);

d1 = veorq_u8(pt1, d1);

d2 = veorq_u8(pt2, d2);

d3 = veorq_u8(pt3, d3);

vst1q_u8(out, d0);

vst1q_u8(out + 16, d1);

vst1q_u8(out + 32, d2);

vst1q_u8(out + 48, d3);

out += 64;

dd = (uint8x16_t)vaddq_u32((uint32x4_t)dd, four);

if (unlikely(len < 64))

break;

dd1 = (uint8x16_t)vaddq_u32((uint32x4_t)dd1, four);

dd2 = (uint8x16_t)vaddq_u32((uint32x4_t)dd2, four);

dd3 = (uint8x16_t)vaddq_u32((uint32x4_t)dd3, four);

}

}

while (len >= 16) {

len -= 16;

uint8x16_t d0 = vrev32q_u8(dd);

uint8x16_t pt = vld1q_u8(in);

in += 16;

dd = (uint8x16_t)vaddq_u32((uint32x4_t)dd, one);

d0 = vaesmcq_u8(vaeseq_u8(d0, k0));

d0 = vaesmcq_u8(vaeseq_u8(d0, k1));

d0 = vaesmcq_u8(vaeseq_u8(d0, k2));

d0 = vaesmcq_u8(vaeseq_u8(d0, k3));

d0 = vaesmcq_u8(vaeseq_u8(d0, k4));

d0 = vaesmcq_u8(vaeseq_u8(d0, k5));

d0 = vaesmcq_u8(vaeseq_u8(d0, k6));

d0 = vaesmcq_u8(vaeseq_u8(d0, k7));

d0 = vaesmcq_u8(vaeseq_u8(d0, k8));

d0 = vaesmcq_u8(vaeseq_u8(d0, k9));

d0 = vaesmcq_u8(vaeseq_u8(d0, k10));

d0 = vaesmcq_u8(vaeseq_u8(d0, k11));

d0 = vaesmcq_u8(vaeseq_u8(d0, k12));

d0 = veorq_u8(vaeseq_u8(d0, k13), k14);

vst1q_u8(out, veorq_u8(pt, d0));

out += 16;

}

// Any remaining input is placed in _out. This will be picked up and crypted

// on subsequent calls to crypt() or finish() as it'll mean _len will not be

// an even multiple of 16.

for (unsigned int i = 0; i < len; ++i)

out[i] = in[i];

vst1q_u8(reinterpret_cast<uint8_t *>(_ctr), vrev32q_u8(dd));

return;

}

#endif // ZT_AES_NEON

uint64_t keyStream[2];

uint32_t ctr = Utils::ntoh(reinterpret_cast<uint32_t *>(_ctr)[3]);

unsigned int totalLen = _len;

if ((totalLen & 15U)) {

for (;;) {

if (!len) {

_len = (totalLen + len);

return;

}

--len;

out[totalLen++] = *(in++);