12#ifndef CRYPTOPP_IMPORTS
13#ifndef CRYPTOPP_GENERATE_X64_MASM
16#if defined(_MSC_VER) && (_MSC_VER < 1400)
17# pragma optimize("", off)
23#if defined(CRYPTOPP_DISABLE_GCM_ASM)
24# undef CRYPTOPP_X86_ASM_AVAILABLE
25# undef CRYPTOPP_X32_ASM_AVAILABLE
26# undef CRYPTOPP_X64_ASM_AVAILABLE
27# undef CRYPTOPP_SSE2_ASM_AVAILABLE
30NAMESPACE_BEGIN(CryptoPP)
32#if (CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
35#if defined(CRYPTOPP_DISABLE_MIXED_ASM)
37# define USE_MOVD_REG32 1
38#elif defined(__GNUC__) || defined(_MSC_VER)
40# define USE_MOVD_REG32_OR_REG64 1
43# define USE_MOV_REG32_OR_REG64 1
47word16 GCM_Base::s_reductionTable[256];
48volatile bool GCM_Base::s_reductionTableInitialized =
false;
50void GCM_Base::GCTR::IncrementCounterBy256()
55static inline void Xor16(
byte *a,
const byte *b,
const byte *c)
57 CRYPTOPP_ASSERT(
IsAlignedOn(a,GetAlignmentOf<word64>()));
58 CRYPTOPP_ASSERT(
IsAlignedOn(b,GetAlignmentOf<word64>()));
59 CRYPTOPP_ASSERT(
IsAlignedOn(c,GetAlignmentOf<word64>()));
60 ((word64 *)(
void *)a)[0] = ((word64 *)(
void *)b)[0] ^ ((word64 *)(
void *)c)[0];
61 ((word64 *)(
void *)a)[1] = ((word64 *)(
void *)b)[1] ^ ((word64 *)(
void *)c)[1];
64#if CRYPTOPP_SSE2_INTRIN_AVAILABLE || CRYPTOPP_SSE2_ASM_AVAILABLE
67extern void GCM_Xor16_SSE2(
byte *a,
const byte *b,
const byte *c);
70#if CRYPTOPP_ARM_NEON_AVAILABLE
71extern void GCM_Xor16_NEON(
byte *a,
const byte *b,
const byte *c);
74#if CRYPTOPP_POWER8_AVAILABLE
75extern void GCM_Xor16_POWER8(
byte *a,
const byte *b,
const byte *c);
78#if CRYPTOPP_CLMUL_AVAILABLE
79extern void GCM_SetKeyWithoutResync_CLMUL(
const byte *hashKey,
byte *mulTable,
unsigned int tableSize);
80extern size_t GCM_AuthenticateBlocks_CLMUL(
const byte *data,
size_t len,
const byte *mtable,
byte *hbuffer);
81const unsigned int s_cltableSizeInBlocks = 8;
82extern void GCM_ReverseHashBufferIfNeeded_CLMUL(
byte *hashBuffer);
85#if CRYPTOPP_ARM_PMULL_AVAILABLE
86extern void GCM_SetKeyWithoutResync_PMULL(
const byte *hashKey,
byte *mulTable,
unsigned int tableSize);
87extern size_t GCM_AuthenticateBlocks_PMULL(
const byte *data,
size_t len,
const byte *mtable,
byte *hbuffer);
88const unsigned int s_cltableSizeInBlocks = 8;
89extern void GCM_ReverseHashBufferIfNeeded_PMULL(
byte *hashBuffer);
92#if CRYPTOPP_POWER8_VMULL_AVAILABLE
93extern void GCM_SetKeyWithoutResync_VMULL(
const byte *hashKey,
byte *mulTable,
unsigned int tableSize);
94extern size_t GCM_AuthenticateBlocks_VMULL(
const byte *data,
size_t len,
const byte *mtable,
byte *hbuffer);
95const unsigned int s_cltableSizeInBlocks = 8;
96extern void GCM_ReverseHashBufferIfNeeded_VMULL(
byte *hashBuffer);
99void GCM_Base::SetKeyWithoutResync(
const byte *userKey,
size_t keylength,
const NameValuePairs ¶ms)
102 blockCipher.
SetKey(userKey, keylength, params);
108 const unsigned int blockSize = blockCipher.
BlockSize();
109 CRYPTOPP_ASSERT(blockSize == REQUIRED_BLOCKSIZE);
110 if (blockCipher.
BlockSize() != REQUIRED_BLOCKSIZE)
113 int tableSize, i, j, k;
115#if CRYPTOPP_CLMUL_AVAILABLE
119 (void)params.
GetIntValue(Name::TableSize(), tableSize);
120 tableSize = s_cltableSizeInBlocks * blockSize;
121 CRYPTOPP_ASSERT(tableSize >
static_cast<int>(blockSize));
124#elif CRYPTOPP_ARM_PMULL_AVAILABLE
128 (void)params.
GetIntValue(Name::TableSize(), tableSize);
129 tableSize = s_cltableSizeInBlocks * blockSize;
130 CRYPTOPP_ASSERT(tableSize >
static_cast<int>(blockSize));
133#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
137 (void)params.
GetIntValue(Name::TableSize(), tableSize);
138 tableSize = s_cltableSizeInBlocks * blockSize;
139 CRYPTOPP_ASSERT(tableSize >
static_cast<int>(blockSize));
144 if (params.
GetIntValue(Name::TableSize(), tableSize))
145 tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
147 tableSize = (GetTablesOption() ==
GCM_64K_Tables) ? 64*1024 : 2*1024;
155 m_buffer.
resize(3*blockSize + tableSize);
156 byte *mulTable = MulTable();
157 byte *hashKey = HashKey();
158 memset(hashKey, 0, REQUIRED_BLOCKSIZE);
161#if CRYPTOPP_CLMUL_AVAILABLE
164 GCM_SetKeyWithoutResync_CLMUL(hashKey, mulTable, tableSize);
167#elif CRYPTOPP_ARM_PMULL_AVAILABLE
170 GCM_SetKeyWithoutResync_PMULL(hashKey, mulTable, tableSize);
173#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
176 GCM_SetKeyWithoutResync_VMULL(hashKey, mulTable, tableSize);
183 Block::Get(hashKey)(V0)(V1);
185 if (tableSize == 64*1024)
187 for (i=0; i<128; i++)
190 Block::Put(NULLPTR, mulTable+(i/8)*256*16+(
size_t(1)<<(11-k)))(V0)(V1);
193 V1 = (V1>>1) | (V0<<63);
194 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
199 memset(mulTable+i*256*16, 0, 16);
200#if CRYPTOPP_SSE2_INTRIN_AVAILABLE || CRYPTOPP_SSE2_ASM_AVAILABLE
202 for (j=2; j<=0x80; j*=2)
204 GCM_Xor16_SSE2(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
206#elif CRYPTOPP_ARM_NEON_AVAILABLE
208 for (j=2; j<=0x80; j*=2)
210 GCM_Xor16_NEON(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
212#elif CRYPTOPP_POWER8_AVAILABLE
214 for (j=2; j<=0x80; j*=2)
216 GCM_Xor16_POWER8(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
219 for (j=2; j<=0x80; j*=2)
221 Xor16(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
226 if (!s_reductionTableInitialized)
228 s_reductionTable[0] = 0;
231 for (
unsigned int ii=2; ii<=0x80; ii*=2)
235 for (
unsigned int jj=1; jj<ii; jj++)
236 s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
238 s_reductionTableInitialized =
true;
241 for (i=0; i<128-24; i++)
245 Block::Put(NULLPTR, mulTable+1024+(i/32)*256+(
size_t(1)<<(7-k)))(V0)(V1);
247 Block::Put(NULLPTR, mulTable+(i/32)*256+(
size_t(1)<<(11-k)))(V0)(V1);
250 V1 = (V1>>1) | (V0<<63);
251 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
256 memset(mulTable+i*256, 0, 16);
257 memset(mulTable+1024+i*256, 0, 16);
258#if CRYPTOPP_SSE2_INTRIN_AVAILABLE || CRYPTOPP_SSE2_ASM_AVAILABLE
260 for (j=2; j<=8; j*=2)
263 GCM_Xor16_SSE2(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
264 GCM_Xor16_SSE2(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
267#elif CRYPTOPP_ARM_NEON_AVAILABLE
269 for (j=2; j<=8; j*=2)
272 GCM_Xor16_NEON(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
273 GCM_Xor16_NEON(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
276#elif CRYPTOPP_POWER8_AVAILABLE
278 for (j=2; j<=8; j*=2)
281 GCM_Xor16_POWER8(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
282 GCM_Xor16_POWER8(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
286 for (j=2; j<=8; j*=2)
289 Xor16(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
290 Xor16(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
296inline void GCM_Base::ReverseHashBufferIfNeeded()
298#if CRYPTOPP_CLMUL_AVAILABLE
301 GCM_ReverseHashBufferIfNeeded_CLMUL(HashBuffer());
303#elif CRYPTOPP_ARM_PMULL_AVAILABLE
306 GCM_ReverseHashBufferIfNeeded_PMULL(HashBuffer());
308#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
311 GCM_ReverseHashBufferIfNeeded_VMULL(HashBuffer());
316void GCM_Base::Resync(
const byte *iv,
size_t len)
319 byte *hashBuffer = HashBuffer();
323 memcpy(hashBuffer, iv, len);
324 memset(hashBuffer+len, 0, 3);
325 hashBuffer[len+3] = 1;
329 size_t origLen = len;
330 memset(hashBuffer, 0, HASH_BLOCKSIZE);
332 if (len >= HASH_BLOCKSIZE)
334 len = GCM_Base::AuthenticateBlocks(iv, len);
335 iv += (origLen - len);
340 memcpy(m_buffer, iv, len);
341 memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
342 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
346 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
348 ReverseHashBufferIfNeeded();
351 if (m_state >= State_IVSet)
356 m_ctr.
Seek(HASH_BLOCKSIZE);
358 memset(hashBuffer, 0, HASH_BLOCKSIZE);
364#if CRYPTOPP_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
366#elif CRYPTOPP_ARM_NEON_AVAILABLE
368#elif CRYPTOPP_POWER8_AVAILABLE
374#if CRYPTOPP_MSC_VERSION
375# pragma warning(disable: 4731)
380#ifdef CRYPTOPP_X64_MASM_AVAILABLE
382void GCM_AuthenticateBlocks_2K_SSE2(
const byte *data,
size_t blocks, word64 *hashBuffer,
const word16 *reductionTable);
383void GCM_AuthenticateBlocks_64K_SSE2(
const byte *data,
size_t blocks, word64 *hashBuffer);
387#ifndef CRYPTOPP_GENERATE_X64_MASM
389size_t GCM_Base::AuthenticateBlocks(
const byte *data,
size_t len)
391#if CRYPTOPP_CLMUL_AVAILABLE
394 return GCM_AuthenticateBlocks_CLMUL(data, len, MulTable(), HashBuffer());
396#elif CRYPTOPP_ARM_PMULL_AVAILABLE
399 return GCM_AuthenticateBlocks_PMULL(data, len, MulTable(), HashBuffer());
401#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
404 return GCM_AuthenticateBlocks_VMULL(data, len, MulTable(), HashBuffer());
409 word64 *hashBuffer = (word64 *)(
void *)HashBuffer();
410 CRYPTOPP_ASSERT(
IsAlignedOn(hashBuffer,GetAlignmentOf<word64>()));
412 switch (2*(m_buffer.
size()>=64*1024)
413#
if CRYPTOPP_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
422 byte *mulTable = MulTable();
423 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
427 word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
428 Block::Get(data)(y0)(y1);
432 data += HASH_BLOCKSIZE;
433 len -= HASH_BLOCKSIZE;
435 #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(void *)(mulTable+(a*1024)+(b*256)+c+d*8)
437 #if (CRYPTOPP_LITTLE_ENDIAN)
438 #if CRYPTOPP_BOOL_SLOW_WORD64
443 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
445 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
447 #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
448 #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
450 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
451 #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
452 #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
455 #define GF_MUL_32BY128(op, a, b, c) \
456 a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0); \
457 a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1); \
458 b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0); \
459 b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1); \
460 c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0); \
461 c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1); \
462 d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0); \
463 d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1); \
465 GF_MUL_32BY128(=, 0, 0, 0)
466 GF_MUL_32BY128(^=, 0, 1, 1)
467 GF_MUL_32BY128(^=, 1, 0, 2)
468 GF_MUL_32BY128(^=, 1, 1, 3)
470 word32 r = (
word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
473 r ^= (
word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
476 r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
479 a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
482 while (len >= HASH_BLOCKSIZE);
484 hashBuffer[0] = x0; hashBuffer[1] = x1;
490 byte *mulTable = MulTable();
491 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
495 word64 y0, y1, a0, a1;
496 Block::Get(data)(y0)(y1);
500 data += HASH_BLOCKSIZE;
501 len -= HASH_BLOCKSIZE;
503 #undef READ_TABLE_WORD64_COMMON
504 #undef READ_TABLE_WORD64
506 #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(void *)(mulTable+(a)*256*16+(c)+(d)*8)
508 #if (CRYPTOPP_LITTLE_ENDIAN)
509 #if CRYPTOPP_BOOL_SLOW_WORD64
514 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
516 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
519 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
522 #define GF_MUL_8BY128(op, b, c, d) \
523 a0 op READ_TABLE_WORD64(b, c, d, 0);\
524 a1 op READ_TABLE_WORD64(b, c, d, 1);\
526 GF_MUL_8BY128(=, 0, 0, 0)
527 GF_MUL_8BY128(^=, 0, 0, 1)
528 GF_MUL_8BY128(^=, 0, 0, 2)
529 GF_MUL_8BY128(^=, 0, 0, 3)
530 GF_MUL_8BY128(^=, 0, 1, 0)
531 GF_MUL_8BY128(^=, 0, 1, 1)
532 GF_MUL_8BY128(^=, 0, 1, 2)
533 GF_MUL_8BY128(^=, 0, 1, 3)
534 GF_MUL_8BY128(^=, 1, 2, 0)
535 GF_MUL_8BY128(^=, 1, 2, 1)
536 GF_MUL_8BY128(^=, 1, 2, 2)
537 GF_MUL_8BY128(^=, 1, 2, 3)
538 GF_MUL_8BY128(^=, 1, 3, 0)
539 GF_MUL_8BY128(^=, 1, 3, 1)
540 GF_MUL_8BY128(^=, 1, 3, 2)
541 GF_MUL_8BY128(^=, 1, 3, 3)
545 while (len >= HASH_BLOCKSIZE);
547 hashBuffer[0] = x0; hashBuffer[1] = x1;
552#ifdef CRYPTOPP_X64_MASM_AVAILABLE
554 GCM_AuthenticateBlocks_2K_SSE2(data, len/16, hashBuffer, s_reductionTable);
557 GCM_AuthenticateBlocks_64K_SSE2(data, len/16, hashBuffer);
561#if CRYPTOPP_SSE2_ASM_AVAILABLE
569 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
571 GCM_AuthenticateBlocks_2K_SSE2 PROC FRAME
579 AS2( mov WORD_REG(cx), data )
580 AS2( mov WORD_REG(dx), len )
581 AS2( mov WORD_REG(si), hashBuffer )
582 AS2( shr WORD_REG(dx), 4 )
585 #
if CRYPTOPP_BOOL_X32
594 AS2( mov AS_REG_7, WORD_REG(di))
595 #elif CRYPTOPP_BOOL_X86
596 AS2( lea AS_REG_7, s_reductionTable)
599 AS2( movdqa xmm0, [WORD_REG(si)] )
601 #define MUL_TABLE_0 WORD_REG(si) + 32
602 #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
603 #define RED_TABLE AS_REG_7
606 AS2( movdqu xmm4, [WORD_REG(cx)] )
607 AS2( pxor xmm0, xmm4 )
609 AS2( movd ebx, xmm0 )
610 AS2( mov eax, AS_HEX(f0f0f0f0) )
613 AS2( and ebx, AS_HEX(f0f0f0f0) )
615 AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
617 AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
620 AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
622 AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
624 #define SSE2_MUL_32BITS(i) \
625 AS2( psrldq xmm0, 4 )\
626 AS2( movd eax, xmm0 )\
627 AS2( and eax, AS_HEX(f0f0f0f0) )\
628 AS2( movzx edi, bh )\
629 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
630 AS2( movzx edi, bl )\
631 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
633 AS2( movzx edi, bh )\
634 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
635 AS2( movzx edi, bl )\
636 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
637 AS2( movd ebx, xmm0 )\
639 AS2( and ebx, AS_HEX(f0f0f0f0) )\
640 AS2( movzx edi, ah )\
641 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
642 AS2( movzx edi, al )\
643 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
645 AS2( movzx edi, ah )\
646 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
647 AS2( movzx edi, al )\
648 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
655 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
657 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
660 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
662 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
664 AS2( movdqa xmm0, xmm3 )
665 AS2( pslldq xmm3, 1 )
666 AS2( pxor xmm2, xmm3 )
667 AS2( movdqa xmm1, xmm2 )
668 AS2( pslldq xmm2, 1 )
669 AS2( pxor xmm5, xmm2 )
671 AS2( psrldq xmm0, 15 )
673 AS2( movd edi, xmm0 )
674#elif USE_MOV_REG32_OR_REG64
675 AS2( mov WORD_REG(di), xmm0 )
677 AS2( movd WORD_REG(di), xmm0 )
679 AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
682 AS2( movdqa xmm0, xmm5 )
683 AS2( pslldq xmm5, 1 )
684 AS2( pxor xmm4, xmm5 )
686 AS2( psrldq xmm1, 15 )
688 AS2( movd edi, xmm1 )
689#elif USE_MOV_REG32_OR_REG64
690 AS2( mov WORD_REG(di), xmm1 )
692 AS2( movd WORD_REG(di), xmm1 )
694 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
697 AS2( psrldq xmm0, 15 )
699 AS2( movd edi, xmm0 )
700#elif USE_MOV_REG32_OR_REG64
701 AS2( mov WORD_REG(di), xmm0 )
703 AS2( movd WORD_REG(di), xmm0 )
705 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
707 AS2( movd xmm0, eax )
708 AS2( pxor xmm0, xmm4 )
710 AS2( add WORD_REG(cx), 16 )
711 AS2( sub WORD_REG(dx), 1 )
715 AS2( movdqa [WORD_REG(si)], xmm0 )
717 #
if CRYPTOPP_BOOL_X32
728 :
"c" (data),
"d" (len/16),
"S" (hashBuffer),
"D" (s_reductionTable)
729 :
"memory",
"cc",
"%eax",
"%ebx", PERCENT_REG(AS_REG_7),
"%xmm0",
730 "%xmm1",
"%xmm2",
"%xmm3",
"%xmm4",
"%xmm5"
732 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
737 GCM_AuthenticateBlocks_2K_SSE2 ENDP
748 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
750 GCM_AuthenticateBlocks_64K_SSE2 PROC FRAME
756 AS2( mov WORD_REG(cx), data )
757 AS2( mov WORD_REG(dx), len )
758 AS2( mov WORD_REG(si), hashBuffer )
759 AS2( shr WORD_REG(dx), 4 )
762 AS2( movdqa xmm0, [WORD_REG(si)] )
765 #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
768 AS2( movdqu xmm1, [WORD_REG(cx)] )
769 AS2( pxor xmm1, xmm0 )
770 AS2( pxor xmm0, xmm0 )
772 #undef SSE2_MUL_32BITS
773 #define SSE2_MUL_32BITS(i) \
774 AS2( movd eax, xmm1 )\
775 AS2( psrldq xmm1, 4 )\
776 AS2( movzx edi, al )\
777 AS2( add WORD_REG(di), WORD_REG(di) )\
778 AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
779 AS2( movzx edi, ah )\
780 AS2( add WORD_REG(di), WORD_REG(di) )\
781 AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
783 AS2( movzx edi, al )\
784 AS2( add WORD_REG(di), WORD_REG(di) )\
785 AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
786 AS2( movzx edi, ah )\
787 AS2( add WORD_REG(di), WORD_REG(di) )\
788 AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
795 AS2( add WORD_REG(cx), 16 )
796 AS2( sub WORD_REG(dx), 1 )
800 AS2( movdqa [WORD_REG(si)], xmm0 )
805 :
"c" (data),
"d" (len/16),
"S" (hashBuffer)
806 :
"memory",
"cc",
"%edi",
"%eax",
"%xmm0",
"%xmm1"
808 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
812 GCM_AuthenticateBlocks_64K_SSE2 ENDP
818#ifndef CRYPTOPP_GENERATE_X64_MASM
824void GCM_Base::AuthenticateLastHeaderBlock()
826 if (m_bufferedDataLength > 0)
828 memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
829 m_bufferedDataLength = 0;
830 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
834void GCM_Base::AuthenticateLastConfidentialBlock()
836 GCM_Base::AuthenticateLastHeaderBlock();
838 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
841void GCM_Base::AuthenticateLastFooterBlock(
byte *mac,
size_t macSize)
844 ReverseHashBufferIfNeeded();
void ProcessData(byte *outString, const byte *inString, size_t length)
Apply keystream to data.
void Seek(lword position)
Seeks to a random position in the stream.
void Resynchronize(const byte *iv, int length=-1)
Resynchronize the cipher.
Interface for one direction (encryption or decryption) of a block cipher.
void SetCipherWithIV(BlockCipher &cipher, const byte *iv, int feedbackSize=0)
Set external block cipher and IV.
unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
std::string AlgorithmName() const
Provides the name of this algorithm.
An invalid argument was detected.
Interface for retrieving values given their names.
CRYPTOPP_DLL bool GetIntValue(const char *name, int &value) const
Get a named value with type int.
Access a block of memory.
size_type size() const
Provides the count of elements in the SecBlock.
void resize(size_type newSize)
Change size and preserve contents.
virtual void SetKey(const byte *key, size_t length, const NameValuePairs ¶ms=g_nullNameValuePairs)
Sets or reset the key of this object.
Library configuration file.
unsigned int word32
32-bit unsigned datatype
unsigned short word16
16-bit unsigned datatype
Functions for CPU features and intrinsics.
GCM block cipher mode of operation.
@ GCM_64K_Tables
Use a table with 64K entries.
byte ByteReverse(byte value)
Reverses bytes in a 8-bit value.
void IncrementCounterByOne(byte *inout, unsigned int size)
Performs an addition with carry on a block of bytes.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Access a block of memory.