123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280 |
- /*
- * AESNI.c: AES using AES-NI instructions
- *
- * Written in 2013 by Sebastian Ramacher <sebastian@ramacher.at>
- *
- * ===================================================================
- * The contents of this file are dedicated to the public domain. To
- * the extent that dedication to the public domain is not available,
- * everyone is granted a worldwide, perpetual, royalty-free,
- * non-exclusive license to exercise all rights associated with the
- * contents of this file for any purpose whatsoever.
- * No rights are reserved.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- * ===================================================================
- */
-
- #include <wmmintrin.h>
- #include <stdlib.h>
- #include <string.h>
- #include <errno.h>
- #if defined(HAVE__ALIGNED_MALLOC)
- #include <malloc.h>
- #endif
-
- #define MODULE_NAME _AESNI
- #define BLOCK_SIZE 16
- #define KEY_SIZE 0
-
- #define MAXKC (256/32)
- #define MAXKB (256/8)
- #define MAXNR 14
-
- typedef unsigned char u8;
-
- typedef struct {
- __m128i* ek;
- __m128i* dk;
- int rounds;
- } block_state;
-
- /* Wrapper functions for malloc and free with memory alignment */
- #if defined(HAVE_ALIGNED_ALLOC) /* aligned_alloc is defined by C11 */
- # define aligned_malloc_wrapper aligned_alloc
- # define aligned_free_wrapper free
- #elif defined(HAVE_POSIX_MEMALIGN) /* posix_memalign is defined by POSIX */
- static void* aligned_malloc_wrapper(size_t alignment, size_t size)
- {
- void* tmp = NULL;
- int err = posix_memalign(&tmp, alignment, size);
- if (err != 0) {
- /* posix_memalign does NOT set errno on failure; the error is returned */
- errno = err;
- return NULL;
- }
- return tmp;
- }
- # define aligned_free_wrapper free
- #elif defined(HAVE__ALIGNED_MALLOC) /* _aligned_malloc is available on Windows */
- static void* aligned_malloc_wrapper(size_t alignment, size_t size)
- {
- /* NB: _aligned_malloc takes its args in the opposite order from aligned_alloc */
- return _aligned_malloc(size, alignment);
- }
- # define aligned_free_wrapper _aligned_free
- #else
- # error "No function to allocate/free aligned memory is available."
- #endif
-
- /* Helper functions to expand keys */
-
- static __m128i aes128_keyexpand(__m128i key)
- {
- key = _mm_xor_si128(key, _mm_slli_si128(key, 4));
- key = _mm_xor_si128(key, _mm_slli_si128(key, 4));
- return _mm_xor_si128(key, _mm_slli_si128(key, 4));
- }
-
- static __m128i aes192_keyexpand_2(__m128i key, __m128i key2)
- {
- key = _mm_shuffle_epi32(key, 0xff);
- key2 = _mm_xor_si128(key2, _mm_slli_si128(key2, 4));
- return _mm_xor_si128(key, key2);
- }
-
- #define KEYEXP128_H(K1, K2, I, S) _mm_xor_si128(aes128_keyexpand(K1), \
- _mm_shuffle_epi32(_mm_aeskeygenassist_si128(K2, I), S))
-
- #define KEYEXP128(K, I) KEYEXP128_H(K, K, I, 0xff)
- #define KEYEXP192(K1, K2, I) KEYEXP128_H(K1, K2, I, 0x55)
- #define KEYEXP192_2(K1, K2) aes192_keyexpand_2(K1, K2)
- #define KEYEXP256(K1, K2, I) KEYEXP128_H(K1, K2, I, 0xff)
- #define KEYEXP256_2(K1, K2) KEYEXP128_H(K1, K2, 0x00, 0xaa)
-
- /* Encryption key setup */
- static void aes_key_setup_enc(__m128i rk[], const u8* cipherKey, int keylen)
- {
- switch (keylen) {
- case 16:
- {
- /* 128 bit key setup */
- rk[0] = _mm_loadu_si128((const __m128i*) cipherKey);
- rk[1] = KEYEXP128(rk[0], 0x01);
- rk[2] = KEYEXP128(rk[1], 0x02);
- rk[3] = KEYEXP128(rk[2], 0x04);
- rk[4] = KEYEXP128(rk[3], 0x08);
- rk[5] = KEYEXP128(rk[4], 0x10);
- rk[6] = KEYEXP128(rk[5], 0x20);
- rk[7] = KEYEXP128(rk[6], 0x40);
- rk[8] = KEYEXP128(rk[7], 0x80);
- rk[9] = KEYEXP128(rk[8], 0x1B);
- rk[10] = KEYEXP128(rk[9], 0x36);
- break;
- }
- case 24:
- {
- /* 192 bit key setup */
- __m128i temp[2];
- rk[0] = _mm_loadu_si128((const __m128i*) cipherKey);
- rk[1] = _mm_loadu_si128((const __m128i*) (cipherKey+16));
- temp[0] = KEYEXP192(rk[0], rk[1], 0x01);
- temp[1] = KEYEXP192_2(temp[0], rk[1]);
- rk[1] = (__m128i)_mm_shuffle_pd((__m128d)rk[1], (__m128d)temp[0], 0);
- rk[2] = (__m128i)_mm_shuffle_pd((__m128d)temp[0], (__m128d)temp[1], 1);
- rk[3] = KEYEXP192(temp[0], temp[1], 0x02);
- rk[4] = KEYEXP192_2(rk[3], temp[1]);
- temp[0] = KEYEXP192(rk[3], rk[4], 0x04);
- temp[1] = KEYEXP192_2(temp[0], rk[4]);
- rk[4] = (__m128i)_mm_shuffle_pd((__m128d)rk[4], (__m128d)temp[0], 0);
- rk[5] = (__m128i)_mm_shuffle_pd((__m128d)temp[0], (__m128d)temp[1], 1);
- rk[6] = KEYEXP192(temp[0], temp[1], 0x08);
- rk[7] = KEYEXP192_2(rk[6], temp[1]);
- temp[0] = KEYEXP192(rk[6], rk[7], 0x10);
- temp[1] = KEYEXP192_2(temp[0], rk[7]);
- rk[7] = (__m128i)_mm_shuffle_pd((__m128d)rk[7], (__m128d)temp[0], 0);
- rk[8] = (__m128i)_mm_shuffle_pd((__m128d)temp[0], (__m128d)temp[1], 1);
- rk[9] = KEYEXP192(temp[0], temp[1], 0x20);
- rk[10] = KEYEXP192_2(rk[9], temp[1]);
- temp[0] = KEYEXP192(rk[9], rk[10], 0x40);
- temp[1] = KEYEXP192_2(temp[0], rk[10]);
- rk[10] = (__m128i)_mm_shuffle_pd((__m128d)rk[10], (__m128d) temp[0], 0);
- rk[11] = (__m128i)_mm_shuffle_pd((__m128d)temp[0],(__m128d) temp[1], 1);
- rk[12] = KEYEXP192(temp[0], temp[1], 0x80);
- break;
- }
- case 32:
- {
- /* 256 bit key setup */
- rk[0] = _mm_loadu_si128((const __m128i*) cipherKey);
- rk[1] = _mm_loadu_si128((const __m128i*) (cipherKey+16));
- rk[2] = KEYEXP256(rk[0], rk[1], 0x01);
- rk[3] = KEYEXP256_2(rk[1], rk[2]);
- rk[4] = KEYEXP256(rk[2], rk[3], 0x02);
- rk[5] = KEYEXP256_2(rk[3], rk[4]);
- rk[6] = KEYEXP256(rk[4], rk[5], 0x04);
- rk[7] = KEYEXP256_2(rk[5], rk[6]);
- rk[8] = KEYEXP256(rk[6], rk[7], 0x08);
- rk[9] = KEYEXP256_2(rk[7], rk[8]);
- rk[10] = KEYEXP256(rk[8], rk[9], 0x10);
- rk[11] = KEYEXP256_2(rk[9], rk[10]);
- rk[12] = KEYEXP256(rk[10], rk[11], 0x20);
- rk[13] = KEYEXP256_2(rk[11], rk[12]);
- rk[14] = KEYEXP256(rk[12], rk[13], 0x40);
- break;
- }
- }
- }
-
- /* Decryption key setup */
- static void aes_key_setup_dec(__m128i dk[], const __m128i ek[], int rounds)
- {
- int i;
- dk[rounds] = ek[0];
- for (i = 1; i < rounds; ++i) {
- dk[rounds - i] = _mm_aesimc_si128(ek[i]);
- }
- dk[0] = ek[rounds];
- }
-
- void block_init_aesni(block_state* self, unsigned char* key, int keylen)
- {
- int nr = 0;
- switch (keylen) {
- case 16: nr = 10; break;
- case 24: nr = 12; break;
- case 32: nr = 14; break;
- default:
- return;
- }
-
- /* ensure that self->ek and self->dk are aligned to 16 byte boundaries */
- void* tek = aligned_malloc_wrapper(16, (nr + 1) * sizeof(__m128i));
- void* tdk = aligned_malloc_wrapper(16, (nr + 1) * sizeof(__m128i));
- if (!tek || !tdk) {
- aligned_free_wrapper(tek);
- aligned_free_wrapper(tdk);
- return;
- }
-
- self->ek = tek;
- self->dk = tdk;
-
- self->rounds = nr;
- aes_key_setup_enc(self->ek, key, keylen);
- aes_key_setup_dec(self->dk, self->ek, nr);
- }
-
- void block_finalize_aesni(block_state* self)
- {
- /* overwrite contents of ek and dk */
- memset(self->ek, 0, (self->rounds + 1) * sizeof(__m128i));
- memset(self->dk, 0, (self->rounds + 1) * sizeof(__m128i));
-
- aligned_free_wrapper(self->ek);
- aligned_free_wrapper(self->dk);
- }
-
- void block_encrypt_aesni(block_state* self, const u8* in, u8* out)
- {
- __m128i m = _mm_loadu_si128((const __m128i*) in);
- /* first 9 rounds */
- m = _mm_xor_si128(m, self->ek[0]);
- m = _mm_aesenc_si128(m, self->ek[1]);
- m = _mm_aesenc_si128(m, self->ek[2]);
- m = _mm_aesenc_si128(m, self->ek[3]);
- m = _mm_aesenc_si128(m, self->ek[4]);
- m = _mm_aesenc_si128(m, self->ek[5]);
- m = _mm_aesenc_si128(m, self->ek[6]);
- m = _mm_aesenc_si128(m, self->ek[7]);
- m = _mm_aesenc_si128(m, self->ek[8]);
- m = _mm_aesenc_si128(m, self->ek[9]);
- if (self->rounds != 10) {
- /* two additional rounds for AES-192/256 */
- m = _mm_aesenc_si128(m, self->ek[10]);
- m = _mm_aesenc_si128(m, self->ek[11]);
- if (self->rounds == 14) {
- /* another two additional rounds for AES-256 */
- m = _mm_aesenc_si128(m, self->ek[12]);
- m = _mm_aesenc_si128(m, self->ek[13]);
- }
- }
- m = _mm_aesenclast_si128(m, self->ek[self->rounds]);
- _mm_storeu_si128((__m128i*) out, m);
- }
-
- void block_decrypt_aesni(block_state* self, const u8* in, u8* out)
- {
- __m128i m = _mm_loadu_si128((const __m128i*) in);
- /* first 9 rounds */
- m = _mm_xor_si128(m, self->dk[0]);
- m = _mm_aesdec_si128(m, self->dk[1]);
- m = _mm_aesdec_si128(m, self->dk[2]);
- m = _mm_aesdec_si128(m, self->dk[3]);
- m = _mm_aesdec_si128(m, self->dk[4]);
- m = _mm_aesdec_si128(m, self->dk[5]);
- m = _mm_aesdec_si128(m, self->dk[6]);
- m = _mm_aesdec_si128(m, self->dk[7]);
- m = _mm_aesdec_si128(m, self->dk[8]);
- m = _mm_aesdec_si128(m, self->dk[9]);
- if (self->rounds != 10) {
- /* two additional rounds for AES-192/256 */
- m = _mm_aesdec_si128(m, self->dk[10]);
- m = _mm_aesdec_si128(m, self->dk[11]);
- if (self->rounds == 14) {
- /* another two additional rounds for AES-256 */
- m = _mm_aesdec_si128(m, self->dk[12]);
- m = _mm_aesdec_si128(m, self->dk[13]);
- }
- }
- m = _mm_aesdeclast_si128(m, self->dk[self->rounds]);
- _mm_storeu_si128((__m128i*) out, m);
- }
|