From cb4f3e4f06bb6e96da999cf5f6bec55a9a5592c0 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Fri, 2 May 2014 15:20:34 -0700 Subject: [PATCH] Use SSE2 or portable scrypt implementation according to what the CPU supports. --- configure.ac | 1 + .../crypto_scrypt-common.c | 264 +++++++++--------- .../scryptxsalsa208sha256/crypto_scrypt.h | 6 + .../scryptxsalsa208sha256/scrypt_platform.c | 1 + src/libsodium/sodium/runtime.c | 10 + 5 files changed, 153 insertions(+), 129 deletions(-) diff --git a/configure.ac b/configure.ac index 095dce23..d25f2d22 100644 --- a/configure.ac +++ b/configure.ac @@ -244,6 +244,7 @@ dnl Checks for headers AS_IF([test "x$EMSCRIPTEN" = "x"],[ AC_CHECK_HEADERS([emmintrin.h], [], [], [#pragma GCC target("sse2")]) + AC_CHECK_HEADERS([pmmintrin.h], [], [], [#pragma GCC target("sse3")]) AC_CHECK_HEADERS([tmmintrin.h], [], [], [#pragma GCC target("ssse3")]) AC_CHECK_HEADERS([smmintrin.h], [], [], [#pragma GCC target("sse4.1")]) AC_CHECK_HEADERS([immintrin.h], [], [], [#pragma GCC target("avx")]) diff --git a/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt-common.c b/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt-common.c index 9791863d..9d5de17a 100644 --- a/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt-common.c +++ b/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt-common.c @@ -22,86 +22,85 @@ #include #include "crypto_scrypt.h" - -#define escrypt_kdf escrypt_kdf_nosse +#include "runtime.h" #define BYTES2CHARS(bytes) \ - ((((bytes) * 8) + 5) / 6) + ((((bytes) * 8) + 5) / 6) #define HASH_SIZE 32 /* bytes */ #define HASH_LEN BYTES2CHARS(HASH_SIZE) /* base-64 chars */ static const char * const itoa64 = - "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; static uint8_t * encode64_uint32(uint8_t * dst, size_t dstlen, uint32_t src, uint32_t srcbits) { - uint32_t bit; + uint32_t bit; - for (bit = 0; bit < srcbits; bit += 6) { - if (dstlen < 1) - return NULL; - *dst++ = itoa64[src & 0x3f]; - dstlen--; - src >>= 6; - } + for (bit = 0; bit < srcbits; bit += 6) { + if (dstlen < 1) + return NULL; + *dst++ = itoa64[src & 0x3f]; + dstlen--; + src >>= 6; + } - return dst; + return dst; } static uint8_t * encode64(uint8_t * dst, size_t dstlen, const uint8_t * src, size_t srclen) { - size_t i; + size_t i; - for (i = 0; i < srclen; ) { - uint8_t * dnext; - uint32_t value = 0, bits = 0; - do { - value |= (uint32_t)src[i++] << bits; - bits += 8; - } while (bits < 24 && i < srclen); - dnext = encode64_uint32(dst, dstlen, value, bits); - if (!dnext) - return NULL; - dstlen -= dnext - dst; - dst = dnext; - } + for (i = 0; i < srclen; ) { + uint8_t * dnext; + uint32_t value = 0, bits = 0; + do { + value |= (uint32_t)src[i++] << bits; + bits += 8; + } while (bits < 24 && i < srclen); + dnext = encode64_uint32(dst, dstlen, value, bits); + if (!dnext) + return NULL; + dstlen -= dnext - dst; + dst = dnext; + } - return dst; + return dst; } static int decode64_one(uint32_t * dst, uint8_t src) { - const char * ptr = strchr(itoa64, src); - if (ptr) { - *dst = ptr - itoa64; - return 0; - } - *dst = 0; - return -1; + const char * ptr = strchr(itoa64, src); + if (ptr) { + *dst = ptr - itoa64; + return 0; + } + *dst = 0; + return -1; } static const uint8_t * decode64_uint32(uint32_t * dst, uint32_t dstbits, const uint8_t * src) { - uint32_t bit; - uint32_t value; + uint32_t bit; + uint32_t value; - value = 0; - for (bit = 0; bit < dstbits; bit += 6) { - uint32_t one; - if (decode64_one(&one, *src)) { - *dst = 0; - return NULL; - } - src++; - value |= one << bit; - } + value = 0; + for (bit = 0; bit < dstbits; bit += 6) { + uint32_t one; + if (decode64_one(&one, *src)) { + *dst = 0; + return NULL; + } + src++; + value |= one << bit; + } - *dst = value; - return src; + *dst = value; + return src; } uint8_t * @@ -110,64 +109,68 @@ escrypt_r(escrypt_local_t * local, const uint8_t * setting, uint8_t * buf, size_t buflen) { - uint8_t hash[HASH_SIZE]; - const uint8_t * src, * salt; - uint8_t * dst; - size_t prefixlen, saltlen, need; - uint64_t N; - uint32_t r, p; + uint8_t hash[HASH_SIZE]; + escrypt_kdf_t escrypt_kdf; + const uint8_t * src, * salt; + uint8_t * dst; + size_t prefixlen, saltlen, need; + uint64_t N; + uint32_t r, p; - if (setting[0] != '$' || setting[1] != '7' || setting[2] != '$') - return NULL; - src = setting + 3; + if (setting[0] != '$' || setting[1] != '7' || setting[2] != '$') + return NULL; + src = setting + 3; - { - uint32_t N_log2; - if (decode64_one(&N_log2, *src)) - return NULL; - src++; - N = (uint64_t)1 << N_log2; - } + { + uint32_t N_log2; + if (decode64_one(&N_log2, *src)) + return NULL; + src++; + N = (uint64_t)1 << N_log2; + } - src = decode64_uint32(&r, 30, src); - if (!src) - return NULL; + src = decode64_uint32(&r, 30, src); + if (!src) + return NULL; - src = decode64_uint32(&p, 30, src); - if (!src) - return NULL; + src = decode64_uint32(&p, 30, src); + if (!src) + return NULL; - prefixlen = src - setting; + prefixlen = src - setting; - salt = src; - src = (uint8_t *)strrchr((char *)salt, '$'); - if (src) - saltlen = src - salt; - else - saltlen = strlen((char *)salt); + salt = src; + src = (uint8_t *)strrchr((char *)salt, '$'); + if (src) + saltlen = src - salt; + else + saltlen = strlen((char *)salt); - need = prefixlen + saltlen + 1 + HASH_LEN + 1; - if (need > buflen || need < saltlen) - return NULL; + need = prefixlen + saltlen + 1 + HASH_LEN + 1; + if (need > buflen || need < saltlen) + return NULL; - if (escrypt_kdf(local, passwd, passwdlen, salt, saltlen, - N, r, p, hash, sizeof(hash))) - return NULL; + escrypt_kdf = + sodium_runtime_has_sse2() ? escrypt_kdf_sse : escrypt_kdf_nosse; + if (escrypt_kdf(local, passwd, passwdlen, salt, saltlen, + N, r, p, hash, sizeof(hash))) { + return NULL; + } - dst = buf; - memcpy(dst, setting, prefixlen + saltlen); - dst += prefixlen + saltlen; - *dst++ = '$'; + dst = buf; + memcpy(dst, setting, prefixlen + saltlen); + dst += prefixlen + saltlen; + *dst++ = '$'; - dst = encode64(dst, buflen - (dst - buf), hash, sizeof(hash)); - /* Could zeroize hash[] here, but escrypt_kdf() doesn't zeroize its - * memory allocations yet anyway. */ - if (!dst || dst >= buf + buflen) /* Can't happen */ - return NULL; + dst = encode64(dst, buflen - (dst - buf), hash, sizeof(hash)); + /* Could zeroize hash[] here, but escrypt_kdf() doesn't zeroize its + * memory allocations yet anyway. */ + if (!dst || dst >= buf + buflen) /* Can't happen */ + return NULL; - *dst = 0; /* NUL termination */ + *dst = 0; /* NUL termination */ - return buf; + return buf; } uint8_t * @@ -175,40 +178,40 @@ escrypt_gensalt_r(uint32_t N_log2, uint32_t r, uint32_t p, const uint8_t * src, size_t srclen, uint8_t * buf, size_t buflen) { - uint8_t * dst; - size_t prefixlen = 3 + 1 + 5 + 5; - size_t saltlen = BYTES2CHARS(srclen); - size_t need; + uint8_t * dst; + size_t prefixlen = 3 + 1 + 5 + 5; + size_t saltlen = BYTES2CHARS(srclen); + size_t need; - need = prefixlen + saltlen + 1; - if (need > buflen || need < saltlen || saltlen < srclen) - return NULL; + need = prefixlen + saltlen + 1; + if (need > buflen || need < saltlen || saltlen < srclen) + return NULL; - if (N_log2 > 63 || ((uint64_t)r * (uint64_t)p >= (1U << 30))) - return NULL; + if (N_log2 > 63 || ((uint64_t)r * (uint64_t)p >= (1U << 30))) + return NULL; - dst = buf; - *dst++ = '$'; - *dst++ = '7'; - *dst++ = '$'; + dst = buf; + *dst++ = '$'; + *dst++ = '7'; + *dst++ = '$'; - *dst++ = itoa64[N_log2]; + *dst++ = itoa64[N_log2]; - dst = encode64_uint32(dst, buflen - (dst - buf), r, 30); - if (!dst) /* Can't happen */ - return NULL; + dst = encode64_uint32(dst, buflen - (dst - buf), r, 30); + if (!dst) /* Can't happen */ + return NULL; - dst = encode64_uint32(dst, buflen - (dst - buf), p, 30); - if (!dst) /* Can't happen */ - return NULL; + dst = encode64_uint32(dst, buflen - (dst - buf), p, 30); + if (!dst) /* Can't happen */ + return NULL; - dst = encode64(dst, buflen - (dst - buf), src, srclen); - if (!dst || dst >= buf + buflen) /* Can't happen */ - return NULL; + dst = encode64(dst, buflen - (dst - buf), src, srclen); + if (!dst || dst >= buf + buflen) /* Can't happen */ + return NULL; - *dst = 0; /* NUL termination */ + *dst = 0; /* NUL termination */ - return buf; + return buf; } int @@ -216,14 +219,17 @@ crypto_scrypt(const uint8_t * passwd, size_t passwdlen, const uint8_t * salt, size_t saltlen, uint64_t N, uint32_t r, uint32_t p, uint8_t * buf, size_t buflen) { - escrypt_local_t local; - int retval; + escrypt_kdf_t escrypt_kdf; + escrypt_local_t local; + int retval; - if (escrypt_init_local(&local)) - return -1; - retval = escrypt_kdf(&local, - passwd, passwdlen, salt, saltlen, N, r, p, buf, buflen); - if (escrypt_free_local(&local)) - return -1; - return retval; + if (escrypt_init_local(&local)) + return -1; + escrypt_kdf = + sodium_runtime_has_sse2() ? escrypt_kdf_sse : escrypt_kdf_nosse; + retval = escrypt_kdf(&local, + passwd, passwdlen, salt, saltlen, N, r, p, buf, buflen); + if (escrypt_free_local(&local)) + return -1; + return retval; } diff --git a/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt.h b/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt.h index 2a1d8d18..1ffc7830 100644 --- a/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt.h +++ b/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/crypto_scrypt.h @@ -60,6 +60,12 @@ extern int escrypt_free_local(escrypt_local_t * __local); extern void *alloc_region(escrypt_region_t * region, size_t size); extern int free_region(escrypt_region_t * region); +typedef int (*escrypt_kdf_t)(escrypt_local_t * __local, + const uint8_t * __passwd, size_t __passwdlen, + const uint8_t * __salt, size_t __saltlen, + uint64_t __N, uint32_t __r, uint32_t __p, + uint8_t * __buf, size_t __buflen); + extern int escrypt_kdf_nosse(escrypt_local_t * __local, const uint8_t * __passwd, size_t __passwdlen, const uint8_t * __salt, size_t __saltlen, diff --git a/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/scrypt_platform.c b/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/scrypt_platform.c index 6dc26c57..f8d8e8bf 100644 --- a/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/scrypt_platform.c +++ b/src/libsodium/crypto_pwhash/scryptxsalsa208sha256/scrypt_platform.c @@ -25,6 +25,7 @@ #include #include "crypto_scrypt.h" +#include "runtime.h" void * alloc_region(escrypt_region_t * region, size_t size) diff --git a/src/libsodium/sodium/runtime.c b/src/libsodium/sodium/runtime.c index 2e4ae191..4bc200bb 100644 --- a/src/libsodium/sodium/runtime.c +++ b/src/libsodium/sodium/runtime.c @@ -66,8 +66,18 @@ _sodium_runtime_intel_cpu_features(CPUFeatures * const cpu_features) return -1; } _cpuid(cpu_info, 0x00000001); + +#ifndef HAVE_EMMINTRIN_H cpu_features->has_sse2 = ((cpu_info[3] & CPUID_SSE2) != 0x0); +#else + cpu_features->has_sse2 = 0; +#endif + +#ifndef HAVE_PMMINTRIN_H cpu_features->has_sse3 = ((cpu_info[2] & CPUIDECX_SSE3) != 0x0); +#else + cpu_features->has_sse3 = 0; +#endif return 0; }