diff --git a/source/build/include/libdivide.h b/source/build/include/libdivide.h index 3b8377a90..c48d9982b 100644 --- a/source/build/include/libdivide.h +++ b/source/build/include/libdivide.h @@ -1,69 +1,36 @@ // libdivide.h -// Copyright 2010 - 2016 ridiculous_fish +// Copyright 2010 - 2018 ridiculous_fish // -// libdivide is dual-licensed under the Boost or zlib -// licenses. You may use libdivide under the terms of -// either of these. See LICENSE.txt for more details. +// libdivide is dual-licensed under the Boost or zlib licenses. +// You may use libdivide under the terms of either of these. +// See LICENSE.txt for more details. -// Modified for EDuke32. - -#ifndef libdivide_h_ -#define libdivide_h_ - -#if defined(_WIN32) || defined(WIN32) -#define LIBDIVIDE_WINDOWS 1 -#endif +#ifndef LIBDIVIDE_H +#define LIBDIVIDE_H #if defined(_MSC_VER) -#define LIBDIVIDE_VC 1 // disable warning C4146: unary minus operator applied to // unsigned type, result still unsigned #pragma warning(disable: 4146) +#define LIBDIVIDE_VC #endif -#if defined __cplusplus && !defined LIBDIVIDE_C_HEADERS +#ifdef __cplusplus #include #include -#include #else #include #include -#include #endif -#if defined(__x86_64__) || defined(_WIN64) || defined(_M_64) -#define LIBDIVIDE_IS_X86_64 1 -#endif - -#if defined(__i386__) -#define LIBDIVIDE_IS_i386 1 -#endif - -#if LIBDIVIDE_IS_X86_64 || defined __SSE2__ || (defined _M_IX86_FP && _M_IX86_FP == 2) -#define LIBDIVIDE_USE_SSE2 1 -#endif - -#if ! LIBDIVIDE_HAS_STDINT_TYPES && (! LIBDIVIDE_VC || _MSC_VER >= 1600) -// Only Visual C++ 2010 and later include stdint.h #include -#define LIBDIVIDE_HAS_STDINT_TYPES 1 + +#if defined(LIBDIVIDE_USE_SSE2) +#include #endif -#if ! LIBDIVIDE_HAS_STDINT_TYPES -typedef __int32 int32_t; -typedef unsigned __int32 uint32_t; -typedef __int64 int64_t; -typedef unsigned __int64 uint64_t; -typedef __int8 int8_t; -typedef unsigned __int8 uint8_t; -#endif - -#if LIBDIVIDE_USE_SSE2 - #include -#endif - -#if LIBDIVIDE_VC - #include +#if defined(LIBDIVIDE_VC) +#include #endif #ifndef __has_builtin @@ -71,18 +38,40 @@ typedef unsigned __int8 uint8_t; #endif #if defined(__SIZEOF_INT128__) -#define HAS_INT128_T 1 +#define HAS_INT128_T #endif -#if __GNUC__ || __clang__ -#define LIBDIVIDE_GCC_STYLE_ASM 1 +#if defined(__x86_64__) || defined(_WIN64) || defined(_M_X64) +#define LIBDIVIDE_IS_X86_64 #endif -#if LIBDIVIDE_ASSERTIONS_ON +#if defined(__i386__) +#define LIBDIVIDE_IS_i386 +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define LIBDIVIDE_GCC_STYLE_ASM +#endif + +#if defined(__cplusplus) || defined(LIBDIVIDE_VC) +#define LIBDIVIDE_FUNCTION __FUNCTION__ +#else +#define LIBDIVIDE_FUNCTION __func__ +#endif + +#define LIBDIVIDE_ERROR(msg) \ + do { \ + fprintf(stderr, "libdivide.h:%d: %s(): Error: %s\n", \ + __LINE__, LIBDIVIDE_FUNCTION, msg); \ + exit(-1); \ + } while (0) + +#if defined(LIBDIVIDE_ASSERTIONS_ON) #define LIBDIVIDE_ASSERT(x) \ do { \ - if (! (x)) { \ - fprintf(stderr, "Assertion failure on line %ld: %s\n", (long)__LINE__, #x); \ + if (!(x)) { \ + fprintf(stderr, "libdivide.h:%d: %s(): Assertion failed: %s\n", \ + __LINE__, LIBDIVIDE_FUNCTION, #x); \ exit(-1); \ } \ } while (0) @@ -98,7 +87,7 @@ typedef unsigned __int8 uint8_t; #include #endif -#if defined __cplusplus && !defined LIBDIVIDE_NONAMESPACE +#ifdef __cplusplus // We place libdivide within the libdivide namespace, and that goes in an // anonymous namespace so that the functions are only visible to files that // #include this header and don't get external linkage. At least that's the @@ -146,6 +135,12 @@ enum { LIBDIVIDE_NEGATIVE_DIVISOR = 0x80 }; +// pack divider structs to prevent compilers from padding. +// This reduces memory usage by up to 43% when using a large +// array of libdivide dividers and improves performance +// by up to 10% because of reduced memory bandwidth. +#pragma pack(push, 1) + struct libdivide_u32_t { uint32_t magic; uint8_t more; @@ -186,8 +181,10 @@ struct libdivide_s64_branchfree_t { uint8_t more; }; +#pragma pack(pop) + #ifndef LIBDIVIDE_API - #if defined __cplusplus || defined LIBDIVIDE_NOINLINE + #ifdef __cplusplus // In C++, we don't want our public functions to be static, because // they are arguments to templates and static functions can't do that. // They get internal linkage through virtue of the anonymous namespace. @@ -252,41 +249,40 @@ LIBDIVIDE_API int64_t libdivide_s64_do_alg2(int64_t numer, const struct libdivid LIBDIVIDE_API int64_t libdivide_s64_do_alg3(int64_t numer, const struct libdivide_s64_t *denom); LIBDIVIDE_API int64_t libdivide_s64_do_alg4(int64_t numer, const struct libdivide_s64_t *denom); -#if LIBDIVIDE_USE_SSE2 -LIBDIVIDE_API __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t * denom); +#if defined(LIBDIVIDE_USE_SSE2) -LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg0(__m128i numers, const struct libdivide_u32_t * denom); -LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg1(__m128i numers, const struct libdivide_u32_t * denom); -LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg2(__m128i numers, const struct libdivide_u32_t * denom); +LIBDIVIDE_API __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom); +LIBDIVIDE_API __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom); +LIBDIVIDE_API __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom); +LIBDIVIDE_API __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg0(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg1(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg2(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg3(__m128i numers, const struct libdivide_s32_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg4(__m128i numers, const struct libdivide_s32_t * denom); +LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg0(__m128i numers, const struct libdivide_u32_t *denom); +LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg1(__m128i numers, const struct libdivide_u32_t *denom); +LIBDIVIDE_API __m128i libdivide_u32_do_vector_alg2(__m128i numers, const struct libdivide_u32_t *denom); -LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg0(__m128i numers, const struct libdivide_u64_t * denom); -LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg1(__m128i numers, const struct libdivide_u64_t * denom); -LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg2(__m128i numers, const struct libdivide_u64_t * denom); +LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg0(__m128i numers, const struct libdivide_s32_t *denom); +LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg1(__m128i numers, const struct libdivide_s32_t *denom); +LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg2(__m128i numers, const struct libdivide_s32_t *denom); +LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg3(__m128i numers, const struct libdivide_s32_t *denom); +LIBDIVIDE_API __m128i libdivide_s32_do_vector_alg4(__m128i numers, const struct libdivide_s32_t *denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg0(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg1(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg2(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg3(__m128i numers, const struct libdivide_s64_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg4(__m128i numers, const struct libdivide_s64_t * denom); +LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg0(__m128i numers, const struct libdivide_u64_t *denom); +LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg1(__m128i numers, const struct libdivide_u64_t *denom); +LIBDIVIDE_API __m128i libdivide_u64_do_vector_alg2(__m128i numers, const struct libdivide_u64_t *denom); -LIBDIVIDE_API __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t * denom); -LIBDIVIDE_API __m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t * denom); -LIBDIVIDE_API __m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t * denom); -LIBDIVIDE_API __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t * denom); +LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg0(__m128i numers, const struct libdivide_s64_t *denom); +LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg1(__m128i numers, const struct libdivide_s64_t *denom); +LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg2(__m128i numers, const struct libdivide_s64_t *denom); +LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg3(__m128i numers, const struct libdivide_s64_t *denom); +LIBDIVIDE_API __m128i libdivide_s64_do_vector_alg4(__m128i numers, const struct libdivide_s64_t *denom); + +LIBDIVIDE_API __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom); +LIBDIVIDE_API __m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom); +LIBDIVIDE_API __m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom); +LIBDIVIDE_API __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom); #endif -#ifndef LIBDIVIDE_HEADER_ONLY - //////// Internal Utility Functions static inline uint32_t libdivide__mullhi_u32(uint32_t x, uint32_t y) { @@ -296,61 +292,61 @@ static inline uint32_t libdivide__mullhi_u32(uint32_t x, uint32_t y) { } static uint64_t libdivide__mullhi_u64(uint64_t x, uint64_t y) { -#if LIBDIVIDE_VC && LIBDIVIDE_IS_X86_64 +#if defined(LIBDIVIDE_VC) && defined(LIBDIVIDE_IS_X86_64) return __umulh(x, y); -#elif HAS_INT128_T +#elif defined(HAS_INT128_T) __uint128_t xl = x, yl = y; __uint128_t rl = xl * yl; return (uint64_t)(rl >> 64); #else // full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) - const uint32_t mask = 0xFFFFFFFF; - const uint32_t x0 = (uint32_t)(x & mask), x1 = (uint32_t)(x >> 32); - const uint32_t y0 = (uint32_t)(y & mask), y1 = (uint32_t)(y >> 32); - const uint32_t x0y0_hi = libdivide__mullhi_u32(x0, y0); - const uint64_t x0y1 = x0 * (uint64_t)y1; - const uint64_t x1y0 = x1 * (uint64_t)y0; - const uint64_t x1y1 = x1 * (uint64_t)y1; - + uint32_t mask = 0xFFFFFFFF; + uint32_t x0 = (uint32_t)(x & mask); + uint32_t x1 = (uint32_t)(x >> 32); + uint32_t y0 = (uint32_t)(y & mask); + uint32_t y1 = (uint32_t)(y >> 32); + uint32_t x0y0_hi = libdivide__mullhi_u32(x0, y0); + uint64_t x0y1 = x0 * (uint64_t)y1; + uint64_t x1y0 = x1 * (uint64_t)y0; + uint64_t x1y1 = x1 * (uint64_t)y1; uint64_t temp = x1y0 + x0y0_hi; - uint64_t temp_lo = temp & mask, temp_hi = temp >> 32; + uint64_t temp_lo = temp & mask; + uint64_t temp_hi = temp >> 32; + return x1y1 + temp_hi + ((temp_lo + x0y1) >> 32); #endif } static inline int64_t libdivide__mullhi_s64(int64_t x, int64_t y) { -#if LIBDIVIDE_VC && LIBDIVIDE_IS_X86_64 +#if defined(LIBDIVIDE_VC) && defined(LIBDIVIDE_IS_X86_64) return __mulh(x, y); -#elif HAS_INT128_T +#elif defined(HAS_INT128_T) __int128_t xl = x, yl = y; __int128_t rl = xl * yl; return (int64_t)(rl >> 64); #else // full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) - const uint32_t mask = 0xFFFFFFFF; - const uint32_t x0 = (uint32_t)(x & mask), y0 = (uint32_t)(y & mask); - const int32_t x1 = (int32_t)(x >> 32), y1 = (int32_t)(y >> 32); - const uint32_t x0y0_hi = libdivide__mullhi_u32(x0, y0); - const int64_t t = x1*(int64_t)y0 + x0y0_hi; - const int64_t w1 = x0*(int64_t)y1 + (t & mask); - return x1*(int64_t)y1 + (t >> 32) + (w1 >> 32); + uint32_t mask = 0xFFFFFFFF; + uint32_t x0 = (uint32_t)(x & mask); + uint32_t y0 = (uint32_t)(y & mask); + int32_t x1 = (int32_t)(x >> 32); + int32_t y1 = (int32_t)(y >> 32); + uint32_t x0y0_hi = libdivide__mullhi_u32(x0, y0); + int64_t t = x1 * (int64_t)y0 + x0y0_hi; + int64_t w1 = x0 * (int64_t)y1 + (t & mask); + + return x1 * (int64_t)y1 + (t >> 32) + (w1 >> 32); #endif } -#if LIBDIVIDE_USE_SSE2 +#if defined(LIBDIVIDE_USE_SSE2) static inline __m128i libdivide__u64_to_m128(uint64_t x) { -#if LIBDIVIDE_VC && ! _WIN64 +#if defined(LIBDIVIDE_VC) && !defined(_WIN64) // 64 bit windows doesn't seem to have an implementation of any of these // load intrinsics, and 32 bit Visual C++ crashes _declspec(align(16)) uint64_t temp[2] = {x, x}; return _mm_load_si128((const __m128i*)temp); -#elif defined(__ICC) - uint64_t __attribute__((aligned(16))) temp[2] = {x,x}; - return _mm_load_si128((const __m128i*)temp); -#elif __clang__ && (2 > __clang_major__ || (2 == __clang_major__ && 7 > __clang_minor__)) - // clang does not provide this intrinsic either - return (__m128i){x, x}; #else // everyone else gets it right return _mm_set1_epi64x(x); @@ -360,14 +356,16 @@ static inline __m128i libdivide__u64_to_m128(uint64_t x) { static inline __m128i libdivide_get_FFFFFFFF00000000(void) { // returns the same as _mm_set1_epi64(0xFFFFFFFF00000000ULL) // without touching memory. - __m128i result = _mm_set1_epi8(-1); // optimizes to pcmpeqd on OS X + // optimizes to pcmpeqd on OS X + __m128i result = _mm_set1_epi8(-1); return _mm_slli_epi64(result, 32); } static inline __m128i libdivide_get_00000000FFFFFFFF(void) { // returns the same as _mm_set1_epi64(0x00000000FFFFFFFFULL) // without touching memory. - __m128i result = _mm_set1_epi8(-1); // optimizes to pcmpeqd on OS X + // optimizes to pcmpeqd on OS X + __m128i result = _mm_set1_epi8(-1); result = _mm_srli_epi64(result, 32); return result; } @@ -390,7 +388,7 @@ static inline __m128i libdivide_u32_to_m128i(uint32_t amt) { } static inline __m128i libdivide_s64_shift_right_vector(__m128i v, int amt) { - // implementation of _mm_sra_epi64. Here we have two 64 bit values which + // implementation of _mm_sra_epi64. Here we have two 64 bit values which // are shifted right to logically become (64 - amt) values, and are then // sign extended from a (64 - amt) bit number. const int b = 64 - amt; @@ -400,32 +398,35 @@ static inline __m128i libdivide_s64_shift_right_vector(__m128i v, int amt) { return result; } -// Here, b is assumed to contain one 32 bit value repeated four times. If it -// did not, the function would not work. +// Here, b is assumed to contain one 32 bit value repeated four times. +// If it did not, the function would not work. static inline __m128i libdivide__mullhi_u32_flat_vector(__m128i a, __m128i b) { __m128i hi_product_0Z2Z = _mm_srli_epi64(_mm_mul_epu32(a, b), 32); __m128i a1X3X = _mm_srli_epi64(a, 32); - __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epu32(a1X3X, b), libdivide_get_FFFFFFFF00000000()); + __m128i mask = libdivide_get_FFFFFFFF00000000(); + __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epu32(a1X3X, b), mask); return _mm_or_si128(hi_product_0Z2Z, hi_product_Z1Z3); // = hi_product_0123 } // Here, y is assumed to contain one 64 bit value repeated twice. static inline __m128i libdivide_mullhi_u64_flat_vector(__m128i x, __m128i y) { // full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) - const __m128i mask = libdivide_get_00000000FFFFFFFF(); + __m128i mask = libdivide_get_00000000FFFFFFFF(); // x0 is low half of 2 64 bit values, x1 is high half in low slots - const __m128i x0 = _mm_and_si128(x, mask), x1 = _mm_srli_epi64(x, 32); - const __m128i y0 = _mm_and_si128(y, mask), y1 = _mm_srli_epi64(y, 32); + __m128i x0 = _mm_and_si128(x, mask); + __m128i x1 = _mm_srli_epi64(x, 32); + __m128i y0 = _mm_and_si128(y, mask); + __m128i y1 = _mm_srli_epi64(y, 32); // x0 happens to have the low half of the two 64 bit values in 32 bit slots // 0 and 2, so _mm_mul_epu32 computes their full product, and then we shift // right by 32 to get just the high values - const __m128i x0y0_hi = _mm_srli_epi64(_mm_mul_epu32(x0, y0), 32); - const __m128i x0y1 = _mm_mul_epu32(x0, y1); - const __m128i x1y0 = _mm_mul_epu32(x1, y0); - const __m128i x1y1 = _mm_mul_epu32(x1, y1); - - const __m128i temp = _mm_add_epi64(x1y0, x0y0_hi); - __m128i temp_lo = _mm_and_si128(temp, mask), temp_hi = _mm_srli_epi64(temp, 32); + __m128i x0y0_hi = _mm_srli_epi64(_mm_mul_epu32(x0, y0), 32); + __m128i x0y1 = _mm_mul_epu32(x0, y1); + __m128i x1y0 = _mm_mul_epu32(x1, y0); + __m128i x1y1 = _mm_mul_epu32(x1, y1); + __m128i temp = _mm_add_epi64(x1y0, x0y0_hi); + __m128i temp_lo = _mm_and_si128(temp, mask); + __m128i temp_hi = _mm_srli_epi64(temp, 32); temp_lo = _mm_srli_epi64(_mm_add_epi64(temp_lo, x0y1), 32); temp_hi = _mm_add_epi64(x1y1, temp_hi); @@ -448,7 +449,8 @@ static inline __m128i libdivide_mullhi_s64_flat_vector(__m128i x, __m128i y) { static inline __m128i libdivide_mullhi_s32_flat_vector(__m128i a, __m128i b) { __m128i hi_product_0Z2Z = _mm_srli_epi64(_mm_mul_epi32(a, b), 32); __m128i a1X3X = _mm_srli_epi64(a, 32); - __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epi32(a1X3X, b), libdivide_get_FFFFFFFF00000000()); + __m128i mask = libdivide_get_FFFFFFFF00000000(); + __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epi32(a1X3X, b), mask); return _mm_or_si128(hi_product_0Z2Z, hi_product_Z1Z3); // = hi_product_0123 } @@ -465,14 +467,16 @@ static inline __m128i libdivide_mullhi_s32_flat_vector(__m128i a, __m128i b) { p = _mm_sub_epi32(p, t2); return p; } -#endif -#endif + +#endif // LIBDIVIDE_USE_SSE4_1 + +#endif // LIBDIVIDE_USE_SSE2 static inline int32_t libdivide__count_leading_zeros32(uint32_t val) { -#if __GNUC__ || __has_builtin(__builtin_clz) +#if defined(__GNUC__) || __has_builtin(__builtin_clz) // Fast way to count leading zeros return __builtin_clz(val); -#elif LIBDIVIDE_VC +#elif defined(LIBDIVIDE_VC) unsigned long result; if (_BitScanReverse(&result, val)) { return 31 - result; @@ -491,10 +495,10 @@ static inline int32_t libdivide__count_leading_zeros32(uint32_t val) { } static inline int32_t libdivide__count_leading_zeros64(uint64_t val) { -#if __GNUC__ || __has_builtin(__builtin_clzll) +#if defined(__GNUC__) || __has_builtin(__builtin_clzll) // Fast way to count leading zeros return __builtin_clzll(val); -#elif LIBDIVIDE_VC && _WIN64 +#elif defined(LIBDIVIDE_VC) && defined(_WIN64) unsigned long result; if (_BitScanReverse64(&result, val)) { return 63 - result; @@ -508,10 +512,12 @@ static inline int32_t libdivide__count_leading_zeros64(uint64_t val) { #endif } +#if (defined(LIBDIVIDE_IS_i386) || defined(LIBDIVIDE_IS_X86_64)) && \ + defined(LIBDIVIDE_GCC_STYLE_ASM) + // libdivide_64_div_32_to_32: divides a 64 bit uint {u1, u0} by a 32 bit // uint {v}. The result must fit in 32 bits. // Returns the quotient directly and the remainder in *r -#if (LIBDIVIDE_IS_i386 || LIBDIVIDE_IS_X86_64) && LIBDIVIDE_GCC_STYLE_ASM static uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { uint32_t result; __asm__("divl %[v]" @@ -520,16 +526,21 @@ static uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, ); return result; } + #else + static uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { uint64_t n = (((uint64_t)u1) << 32) | u0; uint32_t result = (uint32_t)(n / v); *r = (uint32_t)(n - result * (uint64_t)v); return result; } + #endif -#if LIBDIVIDE_IS_X86_64 && LIBDIVIDE_GCC_STYLE_ASM +#if defined(LIBDIVIDE_IS_X86_64) && \ + defined(LIBDIVIDE_GCC_STYLE_ASM) + static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, uint64_t *r) { // u0 -> rax // u1 -> rdx @@ -541,6 +552,7 @@ static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, ); return result; } + #else // Code taken from Hacker's Delight: @@ -549,62 +561,75 @@ static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, // http://www.hackersdelight.org/permissions.htm static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, uint64_t *r) { - const uint64_t b = (1ULL << 32); // Number base (16 bits). - uint64_t un1, un0, // Norm. dividend LSD's. - vn1, vn0, // Norm. divisor digits. - q1, q0, // Quotient digits. - un64, un21, un10, // Dividend digit pairs. - rhat; // A remainder. - int s; // Shift amount for norm. + const uint64_t b = (1ULL << 32); // Number base (16 bits) + uint64_t un1, un0; // Norm. dividend LSD's + uint64_t vn1, vn0; // Norm. divisor digits + uint64_t q1, q0; // Quotient digits + uint64_t un64, un21, un10; // Dividend digit pairs + uint64_t rhat; // A remainder + int s; // Shift amount for norm - if (EDUKE32_PREDICT_FALSE(u1 >= v)) { // If overflow, set rem. - if (r != NULL) // to an impossible value, - *r = (uint64_t) -1; // and return the largest - return (uint64_t) -1; // possible quotient. + // If overflow, set rem. to an impossible value, + // and return the largest possible quotient + if (u1 >= v) { + if (r != NULL) + *r = (uint64_t) -1; + return (uint64_t) -1; } // count leading zeros - s = libdivide__count_leading_zeros64(v); // 0 <= s <= 63. - if (EDUKE32_PREDICT_TRUE(s > 0)) { - v = v << s; // Normalize divisor. + s = libdivide__count_leading_zeros64(v); + if (s > 0) { + // Normalize divisor + v = v << s; un64 = (u1 << s) | ((u0 >> (64 - s)) & (-s >> 31)); - un10 = u0 << s; // Shift dividend left. + un10 = u0 << s; // Shift dividend left } else { - // Avoid undefined behavior. + // Avoid undefined behavior un64 = u1 | u0; un10 = u0; } - vn1 = v >> 32; // Break divisor up into - vn0 = v & 0xFFFFFFFF; // two 32-bit digits. + // Break divisor up into two 32-bit digits + vn1 = v >> 32; + vn0 = v & 0xFFFFFFFF; - un1 = un10 >> 32; // Break right half of - un0 = un10 & 0xFFFFFFFF; // dividend into two digits. + // Break right half of dividend into two digits + un1 = un10 >> 32; + un0 = un10 & 0xFFFFFFFF; - q1 = un64/vn1; // Compute the first - rhat = un64 - q1*vn1; // quotient digit, q1. -again1: - if (q1 >= b || q1*vn0 > b*rhat + un1) { + // Compute the first quotient digit, q1 + q1 = un64 / vn1; + rhat = un64 - q1 * vn1; + + while (q1 >= b || q1 * vn0 > b * rhat + un1) { q1 = q1 - 1; rhat = rhat + vn1; - if (rhat < b) goto again1; + if (rhat >= b) + break; } - un21 = un64*b + un1 - q1*v; // Multiply and subtract. + // Multiply and subtract + un21 = un64 * b + un1 - q1 * v; - q0 = un21/vn1; // Compute the second - rhat = un21 - q0*vn1; // quotient digit, q0. -again2: - if (q0 >= b || q0*vn0 > b*rhat + un0) { + // Compute the second quotient digit + q0 = un21 / vn1; + rhat = un21 - q0 * vn1; + + while (q0 >= b || q0 * vn0 > b * rhat + un0) { q0 = q0 - 1; rhat = rhat + vn1; - if (rhat < b) goto again2; + if (rhat >= b) + break; } - if (r != NULL) // If remainder is wanted, - *r = (un21*b + un0 - q0*v) >> s; // return it. - return q1*b + q0; + // If remainder is wanted, return it + if (r != NULL) + *r = (un21 * b + un0 - q0 * v) >> s; + + return q1 * b + q0; } + #endif // Bitshift a u128 in place, left (signed_shift > 0) or right (signed_shift < 0) @@ -625,21 +650,23 @@ static inline void libdivide_u128_shift(uint64_t *u1, uint64_t *u0, int32_t sign // Computes a 128 / 128 -> 64 bit division, with a 128 bit remainder. static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64_t v_hi, uint64_t v_lo, uint64_t *r_hi, uint64_t *r_lo) { -#if HAS_INT128_T +#if defined(HAS_INT128_T) __uint128_t ufull = u_hi; - ufull = (ufull << 64) | u_lo; __uint128_t vfull = v_hi; + ufull = (ufull << 64) | u_lo; vfull = (vfull << 64) | v_lo; - __uint128_t remainder = ufull % vfull; + uint64_t res = (uint64_t)(ufull / vfull); + __uint128_t remainder = ufull - (vfull * res); *r_lo = (uint64_t)remainder; *r_hi = (uint64_t)(remainder >> 64); - return (uint64_t)(ufull / vfull); + return res; #else // Adapted from "Unsigned Doubleword Division" in Hacker's Delight // We want to compute u / v typedef struct { uint64_t hi; uint64_t lo; } u128_t; u128_t u = {u_hi, u_lo}; u128_t v = {v_hi, v_lo}; + if (v.hi == 0) { // divisor v is a 64 bit value, so we just need one 128/64 division // Note that we are simpler than Hacker's Delight here, because we know @@ -680,8 +707,9 @@ static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64 // Now q0 is correct. // Compute q0 * v as q0v - // = (q0.hi<<64 + q0.lo) * (v.hi<<64 + v.lo) - // = (q0.hi*v.hi<<128 + q0.hi*v.lo<<64 + q0.lo*v.hi<<64 + q0.lo*v.lo) + // = (q0.hi << 64 + q0.lo) * (v.hi << 64 + v.lo) + // = (q0.hi * v.hi << 128) + (q0.hi * v.lo << 64) + + // (q0.lo * v.hi << 64) + q0.lo * v.lo) // Each term is 128 bit // High half of full product (upper 128 bits!) are dropped u128_t q0v = {0, 0}; @@ -696,7 +724,8 @@ static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64 // Check if u_q0v >= v // This checks if our remainder is larger than the divisor - if ((u_q0v.hi > v.hi) || (u_q0v.hi == v.hi && u_q0v.lo >= v.lo)) { + if ((u_q0v.hi > v.hi) || + (u_q0v.hi == v.hi && u_q0v.lo >= v.lo)) { // Increment q0 q0.lo += 1; q0.hi += (q0.lo == 0); // carry @@ -717,20 +746,21 @@ static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64 ////////// UINT32 static inline struct libdivide_u32_t libdivide_internal_u32_gen(uint32_t d, int branchfree) { - // 1 is not supported with branchfree algorithm - LIBDIVIDE_ASSERT(!branchfree || d != 1); + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } struct libdivide_u32_t result; - const uint32_t floor_log_2_d = 31 - libdivide__count_leading_zeros32(d); - if (EDUKE32_PREDICT_FALSE((d & (d - 1)) == 0)) { + uint32_t floor_log_2_d = 31 - libdivide__count_leading_zeros32(d); + if ((d & (d - 1)) == 0) { // Power of 2 if (! branchfree) { result.magic = 0; result.more = floor_log_2_d | LIBDIVIDE_U32_SHIFT_PATH; } else { // We want a magic number of 2**32 and a shift of floor_log_2_d - // but one of the shifts is taken up by LIBDIVIDE_ADD_MARKER, so we - // subtract 1 from the shift + // but one of the shifts is taken up by LIBDIVIDE_ADD_MARKER, + // so we subtract 1 from the shift result.magic = 0; result.more = (floor_log_2_d-1) | LIBDIVIDE_ADD_MARKER; } @@ -773,6 +803,9 @@ struct libdivide_u32_t libdivide_u32_gen(uint32_t d) { } struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } struct libdivide_u32_t tmp = libdivide_internal_u32_gen(d, 1); struct libdivide_u32_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_32_SHIFT_MASK)}; return ret; @@ -780,7 +813,7 @@ struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) { uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) { uint8_t more = denom->more; - if (EDUKE32_PREDICT_FALSE(more & LIBDIVIDE_U32_SHIFT_PATH)) { + if (more & LIBDIVIDE_U32_SHIFT_PATH) { return numer >> (more & LIBDIVIDE_32_SHIFT_MASK); } else { @@ -790,7 +823,8 @@ uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) { return t >> (more & LIBDIVIDE_32_SHIFT_MASK); } else { - return q >> more; // all upper bits are 0 - don't need to mask them off + // all upper bits are 0 - don't need to mask them off + return q >> more; } } } @@ -800,7 +834,7 @@ uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) { uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; if (more & LIBDIVIDE_U32_SHIFT_PATH) { return 1U << shift; - } else if (! (more & LIBDIVIDE_ADD_MARKER)) { + } else if (!(more & LIBDIVIDE_ADD_MARKER)) { // We compute q = n/d = n*m / 2^(32 + shift) // Therefore we have d = 2^(32 + shift) / m // We need to ceil it. @@ -815,7 +849,6 @@ uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) { // Also note that shift may be as high as 31, so shift + 1 will // overflow. So we have to compute it as 2^(32+shift)/(m+2^32), and // then double the quotient and remainder. - // TODO: do something better than 64 bit math uint64_t half_n = 1ULL << (32 + shift); uint64_t d = (1ULL << 32) | denom->magic; // Note that the quotient is guaranteed <= 32 bits, but the remainder @@ -842,7 +875,7 @@ uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_ int libdivide_u32_get_algorithm(const struct libdivide_u32_t *denom) { uint8_t more = denom->more; if (more & LIBDIVIDE_U32_SHIFT_PATH) return 0; - else if (! (more & LIBDIVIDE_ADD_MARKER)) return 1; + else if (!(more & LIBDIVIDE_ADD_MARKER)) return 1; else return 2; } @@ -864,14 +897,15 @@ uint32_t libdivide_u32_do_alg2(uint32_t numer, const struct libdivide_u32_t *den return t >> (denom->more & LIBDIVIDE_32_SHIFT_MASK); } +// same as algo 2 uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom) { - // same as alg 2 uint32_t q = libdivide__mullhi_u32(denom->magic, numer); uint32_t t = ((numer - q) >> 1) + q; return t >> denom->more; } -#if LIBDIVIDE_USE_SSE2 +#if defined(LIBDIVIDE_USE_SSE2) + __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom) { uint8_t more = denom->more; if (more & LIBDIVIDE_U32_SHIFT_PATH) { @@ -908,8 +942,8 @@ __m128i libdivide_u32_do_vector_alg2(__m128i numers, const struct libdivide_u32_ return _mm_srl_epi32(t, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_32_SHIFT_MASK)); } -LIBDIVIDE_API __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t * denom) { - // same as alg 2 +// same as algo 2 +LIBDIVIDE_API __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom) { __m128i q = libdivide__mullhi_u32_flat_vector(numers, _mm_set1_epi32(denom->magic)); __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); return _mm_srl_epi32(t, libdivide_u32_to_m128i(denom->more)); @@ -920,27 +954,29 @@ LIBDIVIDE_API __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const s /////////// UINT64 static inline struct libdivide_u64_t libdivide_internal_u64_gen(uint64_t d, int branchfree) { - // 1 is not supported with branchfree algorithm - LIBDIVIDE_ASSERT(!branchfree || d != 1); + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } struct libdivide_u64_t result; - const uint32_t floor_log_2_d = 63 - libdivide__count_leading_zeros64(d); - if (EDUKE32_PREDICT_FALSE((d & (d - 1)) == 0)) { + uint32_t floor_log_2_d = 63 - libdivide__count_leading_zeros64(d); + if ((d & (d - 1)) == 0) { // Power of 2 if (! branchfree) { result.magic = 0; result.more = floor_log_2_d | LIBDIVIDE_U64_SHIFT_PATH; } else { // We want a magic number of 2**64 and a shift of floor_log_2_d - // but one of the shifts is taken up by LIBDIVIDE_ADD_MARKER, so we - // subtract 1 from the shift + // but one of the shifts is taken up by LIBDIVIDE_ADD_MARKER, + // so we subtract 1 from the shift result.magic = 0; result.more = (floor_log_2_d-1) | LIBDIVIDE_ADD_MARKER; } } else { uint64_t proposed_m, rem; uint8_t more; - proposed_m = libdivide_128_div_64_to_64(1ULL << floor_log_2_d, 0, d, &rem); // == (1 << (64 + floor_log_2_d)) / d + // (1 << (64 + floor_log_2_d)) / d + proposed_m = libdivide_128_div_64_to_64(1ULL << floor_log_2_d, 0, d, &rem); LIBDIVIDE_ASSERT(rem > 0 && rem < d); const uint64_t e = d - rem; @@ -972,13 +1008,14 @@ static inline struct libdivide_u64_t libdivide_internal_u64_gen(uint64_t d, int return result; } -struct libdivide_u64_t libdivide_u64_gen(uint64_t d) -{ +struct libdivide_u64_t libdivide_u64_gen(uint64_t d) { return libdivide_internal_u64_gen(d, 0); } -struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) -{ +struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } struct libdivide_u64_t tmp = libdivide_internal_u64_gen(d, 1); struct libdivide_u64_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_64_SHIFT_MASK)}; return ret; @@ -986,7 +1023,7 @@ struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) { uint8_t more = denom->more; - if (EDUKE32_PREDICT_FALSE(more & LIBDIVIDE_U64_SHIFT_PATH)) { + if (more & LIBDIVIDE_U64_SHIFT_PATH) { return numer >> (more & LIBDIVIDE_64_SHIFT_MASK); } else { @@ -996,7 +1033,8 @@ uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) { return t >> (more & LIBDIVIDE_64_SHIFT_MASK); } else { - return q >> more; // all upper bits are 0 - don't need to mask them off + // all upper bits are 0 - don't need to mask them off + return q >> more; } } } @@ -1006,7 +1044,7 @@ uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) { uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; if (more & LIBDIVIDE_U64_SHIFT_PATH) { return 1ULL << shift; - } else if (! (more & LIBDIVIDE_ADD_MARKER)) { + } else if (!(more & LIBDIVIDE_ADD_MARKER)) { // We compute q = n/d = n*m / 2^(64 + shift) // Therefore we have d = 2^(64 + shift) / m // We need to ceil it. @@ -1059,7 +1097,7 @@ uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_ int libdivide_u64_get_algorithm(const struct libdivide_u64_t *denom) { uint8_t more = denom->more; if (more & LIBDIVIDE_U64_SHIFT_PATH) return 0; - else if (! (more & LIBDIVIDE_ADD_MARKER)) return 1; + else if (!(more & LIBDIVIDE_ADD_MARKER)) return 1; else return 2; } @@ -1078,15 +1116,16 @@ uint64_t libdivide_u64_do_alg2(uint64_t numer, const struct libdivide_u64_t *den return t >> (denom->more & LIBDIVIDE_64_SHIFT_MASK); } +// same as alg 2 uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom) { - // same as alg 2 uint64_t q = libdivide__mullhi_u64(denom->magic, numer); uint64_t t = ((numer - q) >> 1) + q; return t >> denom->more; } -#if LIBDIVIDE_USE_SSE2 -__m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t * denom) { +#if defined(LIBDIVIDE_USE_SSE2) + +__m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom) { uint8_t more = denom->more; if (more & LIBDIVIDE_U64_SHIFT_PATH) { return _mm_srl_epi64(numers, libdivide_u32_to_m128i(more & LIBDIVIDE_64_SHIFT_MASK)); @@ -1121,7 +1160,7 @@ __m128i libdivide_u64_do_vector_alg2(__m128i numers, const struct libdivide_u64_ return _mm_srl_epi64(t, libdivide_u32_to_m128i(denom->more & LIBDIVIDE_64_SHIFT_MASK)); } -__m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t * denom) { +__m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom) { __m128i q = libdivide_mullhi_u64_flat_vector(numers, libdivide__u64_to_m128(denom->magic)); __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); return _mm_srl_epi64(t, libdivide_u32_to_m128i(denom->more)); @@ -1134,12 +1173,14 @@ __m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivid static inline int32_t libdivide__mullhi_s32(int32_t x, int32_t y) { int64_t xl = x, yl = y; int64_t rl = xl * yl; - return (int32_t)(rl >> 32); // needs to be arithmetic shift + // needs to be arithmetic shift + return (int32_t)(rl >> 32); } static inline struct libdivide_s32_t libdivide_internal_s32_gen(int32_t d, int branchfree) { - // branchfree cannot support or -1 - LIBDIVIDE_ASSERT(!branchfree || (d != 1 && d != -1)); + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } struct libdivide_s32_t result; @@ -1150,11 +1191,11 @@ static inline struct libdivide_s32_t libdivide_internal_s32_gen(int32_t d, int b // INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set // and is a power of 2. uint32_t ud = (uint32_t)d; - uint32_t absD = (d < 0 ? -ud : ud); // gcc optimizes this to the fast abs trick - const uint32_t floor_log_2_d = 31 - libdivide__count_leading_zeros32(absD); + uint32_t absD = (d < 0) ? -ud : ud; + uint32_t floor_log_2_d = 31 - libdivide__count_leading_zeros32(absD); // check if exactly one bit is set, // don't care if absD is 0 since that's divide by zero - if (EDUKE32_PREDICT_FALSE((absD & (absD - 1)) == 0)) { + if ((absD & (absD - 1)) == 0) { // Branchfree and normal paths are exactly the same result.magic = 0; result.more = floor_log_2_d | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0) | LIBDIVIDE_S32_SHIFT_PATH; @@ -1190,7 +1231,7 @@ static inline struct libdivide_s32_t libdivide_internal_s32_gen(int32_t d, int b // branchfull case. if (d < 0) { more |= LIBDIVIDE_NEGATIVE_DIVISOR; - if (! branchfree) { + if (!branchfree) { magic = -magic; } } @@ -1206,6 +1247,12 @@ LIBDIVIDE_API struct libdivide_s32_t libdivide_s32_gen(int32_t d) { } LIBDIVIDE_API struct libdivide_s32_branchfree_t libdivide_s32_branchfree_gen(int32_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } + if (d == -1) { + LIBDIVIDE_ERROR("branchfree divider must be != -1"); + } struct libdivide_s32_t tmp = libdivide_internal_s32_gen(d, 1); struct libdivide_s32_branchfree_t result = {tmp.magic, tmp.more}; return result; @@ -1227,7 +1274,7 @@ int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) { // must be arithmetic shift and then sign extend int32_t sign = (int8_t)more >> 7; // q += (more < 0 ? -numer : numer), casts to avoid UB - uq += (((uint32_t)numer ^ sign) - sign); + uq += ((uint32_t)numer ^ sign) - sign; } int32_t q = (int32_t)uq; q >>= more & LIBDIVIDE_32_SHIFT_MASK; @@ -1241,7 +1288,6 @@ int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_br uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; // must be arithmetic shift and then sign extend int32_t sign = (int8_t)more >> 7; - int32_t magic = denom->magic; int32_t q = libdivide__mullhi_s32(magic, numer); q += numer; @@ -1257,7 +1303,7 @@ int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_br q >>= shift; // Negate if needed - q = ((q ^ sign) - sign); + q = (q ^ sign) - sign; return q; } @@ -1280,7 +1326,8 @@ int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) { // the magic number's sign is opposite that of the divisor. // We want to compute the positive magic number. int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); - int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) ? denom->magic > 0 : denom->magic < 0; + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) + ? denom->magic > 0 : denom->magic < 0; // Handle the power of 2 case (including branchfree) if (denom->magic == 0) { @@ -1289,7 +1336,7 @@ int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) { } uint32_t d = (uint32_t)(magic_was_negated ? -denom->magic : denom->magic); - uint64_t n = 1ULL << (32 + shift); // Note that the shift cannot exceed 30 + uint64_t n = 1ULL << (32 + shift); // this shift cannot exceed 30 uint32_t q = (uint32_t)(n / d); int32_t result = (int32_t)q; result += 1; @@ -1303,7 +1350,7 @@ int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t int libdivide_s32_get_algorithm(const struct libdivide_s32_t *denom) { uint8_t more = denom->more; - int positiveDivisor = ! (more & LIBDIVIDE_NEGATIVE_DIVISOR); + int positiveDivisor = !(more & LIBDIVIDE_NEGATIVE_DIVISOR); if (more & LIBDIVIDE_S32_SHIFT_PATH) return (positiveDivisor ? 0 : 1); else if (more & LIBDIVIDE_ADD_MARKER) return (positiveDivisor ? 2 : 3); else return 4; @@ -1344,8 +1391,9 @@ int32_t libdivide_s32_do_alg4(int32_t numer, const struct libdivide_s32_t *denom return q; } -#if LIBDIVIDE_USE_SSE2 -__m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t * denom) { +#if defined(LIBDIVIDE_USE_SSE2) + +__m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom) { uint8_t more = denom->more; if (more & LIBDIVIDE_S32_SHIFT_PATH) { uint32_t shifter = more & LIBDIVIDE_32_SHIFT_MASK; @@ -1362,7 +1410,7 @@ __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t * d __m128i sign = _mm_set1_epi32((int32_t)(int8_t)more >> 7); // must be arithmetic shift q = _mm_add_epi32(q, _mm_sub_epi32(_mm_xor_si128(numers, sign), sign)); // q += ((numer ^ sign) - sign); } - q = _mm_sra_epi32(q, libdivide_u32_to_m128i(more & LIBDIVIDE_32_SHIFT_MASK)); //q >>= shift + q = _mm_sra_epi32(q, libdivide_u32_to_m128i(more & LIBDIVIDE_32_SHIFT_MASK)); // q >>= shift q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); // q += (q < 0) return q; } @@ -1406,11 +1454,12 @@ __m128i libdivide_s32_do_vector_alg4(__m128i numers, const struct libdivide_s32_ return q; } -__m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t * denom) { +__m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom) { int32_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; - __m128i sign = _mm_set1_epi32((int32_t)(int8_t)more >> 7); // must be arithmetic shift + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int32_t)(int8_t)more >> 7); // libdivide__mullhi_s32(numers, magic); __m128i q = libdivide_mullhi_s32_flat_vector(numers, _mm_set1_epi32(magic)); @@ -1421,17 +1470,22 @@ __m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivid // 2, or (2**shift) if it is not a power of 2 uint32_t is_power_of_2 = (magic == 0); __m128i q_sign = _mm_srai_epi32(q, 31); // q_sign = q >> 31 - q = _mm_add_epi32(q, _mm_and_si128(q_sign, _mm_set1_epi32((1 << shift) - is_power_of_2))); // q = q + (q_sign & ((1 << shift) - is_power_of_2) + __m128i mask = _mm_set1_epi32((1 << shift) - is_power_of_2); + q = _mm_add_epi32(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) q = _mm_srai_epi32(q, shift); //q >>= shift q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign return q; } + #endif ///////////// SINT64 static inline struct libdivide_s64_t libdivide_internal_s64_gen(int64_t d, int branchfree) { - LIBDIVIDE_ASSERT(!branchfree || (d != 1 && d != -1)); + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } + struct libdivide_s64_t result; // If d is a power of 2, or negative a power of 2, we have to use a shift. @@ -1440,12 +1494,12 @@ static inline struct libdivide_s64_t libdivide_internal_s64_gen(int64_t d, int b // whether its absolute value has exactly one bit set. This works even for // INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set // and is a power of 2. - const uint64_t ud = (uint64_t)d; - const uint64_t absD = (d < 0 ? -ud : ud); // gcc optimizes this to the fast abs trick - const uint32_t floor_log_2_d = 63 - libdivide__count_leading_zeros64(absD); + uint64_t ud = (uint64_t)d; + uint64_t absD = (d < 0) ? -ud : ud; + uint32_t floor_log_2_d = 63 - libdivide__count_leading_zeros64(absD); // check if exactly one bit is set, // don't care if absD is 0 since that's divide by zero - if (EDUKE32_PREDICT_FALSE((absD & (absD - 1)) == 0)) { + if ((absD & (absD - 1)) == 0) { // Branchfree and non-branchfree cases are the same result.magic = 0; result.more = floor_log_2_d | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); @@ -1481,7 +1535,7 @@ static inline struct libdivide_s64_t libdivide_internal_s64_gen(int64_t d, int b // Mark if we are negative if (d < 0) { more |= LIBDIVIDE_NEGATIVE_DIVISOR; - if (! branchfree) { + if (!branchfree) { magic = -magic; } } @@ -1497,6 +1551,12 @@ struct libdivide_s64_t libdivide_s64_gen(int64_t d) { } struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } + if (d == -1) { + LIBDIVIDE_ERROR("branchfree divider must be != -1"); + } struct libdivide_s64_t tmp = libdivide_internal_s64_gen(d, 1); struct libdivide_s64_branchfree_t ret = {tmp.magic, tmp.more}; return ret; @@ -1519,7 +1579,7 @@ int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) { if (more & LIBDIVIDE_ADD_MARKER) { // must be arithmetic shift and then sign extend int64_t sign = (int8_t)more >> 7; - uq += (((uint64_t)numer ^ sign) - sign); + uq += ((uint64_t)numer ^ sign) - sign; } int64_t q = (int64_t)uq; q >>= more & LIBDIVIDE_64_SHIFT_MASK; @@ -1548,7 +1608,7 @@ int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_br q >>= shift; // Negate if needed - q = ((q ^ sign) - sign); + q = (q ^ sign) - sign; return q; } @@ -1564,7 +1624,8 @@ int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) { } else { // Unsigned math is much easier int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); - int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) ? denom->magic > 0 : denom->magic < 0; + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) + ? denom->magic > 0 : denom->magic < 0; uint64_t d = (uint64_t)(magic_was_negated ? -denom->magic : denom->magic); uint64_t n_hi = 1ULL << shift, n_lo = 0; @@ -1584,7 +1645,7 @@ int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t int libdivide_s64_get_algorithm(const struct libdivide_s64_t *denom) { uint8_t more = denom->more; - int positiveDivisor = ! (more & LIBDIVIDE_NEGATIVE_DIVISOR); + int positiveDivisor = !(more & LIBDIVIDE_NEGATIVE_DIVISOR); if (denom->magic == 0) return (positiveDivisor ? 0 : 1); // shift path else if (more & LIBDIVIDE_ADD_MARKER) return (positiveDivisor ? 2 : 3); else return 4; @@ -1626,8 +1687,9 @@ int64_t libdivide_s64_do_alg4(int64_t numer, const struct libdivide_s64_t *denom return q; } -#if LIBDIVIDE_USE_SSE2 -__m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t * denom) { +#if defined(LIBDIVIDE_USE_SSE2) + +__m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom) { uint8_t more = denom->more; int64_t magic = denom->magic; if (magic == 0) { // shift path @@ -1645,7 +1707,8 @@ __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t * d __m128i sign = _mm_set1_epi32((int32_t)((int8_t)more >> 7)); // must be arithmetic shift q = _mm_add_epi64(q, _mm_sub_epi64(_mm_xor_si128(numers, sign), sign)); // q += ((numer ^ sign) - sign); } - q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); // q >>= denom->mult_path.shift + // q >>= denom->mult_path.shift + q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); // q += (q < 0) return q; } @@ -1690,11 +1753,12 @@ __m128i libdivide_s64_do_vector_alg4(__m128i numers, const struct libdivide_s64_ return q; } -__m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t * denom) { +__m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom) { int64_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; - __m128i sign = _mm_set1_epi32((int32_t)(int8_t)more >> 7); // must be arithmetic shift + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int32_t)(int8_t)more >> 7); // libdivide__mullhi_s64(numers, magic); __m128i q = libdivide_mullhi_s64_flat_vector(numers, libdivide__u64_to_m128(magic)); @@ -1705,7 +1769,8 @@ __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivid // 2, or (2**shift) if it is not a power of 2. uint32_t is_power_of_2 = (magic == 0); __m128i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 - q = _mm_add_epi64(q, _mm_and_si128(q_sign, libdivide__u64_to_m128((1ULL << shift) - is_power_of_2))); // q = q + (q_sign & ((1 << shift) - is_power_of_2) + __m128i mask = libdivide__u64_to_m128((1ULL << shift) - is_power_of_2); + q = _mm_add_epi64(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign return q; @@ -1713,8 +1778,6 @@ __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivid #endif -#endif // LIBDIVIDE_HEADER_ONLY - /////////// C++ stuff #ifdef __cplusplus @@ -1734,38 +1797,57 @@ enum { namespace libdivide_internal { -#if LIBDIVIDE_USE_SSE2 +#if defined(LIBDIVIDE_USE_SSE2) #define MAYBE_VECTOR(X) X #define MAYBE_VECTOR_PARAM(X) __m128i vector_func(__m128i, const X *) - #else #define MAYBE_VECTOR(X) 0 -#define MAYBE_VECTOR_PARAM(X) int vector_func +#define MAYBE_VECTOR_PARAM(X) int unused #endif - // Some bogus unswitch functions for unsigned types so the same - // (presumably templated) code can work for both signed and unsigned. - uint32_t crash_u32(uint32_t, const libdivide_u32_t *); - uint64_t crash_u64(uint64_t, const libdivide_u64_t *); -#if LIBDIVIDE_USE_SSE2 - __m128i crash_u32_vector(__m128i, const libdivide_u32_t *); - __m128i crash_u64_vector(__m128i, const libdivide_u64_t *); -#endif -#ifndef LIBDIVIDE_HEADER_ONLY - uint32_t crash_u32(uint32_t, const libdivide_u32_t *) { abort(); return *(uint32_t *)NULL; } - uint64_t crash_u64(uint64_t, const libdivide_u64_t *) { abort(); return *(uint64_t *)NULL; } -#if LIBDIVIDE_USE_SSE2 - __m128i crash_u32_vector(__m128i, const libdivide_u32_t *) { abort(); return *(__m128i *)NULL; } - __m128i crash_u64_vector(__m128i, const libdivide_u64_t *) { abort(); return *(__m128i *)NULL; } -#endif -#endif +// The following convenience macros are used to build a type of the base +// divider class and give it as template arguments the C functions +// related to the macro name and the macro type paramaters. - // Base divider, which provides storage for the actual divider - template // like libdivide_u32_do_vector +#define BRANCHFULL_DIVIDER(INT, TYPE) \ + typedef base + +#define BRANCHFREE_DIVIDER(INT, TYPE) \ + typedef base + +#define ALGORITHM_DIVIDER(INT, TYPE, ALGO) \ + typedef base + +#define CRASH_DIVIDER(INT, TYPE) \ + typedef base + + // Base divider, provides storage for the actual divider. + // @IntType: e.g. uint32_t + // @DenomType: e.g. libdivide_u32_t + // @gen_func(): e.g. libdivide_u32_gen + // @do_func(): e.g. libdivide_u32_do + // @MAYBE_VECTOR_PARAM: e.g. libdivide_u32_do_vector + template struct base { // Storage for the actual divider DenomType denom; @@ -1773,184 +1855,202 @@ namespace libdivide_internal { // Constructor that takes a divisor value, and applies the gen function base(IntType d) : denom(gen_func(d)) { } - // Copy constructor - base(const DenomType & d) : denom(d) { } - // Default constructor to allow uninitialized uses in e.g. arrays base() {} - // Scalar divide - IntType perform_divide(IntType val) const { return do_func(val, &denom); } + // Needed for unswitch + base(const DenomType& d) : denom(d) { } -#if LIBDIVIDE_USE_SSE2 - // Vector divide - __m128i perform_divide_vector(__m128i val) const { return vector_func(val, &denom); } + IntType perform_divide(IntType val) const { + return do_func(val, &denom); + } + +#if defined(LIBDIVIDE_USE_SSE2) + __m128i perform_divide_vector(__m128i val) const { + return vector_func(val, &denom); + } #endif }; - // Type-specific dispatch + // Functions that will never be called but are required to be able + // to use unswitch in C++ template code. Unsigned has fewer algorithms + // than signed i.e. alg3 and alg4 are not defined for unsigned. In + // order to make templates compile we need to define unsigned alg3 and + // alg4 as crash functions. + uint32_t libdivide_u32_crash(uint32_t, const libdivide_u32_t *) { exit(-1); } + uint64_t libdivide_u64_crash(uint64_t, const libdivide_u64_t *) { exit(-1); } - // uint32 - template - struct denom_u32 { - typedef base divider; - }; - template struct algo_u32 { }; - template<> struct algo_u32 { typedef denom_u32::divider divider; }; - template<> struct algo_u32 { typedef denom_u32::divider divider; }; - template<> struct algo_u32 { typedef denom_u32::divider divider; }; - template<> struct algo_u32 { typedef denom_u32::divider divider; }; - template<> struct algo_u32 { typedef base divider; }; +#if defined(LIBDIVIDE_USE_SSE2) + __m128i libdivide_u32_crash_vector(__m128i, const libdivide_u32_t *) { exit(-1); } + __m128i libdivide_u64_crash_vector(__m128i, const libdivide_u64_t *) { exit(-1); } +#endif - // uint64 - template - struct denom_u64 { - typedef base divider; - }; - template struct algo_u64 { }; - template<> struct algo_u64 { typedef denom_u64::divider divider; }; - template<> struct algo_u64 { typedef denom_u64::divider divider; }; - template<> struct algo_u64 { typedef denom_u64::divider divider; }; - template<> struct algo_u64 { typedef denom_u64::divider divider; }; - template<> struct algo_u64 { typedef base divider; }; - - // int32 - template - struct denom_s32 { - typedef base divider; - }; - template struct algo_s32 { }; - template<> struct algo_s32 { typedef denom_s32::divider divider; }; - template<> struct algo_s32 { typedef denom_s32::divider divider; }; - template<> struct algo_s32 { typedef denom_s32::divider divider; }; - template<> struct algo_s32 { typedef denom_s32::divider divider; }; - template<> struct algo_s32 { typedef denom_s32::divider divider; }; - template<> struct algo_s32 { typedef denom_s32::divider divider; }; - template<> struct algo_s32 { typedef base divider; }; - - // int64 - template - struct denom_s64 { - typedef base divider; - }; - template struct algo_s64 { }; - template<> struct algo_s64 { typedef denom_s64::divider divider; }; - template<> struct algo_s64 { typedef denom_s64::divider divider; }; - template<> struct algo_s64 { typedef denom_s64::divider divider; }; - template<> struct algo_s64 { typedef denom_s64::divider divider; }; - template<> struct algo_s64 { typedef denom_s64::divider divider; }; - template<> struct algo_s64 { typedef denom_s64::divider divider; }; - template<> struct algo_s64 { typedef base divider; }; - - // Bogus versions to allow templated code to operate on int and uint uniformly - template<> struct algo_u32 { typedef denom_u32::divider divider; }; - template<> struct algo_u32 { typedef denom_u32::divider divider; }; - template<> struct algo_u64 { typedef denom_u64::divider divider; }; - template<> struct algo_u64 { typedef denom_u64::divider divider; }; + template struct dispatcher { }; // Templated dispatch using partial specialization - template struct dispatcher{}; - template struct dispatcher { typedef struct algo_u32 algo; }; - template struct dispatcher { typedef struct algo_s32 algo; }; - template struct dispatcher { typedef struct algo_u64 algo; }; - template struct dispatcher { typedef struct algo_s64 algo; }; + template<> struct dispatcher { BRANCHFULL_DIVIDER(int32_t, s32) divider; }; + template<> struct dispatcher { BRANCHFREE_DIVIDER(int32_t, s32) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(int32_t, s32, alg0) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(int32_t, s32, alg1) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(int32_t, s32, alg2) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(int32_t, s32, alg3) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(int32_t, s32, alg4) divider; }; - // Overloads that don't depend on the algorithm. - inline uint32_t recover(const libdivide_u32_t *s) { return libdivide_u32_recover(s); } + template<> struct dispatcher { BRANCHFULL_DIVIDER(uint32_t, u32) divider; }; + template<> struct dispatcher { BRANCHFREE_DIVIDER(uint32_t, u32) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(uint32_t, u32, alg0) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(uint32_t, u32, alg1) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(uint32_t, u32, alg2) divider; }; + template<> struct dispatcher { CRASH_DIVIDER(uint32_t, u32) divider; }; + template<> struct dispatcher { CRASH_DIVIDER(uint32_t, u32) divider; }; + + template<> struct dispatcher { BRANCHFULL_DIVIDER(int64_t, s64) divider; }; + template<> struct dispatcher { BRANCHFREE_DIVIDER(int64_t, s64) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER (int64_t, s64, alg0) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER (int64_t, s64, alg1) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER (int64_t, s64, alg2) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER (int64_t, s64, alg3) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER (int64_t, s64, alg4) divider; }; + + template<> struct dispatcher { BRANCHFULL_DIVIDER(uint64_t, u64) divider; }; + template<> struct dispatcher { BRANCHFREE_DIVIDER(uint64_t, u64) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(uint64_t, u64, alg0) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(uint64_t, u64, alg1) divider; }; + template<> struct dispatcher { ALGORITHM_DIVIDER(uint64_t, u64, alg2) divider; }; + template<> struct dispatcher { CRASH_DIVIDER(uint64_t, u64) divider; }; + template<> struct dispatcher { CRASH_DIVIDER(uint64_t, u64) divider; }; + + // Overloads that don't depend on the algorithm inline int32_t recover(const libdivide_s32_t *s) { return libdivide_s32_recover(s); } - inline uint64_t recover(const libdivide_u64_t *s) { return libdivide_u64_recover(s); } + inline uint32_t recover(const libdivide_u32_t *s) { return libdivide_u32_recover(s); } inline int64_t recover(const libdivide_s64_t *s) { return libdivide_s64_recover(s); } + inline uint64_t recover(const libdivide_u64_t *s) { return libdivide_u64_recover(s); } - inline uint32_t recover(const libdivide_u32_branchfree_t *s) { return libdivide_u32_branchfree_recover(s); } inline int32_t recover(const libdivide_s32_branchfree_t *s) { return libdivide_s32_branchfree_recover(s); } - inline uint64_t recover(const libdivide_u64_branchfree_t *s) { return libdivide_u64_branchfree_recover(s); } + inline uint32_t recover(const libdivide_u32_branchfree_t *s) { return libdivide_u32_branchfree_recover(s); } inline int64_t recover(const libdivide_s64_branchfree_t *s) { return libdivide_s64_branchfree_recover(s); } + inline uint64_t recover(const libdivide_u64_branchfree_t *s) { return libdivide_u64_branchfree_recover(s); } - inline int get_algorithm(const libdivide_u32_t *s) { return libdivide_u32_get_algorithm(s); } inline int get_algorithm(const libdivide_s32_t *s) { return libdivide_s32_get_algorithm(s); } - inline int get_algorithm(const libdivide_u64_t *s) { return libdivide_u64_get_algorithm(s); } + inline int get_algorithm(const libdivide_u32_t *s) { return libdivide_u32_get_algorithm(s); } inline int get_algorithm(const libdivide_s64_t *s) { return libdivide_s64_get_algorithm(s); } + inline int get_algorithm(const libdivide_u64_t *s) { return libdivide_u64_get_algorithm(s); } // Fallback for branchfree variants, which do not support unswitching template int get_algorithm(const T *) { return -1; } } +// This is the main divider class for use by the user (C++ API). +// The divider itself is stored in the div variable who's +// type is chosen by the dispatcher based on the template paramaters. template class divider { - private: +private: // Here's the actual divider - typedef typename libdivide_internal::dispatcher::algo::divider div_t; - div_t sub; + typedef typename libdivide_internal::dispatcher::divider div_t; + div_t div; // unswitch() friend declaration - template friend divider unswitch(const divider & d); + template + friend divider unswitch(const divider & d); // Constructor used by the unswitch friend - divider(const div_t &denom) : sub(denom) { } + divider(const div_t& denom) : div(denom) { } - public: - - // Ordinary constructor, that takes the divisor as a parameter. - divider(T n) : sub(n) { } +public: + // Ordinary constructor that takes the divisor as a parameter + divider(T n) : div(n) { } // Default constructor. We leave this deliberately undefined so that - // creating an array of divider and then initializing them doesn't slow us - // down. + // creating an array of divider and then initializing them + // doesn't slow us down. divider() { } // Divides the parameter by the divisor, returning the quotient - T perform_divide(T val) const { return sub.perform_divide(val); } + T perform_divide(T val) const { + return div.perform_divide(val); + } // Recovers the divisor that was used to initialize the divider - T recover_divisor() const { return libdivide_internal::recover(&sub.denom); } + T recover_divisor() const { + return libdivide_internal::recover(&div.denom); + } -#if LIBDIVIDE_USE_SSE2 +#if defined(LIBDIVIDE_USE_SSE2) // Treats the vector as either two or four packed values (depending on the - // size), and divides each of them by the divisor, returning the packed - // quotients. - __m128i perform_divide_vector(__m128i val) const { return sub.perform_divide_vector(val); } + // size), and divides each of them by the divisor, + // returning the packed quotients. + __m128i perform_divide_vector(__m128i val) const { + return div.perform_divide_vector(val); + } #endif // Returns the index of algorithm, for use in the unswitch function. Does // not apply to branchfree variant. // Returns the algorithm for unswitching. - int get_algorithm() const { return libdivide_internal::get_algorithm(&sub.denom); } + int get_algorithm() const { + return libdivide_internal::get_algorithm(&div.denom); + } - // operator== - bool operator==(const divider & him) const { return sub.denom.magic == him.sub.denom.magic && sub.denom.more == him.sub.denom.more; } - bool operator!=(const divider & him) const { return ! (*this == him); } + bool operator==(const divider& him) const { + return div.denom.magic == him.div.denom.magic && + div.denom.more == him.div.denom.more; + } + + bool operator!=(const divider& him) const { + return !(*this == him); + } }; -// Returns a divider specialized for the given algorithm. -template -divider unswitch(const divider & d) { return divider(d.sub.denom); } +#if __cplusplus >= 201103L || \ + (defined(_MSC_VER) && _MSC_VER >= 1800) -// Overload of the / operator for scalar division. +// libdivdie::branchfree_divider +template +using branchfree_divider = divider; + +#endif + +// Returns a divider specialized for the given algorithm +template +divider unswitch(const divider& d) { + return divider(d.div.denom); +} + +// Overload of the / operator for scalar division template -int_type operator/(int_type numer, const divider & denom) { +int_type operator/(int_type numer, const divider& denom) { return denom.perform_divide(numer); } -#if LIBDIVIDE_USE_SSE2 -// Overload of the / operator for vector division. +// Overload of the /= operator for scalar division template -__m128i operator/(__m128i numer, const divider & denom) { +int_type operator/=(int_type& numer, const divider& denom) { + numer = denom.perform_divide(numer); + return numer; +} + +#if defined(LIBDIVIDE_USE_SSE2) + +// Overload of the / operator for vector division +template +__m128i operator/(__m128i numer, const divider& denom) { return denom.perform_divide_vector(numer); } -#endif + +// Overload of the /= operator for vector division +template +__m128i operator/=(__m128i& numer, const divider& denom) { + numer = denom.perform_divide_vector(numer); + return numer; +} #endif -#if defined __cplusplus && !defined LIBDIVIDE_NONAMESPACE -} // close namespace libdivide -} // close anonymous namespace -#endif +} // namespace libdivide +} // anonymous namespace -#endif +#endif // __cplusplus + +#endif // LIBDIVIDE_H