Ruby: internal/bits.h Source File

Ruby 3.3.4p94 (2024-07-09 revision be1089c8ec5ba40e09b1553e36b3174bf4014d9d)

bits.h
1#ifndef INTERNAL_BITS_H /*-*-C-*-vi:se ft=c:*/
2#define INTERNAL_BITS_H
28#include "ruby/internal/config.h"
29#include <limits.h> /* for CHAR_BITS */
30#include <stdint.h> /* for uintptr_t */
31#include "internal/compilers.h" /* for MSC_VERSION_SINCE */
32
33#if MSC_VERSION_SINCE(1310)
34# include <stdlib.h> /* for _byteswap_uint64 */
35#endif
36
37#if defined(HAVE_X86INTRIN_H)
38# include <x86intrin.h> /* for _lzcnt_u64 */
39#elif MSC_VERSION_SINCE(1310)
40# include <intrin.h> /* for the following intrinsics */
41#endif
42
43#if defined(_MSC_VER) && defined(__AVX__)
44# pragma intrinsic(__popcnt)
45# pragma intrinsic(__popcnt64)
46#endif
47
48#if defined(_MSC_VER) && defined(__AVX2__)
49# pragma intrinsic(__lzcnt)
50# pragma intrinsic(__lzcnt64)
51#endif
52
53#if MSC_VERSION_SINCE(1310)
54# pragma intrinsic(_rotl)
55# pragma intrinsic(_rotr)
56# ifdef _WIN64
57# pragma intrinsic(_rotl64)
58# pragma intrinsic(_rotr64)
59# endif
60#endif
61
62#if MSC_VERSION_SINCE(1400)
63# pragma intrinsic(_BitScanForward)
64# pragma intrinsic(_BitScanReverse)
65# ifdef _WIN64
66# pragma intrinsic(_BitScanForward64)
67# pragma intrinsic(_BitScanReverse64)
68# endif
69#endif
70
71#include "ruby/ruby.h" /* for VALUE */
72#include "internal/static_assert.h" /* for STATIC_ASSERT */
73
74/* The most significant bit of the lower part of half-long integer.
75 * If sizeof(long) == 4, this is 0x8000.
76 * If sizeof(long) == 8, this is 0x80000000.
77 */
78#define HALF_LONG_MSB ((SIGNED_VALUE)1<<((SIZEOF_LONG*CHAR_BIT-1)/2))
79
80#define SIGNED_INTEGER_TYPE_P(T) (0 > ((T)0)-1)
81
82#define SIGNED_INTEGER_MIN(T) \
83 ((sizeof(T) == sizeof(int8_t)) ? ((T)INT8_MIN) : \
84 ((sizeof(T) == sizeof(int16_t)) ? ((T)INT16_MIN) : \
85 ((sizeof(T) == sizeof(int32_t)) ? ((T)INT32_MIN) : \
86 ((sizeof(T) == sizeof(int64_t)) ? ((T)INT64_MIN) : \
87 0))))
88
89#define SIGNED_INTEGER_MAX(T) ((T)(SIGNED_INTEGER_MIN(T) ^ ((T)~(T)0)))
90
91#define UNSIGNED_INTEGER_MAX(T) ((T)~(T)0)
92
93#if __has_builtin(__builtin_mul_overflow_p)
94# define MUL_OVERFLOW_P(a, b) \
95 __builtin_mul_overflow_p((a), (b), (__typeof__(a * b))0)
96#elif __has_builtin(__builtin_mul_overflow)
97# define MUL_OVERFLOW_P(a, b) \
98 __extension__ ({ __typeof__(a) c; __builtin_mul_overflow((a), (b), &c); })
99#endif
100
101#define MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \
102 (a) == 0 ? 0 : \
103 (a) == -1 ? (b) < -(max) : \
104 (a) > 0 ? \
105 ((b) > 0 ? (max) / (a) < (b) : (min) / (a) > (b)) : \
106 ((b) > 0 ? (min) / (a) < (b) : (max) / (a) > (b)))
107
108#if __has_builtin(__builtin_mul_overflow_p)
109/* __builtin_mul_overflow_p can take bitfield */
110/* and GCC permits bitfields for integers other than int */
111# define MUL_OVERFLOW_FIXNUM_P(a, b) \
112 __extension__ ({ \
113 struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \
114 __builtin_mul_overflow_p((a), (b), c.fixnum); \
115 })
116#else
117# define MUL_OVERFLOW_FIXNUM_P(a, b) \
118 MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
119#endif
120
121#if defined(MUL_OVERFLOW_P) && defined(USE___BUILTIN_MUL_OVERFLOW_LONG_LONG)
122# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_P(a, b)
123#else
124# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
125#endif
126
127#ifdef MUL_OVERFLOW_P
128# define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_P(a, b)
129# define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_P(a, b)
130#else
131# define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
132# define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
133#endif
134
135#ifdef HAVE_UINT128_T
136# define bit_length(x) \
137 (unsigned int) \
138 (sizeof(x) <= sizeof(int32_t) ? 32 - nlz_int32((uint32_t)(x)) : \
139 sizeof(x) <= sizeof(int64_t) ? 64 - nlz_int64((uint64_t)(x)) : \
140 128 - nlz_int128((uint128_t)(x)))
141#else
142# define bit_length(x) \
143 (unsigned int) \
144 (sizeof(x) <= sizeof(int32_t) ? 32 - nlz_int32((uint32_t)(x)) : \
145 64 - nlz_int64((uint64_t)(x)))
146#endif
147
148#ifndef swap16
149# define swap16 ruby_swap16
150#endif
151
152#ifndef swap32
153# define swap32 ruby_swap32
154#endif
155
156#ifndef swap64
157# define swap64 ruby_swap64
158#endif
159
160static inline uint16_t ruby_swap16(uint16_t);
161static inline uint32_t ruby_swap32(uint32_t);
162static inline uint64_t ruby_swap64(uint64_t);
163static inline unsigned nlz_int(unsigned x);
164static inline unsigned nlz_long(unsigned long x);
165static inline unsigned nlz_long_long(unsigned long long x);
166static inline unsigned nlz_intptr(uintptr_t x);
167static inline unsigned nlz_int32(uint32_t x);
168static inline unsigned nlz_int64(uint64_t x);
169#ifdef HAVE_UINT128_T
170static inline unsigned nlz_int128(uint128_t x);
171#endif
172static inline unsigned rb_popcount32(uint32_t x);
173static inline unsigned rb_popcount64(uint64_t x);
174static inline unsigned rb_popcount_intptr(uintptr_t x);
175static inline int ntz_int32(uint32_t x);
176static inline int ntz_int64(uint64_t x);
177static inline int ntz_intptr(uintptr_t x);
178static inline VALUE RUBY_BIT_ROTL(VALUE, int);
179static inline VALUE RUBY_BIT_ROTR(VALUE, int);
180
181static inline uint16_t
182ruby_swap16(uint16_t x)
183{
184#if __has_builtin(__builtin_bswap16)
185 return __builtin_bswap16(x);
186
187#elif MSC_VERSION_SINCE(1310)
188 return _byteswap_ushort(x);
189
190#else
191 return (x << 8) | (x >> 8);
192
193#endif
194}
195
196static inline uint32_t
197ruby_swap32(uint32_t x)
198{
199#if __has_builtin(__builtin_bswap32)
200 return __builtin_bswap32(x);
201
202#elif MSC_VERSION_SINCE(1310)
203 return _byteswap_ulong(x);
204
205#else
206 x = ((x & 0x0000FFFF) << 16) | ((x & 0xFFFF0000) >> 16);
207 x = ((x & 0x00FF00FF) << 8) | ((x & 0xFF00FF00) >> 8);
208 return x;
209
210#endif
211}
212
213static inline uint64_t
214ruby_swap64(uint64_t x)
215{
216#if __has_builtin(__builtin_bswap64)
217 return __builtin_bswap64(x);
218
219#elif MSC_VERSION_SINCE(1310)
220 return _byteswap_uint64(x);
221
222#else
223 x = ((x & 0x00000000FFFFFFFFULL) << 32) | ((x & 0xFFFFFFFF00000000ULL) >> 32);
224 x = ((x & 0x0000FFFF0000FFFFULL) << 16) | ((x & 0xFFFF0000FFFF0000ULL) >> 16);
225 x = ((x & 0x00FF00FF00FF00FFULL) << 8) | ((x & 0xFF00FF00FF00FF00ULL) >> 8);
226 return x;
227
228#endif
229}
230
231static inline unsigned int
232nlz_int32(uint32_t x)
233{
234#if defined(_MSC_VER) && defined(__AVX2__)
235 /* Note: It seems there is no such thing like __LZCNT__ predefined in MSVC.
236 * AMD CPUs have had this instruction for decades (since K10) but for
237 * Intel, Haswell is the oldest one. We need to use __AVX2__ for maximum
238 * safety. */
239 return (unsigned int)__lzcnt(x);
240
241#elif defined(__x86_64__) && defined(__LZCNT__)
242 return (unsigned int)_lzcnt_u32(x);
243
244#elif MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */
245 unsigned long r;
246 return _BitScanReverse(&r, x) ? (31 - (int)r) : 32;
247
248#elif __has_builtin(__builtin_clz)
249 STATIC_ASSERT(sizeof_int, sizeof(int) * CHAR_BIT == 32);
250 return x ? (unsigned int)__builtin_clz(x) : 32;
251
252#else
253 uint32_t y;
254 unsigned n = 32;
255 y = x >> 16; if (y) {n -= 16; x = y;}
256 y = x >> 8; if (y) {n -= 8; x = y;}
257 y = x >> 4; if (y) {n -= 4; x = y;}
258 y = x >> 2; if (y) {n -= 2; x = y;}
259 y = x >> 1; if (y) {return n - 2;}
260 return (unsigned int)(n - x);
261#endif
262}
263
264static inline unsigned int
265nlz_int64(uint64_t x)
266{
267#if defined(_MSC_VER) && defined(__AVX2__)
268 return (unsigned int)__lzcnt64(x);
269
270#elif defined(__x86_64__) && defined(__LZCNT__)
271 return (unsigned int)_lzcnt_u64(x);
272
273#elif defined(_WIN64) && MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */
274 unsigned long r;
275 return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64;
276
277#elif __has_builtin(__builtin_clzl)
278 if (x == 0) {
279 return 64;
280 }
281 else if (sizeof(long) * CHAR_BIT == 64) {
282 return (unsigned int)__builtin_clzl((unsigned long)x);
283 }
284 else if (sizeof(long long) * CHAR_BIT == 64) {
285 return (unsigned int)__builtin_clzll((unsigned long long)x);
286 }
287 else {
288 /* :FIXME: Is there a way to make this branch a compile-time error? */
289 UNREACHABLE_RETURN(~0);
290 }
291
292#else
293 uint64_t y;
294 unsigned int n = 64;
295 y = x >> 32; if (y) {n -= 32; x = y;}
296 y = x >> 16; if (y) {n -= 16; x = y;}
297 y = x >> 8; if (y) {n -= 8; x = y;}
298 y = x >> 4; if (y) {n -= 4; x = y;}
299 y = x >> 2; if (y) {n -= 2; x = y;}
300 y = x >> 1; if (y) {return n - 2;}
301 return (unsigned int)(n - x);
302
303#endif
304}
305
306#ifdef HAVE_UINT128_T
307static inline unsigned int
308nlz_int128(uint128_t x)
309{
310 uint64_t y = (uint64_t)(x >> 64);
311
312 if (x == 0) {
313 return 128;
314 }
315 else if (y == 0) {
316 return (unsigned int)nlz_int64(x) + 64;
317 }
318 else {
319 return (unsigned int)nlz_int64(y);
320 }
321}
322#endif
323
324static inline unsigned int
325nlz_int(unsigned int x)
326{
327 if (sizeof(unsigned int) * CHAR_BIT == 32) {
328 return nlz_int32((uint32_t)x);
329 }
330 else if (sizeof(unsigned int) * CHAR_BIT == 64) {
331 return nlz_int64((uint64_t)x);
332 }
333 else {
334 UNREACHABLE_RETURN(~0);
335 }
336}
337
338static inline unsigned int
339nlz_long(unsigned long x)
340{
341 if (sizeof(unsigned long) * CHAR_BIT == 32) {
342 return nlz_int32((uint32_t)x);
343 }
344 else if (sizeof(unsigned long) * CHAR_BIT == 64) {
345 return nlz_int64((uint64_t)x);
346 }
347 else {
348 UNREACHABLE_RETURN(~0);
349 }
350}
351
352static inline unsigned int
353nlz_long_long(unsigned long long x)
354{
355 if (sizeof(unsigned long long) * CHAR_BIT == 64) {
356 return nlz_int64((uint64_t)x);
357 }
358#ifdef HAVE_UINT128_T
359 else if (sizeof(unsigned long long) * CHAR_BIT == 128) {
360 return nlz_int128((uint128_t)x);
361 }
362#endif
363 else {
364 UNREACHABLE_RETURN(~0);
365 }
366}
367
368static inline unsigned int
369nlz_intptr(uintptr_t x)
370{
371 if (sizeof(uintptr_t) == sizeof(unsigned int)) {
372 return nlz_int((unsigned int)x);
373 }
374 if (sizeof(uintptr_t) == sizeof(unsigned long)) {
375 return nlz_long((unsigned long)x);
376 }
377 if (sizeof(uintptr_t) == sizeof(unsigned long long)) {
378 return nlz_long_long((unsigned long long)x);
379 }
380 else {
381 UNREACHABLE_RETURN(~0);
382 }
383}
384
385static inline unsigned int
386rb_popcount32(uint32_t x)
387{
388#if defined(_MSC_VER) && defined(__AVX__)
389 /* Note: CPUs since Nehalem and Barcelona have had this instruction so SSE
390 * 4.2 should suffice, but it seems there is no such thing like __SSE_4_2__
391 * predefined macro in MSVC. They do have __AVX__ so use it instead. */
392 return (unsigned int)__popcnt(x);
393
394#elif __has_builtin(__builtin_popcount)
395 STATIC_ASSERT(sizeof_int, sizeof(int) * CHAR_BIT >= 32);
396 return (unsigned int)__builtin_popcount(x);
397
398#else
399 x = (x & 0x55555555) + (x >> 1 & 0x55555555);
400 x = (x & 0x33333333) + (x >> 2 & 0x33333333);
401 x = (x & 0x07070707) + (x >> 4 & 0x07070707);
402 x = (x & 0x000f000f) + (x >> 8 & 0x000f000f);
403 x = (x & 0x0000001f) + (x >>16 & 0x0000001f);
404 return (unsigned int)x;
405
406#endif
407}
408
409static inline unsigned int
410rb_popcount64(uint64_t x)
411{
412#if defined(_MSC_VER) && defined(__AVX__)
413 return (unsigned int)__popcnt64(x);
414
415#elif __has_builtin(__builtin_popcount)
416 if (sizeof(long) * CHAR_BIT == 64) {
417 return (unsigned int)__builtin_popcountl((unsigned long)x);
418 }
419 else if (sizeof(long long) * CHAR_BIT == 64) {
420 return (unsigned int)__builtin_popcountll((unsigned long long)x);
421 }
422 else {
423 /* :FIXME: Is there a way to make this branch a compile-time error? */
424 UNREACHABLE_RETURN(~0);
425 }
426
427#else
428 x = (x & 0x5555555555555555) + (x >> 1 & 0x5555555555555555);
429 x = (x & 0x3333333333333333) + (x >> 2 & 0x3333333333333333);
430 x = (x & 0x0707070707070707) + (x >> 4 & 0x0707070707070707);
431 x = (x & 0x000f000f000f000f) + (x >> 8 & 0x000f000f000f000f);
432 x = (x & 0x0000001f0000001f) + (x >>16 & 0x0000001f0000001f);
433 x = (x & 0x000000000000003f) + (x >>32 & 0x000000000000003f);
434 return (unsigned int)x;
435
436#endif
437}
438
439static inline unsigned int
440rb_popcount_intptr(uintptr_t x)
441{
442 if (sizeof(uintptr_t) * CHAR_BIT == 64) {
443 return rb_popcount64((uint64_t)x);
444 }
445 else if (sizeof(uintptr_t) * CHAR_BIT == 32) {
446 return rb_popcount32((uint32_t)x);
447 }
448 else {
449 UNREACHABLE_RETURN(~0);
450 }
451}
452
453static inline int
454ntz_int32(uint32_t x)
455{
456#if defined(__x86_64__) && defined(__BMI__)
457 return (unsigned)_tzcnt_u32(x);
458
459#elif MSC_VERSION_SINCE(1400)
460 /* :FIXME: Is there any way to issue TZCNT instead of BSF, apart from using
461 * assembly? Because issuing LZCNT seems possible (see nlz.h). */
462 unsigned long r;
463 return _BitScanForward(&r, x) ? (int)r : 32;
464
465#elif __has_builtin(__builtin_ctz)
466 STATIC_ASSERT(sizeof_int, sizeof(int) * CHAR_BIT == 32);
467 return x ? (unsigned)__builtin_ctz(x) : 32;
468
469#else
470 return rb_popcount32((~x) & (x-1));
471
472#endif
473}
474
475static inline int
476ntz_int64(uint64_t x)
477{
478#if defined(__x86_64__) && defined(__BMI__)
479 return (unsigned)_tzcnt_u64(x);
480
481#elif defined(_WIN64) && MSC_VERSION_SINCE(1400)
482 unsigned long r;
483 return _BitScanForward64(&r, x) ? (int)r : 64;
484
485#elif __has_builtin(__builtin_ctzl)
486 if (x == 0) {
487 return 64;
488 }
489 else if (sizeof(long) * CHAR_BIT == 64) {
490 return (unsigned)__builtin_ctzl((unsigned long)x);
491 }
492 else if (sizeof(long long) * CHAR_BIT == 64) {
493 return (unsigned)__builtin_ctzll((unsigned long long)x);
494 }
495 else {
496 /* :FIXME: Is there a way to make this branch a compile-time error? */
497 UNREACHABLE_RETURN(~0);
498 }
499
500#else
501 return rb_popcount64((~x) & (x-1));
502
503#endif
504}
505
506static inline int
507ntz_intptr(uintptr_t x)
508{
509 if (sizeof(uintptr_t) * CHAR_BIT == 64) {
510 return ntz_int64((uint64_t)x);
511 }
512 else if (sizeof(uintptr_t) * CHAR_BIT == 32) {
513 return ntz_int32((uint32_t)x);
514 }
515 else {
516 UNREACHABLE_RETURN(~0);
517 }
518}
519
520static inline VALUE
521RUBY_BIT_ROTL(VALUE v, int n)
522{
523#if __has_builtin(__builtin_rotateleft32) && (SIZEOF_VALUE * CHAR_BIT == 32)
524 return __builtin_rotateleft32(v, n);
525
526#elif __has_builtin(__builtin_rotateleft64) && (SIZEOF_VALUE * CHAR_BIT == 64)
527 return __builtin_rotateleft64(v, n);
528
529#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32)
530 return _rotl(v, n);
531
532#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64)
533 return _rotl64(v, n);
534
535#elif defined(_lrotl) && (SIZEOF_VALUE == SIZEOF_LONG)
536 return _lrotl(v, n);
537
538#else
539 const int m = (sizeof(VALUE) * CHAR_BIT) - 1;
540 return (v << (n & m)) | (v >> (-n & m));
541#endif
542}
543
544static inline VALUE
545RUBY_BIT_ROTR(VALUE v, int n)
546{
547#if __has_builtin(__builtin_rotateright32) && (SIZEOF_VALUE * CHAR_BIT == 32)
548 return __builtin_rotateright32(v, n);
549
550#elif __has_builtin(__builtin_rotateright64) && (SIZEOF_VALUE * CHAR_BIT == 64)
551 return __builtin_rotateright64(v, n);
552
553#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32)
554 return _rotr(v, n);
555
556#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64)
557 return _rotr64(v, n);
558
559#elif defined(_lrotr) && (SIZEOF_VALUE == SIZEOF_LONG)
560 return _lrotr(v, n);
561
562#else
563 const int m = (sizeof(VALUE) * CHAR_BIT) - 1;
564 return (v << (-n & m)) | (v >> (n & m));
565#endif
566}
567
568#endif /* INTERNAL_BITS_H */
UNREACHABLE_RETURN
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
ruby.h
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40