Line data    Source code

       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_popcount_avx512_choose.c
       4             :  *    Test whether we can use the AVX-512 pg_popcount() implementation.
       5             :  *
       6             :  * Copyright (c) 2024, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/port/pg_popcount_avx512_choose.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : #include "c.h"
      14             : 
      15             : #if defined(HAVE__GET_CPUID) || defined(HAVE__GET_CPUID_COUNT)
      16             : #include <cpuid.h>
      17             : #endif
      18             : 
      19             : #ifdef HAVE_XSAVE_INTRINSICS
      20             : #include <immintrin.h>
      21             : #endif
      22             : 
      23             : #if defined(HAVE__CPUID) || defined(HAVE__CPUIDEX)
      24             : #include <intrin.h>
      25             : #endif
      26             : 
      27             : #include "port/pg_bitutils.h"
      28             : 
      29             : /*
      30             :  * It's probably unlikely that TRY_POPCNT_FAST won't be set if we are able to
      31             :  * use AVX-512 intrinsics, but we check it anyway to be sure.  We piggy-back on
      32             :  * the function pointers that are only used when TRY_POPCNT_FAST is set.
      33             :  */
      34             : #ifdef TRY_POPCNT_FAST
      35             : 
      36             : /*
      37             :  * Does CPUID say there's support for XSAVE instructions?
      38             :  */
      39             : static inline bool
      40        8296 : xsave_available(void)
      41             : {
      42        8296 :     unsigned int exx[4] = {0, 0, 0, 0};
      43             : 
      44             : #if defined(HAVE__GET_CPUID)
      45        8296 :     __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
      46             : #elif defined(HAVE__CPUID)
      47             :     __cpuid(exx, 1);
      48             : #else
      49             : #error cpuid instruction not available
      50             : #endif
      51        8296 :     return (exx[2] & (1 << 27)) != 0; /* osxsave */
      52             : }
      53             : 
      54             : /*
      55             :  * Does XGETBV say the ZMM registers are enabled?
      56             :  *
      57             :  * NB: Caller is responsible for verifying that xsave_available() returns true
      58             :  * before calling this.
      59             :  */
      60             : static inline bool
      61        8296 : zmm_regs_available(void)
      62             : {
      63             : #ifdef HAVE_XSAVE_INTRINSICS
      64        8296 :     return (_xgetbv(0) & 0xe6) == 0xe6;
      65             : #else
      66             :     return false;
      67             : #endif
      68             : }
      69             : 
      70             : /*
      71             :  * Does CPUID say there's support for AVX-512 popcount and byte-and-word
      72             :  * instructions?
      73             :  */
      74             : static inline bool
      75        8296 : avx512_popcnt_available(void)
      76             : {
      77        8296 :     unsigned int exx[4] = {0, 0, 0, 0};
      78             : 
      79             : #if defined(HAVE__GET_CPUID_COUNT)
      80        8296 :     __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]);
      81             : #elif defined(HAVE__CPUIDEX)
      82             :     __cpuidex(exx, 7, 0);
      83             : #else
      84             : #error cpuid instruction not available
      85             : #endif
      86        8296 :     return (exx[2] & (1 << 14)) != 0 && /* avx512-vpopcntdq */
      87           0 :         (exx[1] & (1 << 30)) != 0;    /* avx512-bw */
      88             : }
      89             : 
      90             : /*
      91             :  * Returns true if the CPU supports the instructions required for the AVX-512
      92             :  * pg_popcount() implementation.
      93             :  */
      94             : bool
      95        8296 : pg_popcount_avx512_available(void)
      96             : {
      97       16592 :     return xsave_available() &&
      98       16592 :         zmm_regs_available() &&
      99        8296 :         avx512_popcnt_available();
     100             : }
     101             : 
     102             : #endif                          /* TRY_POPCNT_FAST */