      1
      2
      3
      4
      5
      6
      7
      8
      9
     10
     11
     12
     13
     14
     15
     16
     17
     18
     19
     20
     21
     22
     23
     24
     25
     26
     27
     28
     29
     30
     31
     32
     33
     34
     35
     36
     37
     38
     39
     40
     41
     42
     43
     44
     45
     46
     47
     48
     49
     50
     51
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187

    //! Caches run-time feature detection so that it only needs to be computed
    //! once.

    #![allow(dead_code)] // not used on all platforms

    use crate::sync::atomic::Ordering;

    use crate::sync::atomic::AtomicUsize;

    /// Sets the `bit` of `x`.
    #[inline]
    const fn set_bit(x: u64, bit: u32) -> u64 {
        x | 1 << bit
    }

    /// Tests the `bit` of `x`.
    #[inline]
    const fn test_bit(x: u64, bit: u32) -> bool {
        x & (1 << bit) != 0
    }

    /// Unset the `bit of `x`.
    #[inline]
    const fn unset_bit(x: u64, bit: u32) -> u64 {
        x & !(1 << bit)
    }

    /// Maximum number of features that can be cached.
    const CACHE_CAPACITY: u32 = 62;

    /// This type is used to initialize the cache
    #[derive(Copy, Clone)]
    pub(crate) struct Initializer(u64);

    #[allow(clippy::use_self)]
    impl Default for Initializer {
        fn default() -> Self {
            Initializer(0)
        }
    }

    // NOTE: the `debug_assert!` would catch that we do not add more Features than
    // the one fitting our cache.
    impl Initializer {
        /// Tests the `bit` of the cache.
        #[inline]
        pub(crate) fn test(self, bit: u32) -> bool {
            debug_assert!(
                bit < CACHE_CAPACITY,
                "too many features, time to increase the cache size!"
            );
            test_bit(self.0, bit)
        }

        /// Sets the `bit` of the cache.
        #[inline]
        pub(crate) fn set(&mut self, bit: u32) {
            debug_assert!(
                bit < CACHE_CAPACITY,
                "too many features, time to increase the cache size!"
            );
            let v = self.0;
            self.0 = set_bit(v, bit);
        }

        /// Unsets the `bit` of the cache.
        #[inline]
        pub(crate) fn unset(&mut self, bit: u32) {
            debug_assert!(
                bit < CACHE_CAPACITY,
                "too many features, time to increase the cache size!"
            );
            let v = self.0;
            self.0 = unset_bit(v, bit);
        }
    }

    /// This global variable is a cache of the features supported by the CPU.
    // Note: on x64, we only use the first slot
    static CACHE: [Cache; 2] = [Cache::uninitialized(), Cache::uninitialized()];

    /// Feature cache with capacity for `size_of::<usize::MAX>() * 8 - 1` features.
    ///
    /// Note: 0 is used to represent an uninitialized cache, and (at least) the most
    /// significant bit is set on any cache which has been initialized.
    ///
    /// Note: we use `Relaxed` atomic operations, because we are only interested in
    /// the effects of operations on a single memory location. That is, we only need
    /// "modification order", and not the full-blown "happens before".
    struct Cache(AtomicUsize);

    impl Cache {
        const CAPACITY: u32 = (core::mem::size_of::<usize>() * 8 - 1) as u32;
        const MASK: usize = (1 << Cache::CAPACITY) - 1;
        const INITIALIZED_BIT: usize = 1usize << Cache::CAPACITY;

        /// Creates an uninitialized cache.
        #[allow(clippy::declare_interior_mutable_const)]
        const fn uninitialized() -> Self {
            Cache(AtomicUsize::new(0))
        }

        /// Is the `bit` in the cache set? Returns `None` if the cache has not been initialized.
        #[inline]
        pub(crate) fn test(&self, bit: u32) -> Option<bool> {
            let cached = self.0.load(Ordering::Relaxed);
            if cached == 0 {
                None
            } else {
                Some(test_bit(cached as u64, bit))
            }
        }

        /// Initializes the cache.
        #[inline]
        fn initialize(&self, value: usize) -> usize {
            debug_assert_eq!((value & !Cache::MASK), 0);
            self.0
                .store(value | Cache::INITIALIZED_BIT, Ordering::Relaxed);
            value
        }
    }

    cfg_if::cfg_if! {
        if #[cfg(feature = "std_detect_env_override")] {
            #[inline]
            fn initialize(mut value: Initializer) -> Initializer {
                if let Ok(disable) = crate::env::var("RUST_STD_DETECT_UNSTABLE") {
                    for v in disable.split(" ") {
                        let _ = super::Feature::from_str(v).map(|v| value.unset(v as u32));
                    }
                }
                do_initialize(value);
                value
            }
        } else {
            #[inline]
            fn initialize(value: Initializer) -> Initializer {
                do_initialize(value);
                value
            }
        }
    }

    #[inline]
    fn do_initialize(value: Initializer) {
        CACHE[0].initialize((value.0) as usize & Cache::MASK);
        CACHE[1].initialize((value.0 >> Cache::CAPACITY) as usize & Cache::MASK);
    }

    // We only have to detect features once, and it's fairly costly, so hint to LLVM
    // that it should assume that cache hits are more common than misses (which is
    // the point of caching). It's possibly unfortunate that this function needs to
    // reach across modules like this to call `os::detect_features`, but it produces
    // the best code out of several attempted variants.
    //
    // The `Initializer` that the cache was initialized with is returned, so that
    // the caller can call `test()` on it without having to load the value from the
    // cache again.
    #[cold]
    fn detect_and_initialize() -> Initializer {
        initialize(super::os::detect_features())
    }

    /// Tests the `bit` of the storage. If the storage has not been initialized,
    /// initializes it with the result of `os::detect_features()`.
    ///
    /// On its first invocation, it detects the CPU features and caches them in the
    /// `CACHE` global variable as an `AtomicU64`.
    ///
    /// It uses the `Feature` variant to index into this variable as a bitset. If
    /// the bit is set, the feature is enabled, and otherwise it is disabled.
    ///
    /// If the feature `std_detect_env_override` is enabled looks for the env
    /// variable `RUST_STD_DETECT_UNSTABLE` and uses its its content to disable
    /// Features that would had been otherwise detected.
    #[inline]
    pub(crate) fn test(bit: u32) -> bool {
        let (relative_bit, idx) = if bit < Cache::CAPACITY {
            (bit, 0)
        } else {
            (bit - Cache::CAPACITY, 1)
        };
        CACHE[idx]
            .test(relative_bit)
            .unwrap_or_else(|| detect_and_initialize().test(bit))
    }