Files
a0_basic_app
a1_vehicle
a2_async_sim
ab_glyph
ab_glyph_rasterizer
adler
adler32
agents
aho_corasick
anyhow
approx
aquamarine
ash
atty
bitflags
bytemuck
byteorder
cache_padded
cfg_if
chrono
color_quant
crc32fast
crossbeam_channel
crossbeam_deque
crossbeam_epoch
crossbeam_utils
deflate
draw2d
either
flexi_logger
generic_array
gif
glfw
glfw_sys
glob
image
indoc
itertools
jpeg_decoder
lazy_static
libc
libloading
log
matrixmultiply
memchr
memoffset
miniz_oxide
nalgebra
base
geometry
linalg
third_party
num_complex
num_cpus
num_integer
num_iter
num_rational
num_traits
owned_ttf_parser
paste
png
proc_macro2
proc_macro_error
proc_macro_error_attr
quote
raw_window_handle
rawpointer
rayon
rayon_core
regex
regex_syntax
scoped_threadpool
scopeguard
semver
semver_parser
serde
serde_derive
simba
smawk
spin_sleep
syn
terminal_size
textwrap
thiserror
thiserror_impl
tiff
time
triple_buffer
ttf_parser
typenum
unicode_width
unicode_xid
unindent
vk_sys
weezl
yansi
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
use std::sync::atomic::{AtomicUsize, Ordering};

pub(super) struct AtomicCounters {
    /// Packs together a number of counters. The counters are ordered as
    /// follows, from least to most significant bits (here, we assuming
    /// that [`THREADS_BITS`] is equal to 10):
    ///
    /// * Bits 0..10: Stores the number of **sleeping threads**
    /// * Bits 10..20: Stores the number of **inactive threads**
    /// * Bits 20..: Stores the **job event counter** (JEC)
    ///
    /// This uses 10 bits ([`THREADS_BITS`]) to encode the number of threads. Note
    /// that the total number of bits (and hence the number of bits used for the
    /// JEC) will depend on whether we are using a 32- or 64-bit architecture.
    value: AtomicUsize,
}

#[derive(Copy, Clone)]
pub(super) struct Counters {
    word: usize,
}

/// A value read from the **Jobs Event Counter**.
/// See the [`README.md`](README.md) for more
/// coverage of how the jobs event counter works.
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
pub(super) struct JobsEventCounter(usize);

impl JobsEventCounter {
    pub(super) const DUMMY: JobsEventCounter = JobsEventCounter(std::usize::MAX);

    #[inline]
    pub(super) fn as_usize(self) -> usize {
        self.0
    }

    /// The JEC "is sleepy" if the last thread to increment it was in the
    /// process of becoming sleepy. This is indicated by its value being *even*.
    /// When new jobs are posted, they check if the JEC is sleepy, and if so
    /// they incremented it.
    #[inline]
    pub(super) fn is_sleepy(self) -> bool {
        (self.as_usize() & 1) == 0
    }

    /// The JEC "is active" if the last thread to increment it was posting new
    /// work. This is indicated by its value being *odd*. When threads get
    /// sleepy, they will check if the JEC is active, and increment it.
    #[inline]
    pub(super) fn is_active(self) -> bool {
        !self.is_sleepy()
    }
}

/// Number of bits used for the thread counters.
const THREADS_BITS: usize = 10;

/// Bits to shift to select the sleeping threads
/// (used with `select_bits`).
const SLEEPING_SHIFT: usize = 0 * THREADS_BITS;

/// Bits to shift to select the inactive threads
/// (used with `select_bits`).
const INACTIVE_SHIFT: usize = 1 * THREADS_BITS;

/// Bits to shift to select the JEC
/// (use JOBS_BITS).
const JEC_SHIFT: usize = 2 * THREADS_BITS;

/// Max value for the thread counters.
const THREADS_MAX: usize = (1 << THREADS_BITS) - 1;

/// Constant that can be added to add one sleeping thread.
const ONE_SLEEPING: usize = 1;

/// Constant that can be added to add one inactive thread.
/// An inactive thread is either idle, sleepy, or sleeping.
const ONE_INACTIVE: usize = 1 << INACTIVE_SHIFT;

/// Constant that can be added to add one to the JEC.
const ONE_JEC: usize = 1 << JEC_SHIFT;

impl AtomicCounters {
    #[inline]
    pub(super) fn new() -> AtomicCounters {
        AtomicCounters {
            value: AtomicUsize::new(0),
        }
    }

    /// Load and return the current value of the various counters.
    /// This value can then be given to other method which will
    /// attempt to update the counters via compare-and-swap.
    #[inline]
    pub(super) fn load(&self, ordering: Ordering) -> Counters {
        Counters::new(self.value.load(ordering))
    }

    #[inline]
    fn try_exchange(&self, old_value: Counters, new_value: Counters, ordering: Ordering) -> bool {
        self.value
            .compare_exchange(old_value.word, new_value.word, ordering, Ordering::Relaxed)
            .is_ok()
    }

    /// Adds an inactive thread. This cannot fail.
    ///
    /// This should be invoked when a thread enters its idle loop looking
    /// for work. It is decremented when work is found. Note that it is
    /// not decremented if the thread transitions from idle to sleepy or sleeping;
    /// so the number of inactive threads is always greater-than-or-equal
    /// to the number of sleeping threads.
    #[inline]
    pub(super) fn add_inactive_thread(&self) {
        self.value.fetch_add(ONE_INACTIVE, Ordering::SeqCst);
    }

    /// Increments the jobs event counter if `increment_when`, when applied to
    /// the current value, is true. Used to toggle the JEC from even (sleepy) to
    /// odd (active) or vice versa. Returns the final value of the counters, for
    /// which `increment_when` is guaranteed to return false.
    pub(super) fn increment_jobs_event_counter_if(
        &self,
        increment_when: impl Fn(JobsEventCounter) -> bool,
    ) -> Counters {
        loop {
            let old_value = self.load(Ordering::SeqCst);
            if increment_when(old_value.jobs_counter()) {
                let new_value = old_value.increment_jobs_counter();
                if self.try_exchange(old_value, new_value, Ordering::SeqCst) {
                    return new_value;
                }
            } else {
                return old_value;
            }
        }
    }

    /// Subtracts an inactive thread. This cannot fail. It is invoked
    /// when a thread finds work and hence becomes active. It returns the
    /// number of sleeping threads to wake up (if any).
    ///
    /// See `add_inactive_thread`.
    #[inline]
    pub(super) fn sub_inactive_thread(&self) -> usize {
        let old_value = Counters::new(self.value.fetch_sub(ONE_INACTIVE, Ordering::SeqCst));
        debug_assert!(
            old_value.inactive_threads() > 0,
            "sub_inactive_thread: old_value {:?} has no inactive threads",
            old_value,
        );
        debug_assert!(
            old_value.sleeping_threads() <= old_value.inactive_threads(),
            "sub_inactive_thread: old_value {:?} had {} sleeping threads and {} inactive threads",
            old_value,
            old_value.sleeping_threads(),
            old_value.inactive_threads(),
        );

        // Current heuristic: whenever an inactive thread goes away, if
        // there are any sleeping threads, wake 'em up.
        let sleeping_threads = old_value.sleeping_threads();
        std::cmp::min(sleeping_threads, 2)
    }

    /// Subtracts a sleeping thread. This cannot fail, but it is only
    /// safe to do if you you know the number of sleeping threads is
    /// non-zero (i.e., because you have just awoken a sleeping
    /// thread).
    #[inline]
    pub(super) fn sub_sleeping_thread(&self) {
        let old_value = Counters::new(self.value.fetch_sub(ONE_SLEEPING, Ordering::SeqCst));
        debug_assert!(
            old_value.sleeping_threads() > 0,
            "sub_sleeping_thread: old_value {:?} had no sleeping threads",
            old_value,
        );
        debug_assert!(
            old_value.sleeping_threads() <= old_value.inactive_threads(),
            "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} inactive threads",
            old_value,
            old_value.sleeping_threads(),
            old_value.inactive_threads(),
        );
    }

    #[inline]
    pub(super) fn try_add_sleeping_thread(&self, old_value: Counters) -> bool {
        debug_assert!(
            old_value.inactive_threads() > 0,
            "try_add_sleeping_thread: old_value {:?} has no inactive threads",
            old_value,
        );
        debug_assert!(
            old_value.sleeping_threads() < THREADS_MAX,
            "try_add_sleeping_thread: old_value {:?} has too many sleeping threads",
            old_value,
        );

        let mut new_value = old_value;
        new_value.word += ONE_SLEEPING;

        self.try_exchange(old_value, new_value, Ordering::SeqCst)
    }
}

#[inline]
fn select_thread(word: usize, shift: usize) -> usize {
    ((word >> shift) as usize) & THREADS_MAX
}

#[inline]
fn select_jec(word: usize) -> usize {
    (word >> JEC_SHIFT) as usize
}

impl Counters {
    #[inline]
    fn new(word: usize) -> Counters {
        Counters { word }
    }

    #[inline]
    fn increment_jobs_counter(self) -> Counters {
        // We can freely add to JEC because it occupies the most significant bits.
        // Thus it doesn't overflow into the other counters, just wraps itself.
        Counters {
            word: self.word.wrapping_add(ONE_JEC),
        }
    }

    #[inline]
    pub(super) fn jobs_counter(self) -> JobsEventCounter {
        JobsEventCounter(select_jec(self.word))
    }

    /// The number of threads that are not actively
    /// executing work. They may be idle, sleepy, or asleep.
    #[inline]
    pub(super) fn inactive_threads(self) -> usize {
        select_thread(self.word, INACTIVE_SHIFT)
    }

    #[inline]
    pub(super) fn awake_but_idle_threads(self) -> usize {
        debug_assert!(
            self.sleeping_threads() <= self.inactive_threads(),
            "sleeping threads: {} > raw idle threads {}",
            self.sleeping_threads(),
            self.inactive_threads()
        );
        self.inactive_threads() - self.sleeping_threads()
    }

    #[inline]
    pub(super) fn sleeping_threads(self) -> usize {
        select_thread(self.word, SLEEPING_SHIFT)
    }
}

impl std::fmt::Debug for Counters {
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let word = format!("{:016x}", self.word);
        fmt.debug_struct("Counters")
            .field("word", &word)
            .field("jobs", &self.jobs_counter().0)
            .field("inactive", &self.inactive_threads())
            .field("sleeping", &self.sleeping_threads())
            .finish()
    }
}