1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

use crate::sys::time;

#[inline]
pub(super) fn monotonize(raw: time::Instant) -> time::Instant {
    inner::monotonize(raw)
}

#[cfg(any(all(target_has_atomic = "64", not(target_has_atomic = "128")), target_arch = "aarch64"))]
pub mod inner {
    use crate::sync::atomic::AtomicU64;
    use crate::sync::atomic::Ordering::*;
    use crate::sys::time;
    use crate::time::Duration;

    pub(in crate::time) const ZERO: time::Instant = time::Instant::zero();

    // bits 30 and 31 are never used since the nanoseconds part never exceeds 10^9
    const UNINITIALIZED: u64 = 0b11 << 30;
    static MONO: AtomicU64 = AtomicU64::new(UNINITIALIZED);

    #[inline]
    pub(super) fn monotonize(raw: time::Instant) -> time::Instant {
        monotonize_impl(&MONO, raw)
    }

    #[inline]
    pub(in crate::time) fn monotonize_impl(mono: &AtomicU64, raw: time::Instant) -> time::Instant {
        let delta = raw.checked_sub_instant(&ZERO).unwrap();
        let secs = delta.as_secs();
        // occupies no more than 30 bits (10^9 seconds)
        let nanos = delta.subsec_nanos() as u64;

        // This wraps around every 136 years (2^32 seconds).
        // To detect backsliding we use wrapping arithmetic and declare forward steps smaller
        // than 2^31 seconds as expected and everything else as a backslide which will be
        // monotonized.
        // This could be a problem for programs that call instants at intervals greater
        // than 68 years. Interstellar probes may want to ensure that actually_monotonic() is true.
        let packed = (secs << 32) | nanos;
        let updated = mono.fetch_update(Relaxed, Relaxed, |old| {
            (old == UNINITIALIZED || packed.wrapping_sub(old) < u64::MAX / 2).then_some(packed)
        });
        match updated {
            Ok(_) => raw,
            Err(newer) => {
                // Backslide occurred. We reconstruct monotonized time from the upper 32 bit of the
                // passed in value and the 64bits loaded from the atomic
                let seconds_lower = newer >> 32;
                let mut seconds_upper = secs & 0xffff_ffff_0000_0000;
                if secs & 0xffff_ffff > seconds_lower {
                    // Backslide caused the lower 32bit of the seconds part to wrap.
                    // This must be the case because the seconds part is larger even though
                    // we are in the backslide branch, i.e. the seconds count should be smaller or equal.
                    //
                    // We assume that backslides are smaller than 2^32 seconds
                    // which means we need to add 1 to the upper half to restore it.
                    //
                    // Example:
                    // most recent observed time: 0xA1_0000_0000_0000_0000u128
                    // bits stored in AtomicU64:     0x0000_0000_0000_0000u64
                    // backslide by 1s
                    // caller time is             0xA0_ffff_ffff_0000_0000u128
                    // -> we can fix up the upper half time by adding 1 << 32
                    seconds_upper = seconds_upper.wrapping_add(0x1_0000_0000);
                }
                let secs = seconds_upper | seconds_lower;
                let nanos = newer as u32;
                ZERO.checked_add_duration(&Duration::new(secs, nanos)).unwrap()
            }
        }
    }
}

#[cfg(all(target_has_atomic = "128", not(target_arch = "aarch64")))]
pub mod inner {
    use crate::sync::atomic::AtomicU128;
    use crate::sync::atomic::Ordering::*;
    use crate::sys::time;
    use crate::time::Duration;

    const ZERO: time::Instant = time::Instant::zero();
    static MONO: AtomicU128 = AtomicU128::new(0);

    #[inline]
    pub(super) fn monotonize(raw: time::Instant) -> time::Instant {
        let delta = raw.checked_sub_instant(&ZERO).unwrap();
        // Split into seconds and nanos since Duration doesn't have a
        // constructor that takes a u128
        let secs = delta.as_secs() as u128;
        let nanos = delta.subsec_nanos() as u128;
        let timestamp: u128 = secs << 64 | nanos;
        let timestamp = MONO.fetch_max(timestamp, Relaxed).max(timestamp);
        let secs = (timestamp >> 64) as u64;
        let nanos = timestamp as u32;
        ZERO.checked_add_duration(&Duration::new(secs, nanos)).unwrap()
    }
}

#[cfg(not(any(target_has_atomic = "64", target_has_atomic = "128")))]
pub mod inner {
    use crate::cmp;
    use crate::sys::time;
    use crate::sys_common::mutex::StaticMutex;

    #[inline]
    pub(super) fn monotonize(os_now: time::Instant) -> time::Instant {
        static LOCK: StaticMutex = StaticMutex::new();
        static mut LAST_NOW: time::Instant = time::Instant::zero();
        unsafe {
            let _lock = LOCK.lock();
            let now = cmp::max(LAST_NOW, os_now);
            LAST_NOW = now;
            now
        }
    }
}