Skip to main content

nautilus_common/
throttler.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Message throttling and rate limiting functionality.
17//!
18//! This module provides throttling capabilities to control the rate of message processing
19//! and prevent system overload. The throttler can buffer, drop, or delay messages based
20//! on configured rate limits and time intervals.
21
22use std::{
23    any::Any,
24    cell::{RefCell, UnsafeCell},
25    collections::VecDeque,
26    fmt::Debug,
27    marker::PhantomData,
28    num::{NonZeroU64, NonZeroUsize},
29    rc::Rc,
30};
31
32use nautilus_core::{UnixNanos, correctness::FAILED};
33use serde::{Deserialize, Serialize};
34use ustr::Ustr;
35
36use crate::{
37    actor::{
38        Actor,
39        registry::{register_actor, try_get_actor_unchecked, with_actor_registry},
40    },
41    clock::Clock,
42    msgbus::{self, Endpoint, Handler, MStr, ShareableMessageHandler},
43    timer::{TimeEvent, TimeEventCallback},
44};
45
46const MAX_INITIAL_TIMESTAMPS_CAPACITY: usize = 1024;
47
48/// Represents a throttling limit per interval.
49///
50/// The non-zero field types make a degenerate rate limit unrepresentable: a zero `limit`
51/// underflows the throttler's `limit - 1` indexing, and a zero `interval_ns` disables
52/// throttling entirely.
53#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
54#[serde(deny_unknown_fields)]
55pub struct RateLimit {
56    limit: NonZeroUsize,
57    interval_ns: NonZeroU64,
58}
59
60impl RateLimit {
61    /// Creates a new [`RateLimit`] instance with correctness checking.
62    ///
63    /// # Errors
64    ///
65    /// Returns an error if `limit` or `interval_ns` is zero.
66    pub fn new_checked(limit: usize, interval_ns: u64) -> anyhow::Result<Self> {
67        let limit = NonZeroUsize::new(limit)
68            .ok_or_else(|| anyhow::anyhow!("Invalid limit: {limit} (must be non-zero)"))?;
69        let interval_ns = NonZeroU64::new(interval_ns).ok_or_else(|| {
70            anyhow::anyhow!("Invalid interval_ns: {interval_ns} (must be non-zero)")
71        })?;
72        Ok(Self { limit, interval_ns })
73    }
74
75    /// Creates a new [`RateLimit`] instance.
76    ///
77    /// # Panics
78    ///
79    /// Panics if `limit` or `interval_ns` is zero.
80    #[must_use]
81    pub fn new(limit: usize, interval_ns: u64) -> Self {
82        Self::new_checked(limit, interval_ns).expect(FAILED)
83    }
84
85    /// Maximum number of messages that can be sent within the interval.
86    #[must_use]
87    pub const fn limit(&self) -> usize {
88        self.limit.get()
89    }
90
91    /// Interval between messages in nanoseconds.
92    #[must_use]
93    pub const fn interval_ns(&self) -> u64 {
94        self.interval_ns.get()
95    }
96}
97
98/// Throttler rate limits messages by dropping or buffering them.
99///
100/// Throttler takes messages of type T and callback of type F for dropping
101/// or processing messages.
102///
103/// The throttler stores its limit and interval as non-zero values from
104/// [`RateLimit`]. Internal counters, buffers, and timer state stay private so
105/// callers can observe state without breaking rate-limit invariants.
106///
107/// # Callback contract
108///
109/// The `output_send` and `output_drop` callbacks are invoked inline from
110/// [`Throttler::send`] and the drain handler. They must not reenter the
111/// throttler (for example by calling `send` synchronously), since the
112/// throttler mutates its buffer and window state through `UnsafeCell` without
113/// borrow-check protection. Route side effects through an asynchronous queue
114/// when in doubt.
115///
116/// # Buffered mode contract
117///
118/// Buffered mode (`output_drop` is `None`) drains through a message-bus
119/// endpoint registered by [`Throttler::to_actor`]. An embedded throttler that
120/// is never registered cannot drain: any buffered messages accumulate without
121/// bound. Embedded throttlers that can reach the limit must therefore use
122/// drop mode (provide `output_drop`), where the `send()` auto-reset provides
123/// recovery.
124pub struct Throttler<T, F> {
125    clock: Rc<RefCell<dyn Clock>>,
126    actor_id: Ustr,
127    timer_name: Ustr,
128    limit: NonZeroUsize,
129    interval_ns: NonZeroU64,
130    large_limit: bool,
131    buffer: VecDeque<T>,
132    timestamps: VecDeque<UnixNanos>,
133    is_limiting: bool,
134    recv_count: usize,
135    sent_count: usize,
136    output_send: F,
137    output_drop: Option<F>,
138}
139
140impl<T, F> Actor for Throttler<T, F>
141where
142    T: 'static + Debug,
143    F: Fn(T) + 'static,
144{
145    fn id(&self) -> Ustr {
146        self.actor_id
147    }
148
149    fn handle(&mut self, _msg: &dyn Any) {}
150
151    fn as_any(&self) -> &dyn Any {
152        self
153    }
154}
155
156impl<T, F> Debug for Throttler<T, F>
157where
158    T: Debug,
159{
160    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
161        f.debug_struct(stringify!(Throttler))
162            .field("actor_id", &self.actor_id)
163            .field("timer_name", &self.timer_name)
164            .field("limit", &self.limit())
165            .field("interval_ns", &self.interval_ns())
166            .field("buffer", &self.buffer)
167            .field("timestamps", &self.timestamps)
168            .field("is_limiting", &self.is_limiting)
169            .field("recv_count", &self.recv_count)
170            .field("sent_count", &self.sent_count)
171            .finish()
172    }
173}
174
175impl<T, F> Throttler<T, F>
176where
177    T: Debug,
178{
179    /// Creates a new [`Throttler`] instance.
180    ///
181    /// The timer is registered under a name namespaced by `actor_id` so multiple
182    /// throttlers can share one clock.
183    #[inline]
184    pub fn new(
185        rate_limit: RateLimit,
186        clock: Rc<RefCell<dyn Clock>>,
187        timer_name: &str,
188        output_send: F,
189        output_drop: Option<F>,
190        actor_id: Ustr,
191    ) -> Self {
192        Self {
193            clock,
194            actor_id,
195            timer_name: Ustr::from(format!("{timer_name}-{actor_id}").as_str()),
196            limit: rate_limit.limit,
197            interval_ns: rate_limit.interval_ns,
198            large_limit: rate_limit.limit.get() > MAX_INITIAL_TIMESTAMPS_CAPACITY,
199            buffer: VecDeque::new(),
200            timestamps: VecDeque::with_capacity(
201                rate_limit.limit.get().min(MAX_INITIAL_TIMESTAMPS_CAPACITY),
202            ),
203            is_limiting: false,
204            recv_count: 0,
205            sent_count: 0,
206            output_send,
207            output_drop,
208        }
209    }
210
211    /// Set timer with a callback to be triggered on next interval.
212    ///
213    /// Typically used to register callbacks:
214    /// - to process buffered messages
215    /// - to stop buffering
216    ///
217    /// `allow_past` is set explicitly so a zero `delta_next` clamps to the
218    /// current time and fires immediately instead of returning an error.
219    ///
220    /// # Panics
221    ///
222    /// Panics if setting the time alert on the internal clock fails.
223    #[inline]
224    pub(crate) fn set_timer(&self, callback: Option<TimeEventCallback>) {
225        let delta = self.delta_next();
226        self.set_timer_after(delta, callback);
227    }
228
229    #[inline]
230    fn set_timer_after(&self, delta: u64, callback: Option<TimeEventCallback>) {
231        let mut clock = self.clock.borrow_mut();
232        if clock.timer_exists(&self.timer_name) {
233            clock.cancel_timer(&self.timer_name);
234        }
235        let alert_ts = clock.timestamp_ns() + delta;
236
237        clock
238            .set_time_alert_ns(self.timer_name.as_str(), alert_ts, callback, Some(true))
239            .expect(FAILED);
240    }
241
242    /// Time delta when the next message can be sent.
243    ///
244    /// Uses saturating subtraction so a clock regression or a future-dated
245    /// timestamp yields a zero delta instead of panicking.
246    #[inline]
247    pub fn delta_next(&self) -> u64 {
248        match self.timestamps.get(self.limit.get() - 1) {
249            Some(ts) => {
250                let diff = self
251                    .clock
252                    .borrow()
253                    .timestamp_ns()
254                    .as_u64()
255                    .saturating_sub(ts.as_u64());
256                self.interval_ns.get().saturating_sub(diff)
257            }
258            None => 0,
259        }
260    }
261
262    #[inline]
263    fn delta_next_at(&self, now: UnixNanos) -> u64 {
264        match self.timestamps.get(self.limit.get() - 1) {
265            Some(ts) => {
266                let diff = now.as_u64().saturating_sub(ts.as_u64());
267                self.interval_ns.get().saturating_sub(diff)
268            }
269            None => 0,
270        }
271    }
272
273    /// Reset the throttler which clears internal state and cancels any pending
274    /// timer so no drain or resume callback fires after reset.
275    #[inline]
276    pub fn reset(&mut self) {
277        self.cancel_timer_internal();
278        self.buffer.clear();
279        self.recv_count = 0;
280        self.sent_count = 0;
281        self.is_limiting = false;
282        self.timestamps.clear();
283    }
284}
285
286impl<T, F> Throttler<T, F> {
287    /// Cancels the throttler's timer if one is pending. Silently does nothing
288    /// when the clock is borrowed elsewhere or no timer exists (best-effort,
289    /// e.g. from `Drop`).
290    ///
291    /// Lives in a boundless impl block so `Drop` (which has no `T: Debug` bound)
292    /// can call it.
293    fn cancel_timer_internal(&self) {
294        if let Ok(mut clock) = self.clock.try_borrow_mut() {
295            clock.cancel_timer(&self.timer_name);
296        }
297    }
298
299    /// Counts sent messages whose timestamps fall inside the current interval
300    /// window. Shared by [`Throttler::used`] and [`Throttler::try_reserve`].
301    fn count_in_window(&self, now: UnixNanos) -> usize {
302        let interval_start = now.as_i64() - self.interval_ns.get() as i64;
303
304        if let Some(oldest) = self.timestamps.back()
305            && oldest.as_i64() > interval_start
306        {
307            return self.timestamps.len();
308        }
309
310        match self.timestamps.front() {
311            Some(newest) if newest.as_i64() > interval_start => {}
312            _ => return 0,
313        }
314
315        self.timestamps
316            .iter()
317            .take_while(|&&ts| ts.as_i64() > interval_start)
318            .count()
319    }
320
321    /// Maximum number of messages that can be sent within the interval.
322    #[inline]
323    pub const fn limit(&self) -> usize {
324        self.limit.get()
325    }
326
327    /// Interval between messages in nanoseconds.
328    #[inline]
329    pub const fn interval_ns(&self) -> u64 {
330        self.interval_ns.get()
331    }
332
333    /// Rate limit configured for this throttler.
334    #[inline]
335    pub const fn rate_limit(&self) -> RateLimit {
336        RateLimit {
337            limit: self.limit,
338            interval_ns: self.interval_ns,
339        }
340    }
341
342    /// Number of messages queued in buffer.
343    #[inline]
344    pub fn qsize(&self) -> usize {
345        self.buffer.len()
346    }
347
348    /// Fractional value of rate limit consumed in current interval.
349    #[inline]
350    pub fn used(&self) -> f64 {
351        if self.timestamps.is_empty() {
352            return 0.0;
353        }
354        let messages_in_current_interval = self.count_in_window(self.clock.borrow().timestamp_ns());
355        (messages_in_current_interval as f64) / (self.limit.get() as f64)
356    }
357
358    /// Whether the throttler is currently limiting the message rate.
359    #[inline]
360    pub const fn is_limiting(&self) -> bool {
361        self.is_limiting
362    }
363
364    /// Number of messages received.
365    #[inline]
366    pub const fn recv_count(&self) -> usize {
367        self.recv_count
368    }
369
370    /// Number of messages sent.
371    #[inline]
372    pub const fn sent_count(&self) -> usize {
373        self.sent_count
374    }
375}
376
377impl<T, F> Throttler<T, F>
378where
379    T: 'static + Debug,
380    F: Fn(T) + 'static,
381{
382    /// Registers the throttler's process endpoint and actor state, returning
383    /// the shared handle.
384    pub fn to_actor(self) -> Rc<UnsafeCell<Self>> {
385        // Register process endpoint
386        let process_handler = ThrottlerProcess::<T, F>::new(self.actor_id);
387        msgbus::register_any(
388            process_handler.id().as_str().into(),
389            ShareableMessageHandler::from(Rc::new(process_handler) as Rc<dyn Handler<dyn Any>>),
390        );
391
392        // Register actor state and return the wrapped reference
393        register_actor(self)
394    }
395
396    /// Disposes of the throttler by cancelling its timer, deregistering its
397    /// process endpoint from the message bus, and removing it from the actor
398    /// registry.
399    ///
400    /// Call this before dropping a throttler registered via [`Throttler::to_actor`]
401    /// to avoid leaking the process endpoint. For embedded throttlers (not
402    /// registered) this is still safe: the endpoint and registry removals are
403    /// no-ops.
404    pub fn dispose(&mut self) {
405        self.cancel_timer_internal();
406        msgbus::deregister_any(process_endpoint(self.actor_id));
407        with_actor_registry(|registry| {
408            registry.remove(&self.actor_id);
409        });
410    }
411
412    #[inline]
413    pub(crate) fn send_msg(&mut self, msg: T) {
414        let now = self.clock.borrow().timestamp_ns();
415
416        if self.timestamps.len() >= self.limit.get() {
417            self.timestamps.pop_back();
418        }
419        self.timestamps.push_front(now);
420
421        self.sent_count += 1;
422        (self.output_send)(msg);
423    }
424
425    /// Reserves capacity for `count` messages without sending callbacks.
426    ///
427    /// Returns `false` when the current window cannot accept all messages. No partial
428    /// reservation is made in that case. The resume timer is armed only when the
429    /// window is genuinely full (`delta_next > 0`); when the window already slid
430    /// (`delta_next == 0`) the next call re-evaluates without arming a zero-delta
431    /// timer that would fire immediately and log spam.
432    #[inline]
433    pub fn try_reserve(&mut self, count: usize) -> bool {
434        self.recv_count += count;
435
436        if count == 0 {
437            return true;
438        }
439
440        let now = self.clock.borrow().timestamp_ns();
441        let delta = self.delta_next_at(now);
442        if self.is_limiting && delta == 0 && self.buffer.is_empty() {
443            self.is_limiting = false;
444        }
445
446        if self.is_limiting {
447            return false;
448        }
449
450        let used = self.count_in_window(now);
451
452        if self.limit.get().saturating_sub(used) < count {
453            self.is_limiting = true;
454
455            if delta > 0 {
456                self.set_timer_after(delta, Some(throttler_resume::<T, F>(self.actor_id)));
457            }
458            return false;
459        }
460
461        for _ in 0..count {
462            if self.timestamps.len() >= self.limit.get() {
463                self.timestamps.pop_back();
464            }
465            self.timestamps.push_front(now);
466        }
467        self.sent_count += count;
468        true
469    }
470
471    #[inline]
472    pub(crate) fn limit_msg(&mut self, msg: T) {
473        if self.output_drop.is_none() {
474            self.buffer.push_front(msg);
475            log::debug!("Buffering {}", self.buffer.len());
476
477            if !self.is_limiting {
478                log::debug!("Limiting");
479                let cb = Some(ThrottlerProcess::<T, F>::new(self.actor_id).get_timer_callback());
480                let delta = self.delta_next();
481                self.set_timer_after(delta, cb);
482                self.is_limiting = true;
483            }
484        } else {
485            log::debug!("Dropping");
486
487            if let Some(drop) = &self.output_drop {
488                drop(msg);
489            }
490
491            if !self.is_limiting {
492                log::debug!("Limiting");
493                let delta = self.delta_next();
494                self.set_timer_after(delta, Some(throttler_resume::<T, F>(self.actor_id)));
495                self.is_limiting = true;
496            }
497        }
498    }
499
500    #[inline]
501    pub fn send(&mut self, msg: T)
502    where
503        T: 'static,
504        F: Fn(T) + 'static,
505    {
506        self.recv_count += 1;
507
508        if self.large_limit && self.timestamps.len() < self.limit.get() && !self.is_limiting {
509            self.send_msg(msg);
510            return;
511        }
512
513        let delta = if self.is_limiting && !self.buffer.is_empty() {
514            0
515        } else {
516            self.delta_next()
517        };
518
519        // Auto-reset when the rate window has passed but no timer callback
520        // arrived (e.g. for embedded throttlers not registered as actors).
521        // Gated on an empty buffer so buffered throttlers keep draining via
522        // ThrottlerProcess; only drop-mode throttlers have an empty buffer here.
523        if self.is_limiting && delta == 0 && self.buffer.is_empty() {
524            self.is_limiting = false;
525        }
526
527        if self.is_limiting || delta > 0 {
528            self.limit_msg(msg);
529        } else {
530            self.send_msg(msg);
531        }
532    }
533}
534
535/// Builds the message-bus endpoint used to drive the buffered drain handler.
536/// Centralized so registration, `dispose`, and `Drop` agree on the name.
537fn process_endpoint(actor_id: Ustr) -> MStr<Endpoint> {
538    MStr::endpoint(format!("{actor_id}_process")).expect(FAILED)
539}
540
541/// Process buffered messages for throttler
542///
543/// When limit is reached, schedules a timer event to call self again. The handler
544/// is registered as a separated endpoint on the message bus as `{actor_id}_process`.
545struct ThrottlerProcess<T, F> {
546    actor_id: Ustr,
547    endpoint: MStr<Endpoint>,
548    phantom_t: PhantomData<T>,
549    phantom_f: PhantomData<F>,
550}
551
552impl<T, F> ThrottlerProcess<T, F>
553where
554    T: Debug,
555{
556    pub(crate) fn new(actor_id: Ustr) -> Self {
557        Self {
558            actor_id,
559            endpoint: process_endpoint(actor_id),
560            phantom_t: PhantomData,
561            phantom_f: PhantomData,
562        }
563    }
564
565    pub(crate) fn get_timer_callback(&self) -> TimeEventCallback {
566        let endpoint = self.endpoint;
567        TimeEventCallback::from(move |event: TimeEvent| {
568            msgbus::send_any(endpoint, &(event));
569        })
570    }
571}
572
573impl<T, F> Handler<dyn Any> for ThrottlerProcess<T, F>
574where
575    T: 'static + Debug,
576    F: Fn(T) + 'static,
577{
578    fn id(&self) -> Ustr {
579        *self.endpoint
580    }
581
582    fn handle(&self, _message: &dyn Any) {
583        // Use the fallible lookup so a late timer fire after teardown is a
584        // no-op rather than a panic.
585        let Some(mut throttler) = try_get_actor_unchecked::<Throttler<T, F>>(&self.actor_id) else {
586            return;
587        };
588
589        while let Some(msg) = throttler.buffer.pop_back() {
590            throttler.send_msg(msg);
591
592            // Set timer to process more buffered messages
593            // if interval limit reached and there are more
594            // buffered messages to process
595            if !throttler.buffer.is_empty() && throttler.delta_next() > 0 {
596                throttler.is_limiting = true;
597                throttler.set_timer(Some(self.get_timer_callback()));
598                return;
599            }
600        }
601
602        throttler.is_limiting = false;
603    }
604}
605
606impl<T, F> Drop for Throttler<T, F> {
607    fn drop(&mut self) {
608        // Cancel any pending timer so drain/resume callbacks do not fire after
609        // teardown. Best-effort: skip silently if the shared clock is currently
610        // borrowed (e.g. during a nested drop).
611        self.cancel_timer_internal();
612    }
613}
614
615/// Sets throttler to resume sending messages.
616///
617/// Uses `try_get_actor_unchecked` so that embedded throttlers (not registered
618/// in the actor registry) are handled gracefully. The `send()` auto-reset
619/// ensures such throttlers recover once the rate window passes.
620///
621/// When messages are buffered (possible after a rejected `try_reserve`), the
622/// throttler stays limiting and arms its timer with the drain callback
623/// instead, so buffered messages drain in FIFO order ahead of later sends.
624pub fn throttler_resume<T, F>(actor_id: Ustr) -> TimeEventCallback
625where
626    T: 'static + Debug,
627    F: Fn(T) + 'static,
628{
629    TimeEventCallback::from(move |_event: TimeEvent| {
630        if let Some(mut throttler) = try_get_actor_unchecked::<Throttler<T, F>>(&actor_id) {
631            if throttler.buffer.is_empty() {
632                throttler.is_limiting = false;
633            } else {
634                let cb = Some(ThrottlerProcess::<T, F>::new(actor_id).get_timer_callback());
635                throttler.set_timer(cb);
636            }
637        }
638    })
639}
640
641#[cfg(test)]
642mod tests {
643    use std::{
644        cell::{RefCell, UnsafeCell},
645        rc::Rc,
646    };
647
648    use nautilus_core::UUID4;
649    use rstest::{fixture, rstest};
650    use ustr::Ustr;
651
652    use super::{MAX_INITIAL_TIMESTAMPS_CAPACITY, RateLimit, Throttler, ThrottlerProcess};
653    use crate::{
654        clock::{Clock, TestClock},
655        msgbus::{self, Handler},
656    };
657    type SharedThrottler = Rc<UnsafeCell<Throttler<u64, Box<dyn Fn(u64)>>>>;
658
659    /// Test throttler with default values for testing
660    ///
661    /// - Rate limit is 5 messages in 10 intervals.
662    /// - Message handling is decided by specific fixture
663    #[derive(Clone)]
664    struct TestThrottler {
665        throttler: SharedThrottler,
666        clock: Rc<RefCell<TestClock>>,
667        interval: u64,
668    }
669
670    #[allow(unsafe_code)]
671    impl TestThrottler {
672        #[expect(clippy::mut_from_ref)]
673        pub(crate) fn get_throttler(&self) -> &mut Throttler<u64, Box<dyn Fn(u64)>> {
674            unsafe { &mut *self.throttler.get() }
675        }
676    }
677
678    // Timer names are namespaced as `{base}-{actor_id}` with a random actor_id,
679    // so tests match on the base prefix and the expected count instead of an
680    // exact name.
681    fn timer_count_with_prefix(
682        throttler: &Throttler<u64, Box<dyn Fn(u64)>>,
683        prefix: &str,
684    ) -> usize {
685        throttler
686            .clock
687            .borrow()
688            .timer_names()
689            .iter()
690            .filter(|name| name.starts_with(prefix))
691            .count()
692    }
693
694    #[allow(unsafe_code)]
695    #[expect(clippy::mut_from_ref)]
696    fn access_shared(t: &SharedThrottler) -> &mut Throttler<u64, Box<dyn Fn(u64)>> {
697        unsafe { &mut *t.get() }
698    }
699
700    #[rstest]
701    #[case(0, 1_000)]
702    #[case(1_000, 0)]
703    fn test_rate_limit_new_checked_rejects_zero(#[case] limit: usize, #[case] interval_ns: u64) {
704        assert!(RateLimit::new_checked(limit, interval_ns).is_err());
705    }
706
707    #[rstest]
708    #[case(0, 1_000)]
709    #[case(1_000, 0)]
710    #[should_panic]
711    fn test_rate_limit_new_panics_on_zero(#[case] limit: usize, #[case] interval_ns: u64) {
712        let _ = RateLimit::new(limit, interval_ns);
713    }
714
715    #[rstest]
716    fn test_rate_limit_new_checked_accepts_positive() {
717        let rate = RateLimit::new_checked(5, 10).unwrap();
718
719        assert_eq!(rate.limit(), 5);
720        assert_eq!(rate.interval_ns(), 10);
721    }
722
723    #[fixture]
724    pub fn test_throttler_buffered() -> TestThrottler {
725        let output_send: Box<dyn Fn(u64)> = Box::new(|msg: u64| {
726            log::debug!("Sent: {msg}");
727        });
728        let clock = Rc::new(RefCell::new(TestClock::new()));
729        let inner_clock = Rc::clone(&clock);
730        let rate_limit = RateLimit::new(5, 10);
731        let interval = rate_limit.interval_ns();
732        let actor_id = Ustr::from(UUID4::new().as_str());
733
734        TestThrottler {
735            throttler: Throttler::new(
736                rate_limit,
737                clock,
738                "buffer_timer",
739                output_send,
740                None,
741                actor_id,
742            )
743            .to_actor(),
744            clock: inner_clock,
745            interval,
746        }
747    }
748
749    #[fixture]
750    pub fn test_throttler_unbuffered() -> TestThrottler {
751        let output_send: Box<dyn Fn(u64)> = Box::new(|msg: u64| {
752            log::debug!("Sent: {msg}");
753        });
754        let output_drop: Box<dyn Fn(u64)> = Box::new(|msg: u64| {
755            log::debug!("Dropped: {msg}");
756        });
757        let clock = Rc::new(RefCell::new(TestClock::new()));
758        let inner_clock = Rc::clone(&clock);
759        let rate_limit = RateLimit::new(5, 10);
760        let interval = rate_limit.interval_ns();
761        let actor_id = Ustr::from(UUID4::new().as_str());
762
763        TestThrottler {
764            throttler: Throttler::new(
765                rate_limit,
766                clock,
767                "dropper_timer",
768                output_send,
769                Some(output_drop),
770                actor_id,
771            )
772            .to_actor(),
773            clock: inner_clock,
774            interval,
775        }
776    }
777
778    #[rstest]
779    fn test_buffering_send_to_limit_becomes_throttled(test_throttler_buffered: TestThrottler) {
780        let throttler = test_throttler_buffered.get_throttler();
781        for _ in 0..6 {
782            throttler.send(42);
783        }
784        assert_eq!(throttler.qsize(), 1);
785
786        assert!(throttler.is_limiting);
787        assert_eq!(throttler.recv_count, 6);
788        assert_eq!(throttler.sent_count, 5);
789        assert_eq!(timer_count_with_prefix(throttler, "buffer_timer"), 1);
790    }
791
792    #[rstest]
793    fn test_buffering_used_when_sent_to_limit_returns_one(test_throttler_buffered: TestThrottler) {
794        let throttler = test_throttler_buffered.get_throttler();
795
796        for _ in 0..5 {
797            throttler.send(42);
798        }
799
800        assert_eq!(throttler.used(), 1.0);
801        assert_eq!(throttler.recv_count, 5);
802        assert_eq!(throttler.sent_count, 5);
803    }
804
805    #[rstest]
806    fn test_buffering_used_when_half_interval_from_limit_returns_one(
807        test_throttler_buffered: TestThrottler,
808    ) {
809        let throttler = test_throttler_buffered.get_throttler();
810
811        for _ in 0..5 {
812            throttler.send(42);
813        }
814
815        let half_interval = test_throttler_buffered.interval / 2;
816        // Advance the clock by half the interval
817        {
818            let mut clock = test_throttler_buffered.clock.borrow_mut();
819            clock.advance_time(half_interval.into(), true);
820        }
821
822        assert_eq!(throttler.used(), 1.0);
823        assert_eq!(throttler.recv_count, 5);
824        assert_eq!(throttler.sent_count, 5);
825    }
826
827    #[rstest]
828    fn test_buffering_used_before_limit_when_halfway_returns_half(
829        test_throttler_buffered: TestThrottler,
830    ) {
831        let throttler = test_throttler_buffered.get_throttler();
832
833        for _ in 0..3 {
834            throttler.send(42);
835        }
836
837        assert_eq!(throttler.used(), 0.6);
838        assert_eq!(throttler.recv_count, 3);
839        assert_eq!(throttler.sent_count, 3);
840    }
841
842    #[rstest]
843    fn test_try_reserve_counts_messages_without_output(test_throttler_buffered: TestThrottler) {
844        let throttler = test_throttler_buffered.get_throttler();
845
846        assert!(throttler.try_reserve(3));
847
848        assert_eq!(throttler.used(), 0.6);
849        assert_eq!(throttler.recv_count, 3);
850        assert_eq!(throttler.sent_count, 3);
851        assert_eq!(throttler.qsize(), 0);
852    }
853
854    #[rstest]
855    fn test_try_reserve_rejects_when_full_batch_exceeds_limit(
856        test_throttler_buffered: TestThrottler,
857    ) {
858        let throttler = test_throttler_buffered.get_throttler();
859
860        assert!(throttler.try_reserve(3));
861        assert!(!throttler.try_reserve(3));
862
863        assert_eq!(throttler.used(), 0.6);
864        assert_eq!(throttler.recv_count, 6);
865        assert_eq!(throttler.sent_count, 3);
866        assert_eq!(throttler.qsize(), 0);
867        assert!(throttler.is_limiting);
868        // delta_next == 0 here (only 3 of 5 slots used), so the resume timer is
869        // not armed to avoid an immediate-fire zero-delta timer. The next call
870        // re-evaluates via the auto-reset branch.
871        assert_eq!(timer_count_with_prefix(throttler, "buffer_timer"), 0);
872
873        assert!(throttler.try_reserve(2));
874
875        assert_eq!(throttler.used(), 1.0);
876        assert_eq!(throttler.recv_count, 8);
877        assert_eq!(throttler.sent_count, 5);
878        assert_eq!(throttler.qsize(), 0);
879        assert!(!throttler.is_limiting);
880    }
881
882    #[rstest]
883    fn test_try_reserve_rejects_batch_larger_than_limit() {
884        let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(TestClock::new()));
885        let mut throttler = Throttler::<u64, Box<dyn Fn(u64)>>::new(
886            RateLimit::new(5, 10),
887            clock,
888            "reserve_over_limit",
889            Box::new(|_| ()) as Box<dyn Fn(u64)>,
890            None,
891            Ustr::from("reserve-over-limit-actor"),
892        );
893
894        assert!(!throttler.try_reserve(6));
895
896        assert_eq!(throttler.used(), 0.0);
897        assert_eq!(throttler.recv_count, 6);
898        assert_eq!(throttler.sent_count, 0);
899        assert_eq!(throttler.qsize(), 0);
900        assert!(throttler.is_limiting);
901        assert_eq!(throttler.clock.borrow().timer_count(), 0);
902    }
903
904    #[rstest]
905    fn test_try_reserve_zero_count_is_noop() {
906        let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(TestClock::new()));
907        let mut throttler = Throttler::<u64, Box<dyn Fn(u64)>>::new(
908            RateLimit::new(5, 10),
909            clock,
910            "reserve_zero",
911            Box::new(|_| ()) as Box<dyn Fn(u64)>,
912            None,
913            Ustr::from("reserve-zero-actor"),
914        );
915
916        assert!(throttler.try_reserve(0));
917
918        assert_eq!(throttler.used(), 0.0);
919        assert_eq!(throttler.recv_count, 0);
920        assert_eq!(throttler.sent_count, 0);
921        assert_eq!(throttler.qsize(), 0);
922        assert!(!throttler.is_limiting);
923        assert_eq!(throttler.clock.borrow().timer_count(), 0);
924    }
925
926    #[rstest]
927    fn test_buffering_refresh_when_at_limit_sends_remaining_items(
928        test_throttler_buffered: TestThrottler,
929    ) {
930        let throttler = test_throttler_buffered.get_throttler();
931
932        for _ in 0..6 {
933            throttler.send(42);
934        }
935
936        // Advance time and process events
937        {
938            let mut clock = test_throttler_buffered.clock.borrow_mut();
939            let time_events = clock.advance_time(test_throttler_buffered.interval.into(), true);
940            for each_event in clock.match_handlers(time_events) {
941                drop(clock); // Release the mutable borrow
942
943                each_event.callback.call(each_event.event);
944
945                // Re-borrow the clock for the next iteration
946                clock = test_throttler_buffered.clock.borrow_mut();
947            }
948        }
949
950        // Assert final state
951        assert_eq!(throttler.used(), 0.2);
952        assert_eq!(throttler.recv_count, 6);
953        assert_eq!(throttler.sent_count, 6);
954        assert_eq!(throttler.qsize(), 0);
955    }
956
957    #[rstest]
958    fn test_buffering_send_message_after_buffering_message(test_throttler_buffered: TestThrottler) {
959        let throttler = test_throttler_buffered.get_throttler();
960
961        for _ in 0..6 {
962            throttler.send(43);
963        }
964
965        // Advance time and process events
966        {
967            let mut clock = test_throttler_buffered.clock.borrow_mut();
968            let time_events = clock.advance_time(test_throttler_buffered.interval.into(), true);
969            for each_event in clock.match_handlers(time_events) {
970                drop(clock); // Release the mutable borrow
971
972                each_event.callback.call(each_event.event);
973
974                // Re-borrow the clock for the next iteration
975                clock = test_throttler_buffered.clock.borrow_mut();
976            }
977        }
978
979        for _ in 0..6 {
980            throttler.send(42);
981        }
982
983        // Assert final state
984        assert_eq!(throttler.used(), 1.0);
985        assert_eq!(throttler.recv_count, 12);
986        assert_eq!(throttler.sent_count, 10);
987        assert_eq!(throttler.qsize(), 2);
988    }
989
990    #[rstest]
991    fn test_buffering_send_message_after_halfway_after_buffering_message(
992        test_throttler_buffered: TestThrottler,
993    ) {
994        let throttler = test_throttler_buffered.get_throttler();
995
996        for _ in 0..6 {
997            throttler.send(42);
998        }
999
1000        // Advance time and process events
1001        {
1002            let mut clock = test_throttler_buffered.clock.borrow_mut();
1003            let time_events = clock.advance_time(test_throttler_buffered.interval.into(), true);
1004            for each_event in clock.match_handlers(time_events) {
1005                drop(clock); // Release the mutable borrow
1006
1007                each_event.callback.call(each_event.event);
1008
1009                // Re-borrow the clock for the next iteration
1010                clock = test_throttler_buffered.clock.borrow_mut();
1011            }
1012        }
1013
1014        for _ in 0..3 {
1015            throttler.send(42);
1016        }
1017
1018        // Assert final state
1019        assert_eq!(throttler.used(), 0.8);
1020        assert_eq!(throttler.recv_count, 9);
1021        assert_eq!(throttler.sent_count, 9);
1022        assert_eq!(throttler.qsize(), 0);
1023    }
1024
1025    #[rstest]
1026    fn test_dropping_send_sends_message_to_handler(test_throttler_unbuffered: TestThrottler) {
1027        let throttler = test_throttler_unbuffered.get_throttler();
1028        throttler.send(42);
1029
1030        assert!(!throttler.is_limiting);
1031        assert_eq!(throttler.recv_count, 1);
1032        assert_eq!(throttler.sent_count, 1);
1033    }
1034
1035    #[rstest]
1036    fn test_dropping_send_to_limit_drops_message(test_throttler_unbuffered: TestThrottler) {
1037        let throttler = test_throttler_unbuffered.get_throttler();
1038        for _ in 0..6 {
1039            throttler.send(42);
1040        }
1041        assert_eq!(throttler.qsize(), 0);
1042
1043        assert!(throttler.is_limiting);
1044        assert_eq!(throttler.used(), 1.0);
1045        assert_eq!(throttler.clock.borrow().timer_count(), 1);
1046        assert_eq!(timer_count_with_prefix(throttler, "dropper_timer"), 1);
1047        assert_eq!(throttler.recv_count, 6);
1048        assert_eq!(throttler.sent_count, 5);
1049    }
1050
1051    #[rstest]
1052    fn test_dropping_advance_time_when_at_limit_dropped_message(
1053        test_throttler_unbuffered: TestThrottler,
1054    ) {
1055        let throttler = test_throttler_unbuffered.get_throttler();
1056        for _ in 0..6 {
1057            throttler.send(42);
1058        }
1059
1060        // Advance time and process events
1061        {
1062            let mut clock = test_throttler_unbuffered.clock.borrow_mut();
1063            let time_events = clock.advance_time(test_throttler_unbuffered.interval.into(), true);
1064            for each_event in clock.match_handlers(time_events) {
1065                drop(clock); // Release the mutable borrow
1066
1067                each_event.callback.call(each_event.event);
1068
1069                // Re-borrow the clock for the next iteration
1070                clock = test_throttler_unbuffered.clock.borrow_mut();
1071            }
1072        }
1073
1074        assert_eq!(throttler.clock.borrow().timer_count(), 0);
1075        assert!(!throttler.is_limiting);
1076        assert_eq!(throttler.used(), 0.0);
1077        assert_eq!(throttler.recv_count, 6);
1078        assert_eq!(throttler.sent_count, 5);
1079    }
1080
1081    #[rstest]
1082    fn test_dropping_send_message_after_dropping_message(test_throttler_unbuffered: TestThrottler) {
1083        let throttler = test_throttler_unbuffered.get_throttler();
1084        for _ in 0..6 {
1085            throttler.send(42);
1086        }
1087
1088        // Advance time and process events
1089        {
1090            let mut clock = test_throttler_unbuffered.clock.borrow_mut();
1091            let time_events = clock.advance_time(test_throttler_unbuffered.interval.into(), true);
1092            for each_event in clock.match_handlers(time_events) {
1093                drop(clock); // Release the mutable borrow
1094
1095                each_event.callback.call(each_event.event);
1096
1097                // Re-borrow the clock for the next iteration
1098                clock = test_throttler_unbuffered.clock.borrow_mut();
1099            }
1100        }
1101
1102        throttler.send(42);
1103
1104        assert_eq!(throttler.used(), 0.2);
1105        assert_eq!(throttler.clock.borrow().timer_count(), 0);
1106        assert!(!throttler.is_limiting);
1107        assert_eq!(throttler.recv_count, 7);
1108        assert_eq!(throttler.sent_count, 6);
1109    }
1110
1111    #[rstest]
1112    fn test_embedded_dropping_auto_resets_after_window_without_actor_callback() {
1113        let clock: Rc<RefCell<TestClock>> = Rc::new(RefCell::new(TestClock::new()));
1114        let sent = Rc::new(RefCell::new(0));
1115        let dropped = Rc::new(RefCell::new(0));
1116
1117        let sent_cb = {
1118            let sent = Rc::clone(&sent);
1119            Box::new(move |_| *sent.borrow_mut() += 1) as Box<dyn Fn(u64)>
1120        };
1121        let drop_cb = {
1122            let dropped = Rc::clone(&dropped);
1123            Box::new(move |_| *dropped.borrow_mut() += 1) as Box<dyn Fn(u64)>
1124        };
1125
1126        let mut throttler = Throttler::new(
1127            RateLimit::new(5, 10),
1128            Rc::clone(&clock) as Rc<RefCell<dyn Clock>>,
1129            "embedded_drop_timer",
1130            sent_cb,
1131            Some(drop_cb),
1132            Ustr::from("embedded-drop-actor"),
1133        );
1134
1135        for _ in 0..6 {
1136            throttler.send(42);
1137        }
1138        let events = clock.borrow_mut().advance_time(10.into(), true);
1139        throttler.send(42);
1140
1141        assert_eq!(events.len(), 1);
1142        assert_eq!(*sent.borrow(), 6);
1143        assert_eq!(*dropped.borrow(), 1);
1144        assert_eq!(throttler.recv_count, 7);
1145        assert_eq!(throttler.sent_count, 6);
1146        assert!(!throttler.is_limiting);
1147        assert_eq!(throttler.clock.borrow().timer_count(), 0);
1148    }
1149
1150    #[rstest]
1151    fn test_large_limit_fast_path_admits_until_limit_then_limits() {
1152        let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(TestClock::new()));
1153        let limit = MAX_INITIAL_TIMESTAMPS_CAPACITY + 1;
1154        let mut throttler = Throttler::<u64, Box<dyn Fn(u64)>>::new(
1155            RateLimit::new(limit, 10),
1156            clock,
1157            "large_limit_timer",
1158            Box::new(|_| ()) as Box<dyn Fn(u64)>,
1159            None,
1160            Ustr::from("large-limit-actor"),
1161        );
1162
1163        for _ in 0..limit {
1164            throttler.send(42);
1165        }
1166        throttler.send(42);
1167
1168        assert_eq!(throttler.used(), 1.0);
1169        assert_eq!(throttler.recv_count, limit + 1);
1170        assert_eq!(throttler.sent_count, limit);
1171        assert_eq!(throttler.qsize(), 1);
1172        assert!(throttler.is_limiting);
1173        assert_eq!(throttler.clock.borrow().timer_count(), 1);
1174    }
1175
1176    #[rstest]
1177    fn test_new_preserves_rate_limit() {
1178        let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(TestClock::new()));
1179        let rate_limit = RateLimit::new(5, 10);
1180
1181        let throttler = Throttler::<u64, Box<dyn Fn(u64)>>::new(
1182            rate_limit,
1183            clock,
1184            "rate_limit",
1185            Box::new(|_| ()) as Box<dyn Fn(u64)>,
1186            None,
1187            Ustr::from("rate-limit-actor"),
1188        );
1189
1190        assert_eq!(throttler.rate_limit(), rate_limit);
1191        assert_eq!(throttler.limit(), 5);
1192        assert_eq!(throttler.interval_ns(), 10);
1193    }
1194
1195    #[rstest]
1196    fn test_debug_output_includes_identity_and_state() {
1197        let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(TestClock::new()));
1198        let actor_id = Ustr::from("debug-actor");
1199        let mut throttler = Throttler::<u64, Box<dyn Fn(u64)>>::new(
1200            RateLimit::new(5, 10),
1201            clock,
1202            "debug_timer",
1203            Box::new(|_| ()) as Box<dyn Fn(u64)>,
1204            None,
1205            actor_id,
1206        );
1207
1208        throttler.send(42);
1209
1210        let debug = format!("{throttler:?}");
1211        let timer_name = Ustr::from("debug_timer-debug-actor");
1212
1213        assert!(debug.contains(&format!("actor_id: {actor_id:?}")));
1214        assert!(debug.contains(&format!("timer_name: {timer_name:?}")));
1215        assert!(debug.contains("limit: 5"));
1216        assert!(debug.contains("interval_ns: 10"));
1217        assert!(debug.contains("is_limiting: false"));
1218        assert!(debug.contains("recv_count: 1"));
1219        assert!(debug.contains("sent_count: 1"));
1220    }
1221
1222    #[rstest]
1223    fn test_reset_clears_state_and_cancels_timer(test_throttler_buffered: TestThrottler) {
1224        let throttler = test_throttler_buffered.get_throttler();
1225
1226        for _ in 0..6 {
1227            throttler.send(42);
1228        }
1229        assert_eq!(timer_count_with_prefix(throttler, "buffer_timer"), 1);
1230        assert_eq!(throttler.qsize(), 1);
1231
1232        throttler.reset();
1233
1234        assert_eq!(throttler.qsize(), 0);
1235        assert_eq!(throttler.recv_count, 0);
1236        assert_eq!(throttler.sent_count, 0);
1237        assert!(!throttler.is_limiting);
1238        assert!(throttler.timestamps.is_empty());
1239        assert_eq!(timer_count_with_prefix(throttler, "buffer_timer"), 0);
1240        assert_eq!(throttler.clock.borrow().timer_count(), 0);
1241    }
1242
1243    #[rstest]
1244    fn test_two_throttlers_share_clock_without_timer_collision() {
1245        let clock: Rc<RefCell<TestClock>> = Rc::new(RefCell::new(TestClock::new()));
1246        let interval = 10u64;
1247
1248        let mk = |base: &str| -> SharedThrottler {
1249            let clock: Rc<RefCell<dyn Clock>> = Rc::clone(&clock) as Rc<RefCell<dyn Clock>>;
1250            Throttler::new(
1251                RateLimit::new(5, interval),
1252                clock,
1253                base,
1254                Box::new(|_| ()) as Box<dyn Fn(u64)>,
1255                None,
1256                Ustr::from(UUID4::new().as_str()),
1257            )
1258            .to_actor()
1259        };
1260
1261        let t1 = mk("shared_timer");
1262        let t2 = mk("shared_timer");
1263
1264        // Both throttlers use the same base timer name on a shared clock; the
1265        // namespaced names must keep both timers distinct.
1266        {
1267            let t1 = access_shared(&t1);
1268
1269            for _ in 0..6 {
1270                t1.send(42);
1271            }
1272        }
1273        {
1274            let t2 = access_shared(&t2);
1275
1276            for _ in 0..6 {
1277                t2.send(42);
1278            }
1279        }
1280
1281        let clock_ref = clock.borrow();
1282        let names = clock_ref.timer_names();
1283        let shared_count = names
1284            .iter()
1285            .filter(|n| n.starts_with("shared_timer"))
1286            .count();
1287        assert_eq!(
1288            shared_count, 2,
1289            "two distinct timers expected, found {names:?}"
1290        );
1291    }
1292
1293    #[rstest]
1294    fn test_try_reserve_then_send_interleaved(test_throttler_buffered: TestThrottler) {
1295        let throttler = test_throttler_buffered.get_throttler();
1296
1297        // Reserve 3 of 5 slots, then send one more via the send path. Both
1298        // paths share the same window: 4 of 5 slots should be used.
1299        assert!(throttler.try_reserve(3));
1300        throttler.send(42);
1301
1302        assert_eq!(throttler.recv_count, 4);
1303        assert_eq!(throttler.sent_count, 4);
1304        assert_eq!(throttler.used(), 0.8);
1305        assert!(!throttler.is_limiting);
1306    }
1307
1308    #[rstest]
1309    fn test_dispose_cancels_timer_and_deregisters_endpoint(test_throttler_buffered: TestThrottler) {
1310        let throttler = test_throttler_buffered.get_throttler();
1311
1312        for _ in 0..6 {
1313            throttler.send(42);
1314        }
1315        let actor_id = throttler.actor_id;
1316        let endpoint_name = format!("{actor_id}_process");
1317        assert_eq!(timer_count_with_prefix(throttler, "buffer_timer"), 1);
1318        assert!(msgbus::has_endpoint(&endpoint_name));
1319
1320        throttler.dispose();
1321
1322        assert_eq!(throttler.clock.borrow().timer_count(), 0);
1323        assert!(
1324            !msgbus::has_endpoint(&endpoint_name),
1325            "dispose must deregister the process endpoint"
1326        );
1327    }
1328
1329    #[rstest]
1330    fn test_try_reserve_then_buffered_sends_drain_in_order_after_window() {
1331        let clock: Rc<RefCell<TestClock>> = Rc::new(RefCell::new(TestClock::new()));
1332        let sent: Rc<RefCell<Vec<u64>>> = Rc::new(RefCell::new(Vec::new()));
1333        let sent_cb = {
1334            let sent = Rc::clone(&sent);
1335            Box::new(move |msg| sent.borrow_mut().push(msg)) as Box<dyn Fn(u64)>
1336        };
1337        let throttler = Throttler::new(
1338            RateLimit::new(5, 10),
1339            Rc::clone(&clock) as Rc<RefCell<dyn Clock>>,
1340            "reserve_drain_timer",
1341            sent_cb,
1342            None,
1343            Ustr::from(UUID4::new().as_str()),
1344        )
1345        .to_actor();
1346        let throttler = access_shared(&throttler);
1347
1348        assert!(throttler.try_reserve(5));
1349        assert!(!throttler.try_reserve(1));
1350        assert!(throttler.is_limiting);
1351
1352        throttler.send(1);
1353        throttler.send(2);
1354        assert_eq!(throttler.qsize(), 2);
1355        assert!(sent.borrow().is_empty());
1356
1357        // The rejected reservation armed only the resume timer. Firing it must
1358        // arm the drain callback and keep limiting, not drain yet or resume.
1359        {
1360            let mut clock_ref = clock.borrow_mut();
1361            let time_events = clock_ref.advance_time(10.into(), true);
1362            for each_event in clock_ref.match_handlers(time_events) {
1363                drop(clock_ref);
1364                each_event.callback.call(each_event.event);
1365                clock_ref = clock.borrow_mut();
1366            }
1367        }
1368        assert!(throttler.is_limiting);
1369        assert_eq!(throttler.qsize(), 2);
1370        assert!(sent.borrow().is_empty());
1371
1372        // A send after the resume fires must queue behind the buffered messages
1373        throttler.send(3);
1374        assert_eq!(throttler.qsize(), 3);
1375        assert!(sent.borrow().is_empty());
1376
1377        for _ in 0..10 {
1378            if throttler.qsize() == 0 && !throttler.is_limiting {
1379                break;
1380            }
1381            let mut clock_ref = clock.borrow_mut();
1382            let current_time = clock_ref.get_time_ns();
1383            let time_events = clock_ref.advance_time(current_time + 10, true);
1384            for each_event in clock_ref.match_handlers(time_events) {
1385                drop(clock_ref);
1386                each_event.callback.call(each_event.event);
1387                clock_ref = clock.borrow_mut();
1388            }
1389        }
1390
1391        assert_eq!(*sent.borrow(), vec![1, 2, 3]);
1392        assert_eq!(throttler.qsize(), 0);
1393        assert!(!throttler.is_limiting);
1394        assert_eq!(throttler.sent_count, 8);
1395        assert_eq!(throttler.clock.borrow().timer_count(), 0);
1396    }
1397
1398    ////////////////////////////////////////////////////////////////////////////////
1399    // Property-based testing
1400    ////////////////////////////////////////////////////////////////////////////////
1401
1402    use proptest::prelude::*;
1403
1404    #[derive(Clone, Debug)]
1405    enum ThrottlerInput {
1406        SendMessage(u64),
1407        AdvanceClock(u8),
1408    }
1409
1410    // Custom strategy for ThrottlerInput
1411    fn throttler_input_strategy() -> impl Strategy<Value = ThrottlerInput> {
1412        prop_oneof![
1413            2 => prop::bool::ANY.prop_map(|_| ThrottlerInput::SendMessage(42)),
1414            8 => prop::num::u8::ANY.prop_map(|v| ThrottlerInput::AdvanceClock(v % 5 + 5)),
1415        ]
1416    }
1417
1418    // Custom strategy for ThrottlerTest
1419    fn throttler_test_strategy() -> impl Strategy<Value = Vec<ThrottlerInput>> {
1420        prop::collection::vec(throttler_input_strategy(), 10..=150)
1421    }
1422
1423    fn test_throttler_with_inputs(inputs: Vec<ThrottlerInput>, test_throttler: &TestThrottler) {
1424        let test_clock = test_throttler.clock.clone();
1425        let interval = test_throttler.interval;
1426        let throttler = test_throttler.get_throttler();
1427        let mut sent_count = 0;
1428
1429        for input in inputs {
1430            match input {
1431                ThrottlerInput::SendMessage(msg) => {
1432                    throttler.send(msg);
1433                    sent_count += 1;
1434                }
1435                ThrottlerInput::AdvanceClock(duration) => {
1436                    let mut clock_ref = test_clock.borrow_mut();
1437                    let current_time = clock_ref.get_time_ns();
1438                    let time_events =
1439                        clock_ref.advance_time(current_time + u64::from(duration), true);
1440                    for each_event in clock_ref.match_handlers(time_events) {
1441                        drop(clock_ref);
1442                        each_event.callback.call(each_event.event);
1443                        clock_ref = test_clock.borrow_mut();
1444                    }
1445                }
1446            }
1447
1448            // Check the throttler rate limits on the appropriate conditions
1449            // * At least one message is buffered
1450            // * Timestamp queue is filled upto limit
1451            // * Least recent timestamp in queue exceeds interval
1452            let buffered_messages = throttler.qsize() > 0;
1453            let now = throttler.clock.borrow().timestamp_ns().as_u64();
1454            let limit_filled_within_interval = throttler
1455                .timestamps
1456                .get(throttler.limit() - 1)
1457                .is_some_and(|&ts| (now - ts.as_u64()) < interval);
1458            let expected_limiting = buffered_messages && limit_filled_within_interval;
1459            assert_eq!(throttler.is_limiting, expected_limiting);
1460
1461            // Message conservation
1462            assert_eq!(sent_count, throttler.sent_count + throttler.qsize());
1463        }
1464
1465        // Drain all buffered messages by repeatedly advancing the clock.
1466        // Each timer callback may send up to `limit` messages and schedule
1467        // a new timer for the next batch, so we must keep advancing.
1468        for i in 1..=100u64 {
1469            if throttler.qsize() == 0 {
1470                break;
1471            }
1472            let advance_to = interval * 100 * i;
1473            let time_events = test_clock
1474                .borrow_mut()
1475                .advance_time(advance_to.into(), true);
1476            let mut clock_ref = test_clock.borrow_mut();
1477            for each_event in clock_ref.match_handlers(time_events) {
1478                drop(clock_ref);
1479                each_event.callback.call(each_event.event);
1480                clock_ref = test_clock.borrow_mut();
1481            }
1482        }
1483        assert_eq!(throttler.qsize(), 0);
1484    }
1485
1486    #[rstest]
1487    fn prop_test() {
1488        // Create a fresh throttler for each iteration to ensure clean state,
1489        // even when tests panic (which would skip the reset code)
1490        proptest!(|(inputs in throttler_test_strategy())| {
1491            let test_throttler = test_throttler_buffered();
1492            test_throttler_with_inputs(inputs, &test_throttler);
1493        });
1494    }
1495
1496    #[rstest]
1497    fn prop_test_dropping() {
1498        // Drop-mode coverage: every received message is either sent or dropped,
1499        // and sent_count tracks the send callback exactly. Catches conservation
1500        // violations and panics under random send/advance sequences.
1501        proptest!(|(inputs in throttler_test_strategy())| {
1502            let clock = Rc::new(RefCell::new(TestClock::new()));
1503            let sent: Rc<RefCell<usize>> = Rc::new(RefCell::new(0));
1504            let dropped: Rc<RefCell<usize>> = Rc::new(RefCell::new(0));
1505
1506            let sent_cb = {
1507                let sent = Rc::clone(&sent);
1508                Box::new(move |_| *sent.borrow_mut() += 1) as Box<dyn Fn(u64)>
1509            };
1510            let drop_cb = {
1511                let dropped = Rc::clone(&dropped);
1512                Box::new(move |_| *dropped.borrow_mut() += 1) as Box<dyn Fn(u64)>
1513            };
1514
1515            let interval = 10u64;
1516            let throttler = Throttler::new(
1517                RateLimit::new(5, interval),
1518                Rc::clone(&clock) as Rc<RefCell<dyn Clock>>,
1519                "prop_drop_timer",
1520                sent_cb,
1521                Some(drop_cb),
1522                Ustr::from(UUID4::new().as_str()),
1523            )
1524            .to_actor();
1525            let throttler = access_shared(&throttler);
1526
1527            for input in inputs {
1528                match input {
1529                    ThrottlerInput::SendMessage(msg) => {
1530                        throttler.send(msg);
1531                    }
1532                    ThrottlerInput::AdvanceClock(duration) => {
1533                        let mut clock_ref = clock.borrow_mut();
1534                        let current_time = clock_ref.get_time_ns();
1535                        let time_events =
1536                            clock_ref.advance_time(current_time + u64::from(duration), true);
1537                        for each_event in clock_ref.match_handlers(time_events) {
1538                            drop(clock_ref);
1539                            each_event.callback.call(each_event.event);
1540                            clock_ref = clock.borrow_mut();
1541                        }
1542                    }
1543                }
1544
1545                let sent_now = *sent.borrow();
1546                let dropped_now = *dropped.borrow();
1547                // Conservation: every received message was sent or dropped.
1548                assert_eq!(sent_now + dropped_now, throttler.recv_count);
1549                assert_eq!(throttler.sent_count, sent_now);
1550                assert!(throttler.qsize() == 0, "drop mode must never buffer");
1551            }
1552        });
1553    }
1554
1555    #[derive(Clone, Debug)]
1556    enum ThrottlerReserveInput {
1557        SendMessage(u64),
1558        TryReserve(u8),
1559        AdvanceClock(u8),
1560    }
1561
1562    fn throttler_reserve_input_strategy() -> impl Strategy<Value = ThrottlerReserveInput> {
1563        prop_oneof![
1564            2 => prop::bool::ANY.prop_map(|_| ThrottlerReserveInput::SendMessage(42)),
1565            2 => prop::num::u8::ANY.prop_map(|v| ThrottlerReserveInput::TryReserve(v % 6 + 1)),
1566            6 => prop::num::u8::ANY.prop_map(|v| ThrottlerReserveInput::AdvanceClock(v % 5 + 5)),
1567        ]
1568    }
1569
1570    #[rstest]
1571    fn prop_test_try_reserve_interleaved() {
1572        // Mixing try_reserve with sends: conservation must hold across every
1573        // interleaving, and the buffer must always drain eventually (a
1574        // rejected reservation arms only the resume timer, which must hand off
1575        // to the drain handler when messages are buffered).
1576        proptest!(|(inputs in prop::collection::vec(throttler_reserve_input_strategy(), 10..=150))| {
1577            let test_throttler = test_throttler_buffered();
1578            let test_clock = test_throttler.clock.clone();
1579            let interval = test_throttler.interval;
1580            let throttler = test_throttler.get_throttler();
1581            let mut attempted = 0usize;
1582            let mut reserved = 0usize;
1583
1584            for input in inputs {
1585                match input {
1586                    ThrottlerReserveInput::SendMessage(msg) => {
1587                        throttler.send(msg);
1588                        attempted += 1;
1589                    }
1590                    ThrottlerReserveInput::TryReserve(n) => {
1591                        if throttler.try_reserve(usize::from(n)) {
1592                            reserved += usize::from(n);
1593                        }
1594                    }
1595                    ThrottlerReserveInput::AdvanceClock(duration) => {
1596                        let mut clock_ref = test_clock.borrow_mut();
1597                        let current_time = clock_ref.get_time_ns();
1598                        let time_events =
1599                            clock_ref.advance_time(current_time + u64::from(duration), true);
1600                        for each_event in clock_ref.match_handlers(time_events) {
1601                            drop(clock_ref);
1602                            each_event.callback.call(each_event.event);
1603                            clock_ref = test_clock.borrow_mut();
1604                        }
1605                    }
1606                }
1607
1608                assert_eq!(throttler.sent_count + throttler.qsize(), attempted + reserved);
1609            }
1610
1611            for i in 1..=100u64 {
1612                if throttler.qsize() == 0 {
1613                    break;
1614                }
1615                let advance_to = interval * 100 * i;
1616                let time_events = test_clock
1617                    .borrow_mut()
1618                    .advance_time(advance_to.into(), true);
1619                let mut clock_ref = test_clock.borrow_mut();
1620                for each_event in clock_ref.match_handlers(time_events) {
1621                    drop(clock_ref);
1622                    each_event.callback.call(each_event.event);
1623                    clock_ref = test_clock.borrow_mut();
1624                }
1625            }
1626            assert_eq!(throttler.qsize(), 0);
1627        });
1628    }
1629
1630    #[rstest]
1631    fn test_throttler_process_id_returns_ustr() {
1632        // This test verifies that ThrottlerProcess::id() correctly returns Ustr
1633        // by dereferencing MStr<Endpoint> (tests *self.endpoint -> Ustr conversion)
1634        let actor_id = Ustr::from("test_throttler");
1635        let process = ThrottlerProcess::<String, fn(String)>::new(actor_id);
1636
1637        // Call id() which does *self.endpoint
1638        let handler_id: Ustr = process.id();
1639
1640        // Verify it's a valid Ustr with expected format
1641        assert!(handler_id.as_str().contains("test_throttler_process"));
1642        assert!(!handler_id.is_empty());
1643
1644        // Verify type - this wouldn't compile if id() didn't return Ustr
1645        let _type_check: Ustr = handler_id;
1646    }
1647}