Skip to main content

nautilus_live/
runner.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Async event loop runner for live and sandbox trading nodes.
17//!
18//! `AsyncRunner` owns seven tokio mpsc channel pairs plus a shutdown
19//! signal channel. Construction creates the channels without side
20//! effects. The sender halves are placed into thread-local storage
21//! via [`AsyncRunner::bind_senders`] so that adapters and engine
22//! components can resolve them through the `get_*_sender()` accessors
23//! in `nautilus_common::runner` and `nautilus_common::live::runner`.
24//!
25//! Channel pairs:
26//!
27//! - **Time events**: timer callbacks dispatched by the clock.
28//! - **System events**: system notifications handled by the live node.
29//! - **System commands**: control requests handled by the live node.
30//! - **Execution events**: fills, order updates, and account state from
31//!   execution clients to the execution engine.
32//! - **Trading commands**: deferred order actions routed to their direct endpoint.
33//! - **Data events**: market data from adapters to the data engine.
34//! - **Data commands**: subscribe/unsubscribe requests to data clients.
35//!
36//! Both `AsyncRunner::run` and `LiveNode::run` use a `biased;` select with
37//! system and execution branches polled ahead of data branches. Within each
38//! channel pair, events are polled before commands.
39//!
40//! The runner can drive the event loop in two ways:
41//!
42//! - **Standalone**: call [`AsyncRunner::run`], which binds senders and
43//!   enters a `tokio::select!` loop internally.
44//! - **Integrated**: call [`AsyncRunner::take_channels`] to extract the
45//!   receivers and run the `select!` loop directly inside `LiveNode::run`,
46//!   where it is interleaved with startup, reconciliation, and shutdown
47//!   phases.
48//!
49//! # Invariants
50//!
51//! - `bind_senders` must be called before any code that reads from TLS.
52//!   This includes adapter constructors, clock initialization, and
53//!   execution client start methods. Every path from construction to
54//!   the event loop must bind before the first TLS read.
55//! - The event loop and all TLS consumers must execute on the same
56//!   thread. Senders are cloneable and `Send`, but the `RefCell`-backed
57//!   TLS slots are not accessible from other threads.
58//! - Only one runner at a time should own the TLS slots on a given
59//!   thread. `bind_senders` overwrites any existing TLS contents on the
60//!   thread, so the last caller wins.
61
62use std::{
63    fmt::Debug,
64    sync::Arc,
65    thread::{self, ThreadId},
66};
67
68use nautilus_common::{
69    live::{
70        dispatch::DispatchMessage,
71        runner::{
72            replace_data_event_sender, replace_exec_event_sender, replace_system_command_sender,
73            replace_system_event_sender,
74        },
75        sender::{DispatchSender, EventSender},
76    },
77    messages::{
78        DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent, data::DataCommand,
79        execution::TradingCommand,
80    },
81    msgbus::{self, MessagingSwitchboard},
82    runner::{
83        DataCommandSender, TimeEventMessage, TimeEventSender, TradingCommandMessage,
84        TradingCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
85        replace_time_event_sender,
86    },
87};
88use nautilus_model::events::OrderEventAny;
89
90use crate::dispatch::drain_callbacks;
91#[cfg(feature = "node")]
92use crate::node::{LiveNodeHandle, NodeState};
93
94/// Asynchronous implementation of `DataCommandSender` for live environments.
95#[derive(Debug)]
96pub struct AsyncDataCommandSender {
97    cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>,
98    owner: ThreadId,
99    #[cfg(feature = "node")]
100    node_handle: Option<LiveNodeHandle>,
101}
102
103impl AsyncDataCommandSender {
104    /// Creates a sender owned by the calling thread.
105    ///
106    /// Construct it on the runtime thread so command sends capture callback roots.
107    #[must_use]
108    pub fn new(cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>) -> Self {
109        Self {
110            cmd_tx,
111            owner: thread::current().id(),
112            #[cfg(feature = "node")]
113            node_handle: None,
114        }
115    }
116}
117
118impl DataCommandSender for AsyncDataCommandSender {
119    fn execute(&self, command: DataCommand) {
120        if let Err(e) = self.cmd_tx.send(DispatchMessage::new(command, self.owner)) {
121            // Disposal releases retained subscriptions after the node drops its receivers
122            #[cfg(feature = "node")]
123            if node_stopped(self.node_handle.as_ref()) {
124                return;
125            }
126
127            log::error!("Failed to send data command: {e}");
128        }
129    }
130}
131
132/// Asynchronous implementation of `TimeEventSender` for live environments.
133#[derive(Debug, Clone)]
134pub struct AsyncTimeEventSender {
135    time_tx: DispatchSender<TimeEventMessage>,
136    #[cfg(feature = "node")]
137    node_handle: Option<LiveNodeHandle>,
138}
139
140impl AsyncTimeEventSender {
141    /// Creates a sender owned by the calling runtime thread.
142    #[must_use]
143    pub fn new(
144        time_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TimeEventMessage>>,
145    ) -> Self {
146        Self {
147            time_tx: DispatchSender::new(time_tx),
148            #[cfg(feature = "node")]
149            node_handle: None,
150        }
151    }
152}
153
154impl TimeEventSender for AsyncTimeEventSender {
155    fn send(&self, message: TimeEventMessage) {
156        if let Err(e) = self.time_tx.send(message) {
157            #[cfg(feature = "node")]
158            if node_stopped(self.node_handle.as_ref()) {
159                return;
160            }
161
162            log::error!("Failed to send time event message: {e}");
163        }
164    }
165}
166
167/// Asynchronous implementation of `TradingCommandSender` for live environments.
168#[derive(Debug)]
169pub struct AsyncTradingCommandSender {
170    cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
171    owner: ThreadId,
172}
173
174impl AsyncTradingCommandSender {
175    /// Creates a sender owned by the calling thread.
176    ///
177    /// Construct it on the runtime thread so command sends capture callback roots.
178    #[must_use]
179    pub fn new(
180        cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
181    ) -> Self {
182        Self {
183            cmd_tx,
184            owner: thread::current().id(),
185        }
186    }
187}
188
189impl TradingCommandSender for AsyncTradingCommandSender {
190    fn execute(&self, message: TradingCommandMessage) {
191        if let Err(e) = self.cmd_tx.send(DispatchMessage::new(message, self.owner)) {
192            log::error!("Failed to send trading command: {e}");
193        }
194    }
195}
196
197pub trait Runner {
198    fn run(&mut self);
199}
200
201/// Channel receivers for the async event loop.
202///
203/// These can be extracted from `AsyncRunner` via `take_channels()` to drive
204/// the event loop directly on the same thread as the msgbus endpoints.
205#[derive(Debug)]
206pub struct AsyncRunnerChannels {
207    pub time_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
208    pub system_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
209    pub system_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
210    pub exec_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
211    pub exec_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
212    pub data_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
213    pub data_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
214}
215
216#[cfg(feature = "node")]
217#[allow(
218    clippy::large_enum_variant,
219    reason = "runner events are consumed immediately; boxing would add routing allocations"
220)]
221pub(crate) enum PendingRunnerEvent {
222    TimeEvent(DispatchMessage<TimeEventMessage>),
223    SystemEvent(DispatchMessage<SystemEvent>),
224    SystemCommand(DispatchMessage<SystemCommand>),
225    ExecEvent(DispatchMessage<ExecutionEvent>),
226    ExecCommand(DispatchMessage<TradingCommandMessage>),
227    DataEvent(DispatchMessage<DataEvent>),
228    DataCommand(DispatchMessage<DataCommand>),
229}
230
231pub struct AsyncRunner {
232    channels: AsyncRunnerChannels,
233    time_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TimeEventMessage>>,
234    system_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<SystemEvent>>,
235    system_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<SystemCommand>>,
236    signal_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
237    signal_tx: tokio::sync::mpsc::UnboundedSender<()>,
238    exec_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<ExecutionEvent>>,
239    exec_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
240    data_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataEvent>>,
241    data_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>,
242}
243
244/// Handle for stopping the `AsyncRunner` from another context.
245#[derive(Clone, Debug)]
246pub struct AsyncRunnerHandle {
247    signal_tx: tokio::sync::mpsc::UnboundedSender<()>,
248}
249
250impl AsyncRunnerHandle {
251    /// Signals the runner to stop.
252    pub fn stop(&self) {
253        if let Err(e) = self.signal_tx.send(()) {
254            log::error!("Failed to send shutdown signal: {e}");
255        }
256    }
257}
258
259impl Default for AsyncRunner {
260    fn default() -> Self {
261        Self::new()
262    }
263}
264
265impl Debug for AsyncRunner {
266    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
267        f.debug_struct(stringify!(AsyncRunner)).finish()
268    }
269}
270
271impl AsyncRunner {
272    /// Creates a new [`AsyncRunner`] instance.
273    ///
274    /// Creates channels but does not bind senders to thread-local storage.
275    /// Call [`bind_senders`](Self::bind_senders) before creating clients that
276    /// read from TLS, and again before entering the event loop.
277    #[must_use]
278    #[rustfmt::skip]
279    pub fn new() -> Self {
280        use tokio::sync::mpsc::unbounded_channel; // tokio-import-ok
281
282        let (time_evt_tx, time_evt_rx) = unbounded_channel::<DispatchMessage<TimeEventMessage>>();
283        let (system_evt_tx, system_evt_rx) = unbounded_channel::<DispatchMessage<SystemEvent>>();
284        let (system_cmd_tx, system_cmd_rx) = unbounded_channel::<DispatchMessage<SystemCommand>>();
285        let (signal_tx, signal_rx) = unbounded_channel::<()>();
286        let (exec_evt_tx, exec_evt_rx) = unbounded_channel::<DispatchMessage<ExecutionEvent>>();
287        let (exec_cmd_tx, exec_cmd_rx) = unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
288        let (data_evt_tx, data_evt_rx) = unbounded_channel::<DispatchMessage<DataEvent>>();
289        let (data_cmd_tx, data_cmd_rx) = unbounded_channel::<DispatchMessage<DataCommand>>();
290
291        Self {
292            channels: AsyncRunnerChannels {
293                time_evt_rx,
294                system_evt_rx,
295                system_cmd_rx,
296                exec_evt_rx,
297                exec_cmd_rx,
298                data_evt_rx,
299                data_cmd_rx,
300            },
301            time_evt_tx,
302            system_evt_tx,
303            system_cmd_tx,
304            signal_rx,
305            signal_tx,
306            exec_evt_tx,
307            exec_cmd_tx,
308            data_evt_tx,
309            data_cmd_tx,
310        }
311    }
312
313    /// Binds this runner's channel senders to thread-local storage.
314    ///
315    /// Call before creating clients that read from TLS (e.g., in the builder),
316    /// and again before entering the event loop to reclaim ownership if another
317    /// runner was constructed on this thread in the interim.
318    pub fn bind_senders(&self) {
319        self.bind_senders_with_data_sender(AsyncDataCommandSender::new(self.data_cmd_tx.clone()));
320    }
321
322    #[cfg(feature = "node")]
323    pub(crate) fn bind_senders_for_node(&self, handle: LiveNodeHandle) {
324        self.bind_senders_with_data_sender(AsyncDataCommandSender {
325            cmd_tx: self.data_cmd_tx.clone(),
326            owner: thread::current().id(),
327            node_handle: Some(handle),
328        });
329    }
330
331    #[rustfmt::skip]
332    fn bind_senders_with_data_sender(&self, sender: AsyncDataCommandSender) {
333        replace_time_event_sender(Arc::new(AsyncTimeEventSender {
334            time_tx: DispatchSender::new(self.time_evt_tx.clone()),
335            #[cfg(feature = "node")]
336            node_handle: sender.node_handle.clone(),
337        }));
338        replace_system_event_sender(EventSender::new(self.system_evt_tx.clone()));
339        replace_system_command_sender(DispatchSender::new(self.system_cmd_tx.clone()));
340        replace_exec_event_sender(EventSender::new(self.exec_evt_tx.clone()));
341        replace_exec_cmd_sender(Arc::new(AsyncTradingCommandSender::new(self.exec_cmd_tx.clone())));
342        replace_data_event_sender(EventSender::new(self.data_evt_tx.clone()));
343        replace_data_cmd_sender(Arc::new(sender));
344    }
345
346    /// Stops the runner with an internal shutdown signal.
347    pub fn stop(&self) {
348        if let Err(e) = self.signal_tx.send(()) {
349            log::error!("Failed to send shutdown signal: {e}");
350        }
351    }
352
353    /// Returns a handle that can be used to stop the runner from another context.
354    #[must_use]
355    pub fn handle(&self) -> AsyncRunnerHandle {
356        AsyncRunnerHandle {
357            signal_tx: self.signal_tx.clone(),
358        }
359    }
360
361    /// Consumes the runner and returns the channel receivers for direct event loop driving.
362    ///
363    /// This is used when the event loop needs to run on the same thread as the msgbus
364    /// endpoints (which use thread-local storage).
365    #[must_use]
366    pub fn take_channels(self) -> AsyncRunnerChannels {
367        self.channels
368    }
369
370    /// Flushes all pending data events and commands from the channels.
371    ///
372    /// Loops until both data channels are empty, processing each item
373    /// into the cache immediately. Used in `start()` where channels are
374    /// not extracted.
375    pub fn flush_pending_data(&mut self) {
376        let mut total = 0;
377
378        loop {
379            let mut progressed = false;
380
381            // Events drain before commands here even though the runtime select
382            // prefers the opposite for everything-else: `LiveNode::start()`
383            // calls this after `connect_data_clients()` to push queued
384            // `DataEvent::Instrument` items into the cache. A pending
385            // subscription command (e.g. `SubscribeBars`) processed before the
386            // matching instrument lands would be rejected by the data engine.
387            while let Ok(evt) = self.channels.data_evt_rx.try_recv() {
388                Self::dispatch_data_event(evt);
389                progressed = true;
390                total += 1;
391            }
392
393            while let Ok(cmd) = self.channels.data_cmd_rx.try_recv() {
394                Self::handle_data_command(cmd);
395                progressed = true;
396                total += 1;
397            }
398
399            if !progressed {
400                break;
401            }
402        }
403
404        if total > 0 {
405            log::debug!("Flushed {total} pending data events/commands");
406        }
407    }
408
409    #[cfg(feature = "node")]
410    pub(crate) fn drain_pending_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
411        let mut events = Vec::new();
412
413        while let Ok(event) = self.channels.system_evt_rx.try_recv() {
414            events.push(event);
415        }
416
417        events
418    }
419
420    #[cfg(feature = "node")]
421    pub(crate) fn drain_pending_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
422        let mut commands = Vec::new();
423
424        while let Ok(command) = self.channels.system_cmd_rx.try_recv() {
425            commands.push(command);
426        }
427
428        commands
429    }
430
431    /// Runs the async runner event loop.
432    ///
433    /// This method processes time, system, execution, and data events in an async loop.
434    /// It will run until a signal is received or the event streams are closed.
435    ///
436    /// # Errors
437    ///
438    /// Returns an error if callback dispatch fails. Pending channel messages and the failure latch
439    /// are retained.
440    pub async fn run(&mut self) -> anyhow::Result<()> {
441        self.bind_senders();
442
443        log::info!("AsyncRunner starting");
444
445        loop {
446            let callbacks_pending = drain_callbacks().await?;
447
448            tokio::select! {
449                biased;
450
451                Some(()) = self.signal_rx.recv() => {
452                    log::info!("AsyncRunner received signal, shutting down");
453                    return Ok(());
454                },
455                () = std::future::ready(()), if callbacks_pending => {},
456                Some(handler) = self.channels.time_evt_rx.recv() => {
457                    let _ = Self::handle_time_event(handler);
458                },
459                Some(event) = self.channels.system_evt_rx.recv() => {
460                    log::error!("System event {event} requires the LiveNode runner");
461                },
462                Some(command) = self.channels.system_cmd_rx.recv() => {
463                    log::error!("System command {command} requires the LiveNode runner");
464                },
465                Some(evt) = self.channels.exec_evt_rx.recv() => {
466                    Self::dispatch_exec_event(evt);
467                },
468                Some(cmd) = self.channels.exec_cmd_rx.recv() => {
469                    Self::handle_trading_command(cmd);
470                },
471                Some(evt) = self.channels.data_evt_rx.recv() => {
472                    Self::dispatch_data_event(evt);
473                },
474                Some(cmd) = self.channels.data_cmd_rx.recv() => {
475                    Self::handle_data_command(cmd);
476                },
477                else => {
478                    log::debug!("AsyncRunner all channels closed, exiting");
479                    return Ok(());
480                }
481            };
482        }
483    }
484
485    /// Handles a time event under its originating callback root.
486    ///
487    /// # Panics
488    ///
489    /// Panics if a rooted message is processed outside its owner thread.
490    #[inline]
491    #[must_use]
492    pub fn handle_time_event(message: DispatchMessage<TimeEventMessage>) -> bool {
493        message.dispatch(TimeEventMessage::dispatch)
494    }
495
496    /// Handles a data command under its captured callback root by sending to the `DataEngine`.
497    ///
498    /// # Panics
499    ///
500    /// Panics if a rooted command is processed outside its owner thread.
501    #[inline]
502    pub fn handle_data_command(cmd: DispatchMessage<DataCommand>) {
503        cmd.dispatch(|cmd| {
504            msgbus::send_data_command(MessagingSwitchboard::data_engine_execute(), cmd);
505        });
506    }
507
508    /// Dispatches a received data event under its originating callback root.
509    ///
510    /// # Panics
511    ///
512    /// Panics if a rooted message is processed outside its owner thread.
513    pub fn dispatch_data_event(event: DispatchMessage<DataEvent>) {
514        event.dispatch(Self::handle_data_event);
515    }
516
517    /// Dispatches a received execution event under its originating callback root.
518    ///
519    /// # Panics
520    ///
521    /// Panics if a rooted message is processed outside its owner thread.
522    pub fn dispatch_exec_event(event: DispatchMessage<ExecutionEvent>) {
523        event.dispatch(Self::handle_exec_event);
524    }
525
526    /// Handles a data event by sending to the appropriate `DataEngine` endpoint.
527    #[inline]
528    pub fn handle_data_event(event: DataEvent) {
529        match event {
530            DataEvent::Data(data) => {
531                msgbus::send_data(MessagingSwitchboard::data_engine_process_data(), data);
532            }
533            DataEvent::Instrument(data) => {
534                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &data);
535            }
536            DataEvent::Response(resp) => {
537                msgbus::send_data_response(MessagingSwitchboard::data_engine_response(), resp);
538            }
539            DataEvent::FundingRate(funding_rate) => {
540                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &funding_rate);
541            }
542            DataEvent::InstrumentStatus(status) => {
543                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &status);
544            }
545            DataEvent::OptionGreeks(greeks) => {
546                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &greeks);
547            }
548            #[cfg(feature = "defi")]
549            DataEvent::DeFi(data) => {
550                msgbus::send_defi_data(MessagingSwitchboard::data_engine_process_defi_data(), data);
551            }
552            #[cfg(feature = "defi")]
553            DataEvent::PoolSnapshotResponse(response) => {
554                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &response);
555            }
556            #[cfg(not(feature = "defi"))]
557            #[allow(
558                unreachable_patterns,
559                reason = "DeFi variants can exist without this crate's defi feature"
560            )]
561            other => log::error!(
562                "Cannot handle {other} data event, nautilus-live built without its `defi` feature"
563            ),
564        }
565    }
566
567    /// Dispatches an internal execution command directly to the execution engine.
568    #[inline]
569    pub fn handle_exec_command(cmd: TradingCommand) {
570        msgbus::send_trading_command(MessagingSwitchboard::exec_engine_execute(), cmd);
571    }
572
573    /// Dispatches a trading command and its deferred children under their captured callback roots.
574    ///
575    /// # Panics
576    ///
577    /// Panics if a rooted command is processed outside its owner thread.
578    #[inline]
579    pub fn handle_trading_command(message: DispatchMessage<TradingCommandMessage>) {
580        message.dispatch_trading(|_| {});
581    }
582
583    /// Handles an execution event by sending to the appropriate engine endpoint.
584    #[inline]
585    pub fn handle_exec_event(event: ExecutionEvent) {
586        match event {
587            ExecutionEvent::Order(order_event) => {
588                msgbus::send_order_event(MessagingSwitchboard::exec_engine_process(), order_event);
589            }
590            ExecutionEvent::OrderSubmittedBatch(batch) => {
591                for submitted in batch {
592                    msgbus::send_order_event(
593                        MessagingSwitchboard::exec_engine_process(),
594                        OrderEventAny::Submitted(submitted),
595                    );
596                }
597            }
598            ExecutionEvent::OrderAcceptedBatch(batch) => {
599                for accepted in batch {
600                    msgbus::send_order_event(
601                        MessagingSwitchboard::exec_engine_process(),
602                        OrderEventAny::Accepted(accepted),
603                    );
604                }
605            }
606            ExecutionEvent::OrderCanceledBatch(batch) => {
607                for canceled in batch {
608                    msgbus::send_order_event(
609                        MessagingSwitchboard::exec_engine_process(),
610                        OrderEventAny::Canceled(canceled),
611                    );
612                }
613            }
614            ExecutionEvent::Report(report) => {
615                Self::handle_exec_report(report);
616            }
617            ExecutionEvent::Account(ref account) => {
618                msgbus::send_account_state(
619                    MessagingSwitchboard::portfolio_update_account(),
620                    account,
621                );
622            }
623        }
624    }
625
626    #[inline]
627    pub fn handle_exec_report(report: ExecutionReport) {
628        let endpoint = MessagingSwitchboard::exec_engine_reconcile_execution_report();
629        msgbus::send_execution_report(endpoint, report);
630    }
631}
632
633#[cfg(feature = "node")]
634impl AsyncRunner {
635    pub(crate) fn poll_pending(&mut self, mut process: impl FnMut(PendingRunnerEvent)) -> usize {
636        self.bind_senders();
637
638        let pending = (
639            self.channels.time_evt_rx.len(),
640            self.channels.system_evt_rx.len(),
641            self.channels.system_cmd_rx.len(),
642            self.channels.exec_evt_rx.len(),
643            self.channels.exec_cmd_rx.len(),
644            self.channels.data_evt_rx.len(),
645            self.channels.data_cmd_rx.len(),
646        );
647
648        let mut processed = 0;
649        processed += poll_channel(
650            &mut self.channels.time_evt_rx,
651            pending.0,
652            PendingRunnerEvent::TimeEvent,
653            &mut process,
654        );
655        processed += poll_channel(
656            &mut self.channels.system_evt_rx,
657            pending.1,
658            PendingRunnerEvent::SystemEvent,
659            &mut process,
660        );
661        processed += poll_channel(
662            &mut self.channels.system_cmd_rx,
663            pending.2,
664            PendingRunnerEvent::SystemCommand,
665            &mut process,
666        );
667        processed += poll_channel(
668            &mut self.channels.exec_evt_rx,
669            pending.3,
670            PendingRunnerEvent::ExecEvent,
671            &mut process,
672        );
673        processed += poll_channel(
674            &mut self.channels.exec_cmd_rx,
675            pending.4,
676            PendingRunnerEvent::ExecCommand,
677            &mut process,
678        );
679        processed += poll_channel(
680            &mut self.channels.data_evt_rx,
681            pending.5,
682            PendingRunnerEvent::DataEvent,
683            &mut process,
684        );
685        processed += poll_channel(
686            &mut self.channels.data_cmd_rx,
687            pending.6,
688            PendingRunnerEvent::DataCommand,
689            &mut process,
690        );
691        processed
692    }
693
694    pub(crate) async fn recv(&mut self) -> Option<PendingRunnerEvent> {
695        tokio::select! {
696            biased;
697
698            Some(message) = self.channels.time_evt_rx.recv() => {
699                Some(PendingRunnerEvent::TimeEvent(message))
700            }
701            Some(event) = self.channels.system_evt_rx.recv() => {
702                Some(PendingRunnerEvent::SystemEvent(event))
703            }
704            Some(command) = self.channels.system_cmd_rx.recv() => {
705                Some(PendingRunnerEvent::SystemCommand(command))
706            }
707            Some(event) = self.channels.exec_evt_rx.recv() => {
708                Some(PendingRunnerEvent::ExecEvent(event))
709            }
710            Some(command) = self.channels.exec_cmd_rx.recv() => {
711                Some(PendingRunnerEvent::ExecCommand(command))
712            }
713            Some(event) = self.channels.data_evt_rx.recv() => {
714                Some(PendingRunnerEvent::DataEvent(event))
715            }
716            Some(command) = self.channels.data_cmd_rx.recv() => {
717                Some(PendingRunnerEvent::DataCommand(command))
718            }
719            else => None,
720        }
721    }
722}
723
724#[cfg(feature = "node")]
725fn node_stopped(handle: Option<&LiveNodeHandle>) -> bool {
726    handle.is_some_and(|handle| handle.state() == NodeState::Stopped)
727}
728
729#[cfg(feature = "node")]
730fn poll_channel<T>(
731    receiver: &mut tokio::sync::mpsc::UnboundedReceiver<T>,
732    pending: usize,
733    event: impl Fn(T) -> PendingRunnerEvent,
734    process: &mut impl FnMut(PendingRunnerEvent),
735) -> usize {
736    let mut processed = 0;
737
738    for _ in 0..pending {
739        let Ok(message) = receiver.try_recv() else {
740            break;
741        };
742
743        process(event(message));
744        processed += 1;
745    }
746
747    processed
748}
749
750#[cfg(test)]
751mod tests {
752    #[cfg(not(all(feature = "simulation", madsim)))]
753    use std::num::NonZeroU64;
754    use std::{cell::RefCell, rc::Rc, sync::Arc, time::Duration};
755
756    #[cfg(not(all(feature = "simulation", madsim)))]
757    use nautilus_common::live::LiveTimer;
758    use nautilus_common::{
759        actor,
760        cache::Cache,
761        clock::VirtualClock,
762        live::{
763            dispatch::DispatchMessage,
764            runner::{
765                get_data_event_sender, get_exec_event_sender, get_system_command_sender,
766                get_system_event_sender, try_get_system_command_sender,
767                try_get_system_event_sender,
768            },
769        },
770        messages::{
771            ExecutionEvent, ExecutionReport,
772            data::{SubscribeCommand, SubscribeCustomData},
773            execution::{CancelAllOrders, QueryAccount, TradingCommand},
774            system::{ReconnectSocket, SocketState, SocketStateChange},
775        },
776        msgbus::{TypedIntoHandler, stubs::get_typed_into_message_saving_handler},
777        runner::{
778            SyncTradingCommandSender, TimeEventMessage, drain_trading_cmd_queue,
779            get_data_cmd_sender, get_time_event_sender, get_trading_cmd_sender,
780            replace_exec_cmd_sender, try_get_time_event_sender, try_get_trading_cmd_sender,
781        },
782        timer::{TimeEvent, TimeEventCallback},
783    };
784    #[cfg(not(all(feature = "simulation", madsim)))]
785    use nautilus_core::time::get_atomic_clock_realtime;
786    use nautilus_core::{UUID4, UnixNanos};
787    use nautilus_execution::engine::ExecutionEngine;
788    use nautilus_model::{
789        data::{Data, DataType, quote::QuoteTick},
790        enums::{
791            AccountType, LiquiditySide, OrderSide, OrderStatus, OrderType, PositionSide,
792            TimeInForce,
793        },
794        events::{
795            OrderAcceptedBatch, OrderCanceledBatch, OrderEvent, OrderEventAny, OrderSubmittedBatch,
796            account::state::AccountState,
797            order::spec::{OrderAcceptedSpec, OrderCanceledSpec, OrderSubmittedSpec},
798        },
799        identifiers::{
800            AccountId, ClientId, ClientOrderId, InstrumentId, PositionId, StrategyId, TradeId,
801            TraderId, Venue, VenueOrderId,
802        },
803        reports::{FillReport, OrderStatusReport, PositionStatusReport},
804        types::{Money, Price, Quantity},
805    };
806    use rstest::rstest;
807    use ustr::Ustr;
808
809    use super::*;
810
811    #[rstest]
812    #[case(false)]
813    #[case(true)]
814    #[tokio::test]
815    async fn test_runner_callback_failure_stops_before_next_command(#[case] before_run: bool) {
816        actor::clear_callbacks().unwrap();
817        let mut runner = AsyncRunner::new();
818        let received = Rc::new(RefCell::new(Vec::new()));
819        let observed = received.clone();
820        msgbus::register_trading_command_endpoint(
821            MessagingSwitchboard::risk_engine_execute(),
822            TypedIntoHandler::from(move |command: TradingCommand| {
823                observed.borrow_mut().push(command.ts_init());
824                crate::dispatch::tests::latch_callback_failure();
825            }),
826        );
827
828        let sender = AsyncTradingCommandSender::new(runner.exec_cmd_tx.clone());
829        let seed_endpoint = ustr::Ustr::from("test.callback-root-seed");
830        msgbus::register_trading_command_endpoint(
831            seed_endpoint.into(),
832            TypedIntoHandler::from(move |command: TradingCommand| {
833                sender.execute(TradingCommandMessage::new(
834                    MessagingSwitchboard::risk_engine_execute(),
835                    command,
836                ));
837            }),
838        );
839
840        for timestamp in [17, 23] {
841            SyncTradingCommandSender.execute(TradingCommandMessage::new(
842                seed_endpoint.into(),
843                TradingCommand::QueryAccount(QueryAccount::new(
844                    "TRADER-001".into(),
845                    None,
846                    "SIM-001".into(),
847                    UUID4::new(),
848                    timestamp.into(),
849                    None,
850                    None,
851                )),
852            ));
853        }
854
855        drain_trading_cmd_queue();
856        let first = runner.channels.exec_cmd_rx.try_recv().unwrap();
857        let second = runner.channels.exec_cmd_rx.try_recv().unwrap();
858        assert!(first.is_rooted());
859        assert!(second.is_rooted());
860        runner.exec_cmd_tx.send(first).unwrap();
861        runner.exec_cmd_tx.send(second).unwrap();
862
863        if before_run {
864            crate::dispatch::tests::latch_callback_failure();
865        }
866
867        let error = runner.run().await.unwrap_err();
868
869        assert_eq!(
870            error.downcast_ref::<actor::CallbackDispatchError>(),
871            Some(&actor::CallbackDispatchError::DeliveryUnwound)
872        );
873        assert_eq!(
874            *received.borrow(),
875            if before_run {
876                vec![]
877            } else {
878                vec![UnixNanos::from(17)]
879            }
880        );
881        assert_eq!(
882            runner.channels.exec_cmd_rx.len(),
883            if before_run { 2 } else { 1 }
884        );
885        assert!(!runner.exec_cmd_tx.is_closed());
886        assert_eq!(
887            actor::callback_failure(),
888            Some(actor::CallbackDispatchError::DeliveryUnwound)
889        );
890        drop(runner);
891        assert_eq!(actor::clear_callbacks(), Ok(()));
892    }
893
894    #[tokio::test]
895    async fn test_runner_stop_preserves_messages_and_channel_only_scheduling() {
896        let mut runner = AsyncRunner::new();
897        let received = Rc::new(RefCell::new(Vec::new()));
898        let stop = runner.signal_tx.clone();
899
900        for timestamp in 1..=65 {
901            let received = received.clone();
902            let stop = stop.clone();
903            runner
904                .time_evt_tx
905                .send(
906                    TimeEventMessage::new(
907                        TimeEvent::new(
908                            "runner-resume".into(),
909                            UUID4::new(),
910                            timestamp.into(),
911                            timestamp.into(),
912                        ),
913                        TimeEventCallback::RustLocal(Rc::new(move |_| {
914                            received.borrow_mut().push(timestamp);
915                            if timestamp == 65 {
916                                stop.send(()).unwrap();
917                            }
918                        })),
919                    )
920                    .into(),
921                )
922                .unwrap();
923        }
924
925        stop.send(()).unwrap();
926        runner.run().await.unwrap();
927
928        assert!(received.borrow().is_empty());
929        assert_eq!(runner.channels.time_evt_rx.len(), 65);
930        assert!(!runner.time_evt_tx.is_closed());
931
932        let (result, at_yield) = tokio::join!(biased;
933            runner.run(),
934            async { received.borrow().clone() },
935        );
936        result.unwrap();
937
938        let expected: Vec<u64> = (1..=65).collect();
939        assert_eq!(at_yield, expected);
940        assert_eq!(*received.borrow(), expected);
941        assert_eq!(runner.channels.time_evt_rx.len(), 0);
942        assert!(!runner.time_evt_tx.is_closed());
943    }
944
945    // Test fixture for creating test quotes
946    fn test_quote() -> QuoteTick {
947        QuoteTick {
948            instrument_id: InstrumentId::from("EUR/USD.SIM"),
949            bid_price: Price::from("1.10000"),
950            ask_price: Price::from("1.10001"),
951            bid_size: Quantity::from(1_000_000),
952            ask_size: Quantity::from(1_000_000),
953            ts_event: UnixNanos::default(),
954            ts_init: UnixNanos::default(),
955        }
956    }
957
958    fn test_system_event() -> SystemEvent {
959        SystemEvent::SocketState(SocketStateChange::new(
960            ClientId::from("BINANCE"),
961            Some(Venue::from("BINANCE")),
962            Ustr::from("binance-futures-market-streams"),
963            SocketState::Connected,
964        ))
965    }
966
967    fn test_system_command() -> SystemCommand {
968        SystemCommand::ReconnectSocket(ReconnectSocket::new(
969            TraderId::from("TRADER-001"),
970            ClientId::from("POLYMARKET"),
971            Ustr::from("polymarket-market-streams"),
972            UnixNanos::from(3),
973        ))
974    }
975
976    // Test fixture to create AsyncRunner with manual channels.
977    // Sender halves are dummies (not connected to the test receivers) since
978    // these tests exercise the event loop, not TLS binding.
979    fn create_test_runner(
980        time_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
981        data_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
982        data_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
983        exec_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
984        exec_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
985        signal_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
986        signal_tx: tokio::sync::mpsc::UnboundedSender<()>,
987    ) -> AsyncRunner {
988        let (time_evt_tx, _) = tokio::sync::mpsc::unbounded_channel();
989        let (system_evt_tx, system_evt_rx) = tokio::sync::mpsc::unbounded_channel();
990        let (system_cmd_tx, system_cmd_rx) = tokio::sync::mpsc::unbounded_channel();
991        let (data_evt_tx, _) = tokio::sync::mpsc::unbounded_channel();
992        let (data_cmd_tx, _) = tokio::sync::mpsc::unbounded_channel();
993        let (exec_evt_tx, _) = tokio::sync::mpsc::unbounded_channel();
994        let (exec_cmd_tx, _) = tokio::sync::mpsc::unbounded_channel();
995
996        AsyncRunner {
997            channels: AsyncRunnerChannels {
998                time_evt_rx,
999                system_evt_rx,
1000                system_cmd_rx,
1001                exec_evt_rx,
1002                exec_cmd_rx,
1003                data_evt_rx,
1004                data_cmd_rx,
1005            },
1006            time_evt_tx,
1007            system_evt_tx,
1008            system_cmd_tx,
1009            exec_evt_tx,
1010            exec_cmd_tx,
1011            data_evt_tx,
1012            data_cmd_tx,
1013            signal_rx,
1014            signal_tx,
1015        }
1016    }
1017
1018    #[cfg(feature = "node")]
1019    #[rstest]
1020    fn test_poll_pending_processes_entry_snapshot_across_channels() {
1021        let (time_evt_tx, time_evt_rx) = tokio::sync::mpsc::unbounded_channel();
1022        let (data_evt_tx, data_evt_rx) = tokio::sync::mpsc::unbounded_channel();
1023        let (data_cmd_tx, data_cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1024        let (exec_evt_tx, exec_evt_rx) = tokio::sync::mpsc::unbounded_channel();
1025        let (exec_cmd_tx, exec_cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1026        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel();
1027
1028        let time_event = TimeEvent::new(
1029            Ustr::from("test"),
1030            UUID4::new(),
1031            UnixNanos::from(1),
1032            UnixNanos::from(2),
1033        );
1034        time_evt_tx
1035            .send(
1036                (TimeEventMessage::new(time_event, TimeEventCallback::from(|_: TimeEvent| {})))
1037                    .into(),
1038            )
1039            .unwrap();
1040        exec_evt_tx
1041            .send(
1042                (ExecutionEvent::Order(OrderEventAny::Submitted(
1043                    OrderSubmittedSpec::builder()
1044                        .client_order_id(ClientOrderId::from("O-POLL-001"))
1045                        .build(),
1046                )))
1047                .into(),
1048            )
1049            .unwrap();
1050        exec_cmd_tx
1051            .send(
1052                TradingCommandMessage::new(
1053                    MessagingSwitchboard::exec_engine_execute(),
1054                    TradingCommand::CancelAllOrders(CancelAllOrders::new(
1055                        TraderId::from("TRADER-001"),
1056                        None,
1057                        StrategyId::from("S-POLL-001"),
1058                        InstrumentId::from("EUR/USD.SIM"),
1059                        Some(OrderSide::Buy),
1060                        UUID4::new(),
1061                        UnixNanos::from(3),
1062                        None,
1063                        None,
1064                    )),
1065                )
1066                .into(),
1067            )
1068            .unwrap();
1069        data_evt_tx
1070            .send((DataEvent::Data(Data::Quote(test_quote()))).into())
1071            .unwrap();
1072        data_cmd_tx
1073            .send(
1074                DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1075                    client_id: Some(ClientId::from("POLL")),
1076                    venue: None,
1077                    data_type: DataType::new("QuoteTick", None, None),
1078                    command_id: UUID4::new(),
1079                    ts_init: UnixNanos::from(4),
1080                    correlation_id: None,
1081                    params: None,
1082                }))
1083                .into(),
1084            )
1085            .unwrap();
1086
1087        let mut runner = create_test_runner(
1088            time_evt_rx,
1089            data_evt_rx,
1090            data_cmd_rx,
1091            exec_evt_rx,
1092            exec_cmd_rx,
1093            signal_rx,
1094            signal_tx,
1095        );
1096        runner.bind_senders();
1097        get_system_command_sender()
1098            .send(test_system_command())
1099            .unwrap();
1100        get_system_event_sender().send(test_system_event()).unwrap();
1101        get_system_event_sender().send(test_system_event()).unwrap();
1102        let mut processed_by_channel = [0; 7];
1103        let mut processed_order = Vec::new();
1104
1105        let first = runner.poll_pending(|event| match event {
1106            PendingRunnerEvent::TimeEvent(_) => {
1107                processed_by_channel[0] += 1;
1108                processed_order.push("time");
1109            }
1110            PendingRunnerEvent::SystemEvent(_) => {
1111                processed_by_channel[1] += 1;
1112                processed_order.push("system_event");
1113            }
1114            PendingRunnerEvent::SystemCommand(_) => {
1115                processed_by_channel[2] += 1;
1116                processed_order.push("system_command");
1117            }
1118            PendingRunnerEvent::ExecEvent(_) => {
1119                processed_by_channel[3] += 1;
1120                processed_order.push("exec_event");
1121            }
1122            PendingRunnerEvent::ExecCommand(_) => {
1123                processed_by_channel[4] += 1;
1124                processed_order.push("exec_command");
1125            }
1126            PendingRunnerEvent::DataEvent(_) => {
1127                processed_by_channel[5] += 1;
1128                processed_order.push("data_event");
1129                data_evt_tx
1130                    .send((DataEvent::Data(Data::Quote(test_quote()))).into())
1131                    .unwrap();
1132            }
1133            PendingRunnerEvent::DataCommand(_) => {
1134                processed_by_channel[6] += 1;
1135                processed_order.push("data_command");
1136            }
1137        });
1138
1139        let second = runner.poll_pending(|event| match event {
1140            PendingRunnerEvent::DataEvent(_) => {
1141                processed_by_channel[5] += 1;
1142                processed_order.push("data_event");
1143            }
1144            _ => panic!("Unexpected runner event"),
1145        });
1146
1147        assert_eq!(first, 8);
1148        assert_eq!(second, 1);
1149        assert_eq!(processed_by_channel, [1, 2, 1, 1, 1, 2, 1]);
1150        assert_eq!(
1151            processed_order,
1152            [
1153                "time",
1154                "system_event",
1155                "system_event",
1156                "system_command",
1157                "exec_event",
1158                "exec_command",
1159                "data_event",
1160                "data_command",
1161                "data_event",
1162            ]
1163        );
1164    }
1165
1166    #[cfg(feature = "node")]
1167    #[tokio::test]
1168    async fn test_recv_processes_system_event_before_command() {
1169        let (_time_evt_tx, time_evt_rx) = tokio::sync::mpsc::unbounded_channel();
1170        let (_data_evt_tx, data_evt_rx) = tokio::sync::mpsc::unbounded_channel();
1171        let (_data_cmd_tx, data_cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1172        let (_exec_evt_tx, exec_evt_rx) = tokio::sync::mpsc::unbounded_channel();
1173        let (_exec_cmd_tx, exec_cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1174        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel();
1175        let mut runner = create_test_runner(
1176            time_evt_rx,
1177            data_evt_rx,
1178            data_cmd_rx,
1179            exec_evt_rx,
1180            exec_cmd_rx,
1181            signal_rx,
1182            signal_tx,
1183        );
1184
1185        runner
1186            .system_cmd_tx
1187            .send((test_system_command()).into())
1188            .unwrap();
1189        runner
1190            .system_evt_tx
1191            .send((test_system_event()).into())
1192            .unwrap();
1193
1194        assert!(matches!(
1195            runner.recv().await,
1196            Some(PendingRunnerEvent::SystemEvent(_))
1197        ));
1198        assert!(matches!(
1199            runner.recv().await,
1200            Some(PendingRunnerEvent::SystemCommand(_))
1201        ));
1202    }
1203
1204    #[rstest]
1205    fn test_async_data_command_sender_creation() {
1206        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
1207        let sender = AsyncDataCommandSender::new(tx);
1208        assert!(format!("{sender:?}").contains("AsyncDataCommandSender"));
1209    }
1210
1211    #[cfg(feature = "node")]
1212    #[rstest]
1213    fn test_runner_senders_shutdown_logging() {
1214        struct ErrorCapture(std::sync::Mutex<Vec<String>>);
1215
1216        impl log::Log for ErrorCapture {
1217            fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
1218                metadata.level() == log::Level::Error
1219                    && metadata.target() == "nautilus_live::runner"
1220            }
1221
1222            fn log(&self, record: &log::Record<'_>) {
1223                if self.enabled(record.metadata()) {
1224                    self.0.lock().unwrap().push(record.args().to_string());
1225                }
1226            }
1227
1228            fn flush(&self) {}
1229        }
1230
1231        static ERRORS: ErrorCapture = ErrorCapture(std::sync::Mutex::new(Vec::new()));
1232        log::set_logger(&ERRORS).unwrap();
1233        log::set_max_level(log::LevelFilter::Error);
1234
1235        let runner = AsyncRunner::new();
1236        let handle = LiveNodeHandle::new();
1237        handle.set_starting();
1238        runner.bind_senders_for_node(handle.clone());
1239        let sender = get_data_cmd_sender();
1240        let time_sender = get_time_event_sender();
1241        let mut channels = runner.take_channels();
1242
1243        let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1244            client_id: Some(ClientId::from("TEST")),
1245            venue: None,
1246            data_type: DataType::new("QuoteTick", None, None),
1247            command_id: UUID4::new(),
1248            ts_init: UnixNanos::default(),
1249            correlation_id: None,
1250            params: None,
1251        }));
1252
1253        let event = TimeEvent::new("shutdown-timer".into(), UUID4::new(), 17.into(), 23.into());
1254        let expected_events = vec![event.clone(); 2];
1255        let received = Rc::new(RefCell::new(Vec::new()));
1256        let observed = received.clone();
1257        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1258            observed.borrow_mut().push(event);
1259        }));
1260
1261        // Final draining still delivers commands and timer callbacks while receivers are alive
1262        for stopped in [false, true] {
1263            if stopped {
1264                handle.set_stopped();
1265            } else {
1266                handle.set_shutting_down();
1267            }
1268
1269            sender.execute(command.clone());
1270            assert_eq!(
1271                channels
1272                    .data_cmd_rx
1273                    .try_recv()
1274                    .unwrap()
1275                    .dispatch(|command| command),
1276                command
1277            );
1278
1279            time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1280            assert!(AsyncRunner::handle_time_event(
1281                channels.time_evt_rx.try_recv().unwrap()
1282            ));
1283        }
1284
1285        assert_eq!(*received.borrow(), expected_events);
1286
1287        drop(channels);
1288        sender.execute(command.clone());
1289        time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1290        assert_eq!(*ERRORS.0.lock().unwrap(), Vec::<String>::new());
1291
1292        // A stopped previous node must not hide an unexpected closure in its replacement
1293        let runner = AsyncRunner::new();
1294        let handle = LiveNodeHandle::new();
1295        runner.bind_senders_for_node(handle.clone());
1296        let sender = get_data_cmd_sender();
1297        let time_sender = get_time_event_sender();
1298        drop(runner);
1299
1300        for shutting_down in [false, true] {
1301            if shutting_down {
1302                handle.set_shutting_down();
1303            } else {
1304                handle.set_starting();
1305            }
1306
1307            sender.execute(command.clone());
1308            time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1309        }
1310
1311        let expected_errors = vec![
1312            "Failed to send data command: channel closed",
1313            "Failed to send time event message: channel closed",
1314        ];
1315        assert_eq!(*ERRORS.0.lock().unwrap(), expected_errors.repeat(2));
1316
1317        ERRORS.0.lock().unwrap().clear();
1318        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
1319        drop(rx);
1320        AsyncDataCommandSender::new(tx).execute(command);
1321        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
1322        drop(rx);
1323        AsyncTimeEventSender::new(tx).send(TimeEventMessage::new(event, callback));
1324        assert_eq!(*ERRORS.0.lock().unwrap(), expected_errors);
1325        assert_eq!(*received.borrow(), expected_events);
1326    }
1327
1328    #[rstest]
1329    fn test_async_time_event_sender_creation() {
1330        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
1331        let sender = AsyncTimeEventSender::new(tx);
1332        assert!(format!("{sender:?}").contains("AsyncTimeEventSender"));
1333    }
1334
1335    #[tokio::test]
1336    async fn test_async_data_command_sender_execute() {
1337        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1338        let sender = AsyncDataCommandSender::new(tx);
1339
1340        let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1341            client_id: Some(ClientId::from("TEST")),
1342            venue: None,
1343            data_type: DataType::new("QuoteTick", None, None),
1344            command_id: UUID4::new(),
1345            ts_init: UnixNanos::default(),
1346            correlation_id: None,
1347            params: None,
1348        }));
1349
1350        sender.execute(command.clone());
1351
1352        let received = rx.recv().await.unwrap().dispatch(|command| command);
1353        match (received, command) {
1354            (
1355                DataCommand::Subscribe(SubscribeCommand::Data(r)),
1356                DataCommand::Subscribe(SubscribeCommand::Data(c)),
1357            ) => {
1358                assert_eq!(r.client_id, c.client_id);
1359                assert_eq!(r.data_type, c.data_type);
1360            }
1361            _ => panic!("Command mismatch"),
1362        }
1363    }
1364
1365    #[rstest]
1366    #[case(false)]
1367    #[case(true)]
1368    fn test_async_command_senders_capture_owner_root(#[case] rooted: bool) {
1369        let runner = AsyncRunner::new();
1370        runner.bind_senders();
1371        let mut channels = runner.take_channels();
1372
1373        let data = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1374            client_id: Some(ClientId::from("ROOT")),
1375            venue: None,
1376            data_type: DataType::new("QuoteTick", None, None),
1377            command_id: UUID4::new(),
1378            ts_init: UnixNanos::from(7),
1379            correlation_id: None,
1380            params: None,
1381        }));
1382
1383        let trading = TradingCommand::CancelAllOrders(CancelAllOrders::new(
1384            TraderId::from("TRADER-001"),
1385            None,
1386            StrategyId::from("ROOT-001"),
1387            InstrumentId::from("EUR/USD.SIM"),
1388            Some(OrderSide::Sell),
1389            UUID4::new(),
1390            UnixNanos::from(11),
1391            None,
1392            None,
1393        ));
1394        let expected_data = data.clone();
1395        let expected_trading = trading.clone();
1396        let trigger = trading.clone();
1397
1398        let send = move || {
1399            get_data_cmd_sender().execute(data.clone());
1400            get_trading_cmd_sender().execute(TradingCommandMessage::new(
1401                MessagingSwitchboard::exec_engine_execute(),
1402                trading.clone(),
1403            ));
1404        };
1405
1406        if rooted {
1407            msgbus::register_trading_command_endpoint(
1408                MessagingSwitchboard::risk_engine_execute(),
1409                TypedIntoHandler::from(move |_| send()),
1410            );
1411            SyncTradingCommandSender.execute(TradingCommandMessage::new(
1412                MessagingSwitchboard::risk_engine_execute(),
1413                trigger,
1414            ));
1415            drain_trading_cmd_queue();
1416        } else {
1417            send();
1418        }
1419
1420        let data = channels.data_cmd_rx.try_recv().unwrap();
1421        let trading = channels.exec_cmd_rx.try_recv().unwrap();
1422
1423        let results = thread::spawn(move || {
1424            let data = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1425                data.dispatch(|command| command)
1426            }));
1427
1428            let trading = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1429                trading.dispatch(|message| (message.endpoint(), message.command().clone()))
1430            }));
1431
1432            (data, trading)
1433        })
1434        .join()
1435        .unwrap();
1436
1437        if rooted {
1438            for error in [results.0.unwrap_err(), results.1.unwrap_err()] {
1439                let message = error.downcast_ref::<String>().unwrap();
1440                assert!(message.contains("command context dispatched outside its owner thread"));
1441            }
1442        } else {
1443            assert_eq!(results.0.unwrap(), expected_data);
1444            assert_eq!(
1445                results.1.unwrap(),
1446                (
1447                    MessagingSwitchboard::exec_engine_execute(),
1448                    expected_trading
1449                ),
1450            );
1451        }
1452
1453        assert!(channels.data_cmd_rx.is_empty());
1454        assert!(channels.exec_cmd_rx.is_empty());
1455    }
1456
1457    #[rstest]
1458    #[case(false)]
1459    #[case(true)]
1460    fn test_event_senders_capture_owner_root(
1461        #[case] rooted: bool,
1462        #[values(false, true)] dispatch: bool,
1463    ) {
1464        let runner = AsyncRunner::new();
1465        runner.bind_senders();
1466        let mut channels = runner.take_channels();
1467        let expected_quote = test_quote();
1468        let expected_order = OrderSubmittedSpec::builder()
1469            .client_order_id(ClientOrderId::from("EVENT-017"))
1470            .build();
1471        let order = expected_order.clone();
1472
1473        let send = move || {
1474            get_data_event_sender()
1475                .send(DataEvent::Data(Data::Quote(expected_quote)))
1476                .unwrap();
1477            get_exec_event_sender()
1478                .send(ExecutionEvent::Order(OrderEventAny::Submitted(
1479                    order.clone(),
1480                )))
1481                .unwrap();
1482        };
1483
1484        if rooted {
1485            msgbus::register_trading_command_endpoint(
1486                MessagingSwitchboard::risk_engine_execute(),
1487                TypedIntoHandler::from(move |_| send()),
1488            );
1489            SyncTradingCommandSender.execute(TradingCommandMessage::new(
1490                MessagingSwitchboard::risk_engine_execute(),
1491                TradingCommand::QueryAccount(QueryAccount::new(
1492                    "TRADER-001".into(),
1493                    None,
1494                    "SIM-001".into(),
1495                    UUID4::new(),
1496                    19.into(),
1497                    None,
1498                    None,
1499                )),
1500            ));
1501            drain_trading_cmd_queue();
1502        } else {
1503            send();
1504        }
1505
1506        if dispatch {
1507            msgbus::register_data_endpoint(
1508                MessagingSwitchboard::data_engine_process_data(),
1509                TypedIntoHandler::from(move |data| {
1510                    assert_eq!(data, Data::Quote(expected_quote));
1511                    get_data_event_sender().send(DataEvent::Data(data)).unwrap();
1512                }),
1513            );
1514
1515            let order = expected_order.clone();
1516            msgbus::register_order_event_endpoint(
1517                MessagingSwitchboard::exec_engine_process(),
1518                TypedIntoHandler::from(move |event| {
1519                    assert_eq!(event, OrderEventAny::Submitted(order.clone()));
1520                    get_exec_event_sender()
1521                        .send(ExecutionEvent::Order(event))
1522                        .unwrap();
1523                }),
1524            );
1525
1526            AsyncRunner::dispatch_data_event(channels.data_evt_rx.try_recv().unwrap());
1527            AsyncRunner::dispatch_exec_event(channels.exec_evt_rx.try_recv().unwrap());
1528        }
1529
1530        let data = channels.data_evt_rx.try_recv().unwrap();
1531        let exec = channels.exec_evt_rx.try_recv().unwrap();
1532
1533        let (data, exec) = thread::spawn(move || {
1534            (
1535                std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1536                    data.dispatch(|event| event)
1537                })),
1538                std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1539                    exec.dispatch(|event| event)
1540                })),
1541            )
1542        })
1543        .join()
1544        .unwrap();
1545
1546        if rooted || dispatch {
1547            for error in [data.unwrap_err(), exec.unwrap_err()] {
1548                assert!(
1549                    error
1550                        .downcast_ref::<String>()
1551                        .unwrap()
1552                        .contains("command context dispatched outside its owner thread")
1553                );
1554            }
1555        } else {
1556            let DataEvent::Data(Data::Quote(quote)) = data.unwrap() else {
1557                panic!("expected quote")
1558            };
1559
1560            let ExecutionEvent::Order(OrderEventAny::Submitted(order)) = exec.unwrap() else {
1561                panic!("expected submitted order")
1562            };
1563
1564            assert_eq!(quote, expected_quote);
1565            assert_eq!(order, expected_order);
1566        }
1567
1568        assert!(channels.data_evt_rx.is_empty());
1569        assert!(channels.exec_evt_rx.is_empty());
1570    }
1571
1572    #[rstest]
1573    fn test_system_and_time_senders_capture_owner_root(#[values(false, true)] rooted: bool) {
1574        let runner = AsyncRunner::new();
1575        runner.bind_senders();
1576        let mut channels = runner.take_channels();
1577        let event = TimeEvent::new("system-time".into(), UUID4::new(), 17.into(), 23.into());
1578        let observed = Rc::new(RefCell::new(Vec::new()));
1579        let received = observed.clone();
1580        let expected_event = event.clone();
1581
1582        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1583            received.borrow_mut().push(event);
1584            get_system_command_sender()
1585                .send(test_system_command())
1586                .unwrap();
1587        }));
1588
1589        let send = move || {
1590            get_system_event_sender().send(test_system_event()).unwrap();
1591            get_system_command_sender()
1592                .send(test_system_command())
1593                .unwrap();
1594            get_time_event_sender().send(TimeEventMessage::new(event.clone(), callback.clone()));
1595        };
1596
1597        if rooted {
1598            msgbus::register_trading_command_endpoint(
1599                MessagingSwitchboard::risk_engine_execute(),
1600                TypedIntoHandler::from(move |_| send()),
1601            );
1602            SyncTradingCommandSender.execute(TradingCommandMessage::new(
1603                MessagingSwitchboard::risk_engine_execute(),
1604                TradingCommand::QueryAccount(QueryAccount::new(
1605                    "TRADER-001".into(),
1606                    None,
1607                    "SIM-001".into(),
1608                    UUID4::new(),
1609                    31.into(),
1610                    None,
1611                    None,
1612                )),
1613            ));
1614            drain_trading_cmd_queue();
1615        } else {
1616            send();
1617        }
1618
1619        let system_event = channels.system_evt_rx.try_recv().unwrap();
1620        let system_command = channels.system_cmd_rx.try_recv().unwrap();
1621        let time_event = channels.time_evt_rx.try_recv().unwrap();
1622        assert_eq!(system_event.is_rooted(), rooted);
1623        assert_eq!(system_command.is_rooted(), rooted);
1624        assert_eq!(time_event.is_rooted(), rooted);
1625        assert!(AsyncRunner::handle_time_event(time_event));
1626        let child = channels.system_cmd_rx.try_recv().unwrap();
1627
1628        assert_eq!(system_event.dispatch(|event| event), test_system_event());
1629        assert_eq!(
1630            system_command.dispatch(|command| command),
1631            test_system_command()
1632        );
1633        assert_eq!(*observed.borrow(), vec![expected_event]);
1634        assert!(child.is_rooted());
1635        assert_eq!(child.dispatch(|command| command), test_system_command());
1636        assert!(channels.time_evt_rx.is_empty());
1637        assert!(channels.system_evt_rx.is_empty());
1638        assert!(channels.system_cmd_rx.is_empty());
1639    }
1640
1641    #[cfg(not(all(feature = "simulation", madsim)))]
1642    #[tokio::test]
1643    async fn test_live_timer_started_under_root_emits_independent_event() {
1644        let runner = AsyncRunner::new();
1645        runner.bind_senders();
1646        let mut channels = runner.take_channels();
1647        let now = get_atomic_clock_realtime().get_time_ns();
1648        let name = Ustr::from("ROOTED_TIMER");
1649        let owner = thread::current().id();
1650        let observed = Rc::new(RefCell::new(Vec::new()));
1651        let received = observed.clone();
1652
1653        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1654            received
1655                .borrow_mut()
1656                .push((event.name, event.ts_event, thread::current().id()));
1657        }));
1658
1659        let timer = Rc::new(RefCell::new(None));
1660        let started = timer.clone();
1661        msgbus::register_trading_command_endpoint(
1662            MessagingSwitchboard::risk_engine_execute(),
1663            TypedIntoHandler::from(move |_| {
1664                get_system_command_sender()
1665                    .send(test_system_command())
1666                    .unwrap();
1667
1668                let mut timer = LiveTimer::new(
1669                    name,
1670                    NonZeroU64::new(1_000_000).unwrap(),
1671                    now,
1672                    Some(now),
1673                    callback.clone(),
1674                    true,
1675                    Some(get_time_event_sender()),
1676                );
1677                timer.start();
1678                *started.borrow_mut() = Some(timer);
1679            }),
1680        );
1681
1682        SyncTradingCommandSender.execute(TradingCommandMessage::new(
1683            MessagingSwitchboard::risk_engine_execute(),
1684            TradingCommand::QueryAccount(QueryAccount::new(
1685                "TRADER-001".into(),
1686                None,
1687                "SIM-001".into(),
1688                UUID4::new(),
1689                31.into(),
1690                None,
1691                None,
1692            )),
1693        ));
1694        drain_trading_cmd_queue();
1695        let witness = channels.system_cmd_rx.try_recv().unwrap();
1696        let message = tokio::time::timeout(Duration::from_secs(2), channels.time_evt_rx.recv())
1697            .await
1698            .unwrap()
1699            .unwrap();
1700        timer.borrow_mut().take().unwrap().cancel();
1701
1702        assert!(witness.is_rooted());
1703        assert!(!message.is_rooted());
1704        assert!(observed.borrow().is_empty());
1705        assert!(AsyncRunner::handle_time_event(message));
1706        assert_eq!(*observed.borrow(), vec![(name, now, owner)]);
1707        assert!(channels.time_evt_rx.is_empty());
1708    }
1709
1710    #[tokio::test]
1711    async fn test_async_time_event_sender_send() {
1712        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1713        let sender = AsyncTimeEventSender::new(tx);
1714
1715        let event = TimeEvent::new(
1716            Ustr::from("test"),
1717            UUID4::new(),
1718            UnixNanos::from(1),
1719            UnixNanos::from(2),
1720        );
1721        let callback = TimeEventCallback::from(|_: TimeEvent| {});
1722        let message = TimeEventMessage::new(event, callback);
1723
1724        sender.send(message);
1725
1726        assert!(rx.recv().await.is_some());
1727    }
1728
1729    #[tokio::test]
1730    async fn test_runner_shutdown_signal() {
1731        // Create runner with manual channels to avoid global state
1732        let (_data_tx, data_evt_rx) =
1733            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
1734        let (_cmd_tx, data_cmd_rx) =
1735            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
1736        let (_time_tx, time_evt_rx) =
1737            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
1738        let (_exec_evt_tx, exec_evt_rx) =
1739            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
1740        let (_exec_cmd_tx, exec_cmd_rx) =
1741            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1742        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
1743
1744        let mut runner = create_test_runner(
1745            time_evt_rx,
1746            data_evt_rx,
1747            data_cmd_rx,
1748            exec_evt_rx,
1749            exec_cmd_rx,
1750            signal_rx,
1751            signal_tx.clone(),
1752        );
1753
1754        // Start runner
1755        let runner_handle = tokio::spawn(async move {
1756            runner.run().await.unwrap();
1757        });
1758
1759        // Send shutdown signal
1760        signal_tx.send(()).unwrap();
1761
1762        // Runner should stop quickly
1763        let result = tokio::time::timeout(Duration::from_millis(100), runner_handle).await;
1764        assert!(result.is_ok(), "Runner should stop on signal");
1765    }
1766
1767    #[tokio::test]
1768    async fn test_runner_closes_on_channel_drop() {
1769        let (data_tx, data_evt_rx) =
1770            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
1771        let (_cmd_tx, data_cmd_rx) =
1772            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
1773        let (_time_tx, time_evt_rx) =
1774            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
1775        let (_exec_evt_tx, exec_evt_rx) =
1776            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
1777        let (_exec_cmd_tx, exec_cmd_rx) =
1778            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1779        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
1780
1781        let mut runner = create_test_runner(
1782            time_evt_rx,
1783            data_evt_rx,
1784            data_cmd_rx,
1785            exec_evt_rx,
1786            exec_cmd_rx,
1787            signal_rx,
1788            signal_tx.clone(),
1789        );
1790
1791        // Start runner
1792        let runner_handle = tokio::spawn(async move {
1793            runner.run().await.unwrap();
1794        });
1795
1796        drop(data_tx);
1797
1798        // Yield to let runner enter event loop before stop signal
1799        tokio::task::yield_now().await;
1800        signal_tx.send(()).ok();
1801
1802        // Runner should stop when channels close or on signal
1803        let result = tokio::time::timeout(Duration::from_millis(200), runner_handle).await;
1804        assert!(
1805            result.is_ok(),
1806            "Runner should stop when channels close or on signal"
1807        );
1808    }
1809
1810    #[tokio::test]
1811    async fn test_concurrent_event_sending() {
1812        let (data_evt_tx, data_evt_rx) =
1813            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
1814        let (_data_cmd_tx, data_cmd_rx) =
1815            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
1816        let (_time_evt_tx, time_evt_rx) =
1817            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
1818        let (_exec_evt_tx, exec_evt_rx) =
1819            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
1820        let (_exec_cmd_tx, exec_cmd_rx) =
1821            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1822        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
1823
1824        // Setup runner
1825        let mut runner = create_test_runner(
1826            time_evt_rx,
1827            data_evt_rx,
1828            data_cmd_rx,
1829            exec_evt_rx,
1830            exec_cmd_rx,
1831            signal_rx,
1832            signal_tx.clone(),
1833        );
1834
1835        // Spawn multiple concurrent senders
1836        let mut handles = vec![];
1837
1838        for _ in 0..5 {
1839            let tx_clone = data_evt_tx.clone();
1840
1841            let handle = tokio::spawn(async move {
1842                for _ in 0..20 {
1843                    let quote = test_quote();
1844                    tx_clone
1845                        .send(DataEvent::Data(Data::Quote(quote)).into())
1846                        .unwrap();
1847                    tokio::task::yield_now().await;
1848                }
1849            });
1850
1851            handles.push(handle);
1852        }
1853
1854        // Start runner in background
1855        let runner_handle = tokio::spawn(async move {
1856            runner.run().await.unwrap();
1857        });
1858
1859        // Wait for all senders
1860        for handle in handles {
1861            handle.await.unwrap();
1862        }
1863
1864        // Yield to let runner enter event loop before stop signal
1865        tokio::task::yield_now().await;
1866        signal_tx.send(()).unwrap();
1867
1868        let _ = tokio::time::timeout(Duration::from_millis(200), runner_handle).await;
1869    }
1870
1871    #[rstest]
1872    #[case(10)]
1873    #[case(100)]
1874    #[case(1000)]
1875    fn test_channel_send_performance(#[case] count: usize) {
1876        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
1877        let quote = test_quote();
1878
1879        // Send events
1880        for _ in 0..count {
1881            tx.send(DataEvent::Data(Data::Quote(quote))).unwrap();
1882        }
1883
1884        // Verify all received
1885        let mut received = 0;
1886
1887        while rx.try_recv().is_ok() {
1888            received += 1;
1889        }
1890
1891        assert_eq!(received, count);
1892    }
1893
1894    #[rstest]
1895    fn test_async_trading_command_sender_creation() {
1896        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
1897        let sender = AsyncTradingCommandSender::new(tx);
1898        assert!(format!("{sender:?}").contains("AsyncTradingCommandSender"));
1899    }
1900
1901    #[rstest]
1902    fn test_async_trading_command_sender_preserves_target_endpoints() {
1903        std::thread::spawn(|| {
1904            msgbus::get_message_bus().borrow_mut().dispose();
1905            let (risk_handler, risk_saving_handler) =
1906                get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1907                    "RiskEngine.execute",
1908                )));
1909            msgbus::register_trading_command_endpoint(
1910                MessagingSwitchboard::risk_engine_execute(),
1911                risk_handler,
1912            );
1913            let (exec_handler, exec_saving_handler) =
1914                get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1915                    "ExecEngine.execute",
1916                )));
1917            msgbus::register_trading_command_endpoint(
1918                MessagingSwitchboard::exec_engine_execute(),
1919                exec_handler,
1920            );
1921
1922            let (tx, mut rx) =
1923                tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1924            let sender = AsyncTradingCommandSender::new(tx);
1925            sender.execute(TradingCommandMessage::new(
1926                MessagingSwitchboard::risk_engine_execute(),
1927                TradingCommand::CancelAllOrders(CancelAllOrders::new(
1928                    TraderId::from("TRADER-001"),
1929                    None,
1930                    StrategyId::from("RISK-001"),
1931                    InstrumentId::from("EUR/USD.SIM"),
1932                    Some(OrderSide::Buy),
1933                    UUID4::new(),
1934                    UnixNanos::default(),
1935                    None,
1936                    None,
1937                )),
1938            ));
1939            sender.execute(TradingCommandMessage::new(
1940                MessagingSwitchboard::exec_engine_execute(),
1941                TradingCommand::CancelAllOrders(CancelAllOrders::new(
1942                    TraderId::from("TRADER-001"),
1943                    None,
1944                    StrategyId::from("EXEC-001"),
1945                    InstrumentId::from("EUR/USD.SIM"),
1946                    Some(OrderSide::Sell),
1947                    UUID4::new(),
1948                    UnixNanos::default(),
1949                    None,
1950                    None,
1951                )),
1952            ));
1953
1954            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1955            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1956
1957            let risk_commands = risk_saving_handler.get_messages();
1958            let exec_commands = exec_saving_handler.get_messages();
1959            assert!(rx.try_recv().is_err());
1960            assert_eq!(risk_commands.len(), 1);
1961            assert_eq!(
1962                risk_commands[0].strategy_id(),
1963                Some(StrategyId::from("RISK-001"))
1964            );
1965            assert_eq!(exec_commands.len(), 1);
1966            assert_eq!(
1967                exec_commands[0].strategy_id(),
1968                Some(StrategyId::from("EXEC-001"))
1969            );
1970        })
1971        .join()
1972        .unwrap();
1973    }
1974
1975    #[rstest]
1976    fn test_async_runner_preserves_deferred_follow_up_order() {
1977        std::thread::spawn(|| {
1978            msgbus::get_message_bus().borrow_mut().dispose();
1979            let clock = Rc::new(RefCell::new(VirtualClock::new()));
1980            let cache = Rc::new(RefCell::new(Cache::default()));
1981            let exec_engine = Rc::new(RefCell::new(ExecutionEngine::new(clock, cache, None)));
1982            ExecutionEngine::register_msgbus_handlers(&exec_engine);
1983            msgbus::register_trading_command_endpoint(
1984                MessagingSwitchboard::risk_engine_execute(),
1985                TypedIntoHandler::from(|command: TradingCommand| {
1986                    msgbus::send_trading_command(
1987                        MessagingSwitchboard::exec_engine_queue_execute(),
1988                        command,
1989                    );
1990                }),
1991            );
1992
1993            let (exec_handler, exec_saving_handler) =
1994                get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1995                    "ExecEngine.execute",
1996                )));
1997            msgbus::register_trading_command_endpoint(
1998                MessagingSwitchboard::exec_engine_execute(),
1999                exec_handler,
2000            );
2001
2002            let (tx, mut rx) =
2003                tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2004            let sender = Arc::new(AsyncTradingCommandSender::new(tx));
2005            replace_exec_cmd_sender(sender.clone());
2006            sender.execute(TradingCommandMessage::new(
2007                MessagingSwitchboard::risk_engine_execute(),
2008                TradingCommand::CancelAllOrders(CancelAllOrders::new(
2009                    TraderId::from("TRADER-001"),
2010                    None,
2011                    StrategyId::from("FIRST-001"),
2012                    InstrumentId::from("EUR/USD.SIM"),
2013                    Some(OrderSide::Buy),
2014                    UUID4::new(),
2015                    UnixNanos::default(),
2016                    None,
2017                    None,
2018                )),
2019            ));
2020            sender.execute(TradingCommandMessage::new(
2021                MessagingSwitchboard::exec_engine_execute(),
2022                TradingCommand::CancelAllOrders(CancelAllOrders::new(
2023                    TraderId::from("TRADER-001"),
2024                    None,
2025                    StrategyId::from("SECOND-001"),
2026                    InstrumentId::from("EUR/USD.SIM"),
2027                    Some(OrderSide::Sell),
2028                    UUID4::new(),
2029                    UnixNanos::default(),
2030                    None,
2031                    None,
2032                )),
2033            ));
2034
2035            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
2036            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
2037
2038            let commands = exec_saving_handler.get_messages();
2039            let strategy_ids = commands
2040                .iter()
2041                .map(TradingCommand::strategy_id)
2042                .collect::<Vec<_>>();
2043            assert!(rx.try_recv().is_err());
2044            assert_eq!(commands.len(), 2);
2045            assert_eq!(
2046                strategy_ids,
2047                vec![
2048                    Some(StrategyId::from("FIRST-001")),
2049                    Some(StrategyId::from("SECOND-001"))
2050                ]
2051            );
2052        })
2053        .join()
2054        .unwrap();
2055    }
2056
2057    #[rstest]
2058    fn test_async_runner_dispatches_deferred_exec_command_once() {
2059        std::thread::spawn(|| {
2060            msgbus::get_message_bus().borrow_mut().dispose();
2061            let clock = Rc::new(RefCell::new(VirtualClock::new()));
2062            let cache = Rc::new(RefCell::new(Cache::default()));
2063            let exec_engine = Rc::new(RefCell::new(ExecutionEngine::new(clock, cache, None)));
2064            ExecutionEngine::register_msgbus_handlers(&exec_engine);
2065
2066            let (tx, mut rx) =
2067                tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2068            replace_exec_cmd_sender(Arc::new(AsyncTradingCommandSender::new(tx)));
2069            let command = TradingCommand::CancelAllOrders(CancelAllOrders::new(
2070                TraderId::from("TRADER-001"),
2071                None,
2072                StrategyId::from("EXEC-001"),
2073                InstrumentId::from("EUR/USD.SIM"),
2074                Some(OrderSide::Buy),
2075                UUID4::new(),
2076                UnixNanos::default(),
2077                None,
2078                None,
2079            ));
2080
2081            msgbus::send_trading_command(
2082                MessagingSwitchboard::exec_engine_queue_execute(),
2083                command,
2084            );
2085            assert_eq!(exec_engine.borrow().command_count(), 0);
2086
2087            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
2088
2089            assert!(rx.try_recv().is_err());
2090            assert_eq!(exec_engine.borrow().command_count(), 1);
2091        })
2092        .join()
2093        .unwrap();
2094    }
2095
2096    #[tokio::test]
2097    async fn test_runner_processes_trading_commands() {
2098        let (_data_evt_tx, data_evt_rx) =
2099            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
2100        let (_data_cmd_tx, data_cmd_rx) =
2101            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
2102        let (_time_evt_tx, time_evt_rx) =
2103            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
2104        let (_exec_evt_tx, exec_evt_rx) =
2105            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
2106        let (exec_cmd_tx, exec_cmd_rx) =
2107            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2108        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
2109
2110        let mut runner = create_test_runner(
2111            time_evt_rx,
2112            data_evt_rx,
2113            data_cmd_rx,
2114            exec_evt_rx,
2115            exec_cmd_rx,
2116            signal_rx,
2117            signal_tx.clone(),
2118        );
2119
2120        let runner_handle = tokio::spawn(async move {
2121            runner.run().await.unwrap();
2122        });
2123
2124        let command = TradingCommand::CancelAllOrders(CancelAllOrders::new(
2125            TraderId::from("TRADER-001"),
2126            None,
2127            StrategyId::from("S-001"),
2128            InstrumentId::from("EUR/USD.SIM"),
2129            Some(OrderSide::Buy),
2130            UUID4::new(),
2131            UnixNanos::default(),
2132            None,
2133            None, // correlation_id
2134        ));
2135        exec_cmd_tx
2136            .send(
2137                TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command)
2138                    .into(),
2139            )
2140            .unwrap();
2141
2142        tokio::task::yield_now().await;
2143        signal_tx.send(()).unwrap();
2144
2145        let result = tokio::time::timeout(Duration::from_millis(100), runner_handle).await;
2146        assert!(result.is_ok(), "Runner should process command and stop");
2147    }
2148
2149    #[tokio::test]
2150    async fn test_runner_processes_multiple_trading_commands() {
2151        let (_data_evt_tx, data_evt_rx) =
2152            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
2153        let (_data_cmd_tx, data_cmd_rx) =
2154            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
2155        let (_time_evt_tx, time_evt_rx) =
2156            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
2157        let (_exec_evt_tx, exec_evt_rx) =
2158            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
2159        let (exec_cmd_tx, exec_cmd_rx) =
2160            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2161        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
2162
2163        let mut runner = create_test_runner(
2164            time_evt_rx,
2165            data_evt_rx,
2166            data_cmd_rx,
2167            exec_evt_rx,
2168            exec_cmd_rx,
2169            signal_rx,
2170            signal_tx.clone(),
2171        );
2172
2173        let runner_handle = tokio::spawn(async move {
2174            runner.run().await.unwrap();
2175        });
2176
2177        for i in 0..10 {
2178            let strategy_id = format!("S-{i:03}");
2179            let command = TradingCommand::CancelAllOrders(CancelAllOrders::new(
2180                TraderId::from("TRADER-001"),
2181                None,
2182                StrategyId::from(strategy_id.as_str()),
2183                InstrumentId::from("EUR/USD.SIM"),
2184                Some(OrderSide::Buy),
2185                UUID4::new(),
2186                UnixNanos::default(),
2187                None,
2188                None, // correlation_id
2189            ));
2190            exec_cmd_tx
2191                .send(
2192                    TradingCommandMessage::new(
2193                        MessagingSwitchboard::exec_engine_execute(),
2194                        command,
2195                    )
2196                    .into(),
2197                )
2198                .unwrap();
2199        }
2200
2201        tokio::task::yield_now().await;
2202        signal_tx.send(()).unwrap();
2203
2204        let result = tokio::time::timeout(Duration::from_millis(100), runner_handle).await;
2205        assert!(
2206            result.is_ok(),
2207            "Runner should process all commands and stop"
2208        );
2209    }
2210
2211    #[tokio::test]
2212    async fn test_execution_event_order_channel() {
2213        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2214
2215        let event = OrderSubmittedSpec::builder()
2216            .client_order_id(ClientOrderId::from("O-001"))
2217            .build();
2218
2219        tx.send(ExecutionEvent::Order(OrderEventAny::Submitted(event)))
2220            .unwrap();
2221
2222        let received = rx.recv().await.unwrap();
2223        match received {
2224            ExecutionEvent::Order(OrderEventAny::Submitted(e)) => {
2225                assert_eq!(e.client_order_id(), ClientOrderId::from("O-001"));
2226            }
2227            _ => panic!("Expected OrderSubmitted event"),
2228        }
2229    }
2230
2231    #[tokio::test]
2232    async fn test_execution_report_order_status_channel() {
2233        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2234
2235        let report = OrderStatusReport::new(
2236            AccountId::from("SIM-001"),
2237            InstrumentId::from("EUR/USD.SIM"),
2238            Some(ClientOrderId::from("O-001")),
2239            VenueOrderId::from("V-001"),
2240            OrderSide::Buy.into(),
2241            OrderType::Market,
2242            TimeInForce::Gtc,
2243            OrderStatus::Accepted,
2244            Quantity::from(100_000),
2245            Quantity::from(100_000),
2246            UnixNanos::from(1),
2247            UnixNanos::from(2),
2248            UnixNanos::from(3),
2249            None,
2250        );
2251
2252        tx.send(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
2253            report,
2254        ))))
2255        .unwrap();
2256
2257        let received = rx.recv().await.unwrap();
2258        match received {
2259            ExecutionEvent::Report(ExecutionReport::Order(r)) => {
2260                assert_eq!(r.venue_order_id.as_str(), "V-001");
2261                assert_eq!(r.order_status, OrderStatus::Accepted);
2262            }
2263            _ => panic!("Expected OrderStatusReport"),
2264        }
2265    }
2266
2267    #[tokio::test]
2268    async fn test_execution_report_fill() {
2269        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2270
2271        let report = FillReport::new(
2272            AccountId::from("SIM-001"),
2273            InstrumentId::from("EUR/USD.SIM"),
2274            VenueOrderId::from("V-001"),
2275            TradeId::from("T-001"),
2276            OrderSide::Buy,
2277            Quantity::from(100_000),
2278            Price::from("1.10000"),
2279            Money::from("10 USD"),
2280            LiquiditySide::Taker,
2281            Some(ClientOrderId::from("O-001")),
2282            None,
2283            UnixNanos::from(1),
2284            UnixNanos::from(2),
2285            None,
2286        );
2287
2288        tx.send(ExecutionEvent::Report(ExecutionReport::Fill(Box::new(
2289            report,
2290        ))))
2291        .unwrap();
2292
2293        let received = rx.recv().await.unwrap();
2294        match received {
2295            ExecutionEvent::Report(ExecutionReport::Fill(r)) => {
2296                assert_eq!(r.venue_order_id.as_str(), "V-001");
2297                assert_eq!(r.trade_id.to_string(), "T-001");
2298            }
2299            _ => panic!("Expected FillReport"),
2300        }
2301    }
2302
2303    #[tokio::test]
2304    async fn test_execution_report_position() {
2305        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2306
2307        let report = PositionStatusReport::new(
2308            AccountId::from("SIM-001"),
2309            InstrumentId::from("EUR/USD.SIM"),
2310            PositionSide::Long,
2311            Quantity::from(100_000),
2312            UnixNanos::from(1),
2313            UnixNanos::from(2),
2314            None,
2315            Some(PositionId::from("P-001")),
2316            None,
2317        );
2318
2319        tx.send(ExecutionEvent::Report(ExecutionReport::Position(Box::new(
2320            report,
2321        ))))
2322        .unwrap();
2323
2324        let received = rx.recv().await.unwrap();
2325        match received {
2326            ExecutionEvent::Report(ExecutionReport::Position(r)) => {
2327                assert_eq!(r.venue_position_id.unwrap().as_str(), "P-001");
2328            }
2329            _ => panic!("Expected PositionStatusReport"),
2330        }
2331    }
2332
2333    #[tokio::test]
2334    async fn test_execution_event_account() {
2335        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2336
2337        let account_state = AccountState::new(
2338            AccountId::from("SIM-001"),
2339            AccountType::Cash,
2340            vec![],
2341            vec![],
2342            true,
2343            UUID4::new(),
2344            UnixNanos::from(1),
2345            UnixNanos::from(2),
2346            None,
2347        );
2348
2349        tx.send(ExecutionEvent::Account(account_state)).unwrap();
2350
2351        let received = rx.recv().await.unwrap();
2352        match received {
2353            ExecutionEvent::Account(r) => {
2354                assert_eq!(r.account_id.as_str(), "SIM-001");
2355            }
2356            _ => panic!("Expected AccountState"),
2357        }
2358    }
2359
2360    #[tokio::test]
2361    async fn test_runner_stop_method() {
2362        let (_data_tx, data_evt_rx) =
2363            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
2364        let (_cmd_tx, data_cmd_rx) =
2365            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
2366        let (_time_tx, time_evt_rx) =
2367            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
2368        let (_exec_evt_tx, exec_evt_rx) =
2369            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
2370        let (_exec_cmd_tx, exec_cmd_rx) =
2371            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2372        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
2373
2374        let mut runner = create_test_runner(
2375            time_evt_rx,
2376            data_evt_rx,
2377            data_cmd_rx,
2378            exec_evt_rx,
2379            exec_cmd_rx,
2380            signal_rx,
2381            signal_tx.clone(),
2382        );
2383
2384        let runner_handle = tokio::spawn(async move {
2385            runner.run().await.unwrap();
2386        });
2387
2388        // Use stop via signal_tx directly
2389        signal_tx.send(()).unwrap();
2390
2391        let result = tokio::time::timeout(Duration::from_millis(100), runner_handle).await;
2392        assert!(result.is_ok(), "Runner should stop when stop() is called");
2393    }
2394
2395    #[tokio::test]
2396    async fn test_all_event_types_integration() {
2397        let (data_evt_tx, data_evt_rx) =
2398            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
2399        let (data_cmd_tx, data_cmd_rx) =
2400            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
2401        let (time_evt_tx, time_evt_rx) =
2402            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
2403        let (exec_evt_tx, exec_evt_rx) =
2404            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
2405        let (_exec_cmd_tx, exec_cmd_rx) =
2406            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2407        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
2408
2409        let mut runner = create_test_runner(
2410            time_evt_rx,
2411            data_evt_rx,
2412            data_cmd_rx,
2413            exec_evt_rx,
2414            exec_cmd_rx,
2415            signal_rx,
2416            signal_tx.clone(),
2417        );
2418
2419        let runner_handle = tokio::spawn(async move {
2420            runner.run().await.unwrap();
2421        });
2422
2423        // Send data event
2424        let quote = test_quote();
2425        data_evt_tx
2426            .send((DataEvent::Data(Data::Quote(quote))).into())
2427            .unwrap();
2428
2429        // Send data command
2430        let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
2431            client_id: Some(ClientId::from("TEST")),
2432            venue: None,
2433            data_type: DataType::new("QuoteTick", None, None),
2434            command_id: UUID4::new(),
2435            ts_init: UnixNanos::default(),
2436            correlation_id: None,
2437            params: None,
2438        }));
2439
2440        data_cmd_tx.send(command.into()).unwrap();
2441
2442        // Send time event
2443        let event = TimeEvent::new(
2444            Ustr::from("test"),
2445            UUID4::new(),
2446            UnixNanos::from(1),
2447            UnixNanos::from(2),
2448        );
2449        let callback = TimeEventCallback::from(|_: TimeEvent| {});
2450        let message = TimeEventMessage::new(event, callback);
2451        time_evt_tx.send((message).into()).unwrap();
2452
2453        // Send execution order event
2454        let order_event = OrderSubmittedSpec::builder()
2455            .client_order_id(ClientOrderId::from("O-001"))
2456            .build();
2457        exec_evt_tx
2458            .send((ExecutionEvent::Order(OrderEventAny::Submitted(order_event))).into())
2459            .unwrap();
2460
2461        // Send execution report (OrderStatus)
2462        let order_status = OrderStatusReport::new(
2463            AccountId::from("SIM-001"),
2464            InstrumentId::from("EUR/USD.SIM"),
2465            Some(ClientOrderId::from("O-001")),
2466            VenueOrderId::from("V-001"),
2467            OrderSide::Buy.into(),
2468            OrderType::Market,
2469            TimeInForce::Gtc,
2470            OrderStatus::Accepted,
2471            Quantity::from(100_000),
2472            Quantity::from(100_000),
2473            UnixNanos::from(1),
2474            UnixNanos::from(2),
2475            UnixNanos::from(3),
2476            None,
2477        );
2478        exec_evt_tx
2479            .send((ExecutionEvent::Report(ExecutionReport::Order(Box::new(order_status)))).into())
2480            .unwrap();
2481
2482        // Send execution report (Fill)
2483        let fill = FillReport::new(
2484            AccountId::from("SIM-001"),
2485            InstrumentId::from("EUR/USD.SIM"),
2486            VenueOrderId::from("V-001"),
2487            TradeId::from("T-001"),
2488            OrderSide::Buy,
2489            Quantity::from(100_000),
2490            Price::from("1.10000"),
2491            Money::from("10 USD"),
2492            LiquiditySide::Taker,
2493            Some(ClientOrderId::from("O-001")),
2494            None,
2495            UnixNanos::from(1),
2496            UnixNanos::from(2),
2497            None,
2498        );
2499        exec_evt_tx
2500            .send((ExecutionEvent::Report(ExecutionReport::Fill(Box::new(fill)))).into())
2501            .unwrap();
2502
2503        // Send execution report (Position)
2504        let position = PositionStatusReport::new(
2505            AccountId::from("SIM-001"),
2506            InstrumentId::from("EUR/USD.SIM"),
2507            PositionSide::Long,
2508            Quantity::from(100_000),
2509            UnixNanos::from(1),
2510            UnixNanos::from(2),
2511            None,
2512            Some(PositionId::from("P-001")),
2513            None,
2514        );
2515        exec_evt_tx
2516            .send((ExecutionEvent::Report(ExecutionReport::Position(Box::new(position)))).into())
2517            .unwrap();
2518
2519        // Send account event
2520        let account_state = AccountState::new(
2521            AccountId::from("SIM-001"),
2522            AccountType::Cash,
2523            vec![],
2524            vec![],
2525            true,
2526            UUID4::new(),
2527            UnixNanos::from(1),
2528            UnixNanos::from(2),
2529            None,
2530        );
2531        exec_evt_tx
2532            .send((ExecutionEvent::Account(account_state)).into())
2533            .unwrap();
2534
2535        // Yield to let runner enter event loop before stop signal
2536        tokio::task::yield_now().await;
2537        signal_tx.send(()).unwrap();
2538
2539        let result = tokio::time::timeout(Duration::from_millis(200), runner_handle).await;
2540        assert!(
2541            result.is_ok(),
2542            "Runner should process all event types and stop cleanly"
2543        );
2544    }
2545
2546    #[tokio::test]
2547    async fn test_runner_handle_stops_runner() {
2548        let (_data_tx, data_evt_rx) =
2549            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
2550        let (_cmd_tx, data_cmd_rx) =
2551            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
2552        let (_time_tx, time_evt_rx) =
2553            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
2554        let (_exec_evt_tx, exec_evt_rx) =
2555            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
2556        let (_exec_cmd_tx, exec_cmd_rx) =
2557            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2558        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
2559
2560        let mut runner = create_test_runner(
2561            time_evt_rx,
2562            data_evt_rx,
2563            data_cmd_rx,
2564            exec_evt_rx,
2565            exec_cmd_rx,
2566            signal_rx,
2567            signal_tx.clone(),
2568        );
2569
2570        // Get handle before moving runner
2571        let handle = runner.handle();
2572
2573        let runner_handle = tokio::spawn(async move {
2574            runner.run().await.unwrap();
2575        });
2576
2577        // Use handle to stop
2578        handle.stop();
2579
2580        let result = tokio::time::timeout(Duration::from_millis(100), runner_handle).await;
2581        assert!(result.is_ok(), "Runner should stop via handle");
2582    }
2583
2584    #[tokio::test]
2585    async fn test_runner_handle_is_cloneable() {
2586        let (signal_tx, _signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
2587        let handle = AsyncRunnerHandle { signal_tx };
2588
2589        let handle2 = handle.clone();
2590
2591        // Both handles should be able to send stop signals
2592        assert!(handle.signal_tx.send(()).is_ok());
2593        assert!(handle2.signal_tx.send(()).is_ok());
2594    }
2595
2596    #[tokio::test]
2597    async fn test_runner_processes_events_before_stop() {
2598        let (data_evt_tx, data_evt_rx) =
2599            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
2600        let (_cmd_tx, data_cmd_rx) =
2601            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
2602        let (_time_tx, time_evt_rx) =
2603            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
2604        let (_exec_evt_tx, exec_evt_rx) =
2605            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
2606        let (_exec_cmd_tx, exec_cmd_rx) =
2607            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2608        let (signal_tx, signal_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
2609
2610        let mut runner = create_test_runner(
2611            time_evt_rx,
2612            data_evt_rx,
2613            data_cmd_rx,
2614            exec_evt_rx,
2615            exec_cmd_rx,
2616            signal_rx,
2617            signal_tx.clone(),
2618        );
2619
2620        let handle = runner.handle();
2621
2622        // Send events before starting runner
2623        for _ in 0..10 {
2624            let quote = test_quote();
2625            data_evt_tx
2626                .send((DataEvent::Data(Data::Quote(quote))).into())
2627                .unwrap();
2628        }
2629
2630        let runner_handle = tokio::spawn(async move {
2631            runner.run().await.unwrap();
2632        });
2633
2634        // Yield to let runner enter event loop before stop signal
2635        tokio::task::yield_now().await;
2636        handle.stop();
2637
2638        let result = tokio::time::timeout(Duration::from_millis(200), runner_handle).await;
2639        assert!(result.is_ok(), "Runner should process events and stop");
2640    }
2641
2642    #[rstest]
2643    fn test_new_does_not_bind_tls() {
2644        std::thread::spawn(|| {
2645            let _runner = AsyncRunner::new();
2646            assert!(try_get_time_event_sender().is_none());
2647            assert!(try_get_system_command_sender().is_none());
2648            assert!(try_get_system_event_sender().is_none());
2649            assert!(try_get_trading_cmd_sender().is_none());
2650        })
2651        .join()
2652        .unwrap();
2653    }
2654
2655    #[rstest]
2656    fn test_bind_senders_routes_to_runner_channels() {
2657        std::thread::spawn(|| {
2658            let mut runner = AsyncRunner::new();
2659            runner.bind_senders();
2660
2661            get_data_cmd_sender().execute(DataCommand::Subscribe(SubscribeCommand::Data(
2662                SubscribeCustomData {
2663                    client_id: Some(ClientId::from("TEST")),
2664                    venue: None,
2665                    data_type: DataType::new("test", None, None),
2666                    command_id: UUID4::new(),
2667                    ts_init: UnixNanos::default(),
2668                    correlation_id: None,
2669                    params: None,
2670                },
2671            )));
2672
2673            assert!(runner.channels.data_cmd_rx.try_recv().is_ok());
2674
2675            get_trading_cmd_sender().execute(TradingCommandMessage::new(
2676                MessagingSwitchboard::exec_engine_execute(),
2677                TradingCommand::CancelAllOrders(CancelAllOrders::new(
2678                    TraderId::from("TRADER-001"),
2679                    None,
2680                    StrategyId::from("S-001"),
2681                    InstrumentId::from("EUR/USD.SIM"),
2682                    Some(OrderSide::Buy),
2683                    UUID4::new(),
2684                    UnixNanos::default(),
2685                    None,
2686                    None, // correlation_id
2687                )),
2688            ));
2689            assert!(runner.channels.exec_cmd_rx.try_recv().is_ok());
2690
2691            let event = TimeEvent::new(
2692                Ustr::from("test"),
2693                UUID4::new(),
2694                UnixNanos::from(1),
2695                UnixNanos::from(2),
2696            );
2697            let callback = TimeEventCallback::from(|_: TimeEvent| {});
2698            get_time_event_sender().send(TimeEventMessage::new(event, callback));
2699            assert!(runner.channels.time_evt_rx.try_recv().is_ok());
2700
2701            get_system_event_sender().send(test_system_event()).unwrap();
2702            assert_eq!(
2703                runner
2704                    .channels
2705                    .system_evt_rx
2706                    .try_recv()
2707                    .unwrap()
2708                    .dispatch(|value| value),
2709                test_system_event()
2710            );
2711
2712            get_system_command_sender()
2713                .send(test_system_command())
2714                .unwrap();
2715            assert_eq!(
2716                runner
2717                    .channels
2718                    .system_cmd_rx
2719                    .try_recv()
2720                    .unwrap()
2721                    .dispatch(|value| value),
2722                test_system_command()
2723            );
2724
2725            get_data_event_sender()
2726                .send(DataEvent::Data(Data::Quote(test_quote())))
2727                .unwrap();
2728            assert!(runner.channels.data_evt_rx.try_recv().is_ok());
2729
2730            let account = AccountState::new(
2731                AccountId::from("SIM-001"),
2732                AccountType::Cash,
2733                vec![],
2734                vec![],
2735                true,
2736                UUID4::new(),
2737                UnixNanos::from(1),
2738                UnixNanos::from(2),
2739                None,
2740            );
2741            get_exec_event_sender()
2742                .send(ExecutionEvent::Account(account))
2743                .unwrap();
2744            assert!(runner.channels.exec_evt_rx.try_recv().is_ok());
2745        })
2746        .join()
2747        .unwrap();
2748    }
2749
2750    #[cfg(feature = "node")]
2751    #[rstest]
2752    fn test_drain_pending_system_events_keeps_data_events_separate() {
2753        std::thread::spawn(|| {
2754            let mut runner = AsyncRunner::new();
2755            runner.bind_senders();
2756            let system_event = test_system_event();
2757
2758            get_system_event_sender().send(system_event).unwrap();
2759            get_data_event_sender()
2760                .send(DataEvent::Data(Data::Quote(test_quote())))
2761                .unwrap();
2762
2763            let system_events = runner
2764                .drain_pending_system_events()
2765                .into_iter()
2766                .map(|message| message.dispatch(|value| value))
2767                .collect::<Vec<_>>();
2768
2769            assert_eq!(system_events, vec![system_event]);
2770            assert!(runner.channels.system_evt_rx.try_recv().is_err());
2771            assert!(runner.channels.data_evt_rx.try_recv().is_ok());
2772        })
2773        .join()
2774        .unwrap();
2775    }
2776
2777    #[cfg(feature = "node")]
2778    #[rstest]
2779    fn test_drain_pending_system_commands_keeps_events_separate() {
2780        std::thread::spawn(|| {
2781            let mut runner = AsyncRunner::new();
2782            runner.bind_senders();
2783            let system_command = test_system_command();
2784
2785            get_system_command_sender().send(system_command).unwrap();
2786            get_system_event_sender().send(test_system_event()).unwrap();
2787
2788            let system_commands = runner
2789                .drain_pending_system_commands()
2790                .into_iter()
2791                .map(|message| message.dispatch(|value| value))
2792                .collect::<Vec<_>>();
2793
2794            assert_eq!(system_commands, vec![system_command]);
2795            assert!(runner.channels.system_cmd_rx.try_recv().is_err());
2796            assert!(runner.channels.system_evt_rx.try_recv().is_ok());
2797        })
2798        .join()
2799        .unwrap();
2800    }
2801
2802    #[rstest]
2803    fn test_bind_senders_reclaims_tls_from_previous_runner() {
2804        std::thread::spawn(|| {
2805            let mut runner1 = AsyncRunner::new();
2806            runner1.bind_senders();
2807
2808            let mut runner2 = AsyncRunner::new();
2809            runner2.bind_senders();
2810
2811            get_data_cmd_sender().execute(DataCommand::Subscribe(SubscribeCommand::Data(
2812                SubscribeCustomData {
2813                    client_id: Some(ClientId::from("TEST")),
2814                    venue: None,
2815                    data_type: DataType::new("test", None, None),
2816                    command_id: UUID4::new(),
2817                    ts_init: UnixNanos::default(),
2818                    correlation_id: None,
2819                    params: None,
2820                },
2821            )));
2822
2823            assert!(runner2.channels.data_cmd_rx.try_recv().is_ok());
2824            assert!(runner1.channels.data_cmd_rx.try_recv().is_err());
2825        })
2826        .join()
2827        .unwrap();
2828    }
2829
2830    #[tokio::test]
2831    async fn test_execution_event_order_submitted_batch_channel() {
2832        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2833
2834        let events = vec![
2835            OrderSubmittedSpec::builder()
2836                .client_order_id(ClientOrderId::from("O-001"))
2837                .build(),
2838            OrderSubmittedSpec::builder()
2839                .client_order_id(ClientOrderId::from("O-002"))
2840                .build(),
2841        ];
2842
2843        let batch = OrderSubmittedBatch::new(events);
2844        tx.send(ExecutionEvent::OrderSubmittedBatch(batch)).unwrap();
2845
2846        let received = rx.recv().await.unwrap();
2847        match received {
2848            ExecutionEvent::OrderSubmittedBatch(b) => {
2849                assert_eq!(b.len(), 2);
2850                assert_eq!(b.events[0].client_order_id, ClientOrderId::from("O-001"));
2851                assert_eq!(b.events[1].client_order_id, ClientOrderId::from("O-002"));
2852            }
2853            _ => panic!("Expected OrderSubmittedBatch event"),
2854        }
2855    }
2856
2857    #[tokio::test]
2858    async fn test_execution_event_order_accepted_batch_channel() {
2859        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2860
2861        let events = vec![
2862            OrderAcceptedSpec::builder()
2863                .client_order_id(ClientOrderId::from("O-001"))
2864                .build(),
2865            OrderAcceptedSpec::builder()
2866                .client_order_id(ClientOrderId::from("O-002"))
2867                .build(),
2868        ];
2869
2870        let batch = OrderAcceptedBatch::new(events);
2871        tx.send(ExecutionEvent::OrderAcceptedBatch(batch)).unwrap();
2872
2873        let received = rx.recv().await.unwrap();
2874        match received {
2875            ExecutionEvent::OrderAcceptedBatch(b) => {
2876                assert_eq!(b.len(), 2);
2877                assert_eq!(b.events[0].client_order_id, ClientOrderId::from("O-001"));
2878                assert_eq!(b.events[1].client_order_id, ClientOrderId::from("O-002"));
2879            }
2880            _ => panic!("Expected OrderAcceptedBatch event"),
2881        }
2882    }
2883
2884    #[tokio::test]
2885    async fn test_execution_event_order_canceled_batch_channel() {
2886        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
2887
2888        let events = vec![
2889            OrderCanceledSpec::builder()
2890                .client_order_id(ClientOrderId::from("O-001"))
2891                .build(),
2892            OrderCanceledSpec::builder()
2893                .client_order_id(ClientOrderId::from("O-002"))
2894                .build(),
2895        ];
2896
2897        let batch = OrderCanceledBatch::new(events);
2898        tx.send(ExecutionEvent::OrderCanceledBatch(batch)).unwrap();
2899
2900        let received = rx.recv().await.unwrap();
2901        match received {
2902            ExecutionEvent::OrderCanceledBatch(b) => {
2903                assert_eq!(b.len(), 2);
2904                assert_eq!(b.events[0].client_order_id, ClientOrderId::from("O-001"));
2905                assert_eq!(b.events[1].client_order_id, ClientOrderId::from("O-002"));
2906            }
2907            _ => panic!("Expected OrderCanceledBatch event"),
2908        }
2909    }
2910
2911    #[cfg(feature = "defi")]
2912    #[rstest]
2913    fn test_handle_data_event_routes_pool_snapshot_response_to_data_engine() {
2914        use nautilus_common::{
2915            messages::defi::PoolSnapshotResponse, msgbus::ShareableMessageHandler,
2916        };
2917        use nautilus_model::defi::{
2918            data::block::BlockPosition,
2919            pool_analysis::snapshot::{PoolAnalytics, PoolSnapshot, PoolState},
2920        };
2921
2922        let received = Rc::new(RefCell::new(Vec::new()));
2923        let sink = received.clone();
2924        msgbus::register_any(
2925            MessagingSwitchboard::data_engine_process(),
2926            ShareableMessageHandler::from_typed(move |response: &PoolSnapshotResponse| {
2927                sink.borrow_mut().push(response.correlation_id);
2928            }),
2929        );
2930
2931        let correlation_id = UUID4::new();
2932
2933        let snapshot = PoolSnapshot::new(
2934            InstrumentId::from("0x11b815efB8f581194ae79006d24E0d814B7697F6.Arbitrum:UniswapV3"),
2935            PoolState::default(),
2936            Vec::new(),
2937            Vec::new(),
2938            PoolAnalytics::default(),
2939            BlockPosition::new(1000, "0x0".to_string(), 0, 0),
2940            UnixNanos::default(),
2941            UnixNanos::default(),
2942        );
2943
2944        AsyncRunner::handle_data_event(DataEvent::PoolSnapshotResponse(PoolSnapshotResponse::new(
2945            correlation_id,
2946            snapshot,
2947        )));
2948
2949        assert_eq!(*received.borrow(), vec![correlation_id]);
2950    }
2951}