Skip to main content

nautilus_live/node/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Live trading node built on a single-threaded tokio event loop.
17//!
18//! The node owns system lifecycle and the event loop. Its reconciliation module schedules checks,
19//! manages report futures and deadlines, and dispatches results. The execution manager owns
20//! reconciliation state, discrepancy decisions, and individual reconciliation operations.
21//!
22//! `LiveNode::run()` drives the system through a `tokio::select!` loop that
23//! multiplexes data events, execution events, trading commands, timers, and
24//! periodic maintenance tasks (reconciliation, purge, prune, audit).
25//!
26//! # Threading model
27//!
28//! The core types (`ExecutionManager`, `ExecutionEngine`, `Cache`) use
29//! `Rc<RefCell<..>>` and are `!Send`. All access happens on the same thread.
30//! Pending report futures can retain client borrows while other select branches run.
31//! The client facade defers instrument updates until those borrows are released;
32//! completion and cancellation paths flush the deferred updates. Single-threaded
33//! execution does not by itself prevent conflicting borrows or reentrant callbacks.
34//!
35//! # Startup sequencing
36//!
37//! Startup connects clients in two phases so that instruments are in the
38//! cache before execution clients read them:
39//!
40//! 1. Connect data clients (instruments arrive as buffered `DataEvent`s).
41//! 2. Flush all pending data events and commands into the cache via
42//!    `flush_pending_data`, which loops `try_recv` on the channel receivers
43//!    until no items remain.
44//! 3. Connect execution clients (`load_instruments_from_cache` now finds
45//!    populated instruments).
46//! 4. Drain remaining events, then run reconciliation.
47//!
48//! Both `run()` (integrated event loop) and `start()` (manual lifecycle)
49//! follow this sequence.
50//!
51//! # Reconciliation
52//!
53//! Continuous inflight, open-order, and position checks run on independent intervals. The
54//! shared maintenance timer in the select loop dispatches reconciliation at
55//! the minimum enabled interval. Each dispatch the handler checks which
56//! sub-checks are due based on elapsed nanoseconds and schedules their work.
57//! Continuous checks do not await venue HTTP in the select loop: open-order
58//! and position checks poll bulk venue report futures from the loop.
59//!
60//! # Maintenance dispatcher
61//!
62//! Six periodic tasks share a single coarse `maintenance_timer`:
63//!
64//! - reconciliation (inflight, open, position sub-checks)
65//! - purge closed orders
66//! - purge closed positions
67//! - purge account events
68//! - own-books audit
69//! - recent-fills cache prune
70//!
71//! The runner wakes one timer per loop iteration regardless of how many
72//! maintenance tasks are configured. Each task tracks its own
73//! `next_fire: Instant` and the dispatcher fires the bodies whose deadline
74//! has passed, rescheduling `next = now + interval` (equivalent to
75//! `MissedTickBehavior::Delay`). Disabled tasks anchor on a far-future
76//! `next` that never trips.
77//!
78//! The 100ms timer cadence is the effective floor for any maintenance
79//! interval. Configured intervals below 100ms (the config types allow
80//! `inflight_check_interval_ms` and `own_books_audit_interval_secs` smaller)
81//! become eligible on the next maintenance tick. Event processing and runtime
82//! scheduling can delay dispatch further; the timer does not guarantee a maximum delay.
83
84use std::{
85    any::Any,
86    cell::RefCell,
87    fmt::Debug,
88    rc::{Rc, Weak},
89    time::Duration,
90};
91
92use anyhow::Context;
93use nautilus_common::{
94    actor::{self, Actor, DataActor, DataActorNative},
95    cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
96    clients::ExecutionClient,
97    component::Component,
98    enums::{Environment, LogColor},
99    live::{dispatch::DispatchMessage, dst},
100    log_info,
101    messages::{
102        DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
103        data::DataCommand,
104        execution::TradingCommand,
105        system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
106    },
107    msgbus::{self, BusMessage, MessagingSwitchboard},
108    runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
109};
110use nautilus_core::{
111    UUID4,
112    datetime::{mins_to_secs, secs_to_nanos_unchecked},
113};
114#[cfg(test)]
115use nautilus_model::reports::OrderStatusReport;
116use nautilus_model::{
117    events::OrderEventAny,
118    identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
119    orders::Order,
120};
121use nautilus_network::mode::ReconnectRequestOutcome;
122#[cfg(feature = "python")]
123use nautilus_system::trader::Trader;
124use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
125use nautilus_trading::{
126    ExecutionAlgorithm, ExecutionAlgorithmNative,
127    strategy::{Strategy, StrategyNative},
128};
129use tabled::{builder::Builder, settings::Style};
130
131use crate::{
132    dispatch::drain_callbacks,
133    execution::{
134        client::LiveExecutionClient,
135        manager::{
136            ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery, TargetedOrderReportResult,
137        },
138    },
139    runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
140    socket::{SocketReconnectLookup, SocketReconnectRegistry},
141};
142
143pub mod builder;
144pub mod config;
145
146#[cfg(feature = "plugin")]
147pub mod plugin;
148
149mod metrics;
150mod queue;
151mod reconciliation;
152mod state;
153
154use builder::ExternalMessageBusIngress;
155pub use builder::LiveNodeBuilder;
156use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
157pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
158use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
159use queue::{QueueMonitor, QueueStateTransition};
160use reconciliation::{
161    OpenOrderReportResult, OpenOrderReportTask, PositionReportTask, PositionReportTaskResult,
162    ReconciliationCheckIntervals, ReconciliationCheckState, ReportTaskOutcome,
163    TargetedOrderReportTask,
164};
165use state::{EngineConnectionStatus, RunningTransition};
166pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
167
168/// Dispatches the run loop performs before yielding to the executor.
169///
170/// A saturated channel keeps every select branch ready, so the loop would otherwise never return
171/// `Pending`. Under a host event loop that starves the adapter I/O tasks feeding those channels,
172/// which shows up as lapsed heartbeats and reconnects rather than as backpressure.
173const DISPATCHES_PER_YIELD: usize = 64;
174
175thread_local! {
176    static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
177}
178
179type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
180
181struct StreamProcessor(Box<StreamProcessorCallback>);
182
183impl Debug for StreamProcessor {
184    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
185        f.debug_struct(stringify!(StreamProcessor)).finish()
186    }
187}
188
189/// High-level abstraction for a live Nautilus system node.
190///
191/// Provides a simplified interface for running live systems
192/// with automatic client management and lifecycle handling.
193///
194/// Only one live node may exist on a thread at a time. Drop the node before building another,
195/// including after disposal, because retained nodes can still access thread-local messaging.
196/// Concurrent nodes in one process remain unsupported, even on separate threads.
197#[derive(Debug)]
198pub struct LiveNode {
199    kernel: NautilusKernel,
200    runner: Option<AsyncRunner>,
201    config: LiveNodeConfig,
202    handle: LiveNodeHandle,
203    exec_manager: ExecutionManager,
204    exec_clients: Vec<LiveExecutionClient>,
205    socket_registry: SocketReconnectRegistry,
206    cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
207    external_msgbus: Option<ExternalMessageBusIngress>,
208    stream_processors: Vec<StreamProcessor>,
209    shutdown_deadline: Option<dst::time::Instant>,
210    #[cfg(feature = "plugin")]
211    plugins: plugin::NodePlugins,
212    _thread_owner: Rc<()>,
213}
214
215impl LiveNode {
216    /// Creates a new `LiveNode` from builder components.
217    ///
218    /// This is an internal constructor used by `LiveNodeBuilder`.
219    #[must_use]
220    #[allow(
221        clippy::too_many_arguments,
222        reason = "builder components have distinct lifecycle roles"
223    )]
224    pub(crate) fn new_from_builder(
225        kernel: NautilusKernel,
226        runner: AsyncRunner,
227        config: LiveNodeConfig,
228        handle: LiveNodeHandle,
229        mut exec_manager: ExecutionManager,
230        exec_clients: Vec<LiveExecutionClient>,
231        socket_registry: SocketReconnectRegistry,
232        cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
233        external_msgbus: Option<ExternalMessageBusIngress>,
234        thread_owner: Rc<()>,
235    ) -> Self {
236        for client in &exec_clients {
237            exec_manager.register_submission_retention(client);
238        }
239
240        Self {
241            kernel,
242            runner: Some(runner),
243            config,
244            handle,
245            exec_manager,
246            exec_clients,
247            socket_registry,
248            cache_database_factory,
249            external_msgbus,
250            stream_processors: Vec::new(),
251            shutdown_deadline: None,
252            #[cfg(feature = "plugin")]
253            plugins: plugin::NodePlugins,
254            _thread_owner: thread_owner,
255        }
256    }
257
258    /// Creates a new [`LiveNodeBuilder`] for fluent configuration.
259    ///
260    /// # Errors
261    ///
262    /// Returns an error if the environment is invalid for live trading.
263    pub fn builder(
264        trader_id: TraderId,
265        environment: Environment,
266    ) -> anyhow::Result<LiveNodeBuilder> {
267        LiveNodeBuilder::new(trader_id, environment)
268    }
269
270    /// Creates a new [`LiveNode`] directly from a kernel name and optional configuration.
271    ///
272    /// This is a convenience method for creating a live node with a pre-configured
273    /// kernel configuration, bypassing the builder pattern. If no config is provided,
274    /// a default configuration will be used.
275    ///
276    /// # Errors
277    ///
278    /// Returns an error if kernel construction fails or another live node exists or is being
279    /// built on this thread.
280    pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
281        let thread_owner = Self::acquire_thread()?;
282        let config = config.unwrap_or_default();
283        validate_live_environment(config.environment())?;
284
285        config.validate_runtime_support()?;
286
287        if config.event_store.is_some() {
288            anyhow::bail!(
289                "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
290                 use LiveNodeBuilder::with_event_store(...) instead"
291            );
292        }
293
294        let handle = LiveNodeHandle::new();
295        let runner = AsyncRunner::new();
296        runner.bind_senders_for_node(handle.clone());
297
298        let kernel = NautilusKernel::new(name, config.clone())?;
299        #[cfg(feature = "python")]
300        if let Some(controller) = config.controller.as_ref() {
301            Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
302        }
303
304        #[cfg(not(feature = "python"))]
305        if let Some(controller) = config.controller.as_ref() {
306            anyhow::bail!(
307                "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
308                controller.controller_path
309            );
310        }
311
312        let exec_manager_config =
313            ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
314
315        let exec_manager = ExecutionManager::new(
316            kernel.clock.clone(),
317            kernel.cache.clone(),
318            exec_manager_config,
319        )?;
320
321        let node = Self {
322            kernel,
323            runner: Some(runner),
324            config,
325            handle,
326            exec_manager,
327            exec_clients: Vec::new(),
328            socket_registry: SocketReconnectRegistry::default(),
329            cache_database_factory: None,
330            external_msgbus: None,
331            stream_processors: Vec::new(),
332            shutdown_deadline: None,
333            #[cfg(feature = "plugin")]
334            plugins: plugin::NodePlugins,
335            _thread_owner: thread_owner,
336        };
337
338        node.load_configured_plugins()?;
339
340        log::info!("LiveNode built successfully with kernel config");
341
342        Ok(node)
343    }
344
345    fn acquire_thread() -> anyhow::Result<Rc<()>> {
346        NODE_THREAD_OWNER.with(|slot| {
347            let mut owner = slot.borrow_mut();
348            anyhow::ensure!(
349                owner.upgrade().is_none(),
350                "A LiveNode already exists or is being built on this thread; drop it before building another"
351            );
352            let token = Rc::new(());
353            *owner = Rc::downgrade(&token);
354            Ok(token)
355        })
356    }
357
358    /// Loads and registers plug-ins declared on the node config.
359    ///
360    /// # Errors
361    ///
362    /// Returns an error if plug-ins are configured without host-side support.
363    pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
364        if self.config.plugins.is_empty() {
365            return Ok(());
366        }
367
368        anyhow::bail!(
369            "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
370        )
371    }
372
373    /// Loads and registers one plug-in instance.
374    ///
375    /// # Errors
376    ///
377    /// Returns an error if called. Dynamic plug-in hosting lives in the host-side integration.
378    #[expect(
379        clippy::needless_pass_by_value,
380        reason = "signature mirrors the host-enabled API"
381    )]
382    pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
383        #[cfg(feature = "plugin")]
384        config.validate_runtime_support(self.config.plugins.len())?;
385        #[cfg(not(feature = "plugin"))]
386        let _ = config;
387
388        anyhow::bail!(
389            "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
390        )
391    }
392
393    /// Returns a thread-safe handle to control this node.
394    #[must_use]
395    pub fn handle(&self) -> LiveNodeHandle {
396        self.handle.clone()
397    }
398
399    /// Adds a callback for supported typed external messages.
400    ///
401    /// While [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) services external ingress,
402    /// the node invokes processors in registration order before normal inbound streaming filters
403    /// for JSON or MessagePack payloads when [`msgbus::BusPayloadType::is_typed_message`] returns
404    /// `true`. Other encodings are skipped with a warning. Each callback receives the decoded
405    /// concrete value as [`Any`], allowing it to downcast to the concrete payload type. External
406    /// egress is suppressed while the processors run, so synchronous publications remain local.
407    pub fn add_stream_processor<F>(&mut self, callback: F)
408    where
409        F: Fn(&dyn Any) + 'static,
410    {
411        self.add_stream_processor_with_mapping(move |message, _| {
412            callback(message);
413            Ok(())
414        });
415    }
416
417    pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
418    where
419        F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
420    {
421        self.stream_processors
422            .push(StreamProcessor(Box::new(callback)));
423    }
424
425    /// Starts the live node without entering a select loop.
426    ///
427    /// Connects clients, runs reconciliation, and starts the trader, but does
428    /// not consume the runner or drive channel receivers, so channel traffic arriving after
429    /// startup is never serviced. This is a building block for tests and embedding, not a
430    /// lifecycle: use [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) to run a node.
431    ///
432    /// # Errors
433    ///
434    /// Returns an error if startup fails.
435    pub async fn start(&mut self) -> anyhow::Result<()> {
436        if self.state().is_running() {
437            anyhow::bail!("Already running");
438        }
439
440        if self.external_msgbus.is_some() {
441            log::warn!(
442                "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
443            );
444        }
445
446        self.prepare_cache().await?;
447
448        if let Some(runner) = self.runner.as_ref() {
449            runner.bind_senders_for_node(self.handle.clone());
450        }
451
452        self.handle.set_starting();
453
454        self.kernel.reset_shutdown_flag();
455        self.kernel.start_async().await;
456
457        if self.kernel.is_event_store_replay() {
458            log::info!(
459                "Event-store replay loaded; skipping live client connection and reconciliation",
460            );
461
462            if !self.finish_startup_replay().await? {
463                return Ok(());
464            }
465
466            return Ok(());
467        }
468
469        if self.kernel.is_event_store_replay_configured() {
470            self.abort_startup("Event-store replay did not start")
471                .await?;
472            return Ok(());
473        }
474
475        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
476
477        // Connect data clients first and flush instrument events into cache
478        if let Err(e) = self.connect_data_phase(connection_deadline).await {
479            return self
480                .abort_startup_with_error("Data client connection timed out", e)
481                .await;
482        }
483
484        let (startup_system_events, startup_system_commands) =
485            if let Some(runner) = self.runner.as_mut() {
486                runner.flush_pending_data();
487                (
488                    runner.drain_pending_system_events(),
489                    runner.drain_pending_system_commands(),
490                )
491            } else {
492                (Vec::new(), Vec::new())
493            };
494
495        if let Err(e) = self.connect_exec_clients(connection_deadline).await {
496            return self
497                .abort_startup_with_error("Execution client connection timed out", e)
498                .await;
499        }
500
501        if let Some(reason) = self.startup_abort_reason() {
502            self.abort_startup(reason).await?;
503            return Ok(());
504        }
505
506        match self.await_engines_connected(connection_deadline).await {
507            EngineConnectionStatus::Connected => {}
508            EngineConnectionStatus::TimedOut => {
509                return self
510                    .abort_startup_with_error(
511                        "Engine readiness timed out",
512                        anyhow::anyhow!("readiness timeout while waiting for engine connections"),
513                    )
514                    .await;
515            }
516            EngineConnectionStatus::StopRequested => {
517                self.abort_startup("Stop signal received during startup")
518                    .await?;
519                return Ok(());
520            }
521            EngineConnectionStatus::ShutdownRequested => {
522                self.abort_startup("Shutdown signal received during startup")
523                    .await?;
524                return Ok(());
525            }
526        }
527
528        if let Err(e) = self.perform_startup_reconciliation().await {
529            if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
530                anyhow::bail!(
531                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
532                );
533            }
534
535            return Err(e);
536        }
537
538        if let Some(reason) = self.startup_abort_reason() {
539            self.abort_startup(reason).await?;
540            return Ok(());
541        }
542
543        if let Err(e) = self.kernel.start_trader() {
544            return self.abort_after_trader_start_failure(e).await;
545        }
546
547        #[cfg(feature = "plugin")]
548        if let Err(e) = self.plugins.start_controllers() {
549            return self.abort_after_trader_start_failure(e).await;
550        }
551
552        self.process_system_events(startup_system_events);
553        self.process_system_commands(startup_system_commands);
554
555        if !self.finish_startup_trader(None).await? {
556            return Ok(());
557        }
558
559        Ok(())
560    }
561
562    /// Stop the live node.
563    ///
564    /// This method stops the trader, waits for the configured grace period to allow
565    /// residual events to be processed, then finalizes the shutdown sequence.
566    ///
567    /// # Errors
568    ///
569    /// Returns an error if shutdown fails or retained submissions remain unresolved at the end
570    /// of `delay_post_stop`. Teardown still completes; subsequent events cannot clear this result.
571    pub async fn stop(&mut self) -> anyhow::Result<()> {
572        if !self.state().is_running() {
573            anyhow::bail!("Not running");
574        }
575
576        self.handle.set_shutting_down();
577
578        #[cfg(feature = "plugin")]
579        let controller_stop_result = self.plugins.stop_controllers();
580        #[cfg(not(feature = "plugin"))]
581        let controller_stop_result: anyhow::Result<()> = Ok(());
582
583        self.kernel.stop_trader();
584        let delay = self.kernel.delay_post_stop();
585        log::info!("Awaiting residual events ({delay:?})...");
586
587        let residual_events = self.process_runner_for(delay).await;
588        if residual_events > 0 {
589            log::debug!("Processed {residual_events} residual events during shutdown");
590        }
591
592        let stop_result = self.finalize_stop().await;
593        let drained_events = self.drain_runner_pending();
594        if drained_events > 0 {
595            log::info!("Drained {drained_events} remaining events during shutdown");
596        }
597
598        match (controller_stop_result, stop_result) {
599            (Ok(()), Ok(())) => Ok(()),
600            (Err(controller_err), Ok(())) => Err(controller_err),
601            (Ok(()), Err(stop_err)) => Err(stop_err),
602            (Err(controller_err), Err(stop_err)) => {
603                log::error!("Error stopping plug-in controllers: {controller_err}");
604                Err(stop_err)
605            }
606        }
607    }
608
609    /// Disposes the live node kernel and releases resources.
610    ///
611    /// Discards any retained runner messages and attempts callback cleanup. Logs latched callback
612    /// failures and cleanup rejection; externally retained work can prevent clearing.
613    pub fn dispose(&mut self) {
614        self.close_external_ingress();
615        self.handle.set_stopped();
616        self.kernel.dispose();
617        drop(self.runner.take());
618
619        if let Some(e) = actor::callback_failure() {
620            log::error!("Callback dispatch failed before disposal cleanup: {e}");
621        }
622
623        if let Err(e) = actor::clear_callbacks() {
624            log::error!("Failed to clear callback dispatch during disposal: {e}");
625        }
626    }
627
628    async fn process_runner_for(&mut self, duration: Duration) -> usize {
629        let Some(mut runner) = self.runner.take() else {
630            dst::time::sleep(duration).await;
631            return 0;
632        };
633
634        runner.bind_senders_for_node(self.handle.clone());
635        let deadline = dst::time::Instant::now() + duration;
636        let mut processed = 0;
637
638        loop {
639            tokio::select! {
640                biased;
641
642                () = dst::time::sleep_until(deadline) => break,
643                event = runner.recv() => {
644                    let Some(event) = event else {
645                        dst::time::sleep_until(deadline).await;
646                        break;
647                    };
648
649                    self.process_runner_event(event);
650                    processed += 1;
651                }
652            }
653        }
654
655        self.runner = Some(runner);
656        processed
657    }
658
659    fn drain_runner_pending(&mut self) -> usize {
660        let Some(mut runner) = self.runner.take() else {
661            return 0;
662        };
663
664        let processed = runner.poll_pending(|event| self.process_runner_event(event));
665        self.runner = Some(runner);
666        processed
667    }
668
669    fn process_runner_event(&mut self, event: PendingRunnerEvent) {
670        match event {
671            PendingRunnerEvent::TimeEvent(message) => {
672                let _ = AsyncRunner::handle_time_event(message);
673            }
674            PendingRunnerEvent::SystemEvent(event) => {
675                event.dispatch(|event| self.process_system_event(event));
676            }
677            PendingRunnerEvent::SystemCommand(command) => {
678                command.dispatch(|command| self.process_system_command(command));
679            }
680            PendingRunnerEvent::ExecEvent(event) => {
681                event.dispatch(|event| self.process_exec_event(event));
682            }
683            PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
684            PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
685            PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
686        }
687    }
688
689    fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
690        for event in events {
691            event.dispatch(|event| self.process_system_event(event));
692        }
693    }
694
695    fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
696        for command in commands {
697            command.dispatch(|command| self.process_system_command(command));
698        }
699    }
700
701    fn process_system_command(&self, command: SystemCommand) {
702        match command {
703            SystemCommand::ReconnectSocket(command) => {
704                self.process_socket_reconnect(command);
705            }
706        }
707    }
708
709    fn process_socket_reconnect(&self, command: ReconnectSocket) {
710        let outcome = if command.trader_id == self.config.trader_id {
711            Self::request_socket_reconnect(
712                self.socket_registry
713                    .get(command.client_id, command.endpoint),
714            )
715        } else {
716            SocketReconnectDispatchOutcome::InvalidTrader
717        };
718
719        if outcome == SocketReconnectDispatchOutcome::Accepted {
720            log::info!(
721                "Requested socket reconnect for client {} endpoint {}",
722                command.client_id,
723                command.endpoint
724            );
725        } else {
726            log::warn!(
727                "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
728                command.client_id,
729                command.endpoint
730            );
731        }
732    }
733
734    fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
735        match lookup {
736            SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
737                ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
738                ReconnectRequestOutcome::AlreadyReconnecting => {
739                    SocketReconnectDispatchOutcome::AlreadyReconnecting
740                }
741                ReconnectRequestOutcome::Disconnected => {
742                    SocketReconnectDispatchOutcome::Disconnected
743                }
744                ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
745                ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
746            },
747            SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
748            SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
749            SocketReconnectLookup::EndpointNotFound => {
750                SocketReconnectDispatchOutcome::UnknownEndpoint
751            }
752            SocketReconnectLookup::AmbiguousEndpoint => {
753                SocketReconnectDispatchOutcome::AmbiguousEndpoint
754            }
755        }
756    }
757
758    fn process_system_event(&self, event: SystemEvent) {
759        match event {
760            SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
761        }
762    }
763
764    fn publish_socket_state_change(&self, change: SocketStateChange) {
765        let timestamp = self.kernel.generate_timestamp_ns();
766
767        let event = SocketStateChanged::new(
768            self.config.trader_id,
769            change.client_id,
770            change.venue,
771            change.endpoint,
772            change.state,
773            UUID4::new(),
774            timestamp,
775            timestamp,
776        );
777
778        msgbus::publish_any(
779            MessagingSwitchboard::socket_state_changed_topic(
780                event.client_id,
781                event.endpoint.as_str(),
782            ),
783            event.as_any(),
784        );
785    }
786
787    /// Awaits engine clients to connect with timeout.
788    ///
789    /// Returns the final connection wait status.
790    async fn await_engines_connected(
791        &self,
792        deadline: dst::time::Instant,
793    ) -> EngineConnectionStatus {
794        log::info!(
795            "Awaiting engine connections ({:?} timeout)...",
796            self.config.timeout_connection
797        );
798
799        let interval = Duration::from_millis(100);
800
801        loop {
802            if self.handle.should_stop() {
803                log::warn!("Stop signal received, aborting connection wait");
804                return EngineConnectionStatus::StopRequested;
805            }
806
807            if self.kernel.is_shutdown_requested() {
808                log::warn!("Shutdown signal received, aborting connection wait");
809                return EngineConnectionStatus::ShutdownRequested;
810            }
811
812            if self.kernel.check_engines_connected() {
813                log::info!("All engine clients connected");
814                return EngineConnectionStatus::Connected;
815            }
816
817            let now = dst::time::Instant::now();
818            if now >= deadline {
819                break;
820            }
821
822            dst::time::sleep(interval.min(deadline - now)).await;
823        }
824
825        self.log_connection_status();
826        EngineConnectionStatus::TimedOut
827    }
828
829    /// Awaits engine clients to disconnect with timeout.
830    ///
831    /// Returns an error with client status on timeout.
832    async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
833        log::info!(
834            "Awaiting engine disconnections ({:?} timeout)...",
835            self.config.timeout_disconnection
836        );
837
838        let timeout = self.config.timeout_disconnection;
839        let interval = Duration::from_millis(100);
840
841        loop {
842            if self.kernel.check_engines_disconnected() {
843                log::info!("All engine clients disconnected");
844                return Ok(());
845            }
846
847            let now = dst::time::Instant::now();
848            if now >= deadline {
849                break;
850            }
851
852            dst::time::sleep(interval.min(deadline - now)).await;
853        }
854
855        log::error!(
856            "Timed out ({:?}) waiting for engines to disconnect\n\
857             DataEngine.check_disconnected() == {}\n\
858             ExecEngine.check_disconnected() == {}",
859            timeout,
860            self.kernel.data_engine().check_disconnected(),
861            self.kernel.exec_engine().borrow().check_disconnected(),
862        );
863        anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
864    }
865
866    fn log_connection_status(&self) {
867        let data_status = self.kernel.data_client_connection_status();
868        let exec_status = self.kernel.exec_client_connection_status();
869
870        let mut rows: Vec<ClientStatus> = Vec::new();
871
872        for (client_id, connected) in data_status {
873            rows.push(ClientStatus {
874                client: client_id.to_string(),
875                client_type: "Data",
876                connected,
877            });
878        }
879
880        for (client_id, connected) in exec_status {
881            rows.push(ClientStatus {
882                client: client_id.to_string(),
883                client_type: "Execution",
884                connected,
885            });
886        }
887
888        let table = render_client_statuses(rows);
889
890        log::warn!(
891            "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
892             DataEngine.check_connected() == {}\n\
893             ExecEngine.check_connected() == {}",
894            self.config.timeout_connection,
895            self.kernel.data_engine().check_connected(),
896            self.kernel.exec_engine().borrow().check_connected(),
897        );
898    }
899
900    /// Performs startup reconciliation to align internal state with venue state.
901    ///
902    /// This method queries each execution client for mass status (orders, fills, positions)
903    /// and reconciles any discrepancies with the local cache state.
904    ///
905    /// # Errors
906    ///
907    /// Returns an error if reconciliation fails or times out.
908    #[expect(clippy::await_holding_refcell_ref)] // Single-threaded runtime, intentional design
909    async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
910        if !self.config.exec_engine.reconciliation {
911            log::info!("Startup reconciliation disabled");
912            self.kernel
913                .portfolio
914                .borrow_mut()
915                .initialize_wallet_orders()?;
916            return Ok(());
917        }
918
919        log_info!(
920            "Starting execution state reconciliation...",
921            color = LogColor::Blue
922        );
923
924        let lookback_mins = self
925            .config
926            .exec_engine
927            .reconciliation_lookback_mins
928            .map(u64::from);
929
930        let timeout = self.config.timeout_reconciliation;
931        let start = dst::time::Instant::now();
932        let client_ids = self.kernel.exec_engine.borrow().client_ids();
933
934        for client_id in client_ids {
935            let elapsed = start.elapsed();
936            if elapsed >= timeout {
937                anyhow::bail!("Startup reconciliation timeout reached");
938            }
939
940            let remaining = timeout
941                .checked_sub(elapsed)
942                .expect("elapsed checked against reconciliation timeout");
943
944            log_info!(
945                "Requesting mass status from {}...",
946                client_id,
947                color = LogColor::Blue
948            );
949
950            let mass_status_result = dst::time::timeout(remaining, async {
951                self.kernel
952                    .exec_engine
953                    .borrow_mut()
954                    .generate_mass_status(&client_id, lookback_mins)
955                    .await
956            })
957            .await
958            .map_err(|_| {
959                anyhow::anyhow!(
960                    "Startup reconciliation timeout reached while requesting mass status from {client_id}"
961                )
962            })?;
963
964            match mass_status_result {
965                Ok(Some(mass_status)) => {
966                    log_info!(
967                        "Reconciling ExecutionMassStatus for {}",
968                        client_id,
969                        color = LogColor::Blue
970                    );
971
972                    let result = self
973                        .exec_manager
974                        .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
975
976                    anyhow::ensure!(
977                        self.kernel
978                            .exec_engine
979                            .borrow()
980                            .get_client(&client_id)
981                            .is_some(),
982                        "Execution client {client_id} disappeared during startup reconciliation",
983                    );
984
985                    anyhow::ensure!(
986                        result.unresolved_positions.is_empty(),
987                        "Unresolved positions during startup reconciliation for {client_id}: {}",
988                        result.unresolved_positions.join("; "),
989                    );
990
991                    if result.events.is_empty() {
992                        log_info!(
993                            "Reconciliation for {} succeeded",
994                            client_id,
995                            color = LogColor::Blue
996                        );
997                    } else {
998                        log::info!(
999                            color = LogColor::Blue as u8;
1000                            "Reconciliation for {} processed {} events",
1001                            client_id,
1002                            result.events.len()
1003                        );
1004                    }
1005
1006                    // Register external orders with execution clients for tracking
1007                    if !result.external_orders.is_empty() {
1008                        let exec_engine = self.kernel.exec_engine.borrow();
1009
1010                        let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1011                            anyhow::anyhow!(
1012                                "Execution client {client_id} disappeared during startup reconciliation"
1013                            )
1014                        })?;
1015
1016                        for external in result.external_orders {
1017                            source_client.register_external_order(
1018                                external.client_order_id,
1019                                external.venue_order_id,
1020                                external.instrument_id,
1021                                external.strategy_id,
1022                                external.ts_init,
1023                            );
1024                        }
1025                    }
1026                }
1027                Ok(None) => {
1028                    log::warn!(
1029                        "No mass status available from {client_id} \
1030                         (likely adapter error when generating reports)"
1031                    );
1032                }
1033                Err(e) => {
1034                    return Err(e).context(format!("Failed to get mass status from {client_id}"));
1035                }
1036            }
1037        }
1038
1039        self.kernel.portfolio.borrow_mut().initialize_orders();
1040        self.kernel.portfolio.borrow_mut().initialize_positions();
1041        self.kernel
1042            .portfolio
1043            .borrow_mut()
1044            .initialize_wallet_orders()?;
1045
1046        let elapsed_secs = start.elapsed().as_secs_f64();
1047        log_info!(
1048            "Startup reconciliation completed in {:.2}s",
1049            elapsed_secs,
1050            color = LogColor::Blue
1051        );
1052
1053        Ok(())
1054    }
1055
1056    /// Run the live node with automatic shutdown handling.
1057    ///
1058    /// This method starts the node, runs indefinitely, and handles graceful shutdown
1059    /// on interrupt signals.
1060    ///
1061    /// # Thread Safety
1062    ///
1063    /// The event loop runs directly on the current thread (not spawned) because the
1064    /// msgbus uses thread-local storage. Endpoints registered by the kernel are only
1065    /// accessible from the same thread.
1066    ///
1067    /// # Shutdown Sequence
1068    ///
1069    /// 1. Signal received (SIGINT, SIGTERM, or handle stop).
1070    /// 2. Trader components stopped (triggers order cancellations, etc.).
1071    /// 3. Event loop continues processing residual events for the configured grace period.
1072    /// 4. Kernel finalized, clients disconnected, remaining events drained.
1073    ///
1074    /// # Errors
1075    ///
1076    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1077    /// with retained submissions unresolved at the end of `delay_post_stop`.
1078    pub async fn run(&mut self) -> anyhow::Result<()> {
1079        self.run_with_mode(NodeRunMode::Owned).await
1080    }
1081
1082    /// Run the live node under the given mode.
1083    ///
1084    /// [`NodeRunMode::Hosted`] leaves signal handling to the host application. Every other
1085    /// responsibility, including maintenance, reconciliation, external ingress, and the shutdown
1086    /// sequence, is identical across modes so that hosted and owned nodes cannot diverge.
1087    ///
1088    /// # Errors
1089    ///
1090    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1091    /// with retained submissions unresolved at the end of `delay_post_stop`.
1092    pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1093        if self.state().is_running() {
1094            anyhow::bail!("Already running");
1095        }
1096
1097        if self.runner.is_none() {
1098            anyhow::bail!("Runner already consumed - run() called twice");
1099        }
1100
1101        self.prepare_cache().await?;
1102
1103        let Some(runner) = self.runner.take() else {
1104            anyhow::bail!("Runner already consumed - run() called twice");
1105        };
1106
1107        runner.bind_senders_for_node(self.handle.clone());
1108
1109        let AsyncRunnerChannels {
1110            mut time_evt_rx,
1111            mut system_evt_rx,
1112            mut system_cmd_rx,
1113            mut exec_evt_rx,
1114            mut exec_cmd_rx,
1115            mut data_evt_rx,
1116            mut data_cmd_rx,
1117        } = runner.take_channels();
1118
1119        log::info!("Event loop starting");
1120
1121        self.handle.set_starting();
1122        self.kernel.reset_shutdown_flag();
1123        self.kernel.start_async().await;
1124
1125        if self.kernel.is_event_store_replay() {
1126            log::info!(
1127                "Event-store replay loaded; skipping live client connection and reconciliation",
1128            );
1129
1130            if !self.finish_startup_replay().await? {
1131                return Ok(());
1132            }
1133
1134            return Ok(());
1135        }
1136
1137        if self.kernel.is_event_store_replay_configured() {
1138            self.abort_startup("Event-store replay did not start")
1139                .await?;
1140            return Ok(());
1141        }
1142
1143        let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1144            Ok(rx) => rx,
1145            Err(e) => {
1146                let result = self
1147                    .abort_startup("External message bus ingress failed to start")
1148                    .await;
1149                Self::drain_channels(
1150                    &mut time_evt_rx,
1151                    &mut system_evt_rx,
1152                    &mut system_cmd_rx,
1153                    &mut exec_evt_rx,
1154                    &mut exec_cmd_rx,
1155                    &mut data_evt_rx,
1156                    &mut data_cmd_rx,
1157                );
1158                log::info!("Event loop stopped");
1159
1160                if let Err(finalize_err) = result {
1161                    anyhow::bail!(
1162                        "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1163                    );
1164                }
1165
1166                return Err(e);
1167            }
1168        };
1169
1170        let stop_handle = self.handle.clone();
1171        let mut pending = PendingEvents::default();
1172        let mut startup_system_events = Vec::new();
1173        let mut startup_system_commands = Vec::new();
1174        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1175
1176        // Startup phase 1: Connect data clients and drain instrument events into cache.
1177        // This ensures the cache is populated before execution clients connect.
1178        let data_connect_result = drive_with_event_buffering(
1179            self.connect_data_phase(connection_deadline),
1180            &mut pending,
1181            &mut time_evt_rx,
1182            &mut system_evt_rx,
1183            &mut system_cmd_rx,
1184            &mut exec_evt_rx,
1185            &mut exec_cmd_rx,
1186            &mut data_evt_rx,
1187            &mut data_cmd_rx,
1188        )
1189        .await;
1190
1191        if let Err(e) = data_connect_result {
1192            flush_all_pending(
1193                &mut pending,
1194                &mut time_evt_rx,
1195                &mut system_evt_rx,
1196                &mut system_cmd_rx,
1197                &mut exec_evt_rx,
1198                &mut exec_cmd_rx,
1199                &mut data_evt_rx,
1200                &mut data_cmd_rx,
1201            );
1202            let result = self
1203                .abort_startup_with_error("Data client connection timed out", e)
1204                .await;
1205            Self::drain_channels(
1206                &mut time_evt_rx,
1207                &mut system_evt_rx,
1208                &mut system_cmd_rx,
1209                &mut exec_evt_rx,
1210                &mut exec_cmd_rx,
1211                &mut data_evt_rx,
1212                &mut data_cmd_rx,
1213            );
1214            log::info!("Event loop stopped");
1215            return result;
1216        }
1217
1218        // Flush any data events still queued in the channel receivers that the
1219        // select loop did not capture before the connect future resolved, then
1220        // drain everything into cache.
1221        flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1222        startup_system_events.extend(pending.take_system_events());
1223        startup_system_commands.extend(pending.take_system_commands());
1224        debug_assert!(
1225            pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1226            "data must be drained into cache before exec clients connect",
1227        );
1228
1229        // Startup phase 2: Connect execution clients (instruments now in cache)
1230        let engine_connection_result = drive_with_event_buffering(
1231            self.connect_exec_phase(connection_deadline),
1232            &mut pending,
1233            &mut time_evt_rx,
1234            &mut system_evt_rx,
1235            &mut system_cmd_rx,
1236            &mut exec_evt_rx,
1237            &mut exec_cmd_rx,
1238            &mut data_evt_rx,
1239            &mut data_cmd_rx,
1240        )
1241        .await;
1242
1243        // Flush channel receivers and drain all remaining pending events
1244        flush_all_pending(
1245            &mut pending,
1246            &mut time_evt_rx,
1247            &mut system_evt_rx,
1248            &mut system_cmd_rx,
1249            &mut exec_evt_rx,
1250            &mut exec_cmd_rx,
1251            &mut data_evt_rx,
1252            &mut data_cmd_rx,
1253        );
1254        startup_system_events.extend(pending.take_system_events());
1255        startup_system_commands.extend(pending.take_system_commands());
1256        debug_assert!(
1257            pending.is_empty(),
1258            "all startup events must be processed before reconciliation",
1259        );
1260
1261        let engine_connection_status = match engine_connection_result {
1262            Ok(status) => status,
1263            Err(e) => {
1264                let result = self
1265                    .abort_startup_with_error("Execution client connection timed out", e)
1266                    .await;
1267                Self::drain_channels(
1268                    &mut time_evt_rx,
1269                    &mut system_evt_rx,
1270                    &mut system_cmd_rx,
1271                    &mut exec_evt_rx,
1272                    &mut exec_cmd_rx,
1273                    &mut data_evt_rx,
1274                    &mut data_cmd_rx,
1275                );
1276                log::info!("Event loop stopped");
1277                return result;
1278            }
1279        };
1280
1281        if engine_connection_status == EngineConnectionStatus::TimedOut {
1282            let result = self
1283                .abort_startup_with_error(
1284                    "Engine readiness timed out",
1285                    anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1286                )
1287                .await;
1288            Self::drain_channels(
1289                &mut time_evt_rx,
1290                &mut system_evt_rx,
1291                &mut system_cmd_rx,
1292                &mut exec_evt_rx,
1293                &mut exec_cmd_rx,
1294                &mut data_evt_rx,
1295                &mut data_cmd_rx,
1296            );
1297            log::info!("Event loop stopped");
1298            return result;
1299        }
1300
1301        if let Some(reason) = engine_connection_status
1302            .abort_reason()
1303            .or_else(|| self.startup_abort_reason())
1304        {
1305            self.abort_startup(reason).await?;
1306            Self::drain_channels(
1307                &mut time_evt_rx,
1308                &mut system_evt_rx,
1309                &mut system_cmd_rx,
1310                &mut exec_evt_rx,
1311                &mut exec_cmd_rx,
1312                &mut data_evt_rx,
1313                &mut data_cmd_rx,
1314            );
1315            log::info!("Event loop stopped");
1316            return Ok(());
1317        }
1318
1319        debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1320
1321        // Run reconciliation now that instruments are in cache and start trader
1322        if let Err(e) = self.perform_startup_reconciliation().await {
1323            let result = self.abort_startup("Startup reconciliation failed").await;
1324            Self::drain_channels(
1325                &mut time_evt_rx,
1326                &mut system_evt_rx,
1327                &mut system_cmd_rx,
1328                &mut exec_evt_rx,
1329                &mut exec_cmd_rx,
1330                &mut data_evt_rx,
1331                &mut data_cmd_rx,
1332            );
1333            log::info!("Event loop stopped");
1334
1335            if let Err(finalize_err) = result {
1336                anyhow::bail!(
1337                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1338                );
1339            }
1340
1341            return Err(e);
1342        }
1343
1344        if let Some(reason) = self.startup_abort_reason() {
1345            let result = self.abort_startup(reason).await;
1346            Self::drain_channels(
1347                &mut time_evt_rx,
1348                &mut system_evt_rx,
1349                &mut system_cmd_rx,
1350                &mut exec_evt_rx,
1351                &mut exec_cmd_rx,
1352                &mut data_evt_rx,
1353                &mut data_cmd_rx,
1354            );
1355            log::info!("Event loop stopped");
1356            return result;
1357        }
1358
1359        if let Err(e) = self.kernel.start_trader() {
1360            let result = self.abort_after_trader_start_failure(e).await;
1361            Self::drain_channels(
1362                &mut time_evt_rx,
1363                &mut system_evt_rx,
1364                &mut system_cmd_rx,
1365                &mut exec_evt_rx,
1366                &mut exec_cmd_rx,
1367                &mut data_evt_rx,
1368                &mut data_cmd_rx,
1369            );
1370            log::info!("Event loop stopped");
1371            return result;
1372        }
1373
1374        #[cfg(feature = "plugin")]
1375        if let Err(e) = self.plugins.start_controllers() {
1376            let result = self.abort_after_trader_start_failure(e).await;
1377            Self::drain_channels(
1378                &mut time_evt_rx,
1379                &mut system_evt_rx,
1380                &mut system_cmd_rx,
1381                &mut exec_evt_rx,
1382                &mut exec_cmd_rx,
1383                &mut data_evt_rx,
1384                &mut data_cmd_rx,
1385            );
1386            log::info!("Event loop stopped");
1387            return result;
1388        }
1389
1390        self.process_system_events(startup_system_events);
1391        self.process_system_commands(startup_system_commands);
1392
1393        let finish_result = {
1394            let mut receivers = RunnerReceivers {
1395                time_evt: &mut time_evt_rx,
1396                system_evt: &mut system_evt_rx,
1397                system_cmd: &mut system_cmd_rx,
1398                data_evt: &mut data_evt_rx,
1399                data_cmd: &mut data_cmd_rx,
1400                exec_evt: &mut exec_evt_rx,
1401                exec_cmd: &mut exec_cmd_rx,
1402            };
1403
1404            self.finish_startup_trader(Some(&mut receivers)).await
1405        };
1406
1407        match finish_result {
1408            Ok(true) => {}
1409            result => {
1410                log::info!("Event loop stopped");
1411                return result.map(|_| ());
1412            }
1413        }
1414
1415        let exec_config = &self.config.exec_engine;
1416        let inflight_interval =
1417            Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1418
1419        let open_interval = exec_config
1420            .open_check_interval_secs
1421            .filter(|&s| s > 0.0)
1422            .map_or(Duration::ZERO, |secs| {
1423                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1424            });
1425
1426        let position_interval = exec_config
1427            .position_check_interval_secs
1428            .filter(|&s| s > 0.0)
1429            .map_or(Duration::ZERO, |secs| {
1430                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1431            });
1432
1433        let has_clients = !self
1434            .kernel
1435            .exec_engine
1436            .borrow()
1437            .get_all_clients()
1438            .is_empty();
1439        let recon_enabled = has_clients
1440            && (!inflight_interval.is_zero()
1441                || !open_interval.is_zero()
1442                || !position_interval.is_zero());
1443
1444        let recon_min_interval = if recon_enabled {
1445            let mut intervals = Vec::new();
1446
1447            if !inflight_interval.is_zero() {
1448                intervals.push(inflight_interval);
1449            }
1450
1451            if !open_interval.is_zero() {
1452                intervals.push(open_interval);
1453            }
1454
1455            if !position_interval.is_zero() {
1456                intervals.push(position_interval);
1457            }
1458
1459            intervals
1460                .into_iter()
1461                .min()
1462                .unwrap_or(Duration::from_secs(1))
1463        } else {
1464            Duration::from_secs(1) // Unused, timer won't fire
1465        };
1466
1467        // `reconciliation_startup_delay_secs` is a post-reconciliation grace period:
1468        // startup reconciliation has already completed above, and this delay offsets
1469        // the first periodic tick to let the system stabilize before continuous checks
1470        // begin.
1471        let startup_delay = if self.config.exec_engine.reconciliation {
1472            Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1473        } else {
1474            Duration::ZERO
1475        };
1476
1477        let recon_start = dst::time::Instant::now() + startup_delay;
1478
1479        let mut last_inflight_check = dst::time::Instant::now();
1480        let mut last_open_check = last_inflight_check;
1481        let mut last_position_check = last_inflight_check;
1482
1483        // Per-task `(interval, next_fire)` schedules dispatched by the
1484        // shared `maintenance_timer` below. See module docs for rationale.
1485        let far_future = Duration::from_hours(24 * 365 * 100);
1486
1487        let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1488            let dur = opt_dur.unwrap_or(far_future);
1489            (dur, recon_start + dur)
1490        };
1491
1492        let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1493            Some(recon_min_interval)
1494        } else {
1495            None
1496        });
1497
1498        let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1499            exec_config
1500                .purge_closed_orders_interval_mins
1501                .filter(|&m| m > 0)
1502                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1503        );
1504
1505        let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1506            exec_config
1507                .purge_closed_positions_interval_mins
1508                .filter(|&m| m > 0)
1509                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1510        );
1511
1512        let (purge_account_interval, mut purge_account_next) = make_schedule(
1513            exec_config
1514                .purge_account_events_interval_mins
1515                .filter(|&m| m > 0)
1516                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1517        );
1518
1519        let (own_books_interval, mut own_books_next) = make_schedule(
1520            exec_config
1521                .own_books_audit_interval_secs
1522                .filter(|&s| s > 0.0)
1523                .map(Duration::from_secs_f64),
1524        );
1525
1526        let (prune_fills_interval, mut prune_fills_next) =
1527            make_schedule(Some(Duration::from_mins(1)));
1528
1529        let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1530        maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1531
1532        // Stop-check timer is not subject to the reconciliation startup delay,
1533        // so shutdown signals remain responsive from the moment the node reaches
1534        // `Running`. Set `MissedTickBehavior::Skip` so backlog ticks do not fire
1535        // a burst after the select arm was suspended by other branches.
1536        let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1537        stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1538
1539        // Running phase: runs until shutdown deadline expires
1540        let mut residual_events = 0usize;
1541        let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1542        let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1543        let mut position_report_task: Option<PositionReportTask> = None;
1544
1545        // A hosted node never installs signal handlers, so these futures stay pending and their
1546        // listeners are never registered. Both arms resolve to the same type as the real listeners.
1547        let owns_signals = mode.owns_signals();
1548
1549        let ctrl_c = async move {
1550            if owns_signals {
1551                dst::signal::ctrl_c().await
1552            } else {
1553                std::future::pending::<std::io::Result<()>>().await
1554            }
1555        };
1556
1557        let terminate = async move {
1558            if owns_signals {
1559                dst::signal::terminate().await
1560            } else {
1561                std::future::pending::<std::io::Result<()>>().await
1562            }
1563        };
1564
1565        tokio::pin!(ctrl_c);
1566        tokio::pin!(terminate);
1567
1568        let metrics = self.handle.metrics.clone();
1569        let metrics_start = dst::time::Instant::now();
1570        metrics.reset();
1571
1572        let mut queue_monitor = self
1573            .config
1574            .queue_monitor
1575            .as_ref()
1576            .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1577        let mut dispatches_since_yield = 0usize;
1578
1579        let dispatch_result = loop {
1580            let callbacks_pending = match drain_callbacks().await {
1581                Ok(pending) => pending,
1582                Err(e) => {
1583                    if self.state() == NodeState::Running {
1584                        self.initiate_shutdown();
1585                    }
1586
1587                    log::warn!(
1588                        "Skipping residual events and final buffered dispatch after callback failure"
1589                    );
1590
1591                    break Err(e);
1592                }
1593            };
1594
1595            let shutdown_deadline = self.shutdown_deadline;
1596            let is_shutting_down = self.state() == NodeState::ShuttingDown;
1597            let is_running = self.state() == NodeState::Running;
1598
1599            tokio::select! {
1600                biased;
1601
1602                // Signal branches first so they are always checked
1603                result = &mut ctrl_c, if is_running => {
1604                    match result {
1605                        Ok(()) => log::info!("Received SIGINT, shutting down"),
1606                        Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1607                    }
1608                    self.initiate_shutdown();
1609                }
1610                result = &mut terminate, if is_running => {
1611                    match result {
1612                        Ok(()) => log::info!("Received SIGTERM, shutting down"),
1613                        Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1614                    }
1615                    self.initiate_shutdown();
1616                }
1617                _ = stop_check_timer.tick(), if is_running => {
1618                    if stop_handle.should_stop() {
1619                        log::info!("Received stop signal from handle");
1620                        self.initiate_shutdown();
1621                    } else if self.kernel.is_shutdown_requested() {
1622                        log::info!("Received ShutdownSystem command, shutting down");
1623                        self.initiate_shutdown();
1624                    }
1625                }
1626                () = async {
1627                    match shutdown_deadline {
1628                        Some(deadline) => dst::time::sleep_until(deadline).await,
1629                        None => std::future::pending::<()>().await,
1630                    }
1631                }, if self.state() == NodeState::ShuttingDown => {
1632                    break Ok(());
1633                }
1634                () = std::future::ready(()), if callbacks_pending => {},
1635                result = async {
1636                    match open_order_report_task.as_mut() {
1637                        Some(task) => task.future.as_mut().await,
1638                        None => std::future::pending::<ReportTaskOutcome<OpenOrderReportResult>>().await,
1639                    }
1640                }, if open_order_report_task.is_some() => {
1641                    let maintenance_start = dst::time::Instant::now();
1642
1643                    drop(open_order_report_task.take());
1644
1645                    match result {
1646                        ReportTaskOutcome::Completed(result) => {
1647                            let client_refs = self
1648                                .exec_clients
1649                                .iter()
1650                                .map(|client| client as &dyn ExecutionClient)
1651                                .collect::<Vec<_>>();
1652                            let reconciliation = self.exec_manager.reconcile_open_order_reports(
1653                                &result.check,
1654                                result.reports,
1655                                &result.queried_clients,
1656                                &result.failed_clients,
1657                                &client_refs,
1658                            );
1659                            self.process_reconciliation_events(&reconciliation.events);
1660                            if !reconciliation.targeted_queries.is_empty() {
1661                                if is_shutting_down {
1662                                    let planned_client_order_ids = reconciliation
1663                                        .targeted_queries
1664                                        .iter()
1665                                        .map(TargetedOrderQuery::client_order_id)
1666                                        .collect::<Vec<_>>();
1667                                    self.cleanup_cancelled_report_tasks(
1668                                        &planned_client_order_ids,
1669                                    );
1670                                } else {
1671                                    targeted_order_report_task = Some(
1672                                        self.start_targeted_order_report_check(
1673                                            reconciliation.targeted_queries,
1674                                        ),
1675                                    );
1676                                }
1677                            }
1678                        }
1679                        ReportTaskOutcome::TimedOut => {
1680                            self.cleanup_cancelled_report_tasks(&[]);
1681                            log::warn!(
1682                                "Open-order report collection expired after {:?}",
1683                                self.config.timeout_reconciliation,
1684                            );
1685                        }
1686                    }
1687                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1688                }
1689                result = async {
1690                    match targeted_order_report_task.as_mut() {
1691                        Some(task) => task.future.as_mut().await,
1692                        None => std::future::pending::<ReportTaskOutcome<Vec<TargetedOrderReportResult>>>().await,
1693                    }
1694                }, if targeted_order_report_task.is_some() => {
1695                    let maintenance_start = dst::time::Instant::now();
1696
1697                    let planned_client_order_ids = targeted_order_report_task
1698                        .as_ref()
1699                        .map(|task| task.planned_client_order_ids.clone())
1700                        .unwrap_or_default();
1701                    drop(targeted_order_report_task.take());
1702
1703                    match result {
1704                        ReportTaskOutcome::Completed(result) => {
1705                            let client_refs = self
1706                                .exec_clients
1707                                .iter()
1708                                .map(|client| client as &dyn ExecutionClient)
1709                                .collect::<Vec<_>>();
1710                            let events = self
1711                                .exec_manager
1712                                .reconcile_targeted_order_reports(result, &client_refs);
1713                            self.process_reconciliation_events(&events);
1714                        }
1715                        ReportTaskOutcome::TimedOut => {
1716                            self.cleanup_cancelled_report_tasks(&planned_client_order_ids);
1717                            log::warn!(
1718                                "Targeted order report collection expired after {:?}",
1719                                self.config.timeout_reconciliation,
1720                            );
1721                        }
1722                    }
1723                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1724                }
1725                result = async {
1726                    match position_report_task.as_mut() {
1727                        Some(task) => task.future.as_mut().await,
1728                        None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1729                    }
1730                }, if position_report_task.is_some() => {
1731                    let maintenance_start = dst::time::Instant::now();
1732
1733                    drop(position_report_task.take());
1734
1735                    match result {
1736                        ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1737                            if is_shutting_down {
1738                                self.cleanup_cancelled_report_tasks(&[]);
1739                            } else {
1740                                position_report_task = self.handle_position_report_result(result);
1741                            }
1742                        }
1743                        ReportTaskOutcome::Completed(PositionReportTaskResult::Fills(result)) => {
1744                            self.handle_position_fill_report_result(result);
1745                        }
1746                        ReportTaskOutcome::TimedOut => {
1747                            self.cleanup_cancelled_report_tasks(&[]);
1748                            log::warn!(
1749                                "Position report collection expired after {:?}",
1750                                self.config.timeout_reconciliation,
1751                            );
1752                        }
1753                    }
1754                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1755                }
1756
1757                // Maintenance dispatcher (before event processing to avoid
1758                // starvation). See module docs for design rationale.
1759                _ = maintenance_timer.tick(), if is_running => {
1760                    let maintenance_start = dst::time::Instant::now();
1761                    metrics.publish_queue_depths(
1762                        RunnerChannelQueueDepths::from_receivers(
1763                            &time_evt_rx,
1764                            &exec_evt_rx,
1765                            &exec_cmd_rx,
1766                            &data_evt_rx,
1767                            &data_cmd_rx,
1768                        ),
1769                        metrics_start.elapsed(),
1770                    );
1771
1772                    if let Some(queue_monitor) = queue_monitor.as_mut() {
1773                        let transitions = queue_monitor.evaluate(metrics.snapshot());
1774                        self.publish_queue_state_transitions(&transitions);
1775                    }
1776
1777                    let mut now = dst::time::Instant::now();
1778
1779                    if recon_enabled && now >= recon_next {
1780                        let recon_intervals = ReconciliationCheckIntervals {
1781                            inflight: inflight_interval,
1782                            open: open_interval,
1783                            position: position_interval,
1784                        };
1785                        let mut recon_state = ReconciliationCheckState {
1786                            last_inflight_check: &mut last_inflight_check,
1787                            last_open_check: &mut last_open_check,
1788                            last_position_check: &mut last_position_check,
1789                            open_order_report_task: &mut open_order_report_task,
1790                            targeted_order_report_task: &mut targeted_order_report_task,
1791                            position_report_task: &mut position_report_task,
1792                        };
1793
1794                        self.run_reconciliation_checks(
1795                            now,
1796                            recon_intervals,
1797                            &mut recon_state,
1798                        );
1799
1800                        now = dst::time::Instant::now();
1801                        recon_next = now + recon_interval;
1802                    }
1803
1804                    if now >= purge_orders_next {
1805                        self.exec_manager.purge_closed_orders();
1806                        purge_orders_next = now + purge_orders_interval;
1807                    }
1808
1809                    if now >= purge_positions_next {
1810                        self.exec_manager.purge_closed_positions();
1811                        purge_positions_next = now + purge_positions_interval;
1812                    }
1813
1814                    if now >= purge_account_next {
1815                        self.exec_manager.purge_account_events();
1816                        purge_account_next = now + purge_account_interval;
1817                    }
1818
1819                    if now >= own_books_next {
1820                        self.kernel.cache().borrow_mut().audit_own_order_books();
1821                        own_books_next = now + own_books_interval;
1822                    }
1823
1824                    if now >= prune_fills_next {
1825                        self.exec_manager.prune_recent_fills_cache(60.0);
1826                        self.exec_manager.prune_processed_fills();
1827                        self.exec_manager.prune_order_local_activity();
1828                        prune_fills_next = now + prune_fills_interval;
1829                    }
1830
1831                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1832                }
1833
1834                // Event processing branches. Exec commands and events are
1835                // ordered ahead of data events so a strategy action (cancel,
1836                // submit, etc.) is not delayed behind a market data backlog
1837                // when the biased select polls receivers each iteration.
1838                Some(handler) = time_evt_rx.recv() => {
1839                    let dispatch_start = dst::time::Instant::now();
1840                    let dispatched = AsyncRunner::handle_time_event(handler);
1841
1842                    if dispatched && is_shutting_down {
1843                        log::debug!("Residual time event");
1844                        residual_events += 1;
1845                    }
1846
1847                    if dispatched {
1848                        record_runner_dispatch(
1849                            &metrics,
1850                            SystemChannel::TimeEvents,
1851                            dispatch_start,
1852                            metrics_start,
1853                        );
1854                    }
1855                }
1856                Some(event) = system_evt_rx.recv() => {
1857                    if is_shutting_down {
1858                        log::debug!("Residual system event: {event}");
1859                        residual_events += 1;
1860                    }
1861                    event.dispatch(|event| self.process_system_event(event));
1862                }
1863                Some(command) = system_cmd_rx.recv() => {
1864                    if is_shutting_down {
1865                        log::debug!("Residual system command: {command}");
1866                        residual_events += 1;
1867                    }
1868                    command.dispatch(|command| self.process_system_command(command));
1869                }
1870                Some(evt) = exec_evt_rx.recv() => {
1871                    let dispatch_start = dst::time::Instant::now();
1872
1873                    if is_shutting_down {
1874                        log::debug!("Residual exec event: {evt}");
1875                        residual_events += 1;
1876                    }
1877
1878                    evt.dispatch(|evt| self.process_exec_event(evt));
1879                    record_runner_dispatch(
1880                        &metrics,
1881                        SystemChannel::ExecEvents,
1882                        dispatch_start,
1883                        metrics_start,
1884                    );
1885                }
1886                Some(cmd) = exec_cmd_rx.recv() => {
1887                    let dispatch_start = dst::time::Instant::now();
1888
1889                    if is_shutting_down {
1890                        log::debug!("Residual exec command: {cmd}");
1891                        residual_events += 1;
1892                    }
1893
1894                    self.process_exec_command(cmd);
1895                    record_runner_dispatch(
1896                        &metrics,
1897                        SystemChannel::ExecCommands,
1898                        dispatch_start,
1899                        metrics_start,
1900                    );
1901                }
1902                message = recv_external_msgbus_message(&mut external_msgbus_rx) => {
1903                    let external_msgbus_start = dst::time::Instant::now();
1904
1905                    match message {
1906                        Some(message) => {
1907                            if is_shutting_down {
1908                                log::debug!("Residual external message bus message: {message}");
1909                                residual_events += 1;
1910                            }
1911                            self.process_external_msgbus_message(&message);
1912                        }
1913                        None => {
1914                            log::info!("External message bus ingress closed");
1915                            external_msgbus_rx = None;
1916                            self.close_external_ingress();
1917                        }
1918                    }
1919
1920                    record_runner_external_msgbus(
1921                        &metrics,
1922                        external_msgbus_start,
1923                        metrics_start,
1924                    );
1925                }
1926                Some(evt) = data_evt_rx.recv() => {
1927                    let dispatch_start = dst::time::Instant::now();
1928
1929                    if is_shutting_down {
1930                        log::debug!("Residual data event: {evt:?}");
1931                        residual_events += 1;
1932                    }
1933                    AsyncRunner::dispatch_data_event(evt);
1934                    record_runner_dispatch(
1935                        &metrics,
1936                        SystemChannel::DataEvents,
1937                        dispatch_start,
1938                        metrics_start,
1939                    );
1940                }
1941                Some(cmd) = data_cmd_rx.recv() => {
1942                    let dispatch_start = dst::time::Instant::now();
1943
1944                    if is_shutting_down {
1945                        log::debug!("Residual data command: {cmd:?}");
1946                        residual_events += 1;
1947                    }
1948                    AsyncRunner::handle_data_command(cmd);
1949                    record_runner_dispatch(
1950                        &metrics,
1951                        SystemChannel::DataCommands,
1952                        dispatch_start,
1953                        metrics_start,
1954                    );
1955                }
1956            }
1957
1958            dispatches_since_yield += 1;
1959            if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1960                dispatches_since_yield = 0;
1961                tokio::task::yield_now().await;
1962            }
1963        };
1964
1965        if residual_events > 0 {
1966            log::debug!("Processed {residual_events} residual events during shutdown");
1967        }
1968
1969        self.cancel_report_tasks(
1970            &mut open_order_report_task,
1971            &mut targeted_order_report_task,
1972            &mut position_report_task,
1973        );
1974        drop(external_msgbus_rx.take());
1975        let _ = self.kernel.cache().borrow().check_residuals();
1976
1977        let stop_result = self.finalize_stop().await;
1978
1979        if let Err(e) = dispatch_result {
1980            if let Err(stop_err) = stop_result {
1981                log::error!("Failed to finalize node after callback failure: {stop_err}");
1982                // Python converts the outer display message, so preserve both diagnostics there.
1983                let diagnostic = format!("{e}; {stop_err}");
1984                return Err(anyhow::Error::new(e).context(diagnostic));
1985            }
1986
1987            return Err(e.into());
1988        }
1989
1990        // Handle events that arrived during finalize_stop
1991        Self::drain_channels(
1992            &mut time_evt_rx,
1993            &mut system_evt_rx,
1994            &mut system_cmd_rx,
1995            &mut exec_evt_rx,
1996            &mut exec_cmd_rx,
1997            &mut data_evt_rx,
1998            &mut data_cmd_rx,
1999        );
2000
2001        log::info!("Event loop stopped");
2002
2003        stop_result
2004    }
2005
2006    fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2007        for transition in transitions {
2008            let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2009            let timestamp = self.kernel.generate_timestamp_ns();
2010
2011            let event = QueueStateChanged::new(
2012                self.config.trader_id,
2013                transition.channel,
2014                transition.condition,
2015                transition.state,
2016                transition.queue_depth,
2017                transition.mean_dispatch_ns,
2018                UUID4::new(),
2019                timestamp,
2020                timestamp,
2021            );
2022
2023            msgbus::publish_any(topic, event.as_any());
2024        }
2025    }
2026
2027    #[expect(
2028        clippy::await_holding_refcell_ref,
2029        reason = "cache loading is serialized before the single-threaded live node starts"
2030    )]
2031    async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2032        self.install_cache_database().await?;
2033
2034        let cache = self.kernel.cache();
2035        if !cache.borrow().has_backing() {
2036            return Ok(());
2037        }
2038
2039        if self
2040            .config
2041            .cache
2042            .as_ref()
2043            .is_some_and(|config| config.flush_on_start)
2044        {
2045            cache
2046                .borrow_mut()
2047                .flush_db()
2048                .context("Failed to flush persistent cache")?;
2049            return Ok(());
2050        }
2051
2052        if self.config.exec_engine.load_cache {
2053            self.kernel
2054                .exec_engine()
2055                .borrow_mut()
2056                .load_cache()
2057                .await
2058                .context("Failed to load persistent cache")?;
2059        }
2060
2061        Ok(())
2062    }
2063
2064    /// Returns whether a cache database backing is configured but not yet installed.
2065    #[must_use]
2066    pub const fn has_pending_cache_database(&self) -> bool {
2067        self.cache_database_factory.is_some()
2068    }
2069
2070    /// Constructs the configured cache database backing and installs it on the kernel cache.
2071    ///
2072    /// Construction is deferred to startup so the adapter is built inside the async runtime rather
2073    /// than by blocking the synchronous builder, and so the connection opens only when the node runs.
2074    async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2075        let Some(factory) = self.cache_database_factory.as_ref() else {
2076            return Ok(());
2077        };
2078
2079        let config = self.config.cache.clone().unwrap_or_default();
2080
2081        // Cleared only after a successful install, so a failed startup can be retried rather than
2082        // silently starting without the backing the caller asked for.
2083        let database = factory
2084            .create(self.config.trader_id, self.kernel.instance_id, config)
2085            .await
2086            .context("failed to create cache database backing")?;
2087        self.kernel.cache().borrow_mut().set_database(database);
2088        self.cache_database_factory = None;
2089
2090        Ok(())
2091    }
2092
2093    fn take_external_ingress_receiver(
2094        &mut self,
2095    ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2096        let Some(external_ingress) = self.external_msgbus.as_mut() else {
2097            return Ok(None);
2098        };
2099
2100        let receiver = external_ingress.take_receiver()?;
2101        log::info!("External message bus ingress started");
2102        Ok(Some(receiver))
2103    }
2104
2105    fn republish_external_msgbus_message(message: &BusMessage) {
2106        if let Err(e) = msgbus::republish_external_message(message) {
2107            log::error!(
2108                "Failed to republish external message bus topic '{}': {e:#}",
2109                message.topic
2110            );
2111        }
2112    }
2113
2114    fn process_external_msgbus_message(&self, message: &BusMessage) {
2115        if self.stream_processors.is_empty() {
2116            Self::republish_external_msgbus_message(message);
2117            return;
2118        }
2119
2120        let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2121            for processor in &self.stream_processors {
2122                (processor.0)(value, mapping)?;
2123            }
2124
2125            Ok(())
2126        };
2127
2128        if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2129            log::error!(
2130                "Failed to process external message bus topic '{}': {e:#}",
2131                message.topic
2132            );
2133        }
2134    }
2135
2136    fn close_external_ingress(&mut self) {
2137        if let Some(external_ingress) = self.external_msgbus.as_mut()
2138            && !external_ingress.is_closed()
2139        {
2140            external_ingress.close();
2141        }
2142    }
2143
2144    fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2145        if events.is_empty() {
2146            self.publish_submission_recovery_exhaustions();
2147            return;
2148        }
2149
2150        log::info!(
2151            "Processing {} reconciliation event{}",
2152            events.len(),
2153            if events.len() == 1 { "" } else { "s" }
2154        );
2155
2156        for event in events {
2157            self.exec_manager
2158                .record_local_activity(event.client_order_id());
2159            if let OrderEventAny::Filled(fill) = event {
2160                self.exec_manager
2161                    .record_position_activity(fill.instrument_id, fill.account_id);
2162            }
2163
2164            self.kernel.exec_engine.borrow_mut().process(event);
2165            self.exec_manager
2166                .confirm_submission_outcome(&event.client_order_id());
2167            if let OrderEventAny::Filled(fill) = event {
2168                self.exec_manager.commit_recent_fill_if_applied(fill);
2169            }
2170        }
2171
2172        self.publish_submission_recovery_exhaustions();
2173    }
2174
2175    fn publish_submission_recovery_exhaustions(&mut self) {
2176        for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2177            log::warn!(
2178                "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2179                diagnostic.client_order_id,
2180                diagnostic.source,
2181                diagnostic.retry_count,
2182            );
2183            msgbus::publish_any(
2184                MessagingSwitchboard::submission_recovery_exhausted_topic(),
2185                &diagnostic,
2186            );
2187        }
2188    }
2189
2190    fn process_exec_event(&mut self, event: ExecutionEvent) {
2191        let clear_closed_orders = matches!(
2192            &event,
2193            ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2194        );
2195        let closed_report_id = match &event {
2196            ExecutionEvent::Report(
2197                ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2198            ) if report.order_status.is_closed() => report.client_order_id,
2199            _ => None,
2200        };
2201        let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2202            return;
2203        };
2204
2205        self.dispatch_exec_event_and_commit_fill(event);
2206
2207        for client_order_id in &order_ids {
2208            self.exec_manager
2209                .confirm_submission_outcome(client_order_id);
2210            let is_closed = self
2211                .kernel
2212                .cache()
2213                .borrow()
2214                .order(client_order_id)
2215                .is_some_and(|order| order.is_closed());
2216
2217            if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2218                self.exec_manager
2219                    .clear_recon_tracking(client_order_id, true);
2220            }
2221        }
2222    }
2223
2224    fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2225        message.dispatch_trading(|message| {
2226            if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2227                self.observe_exec_command_before_dispatch(message.command());
2228            }
2229        });
2230    }
2231
2232    /// Dispatches a normal-ingress execution event, then commits a direct
2233    /// `OrderFilled` to the recent-fills dedup cache only once it is present on
2234    /// its canonical order.
2235    ///
2236    /// The fill candidate is captured from `&evt` BEFORE the value is moved into
2237    /// [`AsyncRunner::handle_exec_event`]; the gated commit runs AFTER dispatch,
2238    /// so a fill the execution engine rejects (unknown order, invalid
2239    /// transition) is never marked and its later `Fill` report stays eligible.
2240    fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2241        let recent_fill_candidate = match &evt {
2242            ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2243            _ => None,
2244        };
2245
2246        AsyncRunner::handle_exec_event(evt);
2247
2248        if let Some(fill) = &recent_fill_candidate {
2249            self.exec_manager.commit_recent_fill_if_applied(fill);
2250        }
2251    }
2252
2253    async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2254        // A zero remaining budget still admits an immediately-ready connect (an
2255        // empty/ready client set completes on the first poll); a pending connect
2256        // fails closed on that same poll. This keeps a zero `timeout_connection`
2257        // - a supported "do not wait, but allow ready work" configuration -
2258        // working, while still bounding a hung connect.
2259        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2260        dst::time::timeout(remaining, self.kernel.connect_data_clients())
2261            .await
2262            .map_err(|_| anyhow::anyhow!("data-connect timeout"))
2263    }
2264
2265    async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2266        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2267        dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2268            .await
2269            .map_err(|_| anyhow::anyhow!("exec-connect timeout"))
2270    }
2271
2272    /// Connects execution clients and checks all engines are connected.
2273    ///
2274    /// Returns the final connection wait status.
2275    /// Must be called after data clients are connected and instrument events drained.
2276    async fn connect_exec_phase(
2277        &mut self,
2278        deadline: dst::time::Instant,
2279    ) -> anyhow::Result<EngineConnectionStatus> {
2280        self.connect_exec_clients(deadline).await?;
2281        Ok(self.await_engines_connected(deadline).await)
2282    }
2283
2284    fn startup_abort_reason(&self) -> Option<&'static str> {
2285        if self.handle.should_stop() {
2286            Some("Stop signal received during startup")
2287        } else if self.kernel.is_shutdown_requested() {
2288            Some("Shutdown signal received during startup")
2289        } else {
2290            None
2291        }
2292    }
2293
2294    async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2295        match self.handle.try_set_running() {
2296            RunningTransition::Entered => Ok(true),
2297            RunningTransition::StopRequested => {
2298                self.abort_startup("Stop signal received during startup")
2299                    .await?;
2300                Ok(false)
2301            }
2302            RunningTransition::Invalid(control) => {
2303                self.abort_startup_with_error(
2304                    "Invalid lifecycle state during startup",
2305                    anyhow::anyhow!(
2306                        "Invalid LiveNode control state {control:#04x} while entering Running"
2307                    ),
2308                )
2309                .await?;
2310                Ok(false)
2311            }
2312        }
2313    }
2314
2315    async fn finish_startup_trader(
2316        &mut self,
2317        receivers: Option<&mut RunnerReceivers<'_>>,
2318    ) -> anyhow::Result<bool> {
2319        match self.handle.try_set_running() {
2320            RunningTransition::Entered => Ok(true),
2321            RunningTransition::StopRequested => {
2322                self.abort_started_trader("Stop signal received during startup", receivers)
2323                    .await?;
2324                Ok(false)
2325            }
2326            RunningTransition::Invalid(control) => {
2327                let state_err = anyhow::anyhow!(
2328                    "Invalid LiveNode control state {control:#04x} while entering Running"
2329                );
2330
2331                match self
2332                    .abort_started_trader("Invalid lifecycle state during startup", receivers)
2333                    .await
2334                {
2335                    Ok(()) => Err(state_err),
2336                    Err(finalize_err) => {
2337                        anyhow::bail!(
2338                            "{state_err}; failed to finalize startup abort: {finalize_err}"
2339                        )
2340                    }
2341                }
2342            }
2343        }
2344    }
2345
2346    async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2347        log::info!("{reason}, aborting startup");
2348        self.handle.set_shutting_down();
2349        self.finalize_stop().await
2350    }
2351
2352    async fn abort_startup_with_error(
2353        &mut self,
2354        reason: &str,
2355        startup_err: anyhow::Error,
2356    ) -> anyhow::Result<()> {
2357        match self.abort_startup(reason).await {
2358            Ok(()) => Err(startup_err),
2359            Err(finalize_err) => {
2360                anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2361            }
2362        }
2363    }
2364
2365    async fn abort_started_trader(
2366        &mut self,
2367        reason: &str,
2368        mut receivers: Option<&mut RunnerReceivers<'_>>,
2369    ) -> anyhow::Result<()> {
2370        log::info!("{reason}, aborting startup");
2371        self.handle.set_shutting_down();
2372
2373        #[cfg(feature = "plugin")]
2374        let controller_stop_result = self.plugins.stop_controllers();
2375        #[cfg(not(feature = "plugin"))]
2376        let controller_stop_result: anyhow::Result<()> = Ok(());
2377
2378        let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2379        let delay = self.kernel.delay_post_stop();
2380        log::info!("Awaiting residual events ({delay:?})...");
2381
2382        let residual_events = match receivers.as_mut() {
2383            Some(receivers) => self.process_receivers_for(delay, receivers).await,
2384            None => self.process_runner_for(delay).await,
2385        };
2386
2387        if residual_events > 0 {
2388            log::debug!("Processed {residual_events} residual events during shutdown");
2389        }
2390
2391        let finalize_result = self.finalize_stop().await;
2392
2393        if let Some(receivers) = receivers {
2394            Self::drain_channels(
2395                receivers.time_evt,
2396                receivers.system_evt,
2397                receivers.system_cmd,
2398                receivers.exec_evt,
2399                receivers.exec_cmd,
2400                receivers.data_evt,
2401                receivers.data_cmd,
2402            );
2403        } else {
2404            let drained_events = self.drain_runner_pending();
2405            if drained_events > 0 {
2406                log::info!("Drained {drained_events} remaining events during shutdown");
2407            }
2408        }
2409
2410        let mut errors = Vec::new();
2411
2412        if let Err(e) = controller_stop_result {
2413            errors.push(format!("Failed to stop plug-in controllers: {e}"));
2414        }
2415
2416        if let Err(e) = trader_stop_result {
2417            errors.push(format!("Failed to stop trader: {e}"));
2418        }
2419
2420        if let Err(e) = finalize_result {
2421            errors.push(format!("Failed to finalize startup abort: {e}"));
2422        }
2423
2424        if errors.is_empty() {
2425            Ok(())
2426        } else {
2427            anyhow::bail!("{}", errors.join("; "))
2428        }
2429    }
2430
2431    async fn process_receivers_for(
2432        &mut self,
2433        duration: Duration,
2434        receivers: &mut RunnerReceivers<'_>,
2435    ) -> usize {
2436        let deadline = dst::time::Instant::now() + duration;
2437        let mut processed = 0;
2438
2439        loop {
2440            tokio::select! {
2441                biased;
2442
2443                () = dst::time::sleep_until(deadline) => break,
2444                Some(message) = receivers.time_evt.recv() => {
2445                    let _ = AsyncRunner::handle_time_event(message);
2446                    processed += 1;
2447                }
2448                Some(event) = receivers.system_evt.recv() => {
2449                    event.dispatch(|event| self.process_system_event(event));
2450                    processed += 1;
2451                }
2452                Some(command) = receivers.system_cmd.recv() => {
2453                    command.dispatch(|command| self.process_system_command(command));
2454                    processed += 1;
2455                }
2456                Some(event) = receivers.exec_evt.recv() => {
2457                    event.dispatch(|event| self.process_exec_event(event));
2458                    processed += 1;
2459                }
2460                Some(command) = receivers.exec_cmd.recv() => {
2461                    self.process_exec_command(command);
2462                    processed += 1;
2463                }
2464                Some(event) = receivers.data_evt.recv() => {
2465                    AsyncRunner::dispatch_data_event(event);
2466                    processed += 1;
2467                }
2468                Some(command) = receivers.data_cmd.recv() => {
2469                    AsyncRunner::handle_data_command(command);
2470                    processed += 1;
2471                }
2472            }
2473        }
2474
2475        processed
2476    }
2477
2478    async fn abort_after_trader_start_failure(
2479        &mut self,
2480        start_err: anyhow::Error,
2481    ) -> anyhow::Result<()> {
2482        log::info!("Trader startup failed, aborting startup");
2483        self.handle.set_shutting_down();
2484        let stop_result = self.kernel.stop_trader_after_start_failure();
2485        let finalize_result = self.finalize_stop().await;
2486
2487        match (stop_result, finalize_result) {
2488            (Ok(()), Ok(())) => Err(start_err),
2489            (Err(stop_err), Ok(())) => anyhow::bail!(
2490                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2491                 {stop_err}"
2492            ),
2493            (Ok(()), Err(finalize_err)) => anyhow::bail!(
2494                "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2495                 {finalize_err}"
2496            ),
2497            (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2498                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2499                 {stop_err}; failed to finalize startup abort: {finalize_err}"
2500            ),
2501        }
2502    }
2503
2504    fn initiate_shutdown(&mut self) {
2505        #[cfg(feature = "plugin")]
2506        if let Err(e) = self.plugins.stop_controllers() {
2507            log::error!("Error stopping plug-in controllers: {e}");
2508        }
2509
2510        self.kernel.stop_trader();
2511        let delay = self.kernel.delay_post_stop();
2512        log::info!("Awaiting residual events ({delay:?})...");
2513
2514        self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2515        self.handle.set_shutting_down();
2516    }
2517
2518    async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2519        // Capture recovery at the grace-period boundary, before disconnection can queue more
2520        // evidence. A later drain must not turn incomplete recovery into a successful shutdown.
2521        let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2522        self.close_external_ingress();
2523
2524        let timeout = self.config.timeout_disconnection;
2525        let deadline = dst::time::Instant::now() + timeout;
2526
2527        let mut errors = Vec::new();
2528        let (report_result, disconnect_result) = futures_util::join!(
2529            dst::time::timeout(
2530                timeout,
2531                Self::finish_report_tasks(&self.exec_clients, &mut errors)
2532            ),
2533            dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2534        );
2535
2536        if report_result.is_err() {
2537            errors.push("report collection shutdown timeout".to_string());
2538        }
2539
2540        let disconnect_result = match disconnect_result {
2541            Ok(result) => result,
2542            Err(_) => Err(anyhow::anyhow!(
2543                "disconnect timeout while disconnecting clients"
2544            )),
2545        };
2546
2547        if let Err(ref e) = disconnect_result {
2548            log::error!("Error disconnecting clients: {e}");
2549        }
2550
2551        let readiness_result = self.await_engines_disconnected(deadline).await;
2552        let kernel_result = self.kernel.finalize_stop().await;
2553
2554        self.handle.set_stopped();
2555
2556        if !unresolved_submissions.is_empty() {
2557            errors.push(format!(
2558                "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2559            ));
2560        }
2561
2562        if let Err(e) = disconnect_result {
2563            errors.push(e.to_string());
2564        }
2565
2566        if let Err(e) = readiness_result {
2567            errors.push(format!("failed while awaiting engine disconnection: {e}"));
2568        }
2569
2570        if let Err(e) = kernel_result {
2571            errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2572        }
2573
2574        if errors.is_empty() {
2575            Ok(())
2576        } else {
2577            anyhow::bail!("{}", errors.join("; "))
2578        }
2579    }
2580
2581    fn drain_channels(
2582        time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2583        system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2584        system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2585        exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2586        exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2587            DispatchMessage<TradingCommandMessage>,
2588        >,
2589        data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2590        data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2591    ) {
2592        let mut drained = 0;
2593
2594        while let Ok(handler) = time_evt_rx.try_recv() {
2595            let _ = AsyncRunner::handle_time_event(handler);
2596            drained += 1;
2597        }
2598
2599        while system_evt_rx.try_recv().is_ok() {
2600            drained += 1;
2601        }
2602
2603        while system_cmd_rx.try_recv().is_ok() {
2604            drained += 1;
2605        }
2606
2607        while let Ok(evt) = data_evt_rx.try_recv() {
2608            AsyncRunner::dispatch_data_event(evt);
2609            drained += 1;
2610        }
2611
2612        while let Ok(cmd) = data_cmd_rx.try_recv() {
2613            AsyncRunner::handle_data_command(cmd);
2614            drained += 1;
2615        }
2616
2617        while let Ok(evt) = exec_evt_rx.try_recv() {
2618            AsyncRunner::dispatch_exec_event(evt);
2619            drained += 1;
2620        }
2621
2622        while let Ok(cmd) = exec_cmd_rx.try_recv() {
2623            AsyncRunner::handle_trading_command(cmd);
2624            drained += 1;
2625        }
2626
2627        if drained > 0 {
2628            log::info!("Drained {drained} remaining events during shutdown");
2629        }
2630    }
2631
2632    fn observe_exec_event_before_dispatch(
2633        &mut self,
2634        evt: &ExecutionEvent,
2635    ) -> Option<Vec<ClientOrderId>> {
2636        let mut order_ids = Vec::new();
2637
2638        match evt {
2639            ExecutionEvent::Order(order_evt) => {
2640                self.exec_manager.observe_order_event(order_evt);
2641                order_ids.push(order_evt.client_order_id());
2642            }
2643            ExecutionEvent::OrderSubmittedBatch(batch) => {
2644                for submitted in &batch.events {
2645                    self.exec_manager
2646                        .record_local_activity(submitted.client_order_id);
2647                }
2648            }
2649            ExecutionEvent::OrderAcceptedBatch(batch) => {
2650                for accepted in &batch.events {
2651                    self.exec_manager
2652                        .observe_order_event(&OrderEventAny::Accepted(*accepted));
2653                    order_ids.push(accepted.client_order_id);
2654                }
2655            }
2656            ExecutionEvent::OrderCanceledBatch(batch) => {
2657                for canceled in &batch.events {
2658                    self.exec_manager
2659                        .observe_order_event(&OrderEventAny::Canceled(*canceled));
2660                    order_ids.push(canceled.client_order_id);
2661                }
2662            }
2663            ExecutionEvent::Report(report) => {
2664                if let ExecutionReport::Fill(fill_report) = report
2665                    && self.exec_manager.is_fill_recently_processed(
2666                        fill_report.account_id,
2667                        fill_report.instrument_id,
2668                        fill_report.trade_id,
2669                    )
2670                {
2671                    log::debug!(
2672                        "Skipping recently processed fill report: {}",
2673                        fill_report.trade_id,
2674                    );
2675                    return None;
2676                }
2677
2678                self.exec_manager.observe_execution_report(report);
2679
2680                if let Some(client_order_id) = self.order_report_client_order_id(report) {
2681                    order_ids.push(client_order_id);
2682                }
2683            }
2684            ExecutionEvent::Account(_) => {}
2685        }
2686
2687        Some(order_ids)
2688    }
2689
2690    fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2691        match report {
2692            ExecutionReport::Order(order_report)
2693            | ExecutionReport::OrderWithFills(order_report, _) => {
2694                order_report.client_order_id.or_else(|| {
2695                    self.kernel
2696                        .cache
2697                        .borrow()
2698                        .client_order_id(&order_report.venue_order_id)
2699                        .copied()
2700                })
2701            }
2702            ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2703                self.kernel
2704                    .cache
2705                    .borrow()
2706                    .client_order_id(&fill_report.venue_order_id)
2707                    .copied()
2708            }),
2709            _ => None,
2710        }
2711    }
2712
2713    fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2714        let submission_client_id = if matches!(
2715            cmd,
2716            TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2717        ) {
2718            self.kernel
2719                .exec_engine
2720                .borrow()
2721                .find_client_for_command(cmd)
2722                .map(|client| client.client_id())
2723        } else {
2724            None
2725        };
2726
2727        match cmd {
2728            TradingCommand::SubmitOrder(submit) => {
2729                self.exec_manager.register_submission(
2730                    &submit.order_init,
2731                    submission_client_id.or(submit.client_id),
2732                );
2733            }
2734            TradingCommand::SubmitOrderList(submit) => {
2735                for order_init in &submit.order_inits {
2736                    self.exec_manager
2737                        .register_submission(order_init, submission_client_id.or(submit.client_id));
2738                }
2739            }
2740            TradingCommand::ModifyOrder(modify) => {
2741                self.exec_manager.register_inflight(modify.client_order_id);
2742            }
2743            TradingCommand::ModifyOrders(modify) => {
2744                for child in &modify.modifies {
2745                    self.exec_manager.register_inflight(child.client_order_id);
2746                }
2747            }
2748            TradingCommand::CancelOrder(cancel) => {
2749                self.exec_manager.register_inflight(cancel.client_order_id);
2750            }
2751            TradingCommand::CancelOrders(cancel) => {
2752                for child in &cancel.cancels {
2753                    self.exec_manager.register_inflight(child.client_order_id);
2754                }
2755            }
2756            _ => {}
2757        }
2758    }
2759
2760    /// Gets the node's environment.
2761    #[must_use]
2762    pub fn environment(&self) -> Environment {
2763        self.kernel.environment()
2764    }
2765
2766    /// Gets a reference to the underlying kernel.
2767    #[must_use]
2768    pub const fn kernel(&self) -> &NautilusKernel {
2769        &self.kernel
2770    }
2771
2772    /// Gets an exclusive reference to the underlying kernel.
2773    #[must_use]
2774    pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2775        &mut self.kernel
2776    }
2777
2778    /// Gets the node's trader ID.
2779    #[must_use]
2780    pub fn trader_id(&self) -> TraderId {
2781        self.kernel.trader_id()
2782    }
2783
2784    /// Gets the node's instance ID.
2785    #[must_use]
2786    pub const fn instance_id(&self) -> UUID4 {
2787        self.kernel.instance_id()
2788    }
2789
2790    /// Returns the current node state.
2791    #[must_use]
2792    pub fn state(&self) -> NodeState {
2793        self.handle.state()
2794    }
2795
2796    /// Checks if the live node is currently running.
2797    #[must_use]
2798    pub fn is_running(&self) -> bool {
2799        self.state().is_running()
2800    }
2801
2802    /// Sets the cache database adapter for persistence.
2803    ///
2804    /// This allows setting a database adapter (e.g., PostgreSQL, Redis) after the node
2805    /// is built but before it starts running. The database adapter is used to persist
2806    /// cache data for recovery and state management.
2807    ///
2808    /// # Errors
2809    ///
2810    /// Returns an error if the node is already running.
2811    pub fn set_cache_database(
2812        &mut self,
2813        database: Box<dyn CacheDatabaseAdapter>,
2814    ) -> anyhow::Result<()> {
2815        if self.state() != NodeState::Idle {
2816            anyhow::bail!(
2817                "Cannot set cache database while node is running, set it before running the node"
2818            );
2819        }
2820
2821        self.kernel.cache().borrow_mut().set_database(database);
2822        Ok(())
2823    }
2824
2825    /// Returns the execution manager.
2826    #[must_use]
2827    pub fn exec_manager(&self) -> &ExecutionManager {
2828        &self.exec_manager
2829    }
2830
2831    /// Returns a mutable reference to the execution manager.
2832    #[must_use]
2833    pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2834        &mut self.exec_manager
2835    }
2836
2837    /// Adds an actor to the trader.
2838    ///
2839    /// This method provides a high-level interface for adding actors to the underlying
2840    /// trader without requiring direct access to the kernel. Actors should be added
2841    /// after the node is built but before starting the node.
2842    ///
2843    /// # Errors
2844    ///
2845    /// Returns an error if:
2846    /// - The trader is not in a valid state for adding components.
2847    /// - An actor with the same ID is already registered.
2848    /// - The node is currently running.
2849    pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2850    where
2851        T: DataActor + DataActorNative + Component + Actor + 'static,
2852    {
2853        if self.state() != NodeState::Idle {
2854            anyhow::bail!(
2855                "Cannot add actor while node is running, add actors before running the node"
2856            );
2857        }
2858
2859        self.kernel.trader.borrow_mut().add_actor(actor)
2860    }
2861
2862    /// Adds an actor to the live node using a factory function.
2863    ///
2864    /// The factory function is called at registration time to create the actor,
2865    /// avoiding cloning issues with non-cloneable actor types.
2866    ///
2867    /// # Errors
2868    ///
2869    /// Returns an error if:
2870    /// - The node is currently running.
2871    /// - The factory function fails to create the actor.
2872    /// - The underlying trader registration fails.
2873    pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2874    where
2875        F: FnOnce() -> anyhow::Result<T>,
2876        T: DataActor + DataActorNative + Component + Actor + 'static,
2877    {
2878        if self.state() != NodeState::Idle {
2879            anyhow::bail!(
2880                "Cannot add actor while node is running, add actors before running the node"
2881            );
2882        }
2883
2884        self.kernel
2885            .trader
2886            .borrow_mut()
2887            .add_actor_from_factory(factory)
2888    }
2889
2890    /// Adds a strategy to the trader.
2891    ///
2892    /// Strategies are registered in both the component registry (for lifecycle management)
2893    /// and the actor registry (for data callbacks via msgbus).
2894    ///
2895    /// # Errors
2896    ///
2897    /// Returns an error if:
2898    /// - The node is currently running.
2899    /// - A strategy with the same ID is already registered.
2900    /// - The configured external order instrument IDs repeat an instrument or the cache already
2901    ///   contains a requested claim.
2902    /// - The strategy configures one or more external order instrument IDs and the cache is already
2903    ///   borrowed.
2904    /// - The strategy configures an OMS type override and the execution engine is already borrowed.
2905    pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2906    where
2907        T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2908    {
2909        if self.state() != NodeState::Idle {
2910            anyhow::bail!(
2911                "Cannot add strategy while node is running, add strategies before running the node"
2912            );
2913        }
2914
2915        // Capture strategy-owned values before adding the strategy, which moves it
2916        let strategy_id = self
2917            .kernel
2918            .trader
2919            .borrow()
2920            .prepare_strategy_for_registration(&mut strategy)?;
2921        let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
2922        let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
2923
2924        if !instrument_ids.is_empty() {
2925            self.register_external_order_claims(strategy_id, &instrument_ids)?;
2926        }
2927
2928        let mut exec_engine = match oms_type
2929            .map(|_| {
2930                self.kernel
2931                    .exec_engine
2932                    .try_borrow_mut()
2933                    .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
2934            })
2935            .transpose()
2936        {
2937            Ok(exec_engine) => exec_engine,
2938            Err(e) => {
2939                if !instrument_ids.is_empty()
2940                    && let Err(rollback_error) =
2941                        self.rollback_external_order_claims(strategy_id, &instrument_ids)
2942                {
2943                    anyhow::bail!(
2944                        "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
2945                    );
2946                }
2947
2948                return Err(e);
2949            }
2950        };
2951
2952        if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
2953            drop(exec_engine);
2954
2955            if !instrument_ids.is_empty()
2956                && let Err(rollback_error) =
2957                    self.rollback_external_order_claims(strategy_id, &instrument_ids)
2958            {
2959                anyhow::bail!(
2960                    "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
2961                );
2962            }
2963
2964            return Err(add_error);
2965        }
2966
2967        if let Some(exec_engine) = &mut exec_engine
2968            && let Some(oms_type) = oms_type
2969        {
2970            exec_engine.register_oms_type(strategy_id, oms_type);
2971        }
2972
2973        Ok(())
2974    }
2975
2976    /// Registers external order claims in the shared cache.
2977    ///
2978    /// It can be called while the node is idle, after manual [`start`](Self::start) returns, or
2979    /// after the node stops. A running strategy can update its own claims through
2980    /// `Strategy::set_external_order_instrument_ids`.
2981    ///
2982    /// # Errors
2983    ///
2984    /// Returns an error if the cache is already borrowed, the request repeats an instrument, or any
2985    /// requested instrument already has a claim. The cache remains unchanged.
2986    pub fn register_external_order_claims(
2987        &self,
2988        strategy_id: StrategyId,
2989        instrument_ids: &[InstrumentId],
2990    ) -> anyhow::Result<()> {
2991        self.kernel
2992            .cache
2993            .try_borrow_mut()
2994            .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
2995            .register_external_order_claims(strategy_id, instrument_ids)?;
2996
2997        if !instrument_ids.is_empty() {
2998            log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
2999        }
3000
3001        Ok(())
3002    }
3003
3004    /// Deregisters all external order claims owned by `strategy_id` from the shared cache.
3005    ///
3006    /// The operation is synchronous and can be called while the node is idle, after manual
3007    /// [`start`](Self::start) returns, or after the node stops. It cannot be called while
3008    /// [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) owns the node.
3009    ///
3010    /// # Errors
3011    ///
3012    /// Returns an error if the cache is already borrowed.
3013    pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3014        self.kernel
3015            .cache
3016            .try_borrow_mut()
3017            .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3018            .set_external_order_claims(strategy_id, &[])?;
3019
3020        Ok(())
3021    }
3022
3023    pub(crate) fn rollback_external_order_claims(
3024        &self,
3025        strategy_id: StrategyId,
3026        instrument_ids: &[InstrumentId],
3027    ) -> anyhow::Result<()> {
3028        let mut cache = self
3029            .kernel
3030            .cache
3031            .try_borrow_mut()
3032            .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3033        let retained: Vec<_> = cache
3034            .external_order_claim_instrument_ids(Some(strategy_id))
3035            .into_iter()
3036            .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3037            .collect();
3038        cache.set_external_order_claims(strategy_id, &retained)
3039    }
3040
3041    /// Adds an execution algorithm to the trader.
3042    ///
3043    /// Execution algorithms are registered in both the component registry (for lifecycle
3044    /// management) and the actor registry (for data callbacks via msgbus).
3045    ///
3046    /// # Errors
3047    ///
3048    /// Returns an error if:
3049    /// - The node is currently running.
3050    /// - An execution algorithm with the same ID is already registered.
3051    pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3052    where
3053        T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3054    {
3055        if self.state() != NodeState::Idle {
3056            anyhow::bail!(
3057                "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3058            );
3059        }
3060
3061        self.kernel
3062            .trader
3063            .borrow_mut()
3064            .add_exec_algorithm(exec_algorithm)
3065    }
3066
3067    // Runs reconciliation sub-checks, each gated by its own interval.
3068    // Continuous checks only schedule venue work; they do not await venue I/O
3069    // in the event loop.
3070}
3071
3072#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3073enum SocketReconnectDispatchOutcome {
3074    Accepted,
3075    AlreadyReconnecting,
3076    Disconnected,
3077    Closed,
3078    Unsupported,
3079    InvalidTrader,
3080    UnknownClient,
3081    UnknownEndpoint,
3082    AmbiguousEndpoint,
3083}
3084
3085fn record_runner_dispatch(
3086    metrics: &RunnerMetrics,
3087    channel: SystemChannel,
3088    dispatch_start: dst::time::Instant,
3089    metrics_start: dst::time::Instant,
3090) {
3091    let dispatch_end = dst::time::Instant::now();
3092    metrics.record_dispatch(
3093        channel,
3094        dispatch_end.duration_since(dispatch_start),
3095        dispatch_end.duration_since(metrics_start),
3096    );
3097}
3098
3099fn record_runner_maintenance(
3100    metrics: &RunnerMetrics,
3101    work_start: dst::time::Instant,
3102    metrics_start: dst::time::Instant,
3103) {
3104    let work_end = dst::time::Instant::now();
3105    metrics.record_maintenance(
3106        work_end.duration_since(work_start),
3107        work_end.duration_since(metrics_start),
3108    );
3109}
3110
3111fn record_runner_external_msgbus(
3112    metrics: &RunnerMetrics,
3113    work_start: dst::time::Instant,
3114    metrics_start: dst::time::Instant,
3115) {
3116    let work_end = dst::time::Instant::now();
3117    metrics.record_external_msgbus(
3118        work_end.duration_since(work_start),
3119        work_end.duration_since(metrics_start),
3120    );
3121}
3122
3123async fn recv_external_msgbus_message(
3124    rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3125) -> Option<BusMessage> {
3126    match rx {
3127        Some(rx) => rx.recv().await,
3128        None => std::future::pending::<Option<BusMessage>>().await,
3129    }
3130}
3131
3132struct RunnerReceivers<'a> {
3133    time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3134    system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3135    system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3136    exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3137    exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3138    data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3139    data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3140}
3141
3142/// Flushes data events and commands from both `pending` and the channel receivers
3143/// into the cache, looping until no progress is made.
3144///
3145/// This closes the gap where `drive_with_event_buffering` exits as soon as its
3146/// driven future resolves (biased select), leaving items in the channel receivers
3147/// that were not captured into `pending`.
3148fn flush_pending_data(
3149    pending: &mut PendingEvents,
3150    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3151    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3152) {
3153    loop {
3154        let mut progressed = pending.drain_data();
3155
3156        while let Ok(evt) = data_evt_rx.try_recv() {
3157            AsyncRunner::dispatch_data_event(evt);
3158            progressed = true;
3159        }
3160
3161        while let Ok(cmd) = data_cmd_rx.try_recv() {
3162            AsyncRunner::handle_data_command(cmd);
3163            progressed = true;
3164        }
3165
3166        if !progressed {
3167            break;
3168        }
3169    }
3170}
3171
3172/// Flushes all channel receivers into `pending`, then drains everything.
3173///
3174/// Unlike [`flush_pending_data`] this is a single pass, not a drain-until-quiet
3175/// loop. Sufficient for phase 2 where the goal is to capture items the biased
3176/// select did not poll before the connect future resolved.
3177#[expect(
3178    clippy::too_many_arguments,
3179    reason = "all runner receivers are drained together"
3180)]
3181fn flush_all_pending(
3182    pending: &mut PendingEvents,
3183    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3184    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3185    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3186    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3187    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3188    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3189    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3190) {
3191    // Flush channel receivers into pending
3192    while let Ok(handler) = time_evt_rx.try_recv() {
3193        let _ = AsyncRunner::handle_time_event(handler);
3194    }
3195
3196    while let Ok(event) = system_evt_rx.try_recv() {
3197        pending.system_events.push(event);
3198    }
3199
3200    while let Ok(command) = system_cmd_rx.try_recv() {
3201        pending.system_commands.push(command);
3202    }
3203
3204    while let Ok(evt) = data_evt_rx.try_recv() {
3205        pending.data_evts.push(evt);
3206    }
3207
3208    while let Ok(cmd) = data_cmd_rx.try_recv() {
3209        pending.data_cmds.push(cmd);
3210    }
3211
3212    while let Ok(evt) = exec_evt_rx.try_recv() {
3213        pending.push_exec_event(evt);
3214    }
3215
3216    while let Ok(cmd) = exec_cmd_rx.try_recv() {
3217        pending.exec_cmds.push(cmd);
3218    }
3219
3220    pending.drain();
3221}
3222
3223/// Drives a future to completion while buffering channel events.
3224///
3225/// Time events are handled immediately. Account events are forwarded directly.
3226/// All other events are buffered in `pending` for later processing.
3227#[expect(
3228    clippy::too_many_arguments,
3229    reason = "startup buffering owns one future plus the pending state and all runner receivers"
3230)]
3231async fn drive_with_event_buffering<F: std::future::Future>(
3232    future: F,
3233    pending: &mut PendingEvents,
3234    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3235    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3236    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3237    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3238    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3239    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3240    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3241) -> F::Output {
3242    tokio::pin!(future);
3243
3244    loop {
3245        tokio::select! {
3246            biased;
3247
3248            result = &mut future => {
3249                break result;
3250            }
3251            Some(handler) = time_evt_rx.recv() => {
3252                let _ = AsyncRunner::handle_time_event(handler);
3253            }
3254            Some(event) = system_evt_rx.recv() => {
3255                pending.system_events.push(event);
3256            }
3257            Some(command) = system_cmd_rx.recv() => {
3258                pending.system_commands.push(command);
3259            }
3260            Some(evt) = exec_evt_rx.recv() => {
3261                pending.push_exec_event(evt);
3262            }
3263            Some(cmd) = exec_cmd_rx.recv() => {
3264                pending.exec_cmds.push(cmd);
3265            }
3266            Some(evt) = data_evt_rx.recv() => {
3267                pending.data_evts.push(evt);
3268            }
3269            Some(cmd) = data_cmd_rx.recv() => {
3270                pending.data_cmds.push(cmd);
3271            }
3272        }
3273    }
3274}
3275
3276#[derive(Default)]
3277struct PendingEvents {
3278    system_events: Vec<DispatchMessage<SystemEvent>>,
3279    system_commands: Vec<DispatchMessage<SystemCommand>>,
3280    data_evts: Vec<DispatchMessage<DataEvent>>,
3281    data_cmds: Vec<DispatchMessage<DataCommand>>,
3282    exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3283    order_evts: Vec<DispatchMessage<OrderEventAny>>,
3284    exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3285}
3286
3287impl PendingEvents {
3288    fn is_empty(&self) -> bool {
3289        self.system_events.is_empty()
3290            && self.system_commands.is_empty()
3291            && self.data_evts.is_empty()
3292            && self.data_cmds.is_empty()
3293            && self.exec_reports.is_empty()
3294            && self.order_evts.is_empty()
3295            && self.exec_cmds.is_empty()
3296    }
3297
3298    /// Drains only data events and commands into the cache.
3299    ///
3300    /// Returns `true` if any events or commands were drained.
3301    fn drain_data(&mut self) -> bool {
3302        let total = self.data_evts.len() + self.data_cmds.len();
3303
3304        if total > 0 {
3305            log::debug!(
3306                "Draining {total} data events/commands into cache \
3307                 (data_evts={}, data_cmds={})",
3308                self.data_evts.len(),
3309                self.data_cmds.len(),
3310            );
3311        }
3312
3313        for evt in self.data_evts.drain(..) {
3314            AsyncRunner::dispatch_data_event(evt);
3315        }
3316
3317        for cmd in self.data_cmds.drain(..) {
3318            AsyncRunner::handle_data_command(cmd);
3319        }
3320
3321        total > 0
3322    }
3323
3324    /// Drains all remaining pending events.
3325    fn drain(&mut self) {
3326        let total = self.data_evts.len()
3327            + self.data_cmds.len()
3328            + self.exec_reports.len()
3329            + self.order_evts.len()
3330            + self.exec_cmds.len();
3331
3332        if total > 0 {
3333            log::debug!(
3334                "Processing {total} events/commands queued during startup \
3335                 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3336                self.data_evts.len(),
3337                self.data_cmds.len(),
3338                self.exec_reports.len(),
3339                self.order_evts.len(),
3340                self.exec_cmds.len()
3341            );
3342        }
3343
3344        for evt in self.data_evts.drain(..) {
3345            AsyncRunner::dispatch_data_event(evt);
3346        }
3347
3348        for cmd in self.data_cmds.drain(..) {
3349            AsyncRunner::handle_data_command(cmd);
3350        }
3351
3352        for report in self.exec_reports.drain(..) {
3353            report
3354                .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3355        }
3356
3357        for evt in self.order_evts.drain(..) {
3358            evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3359        }
3360
3361        for cmd in self.exec_cmds.drain(..) {
3362            AsyncRunner::handle_trading_command(cmd);
3363        }
3364    }
3365
3366    fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3367        let rooted = event.is_rooted();
3368        event.dispatch(|event| {
3369            let owner = std::thread::current().id();
3370
3371            let order = |event| {
3372                if rooted {
3373                    DispatchMessage::new(event, owner)
3374                } else {
3375                    DispatchMessage::from(event)
3376                }
3377            };
3378
3379            // Account events are safe to process immediately. Reports and orders need
3380            // ExecEngine access, which may conflict with the driven startup future's borrow.
3381            match event {
3382                ExecutionEvent::Account(_) => AsyncRunner::handle_exec_event(event),
3383                ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3384                    DispatchMessage::new(report, owner)
3385                } else {
3386                    report.into()
3387                }),
3388                ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3389                ExecutionEvent::OrderSubmittedBatch(batch) => {
3390                    for event in batch {
3391                        self.order_evts.push(order(OrderEventAny::Submitted(event)));
3392                    }
3393                }
3394                ExecutionEvent::OrderAcceptedBatch(batch) => {
3395                    for event in batch {
3396                        self.order_evts.push(order(OrderEventAny::Accepted(event)));
3397                    }
3398                }
3399                ExecutionEvent::OrderCanceledBatch(batch) => {
3400                    for event in batch {
3401                        self.order_evts.push(order(OrderEventAny::Canceled(event)));
3402                    }
3403                }
3404            }
3405        });
3406    }
3407
3408    fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3409        std::mem::take(&mut self.system_events)
3410    }
3411
3412    fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3413        std::mem::take(&mut self.system_commands)
3414    }
3415}
3416
3417struct ClientStatus {
3418    client: String,
3419    client_type: &'static str,
3420    connected: bool,
3421}
3422
3423fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3424    let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3425    builder.push_record(["Client", "Type", "Connected"]);
3426
3427    for row in rows {
3428        builder.push_record([
3429            row.client,
3430            row.client_type.to_string(),
3431            row.connected.to_string(),
3432        ]);
3433    }
3434
3435    builder.build().with(Style::rounded()).to_string()
3436}
3437
3438#[cfg(test)]
3439mod tests {
3440    mod shutdown;
3441
3442    use std::{
3443        cell::{Cell, RefCell},
3444        fmt::Debug,
3445        rc::Rc,
3446        sync::{
3447            Arc,
3448            atomic::{AtomicBool, Ordering},
3449        },
3450    };
3451
3452    use bytes::Bytes;
3453    use indexmap::{IndexMap, IndexSet};
3454    use log::{Level, LevelFilter, Log, Metadata, Record};
3455    #[cfg(feature = "python")]
3456    use nautilus_common::runner::{
3457        SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3458    };
3459    use nautilus_common::{
3460        actor::{
3461            self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3462        },
3463        cache::Cache,
3464        clock::{Clock, VirtualClock},
3465        enums::SerializationEncoding,
3466        live::{
3467            runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3468            sender::DispatchSender,
3469        },
3470        logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3471        messages::{
3472            data::{SubscribeCommand, SubscribeQuotes},
3473            execution::{
3474                GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3475                GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3476                TradingCommand,
3477            },
3478            system::{
3479                QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3480            },
3481        },
3482        msgbus::{
3483            self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3484            MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3485            MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3486        },
3487        nautilus_actor,
3488        runner::{SyncTradingCommandSender, TradingCommandSender},
3489        testing::wait_until_async,
3490        timer::{TimeEvent, TimeEventCallback},
3491    };
3492    use nautilus_core::{Params, UUID4, UnixNanos};
3493    use nautilus_execution::{
3494        engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3495        reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3496    };
3497    use nautilus_model::{
3498        accounts::{AccountAny, MarginAccount},
3499        data::QuoteTick,
3500        enums::{
3501            AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3502            TimeInForce,
3503        },
3504        events::{
3505            AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3506            order::spec::{
3507                OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3508            },
3509        },
3510        identifiers::{
3511            AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3512            TradeId, TraderId, Venue, VenueOrderId,
3513        },
3514        instruments::{
3515            Instrument, InstrumentAny,
3516            stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3517        },
3518        orders::{
3519            Order, OrderAny, OrderList, OrderTestBuilder,
3520            stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3521        },
3522        reports::{FillReport, PositionStatusReport},
3523        types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3524    };
3525    use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3526    use nautilus_testkit::{
3527        cache::TestCacheDatabaseControl,
3528        components::{StateActor, StateStrategy},
3529    };
3530    use nautilus_trading::{
3531        nautilus_strategy,
3532        strategy::{config::StrategyConfig, core::StrategyCore},
3533    };
3534    use parking_lot::Mutex;
3535    use rstest::*;
3536    use rust_decimal::Decimal;
3537    use rust_decimal_macros::dec;
3538    use ustr::Ustr;
3539
3540    use super::{
3541        reconciliation::{
3542            PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3543            request_position_fill_reports,
3544        },
3545        *,
3546    };
3547    use crate::{
3548        execution::{
3549            client::LiveExecutionClient,
3550            manager::{PositionFillReportQuery, ReportClientCoverage},
3551            submission::{
3552                SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3553            },
3554        },
3555        socket::SocketControl,
3556    };
3557
3558    struct ExternalIngressLogCapture {
3559        messages: Mutex<Vec<String>>,
3560    }
3561
3562    static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3563        messages: Mutex::new(Vec::new()),
3564    };
3565
3566    #[derive(Debug)]
3567    enum FillReportClientOutcome {
3568        Reports(Vec<FillReport>),
3569        Failure,
3570    }
3571
3572    #[derive(Debug)]
3573    struct FillReportClient {
3574        client_id: ClientId,
3575        account_id: AccountId,
3576        venue: Venue,
3577        outcome: FillReportClientOutcome,
3578        commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3579    }
3580
3581    #[async_trait::async_trait(?Send)]
3582    impl ExecutionClient for FillReportClient {
3583        fn is_connected(&self) -> bool {
3584            true
3585        }
3586
3587        fn client_id(&self) -> ClientId {
3588            self.client_id
3589        }
3590
3591        fn account_id(&self) -> AccountId {
3592            self.account_id
3593        }
3594
3595        fn venue(&self) -> Venue {
3596            self.venue
3597        }
3598
3599        fn oms_type(&self) -> OmsType {
3600            OmsType::Netting
3601        }
3602
3603        fn get_account(&self) -> Option<AccountAny> {
3604            None
3605        }
3606
3607        fn generate_account_state(
3608            &self,
3609            _balances: Vec<AccountBalance>,
3610            _margins: Vec<MarginBalance>,
3611            _reported: bool,
3612            _ts_event: UnixNanos,
3613            _info: Option<Params>,
3614        ) -> anyhow::Result<()> {
3615            Ok(())
3616        }
3617
3618        fn start(&mut self) -> anyhow::Result<()> {
3619            Ok(())
3620        }
3621
3622        fn stop(&mut self) -> anyhow::Result<()> {
3623            Ok(())
3624        }
3625
3626        async fn generate_order_status_reports(
3627            &self,
3628            _cmd: &GenerateOrderStatusReports,
3629        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3630            match self.outcome {
3631                FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3632                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3633            }
3634        }
3635
3636        async fn generate_position_status_reports(
3637            &self,
3638            _cmd: &GeneratePositionStatusReports,
3639        ) -> anyhow::Result<Vec<PositionStatusReport>> {
3640            match self.outcome {
3641                FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3642                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3643            }
3644        }
3645
3646        async fn generate_fill_reports(
3647            &self,
3648            cmd: GenerateFillReports,
3649        ) -> anyhow::Result<Vec<FillReport>> {
3650            self.commands.borrow_mut().push(cmd);
3651
3652            match &self.outcome {
3653                FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3654                FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3655            }
3656        }
3657    }
3658
3659    #[derive(Debug)]
3660    struct OrderReportClient {
3661        client_id: ClientId,
3662        report: OrderStatusReport,
3663        targeted: bool,
3664        queries: Rc<RefCell<Vec<&'static str>>>,
3665    }
3666
3667    #[async_trait::async_trait(?Send)]
3668    impl ExecutionClient for OrderReportClient {
3669        fn is_connected(&self) -> bool {
3670            true
3671        }
3672
3673        fn client_id(&self) -> ClientId {
3674            self.client_id
3675        }
3676
3677        fn account_id(&self) -> AccountId {
3678            self.report.account_id
3679        }
3680
3681        fn venue(&self) -> Venue {
3682            self.report.instrument_id.venue
3683        }
3684
3685        fn oms_type(&self) -> OmsType {
3686            OmsType::Netting
3687        }
3688
3689        fn get_account(&self) -> Option<AccountAny> {
3690            None
3691        }
3692
3693        fn generate_account_state(
3694            &self,
3695            _balances: Vec<AccountBalance>,
3696            _margins: Vec<MarginBalance>,
3697            _reported: bool,
3698            _ts_event: UnixNanos,
3699            _info: Option<Params>,
3700        ) -> anyhow::Result<()> {
3701            Ok(())
3702        }
3703
3704        fn start(&mut self) -> anyhow::Result<()> {
3705            Ok(())
3706        }
3707
3708        fn stop(&mut self) -> anyhow::Result<()> {
3709            Ok(())
3710        }
3711
3712        async fn generate_order_status_reports(
3713            &self,
3714            _command: &GenerateOrderStatusReports,
3715        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3716            self.queries.borrow_mut().push("bulk");
3717            Ok(if self.targeted {
3718                Vec::new()
3719            } else {
3720                vec![self.report.clone()]
3721            })
3722        }
3723
3724        async fn generate_order_status_report(
3725            &self,
3726            _command: &GenerateOrderStatusReport,
3727        ) -> anyhow::Result<Option<OrderStatusReport>> {
3728            self.queries.borrow_mut().push("targeted");
3729            Ok(Some(self.report.clone()))
3730        }
3731    }
3732
3733    #[derive(Debug)]
3734    struct StartupSocketActor {
3735        core: DataActorCore,
3736        received: Rc<RefCell<Vec<SocketStateChanged>>>,
3737    }
3738
3739    impl StartupSocketActor {
3740        fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3741            Self {
3742                core: DataActorCore::new(DataActorConfig {
3743                    actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3744                    ..Default::default()
3745                }),
3746                received,
3747            }
3748        }
3749    }
3750
3751    impl DataActor for StartupSocketActor {
3752        fn on_start(&mut self) -> anyhow::Result<()> {
3753            self.subscribe_socket_state(None, None, None);
3754            Ok(())
3755        }
3756
3757        fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3758            self.received.borrow_mut().push(event.clone());
3759            Ok(())
3760        }
3761    }
3762
3763    nautilus_actor!(StartupSocketActor);
3764
3765    impl Log for ExternalIngressLogCapture {
3766        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3767            metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3768        }
3769
3770        fn log(&self, record: &Record<'_>) {
3771            if self.enabled(record.metadata()) {
3772                self.messages.lock().push(record.args().to_string());
3773            }
3774        }
3775
3776        fn flush(&self) {}
3777    }
3778
3779    #[derive(Debug)]
3780    struct FailingTimerActor {
3781        core: DataActorCore,
3782        received: Rc<RefCell<Vec<u64>>>,
3783    }
3784
3785    nautilus_actor!(FailingTimerActor);
3786
3787    impl DataActor for FailingTimerActor {
3788        fn on_start(&mut self) -> anyhow::Result<()> {
3789            for timestamp in [17, 23] {
3790                let received = self.received.clone();
3791
3792                let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3793                    received.borrow_mut().push(timestamp);
3794
3795                    if timestamp == 17 {
3796                        crate::dispatch::tests::latch_callback_failure();
3797                    }
3798                }));
3799
3800                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3801                    TimeEvent::new(
3802                        "callback-failure".into(),
3803                        UUID4::new(),
3804                        timestamp.into(),
3805                        timestamp.into(),
3806                    ),
3807                    callback,
3808                ));
3809            }
3810
3811            Ok(())
3812        }
3813    }
3814
3815    #[rstest]
3816    #[tokio::test]
3817    async fn test_callback_failure_stops_later_live_events(
3818        #[values(false, true)] incomplete_submission: bool,
3819    ) {
3820        actor::clear_callbacks().unwrap();
3821
3822        let config = LiveNodeConfig {
3823            exec_engine: crate::config::LiveExecutionEngineConfig {
3824                reconciliation: false,
3825                submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3826                ..Default::default()
3827            },
3828            timeout_connection: Duration::ZERO,
3829            timeout_reconciliation: Duration::ZERO,
3830            timeout_portfolio: Duration::ZERO,
3831            timeout_disconnection: Duration::ZERO,
3832            delay_post_stop: Duration::ZERO,
3833            timeout_shutdown: Duration::ZERO,
3834            ..Default::default()
3835        };
3836
3837        let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3838
3839        if incomplete_submission {
3840            let order = OrderTestBuilder::new(OrderType::Market)
3841                .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3842                .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3843                .quantity(Quantity::from("1"))
3844                .build();
3845            node.exec_manager
3846                .register_submission(order.init_event(), None);
3847        }
3848        let received = Rc::new(RefCell::new(Vec::new()));
3849        node.add_actor(FailingTimerActor {
3850            core: DataActorCore::new(DataActorConfig {
3851                actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3852                ..Default::default()
3853            }),
3854            received: received.clone(),
3855        })
3856        .unwrap();
3857
3858        let result = node.run_with_mode(NodeRunMode::Hosted).await;
3859
3860        let error = result.unwrap_err();
3861        assert!(error.to_string().contains("Callback delivery unwound"));
3862        if incomplete_submission {
3863            assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3864        }
3865        let state = node.state();
3866        let trader_stopped = node.kernel.trader.borrow().is_stopped();
3867        let failure = actor::callback_failure();
3868        node.dispose();
3869
3870        assert_eq!(
3871            error.downcast_ref::<CallbackDispatchError>(),
3872            Some(&CallbackDispatchError::DeliveryUnwound)
3873        );
3874        assert_eq!(*received.borrow(), [17]);
3875        assert_eq!(state, NodeState::Stopped);
3876        assert!(trader_stopped);
3877        assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
3878        assert_eq!(actor::callback_failure(), None);
3879        assert_eq!(actor::clear_callbacks(), Ok(()));
3880    }
3881
3882    #[rstest]
3883    fn test_render_client_statuses() {
3884        let rows = vec![
3885            ClientStatus {
3886                client: "BINANCE".to_string(),
3887                client_type: "Data",
3888                connected: true,
3889            },
3890            ClientStatus {
3891                client: "SIM".to_string(),
3892                client_type: "Execution",
3893                connected: false,
3894            },
3895        ];
3896
3897        let output = render_client_statuses(rows);
3898        let expected = "╭─────────┬───────────┬───────────╮\n\
3899│ Client  │ Type      │ Connected │\n\
3900├─────────┼───────────┼───────────┤\n\
3901│ BINANCE │ Data      │ true      │\n\
3902│ SIM     │ Execution │ false     │\n\
3903╰─────────┴───────────┴───────────╯";
3904
3905        assert_eq!(output, expected);
3906    }
3907
3908    #[rstest]
3909    fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
3910        log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
3911        log::set_max_level(LevelFilter::Error);
3912        EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
3913        let message = BusMessage::with_str_topic(
3914            "data.quotes.AUDUSD.SIM*",
3915            BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
3916            Bytes::new(),
3917            SerializationEncoding::Json,
3918        );
3919
3920        LiveNode::republish_external_msgbus_message(&message);
3921
3922        assert_eq!(
3923            *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
3924            vec![
3925                "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
3926                 external message topic: Topic `value` contained invalid characters, was \
3927                 data.quotes.AUDUSD.SIM*"
3928                    .to_string()
3929            ],
3930        );
3931    }
3932
3933    #[rstest]
3934    #[case(None, 2)]
3935    #[case(Some(SystemChannel::DataEvents), 1)]
3936    #[case(Some(SystemChannel::ExecCommands), 1)]
3937    fn test_publish_queue_state_transitions_reaches_typed_subscriber(
3938        #[case] channel: Option<SystemChannel>,
3939        #[case] expected_count: usize,
3940    ) {
3941        let config = LiveNodeConfig {
3942            trader_id: TraderId::from("QUEUE-001"),
3943            exec_engine: crate::config::LiveExecutionEngineConfig {
3944                reconciliation: false,
3945                ..Default::default()
3946            },
3947            ..Default::default()
3948        };
3949
3950        let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
3951        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
3952
3953        let handler = ShareableMessageHandler::from_typed({
3954            let received = received.clone();
3955            move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
3956        });
3957
3958        msgbus::subscribe_any(
3959            MessagingSwitchboard::queue_state_changed_pattern(channel),
3960            handler,
3961            None,
3962        );
3963
3964        let transitions = [
3965            QueueStateTransition {
3966                channel: SystemChannel::DataEvents,
3967                condition: QueueCondition::Backlogged,
3968                state: QueueState::Triggered,
3969                queue_depth: 17,
3970                mean_dispatch_ns: 23,
3971            },
3972            QueueStateTransition {
3973                channel: SystemChannel::ExecCommands,
3974                condition: QueueCondition::Slow,
3975                state: QueueState::Triggered,
3976                queue_depth: 17,
3977                mean_dispatch_ns: 23,
3978            },
3979        ];
3980
3981        node.publish_queue_state_transitions(&transitions);
3982
3983        let events = received.borrow();
3984        assert_eq!(events.len(), expected_count);
3985
3986        let expected = transitions
3987            .into_iter()
3988            .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
3989
3990        for (event, transition) in events.iter().zip(expected) {
3991            assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
3992            assert_eq!(event.channel, transition.channel);
3993            assert_eq!(event.condition, transition.condition);
3994            assert_eq!(event.state, transition.state);
3995            assert_eq!(event.queue_depth, transition.queue_depth);
3996            assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
3997            assert_ne!(event.event_id, UUID4::default());
3998            assert_ne!(event.ts_event, UnixNanos::default());
3999            assert_eq!(event.ts_init, event.ts_event);
4000        }
4001
4002        if channel.is_none() {
4003            assert_ne!(events[0].event_id, events[1].event_id);
4004        }
4005
4006        drop(events);
4007        msgbus::get_message_bus().borrow_mut().dispose();
4008    }
4009
4010    #[rstest]
4011    #[case(None, None)]
4012    #[case(Some(ClientId::from("BINANCE")), None)]
4013    #[case(None, Some("binance-futures-market-streams"))]
4014    #[case(
4015        Some(ClientId::from("BINANCE")),
4016        Some("binance-futures-market-streams")
4017    )]
4018    fn test_process_socket_state_change_reaches_typed_subscriber(
4019        #[case] client_id: Option<ClientId>,
4020        #[case] endpoint: Option<&str>,
4021    ) {
4022        let config = LiveNodeConfig {
4023            trader_id: TraderId::from("SOCKET-001"),
4024            exec_engine: crate::config::LiveExecutionEngineConfig {
4025                reconciliation: false,
4026                ..Default::default()
4027            },
4028            ..Default::default()
4029        };
4030
4031        let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4032        let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4033
4034        let handler = ShareableMessageHandler::from_typed({
4035            let received = received.clone();
4036            move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4037        });
4038
4039        msgbus::subscribe_any(
4040            MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4041            handler,
4042            None,
4043        );
4044
4045        let change = SocketStateChange::new(
4046            ClientId::from("BINANCE"),
4047            Some(Venue::from("BINANCE")),
4048            Ustr::from("binance-futures-market-streams"),
4049            SocketState::Disconnected,
4050        );
4051
4052        node.process_system_event(SystemEvent::SocketState(change));
4053
4054        let events = received.borrow();
4055        assert_eq!(events.len(), 1);
4056        assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4057        assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4058        assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4059        assert_eq!(
4060            events[0].endpoint,
4061            Ustr::from("binance-futures-market-streams")
4062        );
4063        assert_eq!(events[0].state, SocketState::Disconnected);
4064        assert_ne!(events[0].event_id, UUID4::default());
4065        assert_ne!(events[0].ts_event, UnixNanos::default());
4066        assert_eq!(events[0].ts_init, events[0].ts_event);
4067        drop(events);
4068        msgbus::get_message_bus().borrow_mut().dispose();
4069    }
4070
4071    #[rstest]
4072    #[case::accepted(
4073        ReconnectRequestOutcome::Accepted,
4074        SocketReconnectDispatchOutcome::Accepted
4075    )]
4076    #[case::already_reconnecting(
4077        ReconnectRequestOutcome::AlreadyReconnecting,
4078        SocketReconnectDispatchOutcome::AlreadyReconnecting
4079    )]
4080    #[case::disconnected(
4081        ReconnectRequestOutcome::Disconnected,
4082        SocketReconnectDispatchOutcome::Disconnected
4083    )]
4084    #[case::closed(
4085        ReconnectRequestOutcome::Closed,
4086        SocketReconnectDispatchOutcome::Closed
4087    )]
4088    #[case::unsupported(
4089        ReconnectRequestOutcome::Unsupported,
4090        SocketReconnectDispatchOutcome::Unsupported
4091    )]
4092    fn test_request_socket_reconnect_maps_transport_outcome(
4093        #[case] transport: ReconnectRequestOutcome,
4094        #[case] expected: SocketReconnectDispatchOutcome,
4095    ) {
4096        let registry = SocketReconnectRegistry::default();
4097        let client_id = ClientId::from("TEST");
4098        let endpoint = Ustr::from("test-streams");
4099        let control = SocketControl::with_registry(client_id, None, endpoint, &registry);
4100        let _sink = control.sink();
4101        control.register(move || transport);
4102
4103        let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4104
4105        assert_eq!(outcome, expected);
4106    }
4107
4108    #[rstest]
4109    #[case::client_not_found(
4110        SocketReconnectLookup::ClientNotFound,
4111        SocketReconnectDispatchOutcome::UnknownClient
4112    )]
4113    #[case::unsupported(
4114        SocketReconnectLookup::Unsupported,
4115        SocketReconnectDispatchOutcome::Unsupported
4116    )]
4117    #[case::endpoint_not_found(
4118        SocketReconnectLookup::EndpointNotFound,
4119        SocketReconnectDispatchOutcome::UnknownEndpoint
4120    )]
4121    #[case::ambiguous(
4122        SocketReconnectLookup::AmbiguousEndpoint,
4123        SocketReconnectDispatchOutcome::AmbiguousEndpoint
4124    )]
4125    fn test_request_socket_reconnect_maps_lookup_failure(
4126        #[case] lookup: SocketReconnectLookup,
4127        #[case] expected: SocketReconnectDispatchOutcome,
4128    ) {
4129        assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4130    }
4131
4132    #[rstest]
4133    fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4134        let trader_id = TraderId::from("SOCKET-001");
4135
4136        let config = LiveNodeConfig {
4137            trader_id,
4138            ..Default::default()
4139        };
4140
4141        let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4142        let client_id = ClientId::from("TEST");
4143        let endpoint = Ustr::from("test-streams");
4144        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4145        let sender = DispatchSender::new(tx);
4146        let events = sender.clone();
4147        msgbus::subscribe_any(
4148            MessagingSwitchboard::socket_state_changed_pattern(
4149                Some(client_id),
4150                Some(endpoint.as_str()),
4151            ),
4152            ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4153                events.send(17u32).unwrap();
4154            }),
4155            None,
4156        );
4157
4158        let control =
4159            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4160        let _sink = control.sink();
4161        control.register(move || {
4162            sender.send(23u32).unwrap();
4163            ReconnectRequestOutcome::Accepted
4164        });
4165
4166        let event = SystemEvent::SocketState(SocketStateChange::new(
4167            client_id,
4168            None,
4169            endpoint,
4170            SocketState::Connected,
4171        ));
4172        let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4173            trader_id,
4174            client_id,
4175            endpoint,
4176            31.into(),
4177        ));
4178
4179        if startup {
4180            let mut pending = PendingEvents::default();
4181            pending.system_events.push(event.into());
4182            pending.system_commands.push(command.into());
4183            node.process_system_events(pending.take_system_events());
4184            node.process_system_commands(pending.take_system_commands());
4185            assert!(pending.is_empty());
4186        } else {
4187            node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4188            node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4189        }
4190
4191        let event_child = rx.try_recv().unwrap();
4192        let command_child = rx.try_recv().unwrap();
4193        assert!(event_child.is_rooted());
4194        assert!(command_child.is_rooted());
4195        assert_eq!(event_child.dispatch(|value| value), 17);
4196        assert_eq!(command_child.dispatch(|value| value), 23);
4197        assert!(rx.is_empty());
4198        msgbus::get_message_bus().borrow_mut().dispose();
4199    }
4200
4201    #[rstest]
4202    fn test_process_socket_reconnect_routes_only_matching_trader() {
4203        let trader_id = TraderId::from("SOCKET-001");
4204
4205        let config = LiveNodeConfig {
4206            trader_id,
4207            exec_engine: crate::config::LiveExecutionEngineConfig {
4208                reconciliation: false,
4209                ..Default::default()
4210            },
4211            ..Default::default()
4212        };
4213
4214        let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4215        let client_id = ClientId::from("TEST");
4216        let endpoint = Ustr::from("test-streams");
4217        let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4218        let request_count = Arc::clone(&requests);
4219        let control =
4220            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4221        let _sink = control.sink();
4222        control.register(move || {
4223            request_count.fetch_add(1, Ordering::SeqCst);
4224            ReconnectRequestOutcome::Accepted
4225        });
4226
4227        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4228            TraderId::from("OTHER-001"),
4229            client_id,
4230            endpoint,
4231            UnixNanos::default(),
4232        )));
4233        assert_eq!(requests.load(Ordering::SeqCst), 0);
4234
4235        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4236            trader_id,
4237            client_id,
4238            endpoint,
4239            UnixNanos::default(),
4240        )));
4241        assert_eq!(requests.load(Ordering::SeqCst), 1);
4242    }
4243
4244    #[rstest]
4245    #[tokio::test]
4246    async fn test_start_publishes_socket_change_after_actor_subscribes() {
4247        let config = LiveNodeConfig {
4248            trader_id: TraderId::from("SOCKET-STARTUP-001"),
4249            exec_engine: crate::config::LiveExecutionEngineConfig {
4250                reconciliation: false,
4251                ..Default::default()
4252            },
4253            timeout_connection: Duration::ZERO,
4254            timeout_reconciliation: Duration::ZERO,
4255            timeout_portfolio: Duration::ZERO,
4256            timeout_disconnection: Duration::ZERO,
4257            delay_post_stop: Duration::ZERO,
4258            timeout_shutdown: Duration::ZERO,
4259            ..Default::default()
4260        };
4261
4262        let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4263        let received = Rc::new(RefCell::new(Vec::new()));
4264        node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4265            .unwrap();
4266        node.runner.as_ref().unwrap().bind_senders();
4267
4268        let change = SocketStateChange::new(
4269            ClientId::from("BINANCE"),
4270            Some(Venue::from("BINANCE")),
4271            Ustr::from("binance-futures-market-streams"),
4272            SocketState::Connected,
4273        );
4274        get_system_event_sender()
4275            .send(SystemEvent::SocketState(change))
4276            .unwrap();
4277
4278        node.start().await.unwrap();
4279
4280        {
4281            let events = received.borrow();
4282            assert_eq!(events.len(), 1);
4283            assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4284            assert_eq!(events[0].client_id, change.client_id);
4285            assert_eq!(events[0].venue, change.venue);
4286            assert_eq!(events[0].endpoint, change.endpoint);
4287            assert_eq!(events[0].state, change.state);
4288            assert_ne!(events[0].event_id, UUID4::default());
4289            assert_ne!(events[0].ts_event, UnixNanos::default());
4290            assert_eq!(events[0].ts_init, events[0].ts_event);
4291        }
4292
4293        node.stop().await.unwrap();
4294        node.dispose();
4295    }
4296
4297    #[rstest]
4298    #[tokio::test(flavor = "current_thread")]
4299    async fn test_run_publishes_queue_state_after_dispatch_sample() {
4300        let config = LiveNodeConfig {
4301            trader_id: TraderId::from("QUEUE-RUN-001"),
4302            queue_monitor: Some(crate::config::QueueMonitorConfig {
4303                queue_depth_trigger: usize::MAX,
4304                queue_depth_clear: 0,
4305                mean_dispatch_ns_trigger: 1,
4306                mean_dispatch_ns_clear: 0,
4307            }),
4308            exec_engine: crate::config::LiveExecutionEngineConfig {
4309                reconciliation: false,
4310                ..Default::default()
4311            },
4312            delay_post_stop: Duration::ZERO,
4313            ..Default::default()
4314        };
4315
4316        let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4317        let handle = node.handle();
4318        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4319
4320        let handler = ShareableMessageHandler::from_typed({
4321            let received = received.clone();
4322            let stop_handle = handle.clone();
4323
4324            move |event: &QueueStateChanged| {
4325                received.borrow_mut().push(event.clone());
4326                stop_handle.stop();
4327            }
4328        });
4329
4330        msgbus::subscribe_any(
4331            MessagingSwitchboard::queue_state_changed_pattern(None),
4332            handler,
4333            None,
4334        );
4335        let drive_handle = handle.clone();
4336
4337        let result = tokio::time::timeout(Duration::from_secs(5), async {
4338            let run = node.run();
4339            tokio::pin!(run);
4340
4341            let drive = async move {
4342                wait_until_async(
4343                    || async { drive_handle.is_running() },
4344                    Duration::from_secs(2),
4345                )
4346                .await;
4347                get_data_event_sender().send(stub_data_event()).unwrap();
4348            };
4349
4350            let (run_result, ()) = tokio::join!(run, drive);
4351            run_result
4352        })
4353        .await;
4354
4355        assert!(
4356            result.is_ok(),
4357            "queue state event should arrive before timeout"
4358        );
4359        assert!(result.unwrap().is_ok(), "run() should succeed");
4360        let events = received.borrow();
4361        assert_eq!(events.len(), 1);
4362        assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4363        assert_eq!(events[0].channel, SystemChannel::DataEvents);
4364        assert_eq!(events[0].condition, QueueCondition::Slow);
4365        assert_eq!(events[0].state, QueueState::Triggered);
4366        assert_eq!(events[0].queue_depth, 0);
4367        assert!(events[0].mean_dispatch_ns > 0);
4368        assert_ne!(events[0].event_id, UUID4::default());
4369        assert_ne!(events[0].ts_event, UnixNanos::default());
4370        assert_eq!(events[0].ts_init, events[0].ts_event);
4371        drop(events);
4372        msgbus::get_message_bus().borrow_mut().dispose();
4373    }
4374
4375    #[rstest]
4376    fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4377        let config = LiveNodeConfig {
4378            exec_engine: crate::config::LiveExecutionEngineConfig {
4379                reconciliation: false,
4380                ..Default::default()
4381            },
4382            ..Default::default()
4383        };
4384
4385        let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4386        let event = stub_exec_event();
4387        let account_id = AccountId::from("TEST-001");
4388        let instrument_id = InstrumentId::from("TEST.VENUE");
4389        let trade_id = TradeId::from("T-001");
4390
4391        let close_ids = node.observe_exec_event_before_dispatch(&event);
4392        assert_eq!(close_ids, Some(Vec::new()));
4393        assert!(
4394            !node
4395                .exec_manager
4396                .is_fill_recently_processed(account_id, instrument_id, trade_id)
4397        );
4398
4399        node.exec_manager
4400            .mark_fill_processed(account_id, instrument_id, trade_id);
4401
4402        let close_ids = node.observe_exec_event_before_dispatch(&event);
4403        assert_eq!(close_ids, None);
4404    }
4405
4406    #[rstest]
4407    #[case(false, false, OrderStatus::Canceled, 1)]
4408    #[case(false, true, OrderStatus::Accepted, 0)]
4409    #[case(true, false, OrderStatus::Canceled, 1)]
4410    #[case(true, true, OrderStatus::Accepted, 0)]
4411    fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4412        #[case] with_fills: bool,
4413        #[case] superseded: bool,
4414        #[case] expected_status: OrderStatus,
4415        #[case] expected_query_count: usize,
4416    ) {
4417        let config = LiveNodeConfig {
4418            exec_engine: crate::config::LiveExecutionEngineConfig {
4419                reconciliation: true,
4420                open_check_threshold_ms: 5_000,
4421                single_order_query_delay_ms: 0,
4422                ..Default::default()
4423            },
4424            ..Default::default()
4425        };
4426
4427        let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4428        let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4429        let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4430        let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4431        let account_id = AccountId::from("TEST-001");
4432        let client_id = ClientId::from("TEST");
4433        let instrument = crypto_perpetual_ethusdt();
4434        let instrument_id = instrument.id();
4435        let account = AccountAny::Margin(MarginAccount::new(
4436            AccountState::new(
4437                account_id,
4438                AccountType::Margin,
4439                vec![AccountBalance::new(
4440                    Money::from("1000000 USDT"),
4441                    Money::from("0 USDT"),
4442                    Money::from("1000000 USDT"),
4443                )],
4444                Vec::new(),
4445                true,
4446                UUID4::new(),
4447                UnixNanos::default(),
4448                UnixNanos::default(),
4449                Some(Currency::USDT()),
4450            ),
4451            true,
4452        ));
4453        node.kernel.cache.borrow_mut().add_account(account).unwrap();
4454        node.kernel
4455            .cache
4456            .borrow_mut()
4457            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4458            .unwrap();
4459        insert_accepted_limit_order_in_node(
4460            &node,
4461            account_id,
4462            client_id,
4463            instrument_id,
4464            client_order_id,
4465            old_venue_order_id,
4466        );
4467
4468        if superseded {
4469            let order = node
4470                .kernel
4471                .cache
4472                .borrow()
4473                .order_owned(&client_order_id)
4474                .unwrap();
4475            let pending_update = OrderPendingUpdateSpec::builder()
4476                .trader_id(order.trader_id())
4477                .strategy_id(order.strategy_id())
4478                .instrument_id(order.instrument_id())
4479                .client_order_id(client_order_id)
4480                .account_id(account_id)
4481                .venue_order_id(old_venue_order_id)
4482                .build();
4483            node.kernel
4484                .cache
4485                .borrow_mut()
4486                .update_order(&OrderEventAny::PendingUpdate(pending_update))
4487                .unwrap();
4488            let order = node
4489                .kernel
4490                .cache
4491                .borrow()
4492                .order_owned(&client_order_id)
4493                .unwrap();
4494            let updated = OrderUpdatedSpec::builder()
4495                .trader_id(order.trader_id())
4496                .strategy_id(order.strategy_id())
4497                .instrument_id(order.instrument_id())
4498                .client_order_id(client_order_id)
4499                .quantity(order.quantity())
4500                .venue_order_id(new_venue_order_id)
4501                .account_id(account_id)
4502                .build();
4503            node.kernel
4504                .cache
4505                .borrow_mut()
4506                .update_order(&OrderEventAny::Updated(updated))
4507                .unwrap();
4508        }
4509
4510        let report = OrderStatusReport::new(
4511            account_id,
4512            instrument_id,
4513            Some(client_order_id),
4514            old_venue_order_id,
4515            OrderSide::Buy.into(),
4516            OrderType::Limit,
4517            TimeInForce::Gtc,
4518            OrderStatus::Canceled,
4519            Quantity::from("10.0"),
4520            Quantity::from("0.0"),
4521            UnixNanos::from(1_000),
4522            UnixNanos::from(2_000),
4523            UnixNanos::from(3_000),
4524            None,
4525        );
4526
4527        let report = if with_fills {
4528            ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4529        } else {
4530            ExecutionReport::Order(Box::new(report))
4531        };
4532
4533        let event = ExecutionEvent::Report(report);
4534
4535        node.process_exec_event(event);
4536
4537        let order = node
4538            .kernel
4539            .cache
4540            .borrow()
4541            .order_owned(&client_order_id)
4542            .unwrap();
4543
4544        let expected_venue_order_id = if superseded {
4545            new_venue_order_id
4546        } else {
4547            old_venue_order_id
4548        };
4549
4550        assert_eq!(order.status(), expected_status);
4551        assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4552
4553        if !superseded {
4554            let replacement = OrderTestBuilder::new(OrderType::Limit)
4555                .client_order_id(client_order_id)
4556                .instrument_id(instrument_id)
4557                .quantity(Quantity::from("20.0"))
4558                .price(Price::from("200.0"))
4559                .build();
4560            let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4561            node.kernel
4562                .cache
4563                .borrow_mut()
4564                .add_order(replacement, None, Some(client_id), true)
4565                .unwrap();
4566            let replacement = node
4567                .kernel
4568                .cache
4569                .borrow_mut()
4570                .update_order(&submitted)
4571                .unwrap();
4572            let accepted =
4573                TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4574            node.kernel
4575                .cache
4576                .borrow_mut()
4577                .update_order(&accepted)
4578                .unwrap();
4579        }
4580
4581        assert_eq!(
4582            node.exec_manager.check_open_order_queries().len(),
4583            expected_query_count,
4584        );
4585    }
4586
4587    #[rstest]
4588    fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4589        let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4590
4591        let OrderEventAny::Filled(fill) = &mut fill_event else {
4592            unreachable!();
4593        };
4594
4595        fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4596        fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4597        let fill = fill.clone();
4598        let report_event = fill_report_event(&fill);
4599        let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4600
4601        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4602        let marked_before_dispatch = is_recent_fill(&node, &fill);
4603
4604        node.dispatch_exec_event_and_commit_fill(event);
4605
4606        let marked_after_dispatch = is_recent_fill(&node, &fill);
4607        let later_report_is_eligible = node
4608            .observe_exec_event_before_dispatch(&report_event)
4609            .is_some();
4610        assert_eq!(
4611            (
4612                marked_before_dispatch,
4613                marked_after_dispatch,
4614                later_report_is_eligible,
4615            ),
4616            (false, false, true),
4617        );
4618    }
4619
4620    #[rstest]
4621    fn test_applied_direct_fill_commits_and_skips_later_report() {
4622        let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
4623
4624        let OrderEventAny::Filled(fill) = &fill_event else {
4625            unreachable!();
4626        };
4627
4628        let fill = fill.clone();
4629        let report_event = fill_report_event(&fill);
4630        let event = ExecutionEvent::Order(fill_event);
4631
4632        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4633        assert!(!is_recent_fill(&node, &fill));
4634
4635        node.dispatch_exec_event_and_commit_fill(event);
4636
4637        assert!(is_recent_fill(&node, &fill));
4638        assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
4639    }
4640
4641    #[rstest]
4642    fn test_canonical_duplicate_fill_counts_as_applied() {
4643        let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
4644
4645        let OrderEventAny::Filled(fill) = &fill_event else {
4646            unreachable!();
4647        };
4648
4649        let mut fill = fill.clone();
4650        node.kernel
4651            .cache
4652            .borrow_mut()
4653            .update_order(&fill_event)
4654            .unwrap();
4655        fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
4656
4657        node.exec_manager.commit_recent_fill_if_applied(&fill);
4658
4659        assert!(is_recent_fill(&node, &fill));
4660    }
4661
4662    #[rstest]
4663    #[case(false)]
4664    #[case(true)]
4665    fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
4666        let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
4667            "AppliedContinuousFillNode"
4668        } else {
4669            "RejectedContinuousFillNode"
4670        });
4671
4672        if !applied {
4673            let OrderEventAny::Filled(fill) = &mut fill_event else {
4674                unreachable!();
4675            };
4676
4677            fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
4678            fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
4679        }
4680
4681        let OrderEventAny::Filled(fill) = &fill_event else {
4682            unreachable!();
4683        };
4684
4685        let fill = fill.clone();
4686
4687        node.process_reconciliation_events(&[fill_event]);
4688
4689        assert_eq!(is_recent_fill(&node, &fill), applied);
4690    }
4691
4692    #[rstest]
4693    fn test_recent_fill_commit_requires_account_and_instrument_match() {
4694        let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
4695        node.kernel
4696            .cache
4697            .borrow_mut()
4698            .update_order(&fill_event)
4699            .unwrap();
4700
4701        let OrderEventAny::Filled(fill) = fill_event else {
4702            unreachable!();
4703        };
4704
4705        let mut account_mismatch = fill.clone();
4706        account_mismatch.account_id = AccountId::from("OTHER-001");
4707        let mut instrument_mismatch = fill;
4708        instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
4709
4710        node.exec_manager
4711            .commit_recent_fill_if_applied(&account_mismatch);
4712        node.exec_manager
4713            .commit_recent_fill_if_applied(&instrument_mismatch);
4714
4715        assert!(!is_recent_fill(&node, &account_mismatch));
4716        assert!(!is_recent_fill(&node, &instrument_mismatch));
4717    }
4718
4719    #[rstest]
4720    fn test_applied_inferred_fill_remains_recently_processed() {
4721        let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
4722        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
4723        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
4724        let account_id = AccountId::from("TEST-001");
4725        let order = node
4726            .kernel
4727            .cache
4728            .borrow()
4729            .order_owned(&client_order_id)
4730            .unwrap();
4731        let report = OrderStatusReport::new(
4732            account_id,
4733            instrument.id(),
4734            Some(client_order_id),
4735            venue_order_id,
4736            OrderSide::Buy.into(),
4737            OrderType::Limit,
4738            TimeInForce::Gtc,
4739            OrderStatus::PartiallyFilled,
4740            Quantity::from("10.0"),
4741            Quantity::from("1.0"),
4742            UnixNanos::from(1_000),
4743            UnixNanos::from(1_000),
4744            UnixNanos::from(1_000),
4745            None,
4746        )
4747        .with_avg_px(dec!(100.0));
4748        let inferred = create_inferred_fill_for_qty(
4749            &order,
4750            &report,
4751            &account_id,
4752            &instrument,
4753            Quantity::from("1.0"),
4754            UnixNanos::from(1_000),
4755            None,
4756        )
4757        .unwrap();
4758
4759        let OrderEventAny::Filled(fill) = &inferred else {
4760            unreachable!();
4761        };
4762
4763        let fill = fill.clone();
4764
4765        node.process_reconciliation_events(&[inferred]);
4766
4767        assert!(fill.reconciliation);
4768        assert!(is_recent_fill(&node, &fill));
4769    }
4770
4771    #[rstest]
4772    #[tokio::test]
4773    async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
4774        let client_id = ClientId::from("POSITION-FILLS");
4775        let account_id = AccountId::from("POSITION-FILLS-001");
4776        let instrument_id = crypto_perpetual_ethusdt().id();
4777        let commands = Rc::new(RefCell::new(Vec::new()));
4778
4779        let client = LiveExecutionClient::new(Box::new(FillReportClient {
4780            client_id,
4781            account_id,
4782            venue: instrument_id.venue,
4783            outcome: FillReportClientOutcome::Failure,
4784            commands: commands.clone(),
4785        }));
4786
4787        let command = GenerateFillReports::new(
4788            UUID4::new(),
4789            UnixNanos::from(2_000),
4790            Some(instrument_id),
4791            None,
4792            Some(UnixNanos::from(1_000)),
4793            Some(UnixNanos::from(2_000)),
4794            None,
4795            None,
4796        );
4797
4798        let result = request_position_fill_reports(
4799            vec![client],
4800            vec![PositionFillReportQuery {
4801                key: (instrument_id, account_id),
4802                client_id,
4803                command: command.clone(),
4804            }],
4805        )
4806        .await;
4807
4808        assert_eq!(*commands.borrow(), vec![command]);
4809        assert!(result.successful_keys.is_empty());
4810        assert!(result.reports.is_empty());
4811    }
4812
4813    #[rstest]
4814    #[tokio::test]
4815    async fn test_request_position_fill_reports_accepts_scoped_response() {
4816        let client_id = ClientId::from("POSITION-FILLS");
4817        let account_id = AccountId::from("POSITION-FILLS-001");
4818        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
4819        let instrument_id = instrument.id();
4820
4821        let report_b = FillReport::new(
4822            account_id,
4823            instrument_id,
4824            VenueOrderId::from("V-POSITION-FILLS"),
4825            TradeId::from("T-POSITION-FILLS-B"),
4826            OrderSide::Buy,
4827            Quantity::from("1.0"),
4828            Price::from("100.0"),
4829            Money::zero(instrument.quote_currency()),
4830            LiquiditySide::Taker,
4831            Some(ClientOrderId::from("O-POSITION-FILLS")),
4832            None,
4833            UnixNanos::from(1_500),
4834            UnixNanos::from(2_000),
4835            None,
4836        );
4837        let mut report_a = report_b.clone();
4838        report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
4839        let commands = Rc::new(RefCell::new(Vec::new()));
4840
4841        let client = LiveExecutionClient::new(Box::new(FillReportClient {
4842            client_id,
4843            account_id,
4844            venue: instrument_id.venue,
4845            outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
4846            commands,
4847        }));
4848
4849        let command = GenerateFillReports::new(
4850            UUID4::new(),
4851            UnixNanos::from(2_000),
4852            Some(instrument_id),
4853            None,
4854            Some(UnixNanos::from(1_000)),
4855            Some(UnixNanos::from(2_000)),
4856            None,
4857            None,
4858        );
4859        let key = (instrument_id, account_id);
4860
4861        let result = request_position_fill_reports(
4862            vec![client],
4863            vec![PositionFillReportQuery {
4864                key,
4865                client_id,
4866                command,
4867            }],
4868        )
4869        .await;
4870
4871        assert_eq!(result.successful_keys, IndexSet::from([key]));
4872        assert_eq!(
4873            result.reports,
4874            IndexMap::from([(key, vec![report_a, report_b])])
4875        );
4876    }
4877
4878    #[rstest]
4879    #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
4880    #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
4881    #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
4882    #[tokio::test]
4883    async fn test_request_position_fill_reports_rejects_out_of_scope_response(
4884        #[case] report_account: &str,
4885        #[case] report_instrument: &str,
4886        #[case] ts_event: u64,
4887        #[case] quantity: &str,
4888    ) {
4889        let client_id = ClientId::from("POSITION-FILLS");
4890        let account_id = AccountId::from("POSITION-FILLS-001");
4891        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
4892        let instrument_id = instrument.id();
4893
4894        let report = FillReport::new(
4895            AccountId::from(report_account),
4896            InstrumentId::from(report_instrument),
4897            VenueOrderId::from("V-POSITION-FILLS"),
4898            TradeId::from("T-POSITION-FILLS"),
4899            OrderSide::Buy,
4900            Quantity::from(quantity),
4901            Price::from("100.0"),
4902            Money::zero(instrument.quote_currency()),
4903            LiquiditySide::Taker,
4904            Some(ClientOrderId::from("O-POSITION-FILLS")),
4905            None,
4906            UnixNanos::from(ts_event),
4907            UnixNanos::from(2_000),
4908            None,
4909        );
4910
4911        let client = LiveExecutionClient::new(Box::new(FillReportClient {
4912            client_id,
4913            account_id,
4914            venue: instrument_id.venue,
4915            outcome: FillReportClientOutcome::Reports(vec![report]),
4916            commands: Rc::new(RefCell::new(Vec::new())),
4917        }));
4918
4919        let command = GenerateFillReports::new(
4920            UUID4::new(),
4921            UnixNanos::from(2_000),
4922            Some(instrument_id),
4923            None,
4924            Some(UnixNanos::from(1_000)),
4925            Some(UnixNanos::from(2_000)),
4926            None,
4927            None,
4928        );
4929        let key = (instrument_id, account_id);
4930
4931        let result = request_position_fill_reports(
4932            vec![client],
4933            vec![PositionFillReportQuery {
4934                key,
4935                client_id,
4936                command,
4937            }],
4938        )
4939        .await;
4940
4941        assert!(result.successful_keys.is_empty());
4942        assert!(result.reports.is_empty());
4943    }
4944
4945    #[rstest]
4946    #[case(999)]
4947    #[case(2_001)]
4948    #[tokio::test]
4949    async fn test_request_position_fill_reports_filters_time_window_superset(
4950        #[case] ts_event: u64,
4951    ) {
4952        let client_id = ClientId::from("POSITION-FILLS");
4953        let account_id = AccountId::from("POSITION-FILLS-001");
4954        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
4955        let instrument_id = instrument.id();
4956
4957        let report = FillReport::new(
4958            account_id,
4959            instrument_id,
4960            VenueOrderId::from("V-POSITION-FILLS"),
4961            TradeId::from("T-POSITION-FILLS"),
4962            OrderSide::Buy,
4963            Quantity::from("1.0"),
4964            Price::from("100.0"),
4965            Money::zero(instrument.quote_currency()),
4966            LiquiditySide::Taker,
4967            Some(ClientOrderId::from("O-POSITION-FILLS")),
4968            None,
4969            UnixNanos::from(ts_event),
4970            UnixNanos::from(2_000),
4971            None,
4972        );
4973
4974        let client = LiveExecutionClient::new(Box::new(FillReportClient {
4975            client_id,
4976            account_id,
4977            venue: instrument_id.venue,
4978            outcome: FillReportClientOutcome::Reports(vec![report]),
4979            commands: Rc::new(RefCell::new(Vec::new())),
4980        }));
4981
4982        let command = GenerateFillReports::new(
4983            UUID4::new(),
4984            UnixNanos::from(2_000),
4985            Some(instrument_id),
4986            None,
4987            Some(UnixNanos::from(1_000)),
4988            Some(UnixNanos::from(2_000)),
4989            None,
4990            None,
4991        );
4992        let key = (instrument_id, account_id);
4993
4994        let result = request_position_fill_reports(
4995            vec![client],
4996            vec![PositionFillReportQuery {
4997                key,
4998                client_id,
4999                command,
5000            }],
5001        )
5002        .await;
5003
5004        assert_eq!(result.successful_keys, IndexSet::from([key]));
5005        assert_eq!(result.reports, IndexMap::from([(key, Vec::new())]));
5006    }
5007
5008    #[rstest]
5009    #[case::query_error(false)]
5010    #[case::missing_client(true)]
5011    #[tokio::test]
5012    async fn test_position_fill_collection_discards_partial_failure(
5013        #[case] missing_client: bool,
5014        #[values(false, true)] failure_first: bool,
5015        position_fill_query: (FillReport, PositionFillReportQuery),
5016    ) {
5017        let (report, query) = position_fill_query;
5018        let failed_client_id = ClientId::from("FAILED-FILLS");
5019        let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5020        let mut healthy_report = report.clone();
5021        healthy_report.account_id = healthy_key.1;
5022        let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5023        let mut clients = vec![
5024            LiveExecutionClient::new(Box::new(FillReportClient {
5025                client_id: query.client_id,
5026                account_id: query.key.1,
5027                venue: query.key.0.venue,
5028                outcome: FillReportClientOutcome::Reports(vec![report]),
5029                commands: Rc::new(RefCell::new(Vec::new())),
5030            })),
5031            LiveExecutionClient::new(Box::new(FillReportClient {
5032                client_id: healthy_client_id,
5033                account_id: healthy_key.1,
5034                venue: healthy_key.0.venue,
5035                outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5036                commands: Rc::new(RefCell::new(Vec::new())),
5037            })),
5038        ];
5039
5040        if !missing_client {
5041            clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5042                client_id: failed_client_id,
5043                account_id: query.key.1,
5044                venue: query.key.0.venue,
5045                outcome: FillReportClientOutcome::Failure,
5046                commands: Rc::new(RefCell::new(Vec::new())),
5047            })));
5048        }
5049
5050        let failed_query = PositionFillReportQuery {
5051            client_id: failed_client_id,
5052            key: query.key,
5053            command: query.command.clone(),
5054        };
5055
5056        let healthy_query = PositionFillReportQuery {
5057            key: healthy_key,
5058            client_id: healthy_client_id,
5059            command: query.command.clone(),
5060        };
5061
5062        let mut queries = vec![query, failed_query];
5063
5064        if failure_first {
5065            queries.reverse();
5066        }
5067
5068        queries.push(healthy_query);
5069
5070        let result = request_position_fill_reports(clients, queries).await;
5071
5072        assert_eq!(result.successful_keys, IndexSet::from([healthy_key]));
5073        assert_eq!(
5074            result.reports,
5075            IndexMap::from([(healthy_key, vec![healthy_report])])
5076        );
5077    }
5078
5079    #[rstest]
5080    #[case::same_economics("metadata", true)]
5081    #[case::venue_order("venue_order", false)]
5082    #[case::side("side", false)]
5083    #[case::quantity("quantity", false)]
5084    #[case::price("price", false)]
5085    #[case::commission("commission", false)]
5086    #[case::liquidity("liquidity", false)]
5087    #[case::average("average", false)]
5088    #[case::event_time("event_time", false)]
5089    #[case::client_order("client_order", false)]
5090    #[case::position("position", false)]
5091    #[tokio::test]
5092    async fn test_position_fill_collection_validates_duplicate_economics(
5093        #[case] changed: &str,
5094        #[case] accepted: bool,
5095        position_fill_query: (FillReport, PositionFillReportQuery),
5096    ) {
5097        let (report, query) = position_fill_query;
5098        let mut duplicate = report.clone();
5099
5100        match changed {
5101            "metadata" => {
5102                duplicate.report_id = UUID4::new();
5103                duplicate.ts_init = UnixNanos::from(3_000);
5104            }
5105            "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5106            "side" => duplicate.order_side = OrderSide::Sell,
5107            "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5108            "price" => duplicate.last_px = Price::from("101.0"),
5109            "commission" => duplicate.commission = Money::from("0.25 USDT"),
5110            "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5111            "average" => duplicate.avg_px = Some(dec!(100.5)),
5112            "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5113            "client_order" => duplicate.client_order_id = None,
5114            "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5115            _ => unreachable!(),
5116        }
5117
5118        let key = query.key;
5119
5120        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5121            client_id: query.client_id,
5122            account_id: key.1,
5123            venue: key.0.venue,
5124            outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5125            commands: Rc::new(RefCell::new(Vec::new())),
5126        }));
5127
5128        let result = request_position_fill_reports(vec![client], vec![query]).await;
5129
5130        if accepted {
5131            assert_eq!(result.successful_keys, IndexSet::from([key]));
5132            assert_eq!(result.reports, IndexMap::from([(key, vec![report])]));
5133        } else {
5134            assert!(result.successful_keys.is_empty());
5135            assert!(result.reports.is_empty());
5136        }
5137    }
5138
5139    #[fixture]
5140    fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5141        let account_id = AccountId::from("POSITION-FILLS-001");
5142        let instrument_id = crypto_perpetual_ethusdt().id();
5143
5144        let report = FillReport::new(
5145            account_id,
5146            instrument_id,
5147            VenueOrderId::from("V-POSITION-FILLS"),
5148            TradeId::from("T-POSITION-FILLS"),
5149            OrderSide::Buy,
5150            Quantity::from("1.0"),
5151            Price::from("100.0"),
5152            Money::from("0.10 USDT"),
5153            LiquiditySide::Taker,
5154            Some(ClientOrderId::from("O-POSITION-FILLS")),
5155            None,
5156            UnixNanos::from(1_500),
5157            UnixNanos::from(2_000),
5158            None,
5159        );
5160
5161        let query = PositionFillReportQuery {
5162            key: (instrument_id, account_id),
5163            client_id: ClientId::from("POSITION-FILLS"),
5164            command: GenerateFillReports::new(
5165                UUID4::new(),
5166                UnixNanos::from(2_000),
5167                Some(instrument_id),
5168                None,
5169                Some(UnixNanos::from(1_000)),
5170                Some(UnixNanos::from(2_000)),
5171                None,
5172                None,
5173            ),
5174        };
5175
5176        (report, query)
5177    }
5178
5179    #[rstest]
5180    #[case::failed_query(false)]
5181    #[case::local_activity(true)]
5182    fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5183        let (mut node, report, _) =
5184            position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5185        let active_key = (report.instrument_id, report.account_id);
5186        let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5187        let deferred_reports = vec![
5188            PositionStatusReport::new(
5189                deferred_key.1,
5190                deferred_key.0,
5191                PositionSide::Long,
5192                Quantity::from("2.0"),
5193                UnixNanos::from(1_000),
5194                UnixNanos::from(1_000),
5195                None,
5196                None,
5197                Some(dec!(100.0)),
5198            ),
5199            PositionStatusReport::new(
5200                deferred_key.1,
5201                deferred_key.0,
5202                PositionSide::Short,
5203                Quantity::from("1.0"),
5204                UnixNanos::from(1_000),
5205                UnixNanos::from(1_000),
5206                None,
5207                None,
5208                Some(dec!(100.0)),
5209            ),
5210        ];
5211        let mut check = node
5212            .exec_manager
5213            .prepare_position_report_check(UUID4::new(), &[]);
5214        check.client_coverage.clear();
5215        let events = node.exec_manager.reconcile_position_reports(
5216            &check,
5217            deferred_reports.clone(),
5218            &IndexSet::new(),
5219            &IndexSet::new(),
5220        );
5221        assert!(events.is_empty());
5222        assert_eq!(
5223            node.exec_manager.position_recon_retry_count(&deferred_key),
5224            1
5225        );
5226
5227        let mut position_result = position_report_result(&node, report);
5228        position_result.reports.extend(deferred_reports);
5229        position_result.check.client_coverage.insert(
5230            deferred_key,
5231            ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5232        );
5233        position_result
5234            .check
5235            .activity_revisions
5236            .insert(deferred_key, 0);
5237        position_result
5238            .queried_clients
5239            .insert(ClientId::from("SECOND"));
5240        let mut successful_keys = IndexSet::from([active_key]);
5241
5242        if local_activity {
5243            successful_keys.insert(deferred_key);
5244            node.exec_manager
5245                .record_position_activity(deferred_key.0, deferred_key.1);
5246        }
5247
5248        node.handle_position_fill_report_result(PositionFillReportResult {
5249            position_result,
5250            reports: IndexMap::from([(active_key, Vec::new())]),
5251            successful_keys,
5252        });
5253
5254        assert_eq!(
5255            node.exec_manager.position_recon_retry_count(&deferred_key),
5256            1
5257        );
5258        {
5259            let cache = node.kernel.cache.borrow();
5260            let positions =
5261                cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5262            assert_eq!(
5263                positions
5264                    .iter()
5265                    .map(|position| position.quantity)
5266                    .collect::<Vec<_>>(),
5267                vec![Quantity::from("1.0"), Quantity::from("1.0")],
5268            );
5269            assert_eq!(
5270                cache
5271                    .positions_open(
5272                        None,
5273                        Some(&deferred_key.0),
5274                        None,
5275                        Some(&deferred_key.1),
5276                        None,
5277                    )
5278                    .len(),
5279                0
5280            );
5281        }
5282
5283        let mut fresh_check = node
5284            .exec_manager
5285            .prepare_position_report_check(UUID4::new(), &[]);
5286        node.exec_manager.plan_position_fill_reports(
5287            &mut fresh_check,
5288            &[],
5289            &IndexSet::new(),
5290            &IndexSet::new(),
5291            &[],
5292        );
5293
5294        assert_eq!(
5295            node.exec_manager.position_recon_retry_count(&deferred_key),
5296            0
5297        );
5298    }
5299
5300    #[rstest]
5301    fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5302        let (mut node, venue_report, fill_report) =
5303            position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5304        let key = (venue_report.instrument_id, venue_report.account_id);
5305        let revision = node.exec_manager.position_activity_revision(&key);
5306        let position_result = position_report_result(&node, venue_report);
5307
5308        node.handle_position_fill_report_result(PositionFillReportResult {
5309            position_result,
5310            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5311            successful_keys: IndexSet::from([key]),
5312        });
5313
5314        let cache = node.kernel.cache.borrow();
5315        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5316        assert_eq!(
5317            positions
5318                .iter()
5319                .map(|position| position.quantity)
5320                .sum::<Quantity>(),
5321            Quantity::from("2.0")
5322        );
5323        assert_eq!(
5324            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5325            1
5326        );
5327        drop(positions);
5328        drop(cache);
5329        assert!(
5330            node.exec_manager
5331                .position_contains_fill_report(&fill_report)
5332        );
5333        assert_eq!(
5334            node.exec_manager.position_activity_revision(&key),
5335            revision + 1
5336        );
5337    }
5338
5339    #[rstest]
5340    fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5341        let (mut node, venue_report, fill_report) =
5342            position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5343        let key = (venue_report.instrument_id, venue_report.account_id);
5344        let position_result = position_report_result(&node, venue_report);
5345
5346        node.handle_position_fill_report_result(PositionFillReportResult {
5347            position_result,
5348            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5349            successful_keys: IndexSet::from([key]),
5350        });
5351
5352        let mut rest_report = fill_report;
5353        rest_report.ts_event = UnixNanos::from(2_000);
5354
5355        assert!(
5356            node.exec_manager
5357                .position_contains_fill_report(&rest_report)
5358        );
5359    }
5360
5361    #[rstest]
5362    fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5363        let (mut node, venue_report, fill_report) =
5364            position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5365        let key = (venue_report.instrument_id, venue_report.account_id);
5366        let client_order_id = fill_report.client_order_id.unwrap();
5367        apply_inferred_position_fill(&mut node, &fill_report);
5368
5369        let venue_report = PositionStatusReport::new(
5370            key.1,
5371            key.0,
5372            PositionSide::Long,
5373            Quantity::from("3.0"),
5374            venue_report.ts_last,
5375            venue_report.ts_init,
5376            None,
5377            None,
5378            venue_report.avg_px_open,
5379        );
5380        let mut later_fill_report = fill_report.clone();
5381        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5382        later_fill_report.ts_event = UnixNanos::from(1_001);
5383        let position_result = position_report_result(&node, venue_report);
5384
5385        node.handle_position_fill_report_result(PositionFillReportResult {
5386            position_result,
5387            reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5388            successful_keys: IndexSet::from([key]),
5389        });
5390
5391        let cache = node.kernel.cache.borrow();
5392        let order = cache.order(&client_order_id).unwrap();
5393        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5394        assert_eq!(order.filled_qty(), Quantity::from("2.0"));
5395        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
5396        assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
5397        assert_eq!(
5398            positions
5399                .iter()
5400                .map(|position| position.quantity)
5401                .sum::<Quantity>(),
5402            Quantity::from("3.0")
5403        );
5404        assert_eq!(
5405            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5406            2
5407        );
5408    }
5409
5410    #[rstest]
5411    fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
5412        let (mut node, _, _) =
5413            position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
5414        let account_id = AccountId::from("TEST-001");
5415        let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
5416        let key = (instrument.id(), account_id);
5417        let commission = Money::zero(instrument.quote_currency());
5418        node.kernel
5419            .cache
5420            .borrow_mut()
5421            .add_instrument(instrument.clone())
5422            .unwrap();
5423
5424        let order = OrderTestBuilder::new(OrderType::Market)
5425            .strategy_id(StrategyId::external())
5426            .instrument_id(instrument.id())
5427            .client_order_id(ClientOrderId::from("O-RECONCILED"))
5428            .quantity(Quantity::from("1.000000"))
5429            .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
5430            .build();
5431        let submitted = TestOrderEventStubs::submitted(&order, account_id);
5432        let accepted =
5433            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
5434        node.kernel
5435            .cache
5436            .borrow_mut()
5437            .add_order(
5438                order.clone(),
5439                None,
5440                Some(ClientId::from("POSITION-FILLS")),
5441                false,
5442            )
5443            .unwrap();
5444        node.process_reconciliation_events(&[submitted, accepted]);
5445        let order = node
5446            .kernel
5447            .cache
5448            .borrow()
5449            .order_owned(&order.client_order_id())
5450            .unwrap();
5451        let opening_fill = TestOrderEventStubs::filled(
5452            &order,
5453            &instrument,
5454            Some(TradeId::from("T-RECONCILED")),
5455            None,
5456            Some(Price::from("100.00")),
5457            Some(Quantity::from("1.000000")),
5458            Some(LiquiditySide::Taker),
5459            Some(commission),
5460            Some(UnixNanos::from(2_000)),
5461            Some(account_id),
5462        );
5463        node.process_reconciliation_events(&[opening_fill]);
5464
5465        // A real fill between the position report query and reconciliation stays off the position
5466        let held_fill_report = FillReport::new(
5467            account_id,
5468            instrument.id(),
5469            VenueOrderId::from("V-HELD"),
5470            TradeId::from("T-HELD"),
5471            OrderSide::Buy,
5472            Quantity::from("1.000000"),
5473            Price::from("100.00"),
5474            commission,
5475            LiquiditySide::Taker,
5476            None,
5477            None,
5478            UnixNanos::from(1_000),
5479            UnixNanos::from(1_000),
5480            None,
5481        );
5482        node.kernel
5483            .exec_engine
5484            .borrow_mut()
5485            .reconcile_fill_report(&held_fill_report);
5486
5487        let venue_report = PositionStatusReport::new(
5488            account_id,
5489            instrument.id(),
5490            PositionSide::Long,
5491            Quantity::from("2.000000"),
5492            UnixNanos::from(3_000),
5493            UnixNanos::from(3_000),
5494            None,
5495            None,
5496            Some(dec!(100.00)),
5497        );
5498        let position_result = position_report_result(&node, venue_report);
5499
5500        node.handle_position_fill_report_result(PositionFillReportResult {
5501            position_result,
5502            reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
5503            successful_keys: IndexSet::from([key]),
5504        });
5505
5506        let cache = node.kernel.cache.borrow();
5507        let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
5508        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5509        assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
5510        assert_eq!(
5511            positions
5512                .iter()
5513                .map(|position| position.quantity)
5514                .sum::<Quantity>(),
5515            Quantity::from("2.000000")
5516        );
5517        assert!(
5518            positions
5519                .iter()
5520                .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
5521        );
5522    }
5523
5524    #[rstest]
5525    fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
5526        let (mut node, _, mut fill_report) =
5527            position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
5528        apply_inferred_position_fill(&mut node, &fill_report);
5529        let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
5530        fill_report.venue_position_id = Some(conflicting_position_id);
5531
5532        let error = node
5533            .exec_manager
5534            .prepare_position_fill_report(&mut fill_report, &[])
5535            .unwrap_err();
5536
5537        assert!(
5538            error.to_string().contains(&format!(
5539                "position ID {conflicting_position_id} conflicts with cached order position"
5540            )),
5541            "{error:#}"
5542        );
5543    }
5544
5545    #[rstest]
5546    #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
5547    #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
5548    fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
5549        #[case] authoritative_qty: &str,
5550        #[case] client_order_id: &str,
5551    ) {
5552        let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
5553            "UnappliedPositionFillNode",
5554            Quantity::from(authoritative_qty),
5555        );
5556        fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
5557        let reports = [fill_report.clone()];
5558
5559        let quantities = (0..2)
5560            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
5561            .collect::<Vec<_>>();
5562
5563        assert_eq!(
5564            quantities,
5565            vec![Quantity::from("1.0"), Quantity::from("2.0")]
5566        );
5567        assert!(
5568            !node
5569                .exec_manager
5570                .position_contains_fill_report(&fill_report)
5571        );
5572    }
5573
5574    #[rstest]
5575    fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
5576        let (mut node, venue_report, fill_report) =
5577            position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
5578        let mut later_fill_report = fill_report.clone();
5579        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5580        later_fill_report.ts_event = UnixNanos::from(1_001);
5581        let reports = [fill_report.clone(), later_fill_report.clone()];
5582
5583        let quantities = (0..3)
5584            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
5585            .collect::<Vec<_>>();
5586
5587        assert_eq!(
5588            quantities,
5589            vec![
5590                Quantity::from("1.0"),
5591                Quantity::from("1.0"),
5592                Quantity::from("2.0"),
5593            ]
5594        );
5595        assert!(
5596            !node
5597                .exec_manager
5598                .position_contains_fill_report(&fill_report)
5599        );
5600        assert!(
5601            !node
5602                .exec_manager
5603                .position_contains_fill_report(&later_fill_report)
5604        );
5605    }
5606
5607    #[rstest]
5608    #[case::within_grace(60_000, true)]
5609    #[case::after_grace(0, false)]
5610    fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
5611        #[case] position_check_threshold_ms: u32,
5612        #[case] expected_applied: bool,
5613    ) {
5614        let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
5615            "TransientPositionFillNode",
5616            Quantity::from("1.0"),
5617            position_check_threshold_ms,
5618        );
5619        let mut conflicting_report = fill_report.clone();
5620        conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
5621
5622        let quantities = [conflicting_report, fill_report.clone()]
5623            .into_iter()
5624            .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
5625            .collect::<Vec<_>>();
5626
5627        assert_eq!(
5628            quantities,
5629            vec![Quantity::from("1.0"), Quantity::from("2.0")]
5630        );
5631        assert_eq!(
5632            node.exec_manager
5633                .position_contains_fill_report(&fill_report),
5634            expected_applied
5635        );
5636    }
5637
5638    #[rstest]
5639    fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
5640        let (mut node, venue_report, fill_report) =
5641            position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
5642        let key = (venue_report.instrument_id, venue_report.account_id);
5643        let first_result = position_report_result(&node, venue_report.clone());
5644
5645        node.handle_position_fill_report_result(PositionFillReportResult {
5646            position_result: first_result,
5647            reports: IndexMap::from([(key, vec![fill_report])]),
5648            successful_keys: IndexSet::from([key]),
5649        });
5650
5651        let fresh_result = position_report_result(&node, venue_report);
5652        node.handle_position_fill_report_result(PositionFillReportResult {
5653            position_result: fresh_result,
5654            reports: IndexMap::from([(key, Vec::new())]),
5655            successful_keys: IndexSet::from([key]),
5656        });
5657
5658        let cache = node.kernel.cache.borrow();
5659        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5660        assert_eq!(
5661            positions
5662                .iter()
5663                .map(|position| position.quantity)
5664                .sum::<Quantity>(),
5665            Quantity::from("2.0")
5666        );
5667        assert_eq!(
5668            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5669            2
5670        );
5671    }
5672
5673    #[rstest]
5674    #[case::below_limit(63)]
5675    #[case::at_limit(64)]
5676    #[case::above_limit(65)]
5677    fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
5678        let (mut node, venue_report, fill_report) =
5679            position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
5680        let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
5681
5682        let venue_report = PositionStatusReport::new(
5683            venue_report.account_id,
5684            venue_report.instrument_id,
5685            PositionSide::Long,
5686            Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
5687            venue_report.ts_last,
5688            venue_report.ts_init,
5689            None,
5690            None,
5691            venue_report.avg_px_open,
5692        );
5693        let key = (venue_report.instrument_id, venue_report.account_id);
5694        let client_order_id = fill_report.client_order_id.unwrap();
5695
5696        let reports = (0..count)
5697            .map(|index| {
5698                let mut report = fill_report.clone();
5699                report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
5700                report.ts_event = UnixNanos::from(1_000 + index as u64);
5701                report
5702            })
5703            .collect::<Vec<_>>();
5704
5705        let position_result = position_report_result(&node, venue_report.clone());
5706
5707        node.handle_position_fill_report_result(PositionFillReportResult {
5708            position_result,
5709            reports: IndexMap::from([(key, reports.clone())]),
5710            successful_keys: IndexSet::from([key]),
5711        });
5712
5713        {
5714            let cache = node.kernel.cache.borrow();
5715            let order = cache.order(&client_order_id).unwrap();
5716            let applied = count.min(64);
5717            assert_eq!(
5718                order.filled_qty().as_decimal(),
5719                dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
5720            );
5721            assert_eq!(order.trade_ids().len(), applied + 1);
5722
5723            for (index, report) in reports.iter().enumerate() {
5724                assert_eq!(
5725                    order.trade_ids().contains(&&report.trade_id),
5726                    index < applied
5727                );
5728            }
5729
5730            assert_eq!(
5731                cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5732                1
5733            );
5734        }
5735
5736        // A fresh cycle replays the full response; only the deferred fill may change exposure
5737        let position_result = position_report_result(&node, venue_report);
5738        node.handle_position_fill_report_result(PositionFillReportResult {
5739            position_result,
5740            reports: IndexMap::from([(key, reports)]),
5741            successful_keys: IndexSet::from([key]),
5742        });
5743
5744        let cache = node.kernel.cache.borrow();
5745        let order = cache.order(&client_order_id).unwrap();
5746        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5747        assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
5748        assert_eq!(order.trade_ids().len(), count + 1);
5749        assert_eq!(positions.len(), 1);
5750        assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
5751        assert_eq!(positions[0].side, PositionSide::Long);
5752        assert_eq!(
5753            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5754            1
5755        );
5756    }
5757
5758    #[rstest]
5759    #[case::client_order(0)]
5760    #[case::venue_order(1)]
5761    #[case::side(2)]
5762    #[case::position(3)]
5763    fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
5764        let (mut node, venue_report, valid_report) =
5765            position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
5766        let key = (venue_report.instrument_id, venue_report.account_id);
5767        let client_order_id = valid_report.client_order_id.unwrap();
5768        let revision = node.exec_manager.position_activity_revision(&key);
5769        let position_result = position_report_result(&node, venue_report);
5770        let mut conflicting_report = valid_report.clone();
5771        conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
5772
5773        match conflict {
5774            0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
5775            1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
5776            2 => conflicting_report.order_side = OrderSide::Sell,
5777            3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
5778            _ => unreachable!(),
5779        }
5780
5781        node.handle_position_fill_report_result(PositionFillReportResult {
5782            position_result,
5783            reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
5784            successful_keys: IndexSet::from([key]),
5785        });
5786
5787        let cache = node.kernel.cache.borrow();
5788        let order = cache.order(&client_order_id).unwrap();
5789        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5790        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
5791        assert_eq!(
5792            order.trade_ids(),
5793            vec![&TradeId::from("T-POSITION-INITIAL")]
5794        );
5795        assert_eq!(positions.len(), 1);
5796        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5797        assert_eq!(
5798            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5799            1
5800        );
5801        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
5802    }
5803
5804    #[rstest]
5805    fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
5806        let (mut node, venue_report, _) =
5807            position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
5808        let key = (venue_report.instrument_id, venue_report.account_id);
5809        let position_result = position_report_result(&node, venue_report);
5810
5811        node.handle_position_fill_report_result(PositionFillReportResult {
5812            position_result,
5813            reports: IndexMap::new(),
5814            successful_keys: IndexSet::new(),
5815        });
5816
5817        let cache = node.kernel.cache.borrow();
5818        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5819        assert_eq!(positions.len(), 1);
5820        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5821        assert_eq!(
5822            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5823            1
5824        );
5825    }
5826
5827    #[rstest]
5828    fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
5829        let (mut node, mut venue_report, mut fill_report) =
5830            position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
5831        node.kernel
5832            .exec_engine
5833            .borrow_mut()
5834            .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
5835        let key = (venue_report.instrument_id, venue_report.account_id);
5836
5837        let position_id = {
5838            let cache = node.kernel.cache.borrow();
5839            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5840            assert_eq!(positions.len(), 1);
5841            positions[0].id
5842        };
5843
5844        venue_report.venue_position_id = Some(position_id);
5845        fill_report.client_order_id = None;
5846        fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
5847        let position_result = position_report_result(&node, venue_report);
5848
5849        node.handle_position_fill_report_result(PositionFillReportResult {
5850            position_result,
5851            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5852            successful_keys: IndexSet::from([key]),
5853        });
5854
5855        let cache = node.kernel.cache.borrow();
5856        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5857        assert_eq!(positions.len(), 1);
5858        assert_eq!(positions[0].id, position_id);
5859        assert_eq!(positions[0].quantity, Quantity::from("2.0"));
5860        assert_eq!(
5861            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5862            2
5863        );
5864        drop(positions);
5865        drop(cache);
5866        assert!(
5867            !node
5868                .exec_manager
5869                .position_contains_fill_report(&fill_report)
5870        );
5871    }
5872
5873    #[rstest]
5874    fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
5875        let (mut node, venue_report, fill_report) =
5876            position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
5877        let key = (venue_report.instrument_id, venue_report.account_id);
5878        let client_order_id = fill_report.client_order_id.unwrap();
5879        let position_result = position_report_result(&node, venue_report);
5880        let revision = node.exec_manager.position_activity_revision(&key);
5881
5882        let order_report = OrderStatusReport::new(
5883            key.1,
5884            key.0,
5885            Some(client_order_id),
5886            fill_report.venue_order_id,
5887            OrderSide::Buy.into(),
5888            OrderType::Limit,
5889            TimeInForce::Gtc,
5890            OrderStatus::PartiallyFilled,
5891            Quantity::from("10.0"),
5892            Quantity::from("2.0"),
5893            UnixNanos::from(1_000),
5894            UnixNanos::from(1_000),
5895            UnixNanos::from(2_000),
5896            None,
5897        );
5898        let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
5899            Box::new(order_report),
5900            vec![fill_report.clone()],
5901        ));
5902
5903        assert_eq!(
5904            node.observe_exec_event_before_dispatch(&event),
5905            Some(vec![client_order_id])
5906        );
5907
5908        assert_eq!(
5909            node.exec_manager.position_activity_revision(&key),
5910            revision + 1
5911        );
5912        assert!(
5913            !node
5914                .exec_manager
5915                .position_report_check_is_current(&position_result.check, &key)
5916        );
5917
5918        node.handle_position_fill_report_result(PositionFillReportResult {
5919            position_result,
5920            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5921            successful_keys: IndexSet::from([key]),
5922        });
5923
5924        let cache = node.kernel.cache.borrow();
5925        let order = cache.order(&client_order_id).unwrap();
5926        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5927        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
5928        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
5929        assert_eq!(positions.len(), 1);
5930        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5931        assert_eq!(positions[0].side, PositionSide::Long);
5932        assert_eq!(
5933            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5934            1
5935        );
5936    }
5937
5938    #[rstest]
5939    fn test_position_fill_report_result_defers_after_local_position_activity() {
5940        let (mut node, venue_report, fill_report) =
5941            position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
5942        let key = (venue_report.instrument_id, venue_report.account_id);
5943        let position_result = position_report_result(&node, venue_report);
5944        node.exec_manager.record_position_activity(key.0, key.1);
5945
5946        node.handle_position_fill_report_result(PositionFillReportResult {
5947            position_result,
5948            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5949            successful_keys: IndexSet::from([key]),
5950        });
5951
5952        let cache = node.kernel.cache.borrow();
5953        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5954        assert_eq!(positions.len(), 1);
5955        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
5956        assert_eq!(
5957            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5958            1
5959        );
5960        drop(positions);
5961        drop(cache);
5962        assert!(
5963            !node
5964                .exec_manager
5965                .position_contains_fill_report(&fill_report)
5966        );
5967    }
5968
5969    #[rstest]
5970    fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
5971        let config = LiveNodeConfig {
5972            exec_engine: crate::config::LiveExecutionEngineConfig {
5973                reconciliation: true,
5974                open_check_threshold_ms: 5_000,
5975                single_order_query_delay_ms: 0,
5976                ..Default::default()
5977            },
5978            ..Default::default()
5979        };
5980
5981        let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
5982        let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
5983        let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
5984        let instrument = crypto_perpetual_ethusdt();
5985        let instrument_id = instrument.id();
5986        let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
5987        let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
5988
5989        node.kernel
5990            .cache
5991            .borrow_mut()
5992            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
5993            .unwrap();
5994        insert_accepted_limit_order_in_node(
5995            &node,
5996            account_id,
5997            client_id,
5998            instrument_id,
5999            client_order_id,
6000            venue_order_id,
6001        );
6002
6003        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6004
6005        let accepted = OrderAcceptedSpec::builder()
6006            .instrument_id(instrument_id)
6007            .client_order_id(client_order_id)
6008            .venue_order_id(venue_order_id)
6009            .account_id(account_id)
6010            .build();
6011        let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6012
6013        let close_ids = node.observe_exec_event_before_dispatch(&event);
6014
6015        assert_eq!(close_ids, Some(vec![client_order_id]));
6016        assert!(node.exec_manager.check_open_order_queries().is_empty());
6017    }
6018
6019    #[rstest]
6020    #[cfg_attr(
6021        not(all(feature = "simulation", madsim)),
6022        tokio::test(start_paused = true)
6023    )]
6024    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6025    async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6026        use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6027        use nautilus_model::{events::OrderPendingCancel, identifiers::ClientOrderId};
6028
6029        let config = LiveNodeConfig {
6030            exec_engine: crate::config::LiveExecutionEngineConfig {
6031                reconciliation: true,
6032                inflight_check_threshold_ms: 100,
6033                inflight_check_retries: 1,
6034                ..Default::default()
6035            },
6036            ..Default::default()
6037        };
6038
6039        let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6040        let trader_id = TraderId::from("TESTER-001");
6041        let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6042        let account_id = AccountId::from("TEST-001");
6043        let instrument = crypto_perpetual_ethusdt();
6044        let instrument_id = instrument.id();
6045        let child_ids = [
6046            ClientOrderId::from("O-BATCH-CANCEL-1"),
6047            ClientOrderId::from("O-BATCH-CANCEL-2"),
6048        ];
6049        node.kernel
6050            .cache
6051            .borrow_mut()
6052            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6053            .unwrap();
6054
6055        for client_order_id in child_ids {
6056            let order = OrderTestBuilder::new(OrderType::Limit)
6057                .trader_id(trader_id)
6058                .strategy_id(strategy_id)
6059                .client_order_id(client_order_id)
6060                .instrument_id(instrument_id)
6061                .quantity(Quantity::from("10.0"))
6062                .price(Price::from("100.0"))
6063                .build();
6064            let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6065            // Model the production batch-cancel path: each child is accepted, then
6066            // moved to PendingCancel, so an inflight timeout must emit a Canceled
6067            // event (not a Submitted-order rejection).
6068            let submitted = TestOrderEventStubs::submitted(&order, account_id);
6069            let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6070            let pending_cancel = OrderEventAny::PendingCancel(OrderPendingCancel::new(
6071                trader_id,
6072                strategy_id,
6073                instrument_id,
6074                client_order_id,
6075                Some(account_id),
6076                UUID4::new(),
6077                UnixNanos::default(),
6078                UnixNanos::default(),
6079                false,
6080                Some(venue_order_id),
6081            ));
6082            let mut cache = node.kernel.cache.borrow_mut();
6083            cache.add_order(order, None, None, false).unwrap();
6084            cache.update_order(&submitted).unwrap();
6085            cache.update_order(&accepted).unwrap();
6086            cache.update_order(&pending_cancel).unwrap();
6087        }
6088
6089        let cancels = child_ids
6090            .into_iter()
6091            .map(|client_order_id| {
6092                CancelOrder::new(
6093                    trader_id,
6094                    None,
6095                    strategy_id,
6096                    instrument_id,
6097                    client_order_id,
6098                    None,
6099                    UUID4::new(),
6100                    UnixNanos::default(),
6101                    None,
6102                    None,
6103                )
6104            })
6105            .collect();
6106
6107        let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6108            trader_id,
6109            None,
6110            strategy_id,
6111            instrument_id,
6112            cancels,
6113            UUID4::new(),
6114            UnixNanos::default(),
6115            None,
6116            None,
6117        ));
6118
6119        node.observe_exec_command_before_dispatch(&command);
6120        advance_clock(Duration::from_millis(101)).await;
6121        let result = node.exec_manager.check_inflight_orders();
6122        let timed_out_ids = result
6123            .events
6124            .iter()
6125            .map(OrderEventAny::client_order_id)
6126            .collect::<IndexSet<_>>();
6127
6128        assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6129        assert_eq!(result.events.len(), child_ids.len());
6130        assert!(
6131            result
6132                .events
6133                .iter()
6134                .all(|event| matches!(event, OrderEventAny::Canceled(_))),
6135            "batch-cancel children must time out as Canceled events",
6136        );
6137    }
6138
6139    #[rstest]
6140    #[cfg_attr(
6141        not(all(feature = "simulation", madsim)),
6142        tokio::test(start_paused = true)
6143    )]
6144    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6145    async fn test_risk_bound_command_does_not_register_inflight() {
6146        let config = LiveNodeConfig {
6147            exec_engine: crate::config::LiveExecutionEngineConfig {
6148                reconciliation: true,
6149                inflight_check_threshold_ms: 100,
6150                inflight_check_retries: 2,
6151                ..Default::default()
6152            },
6153            ..Default::default()
6154        };
6155
6156        let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6157        msgbus::register_trading_command_endpoint(
6158            MessagingSwitchboard::risk_engine_execute(),
6159            TypedIntoHandler::from(|_: TradingCommand| {}),
6160        );
6161        let instrument = crypto_perpetual_ethusdt();
6162        let instrument_id = instrument.id();
6163        let order = OrderTestBuilder::new(OrderType::Limit)
6164            .trader_id(node.trader_id())
6165            .strategy_id(StrategyId::from("S-RISK-DENIED"))
6166            .instrument_id(instrument_id)
6167            .side(OrderSide::Buy)
6168            .quantity(Quantity::from("1.000"))
6169            .price(Price::from("100.00"))
6170            .build();
6171        let client_order_id = order.client_order_id();
6172
6173        {
6174            let mut cache = node.kernel.cache.borrow_mut();
6175            cache
6176                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6177                .unwrap();
6178            cache.add_order(order.clone(), None, None, false).unwrap();
6179        }
6180
6181        let submit_order = SubmitOrder::new(
6182            order.trader_id(),
6183            None,
6184            order.strategy_id(),
6185            instrument_id,
6186            client_order_id,
6187            order.init_event().clone(),
6188            None,
6189            None,
6190            None,
6191            UUID4::new(),
6192            UnixNanos::default(),
6193            None,
6194        );
6195        node.process_exec_command(
6196            TradingCommandMessage::new(
6197                MessagingSwitchboard::risk_engine_execute(),
6198                TradingCommand::SubmitOrder(submit_order),
6199            )
6200            .into(),
6201        );
6202
6203        advance_clock(Duration::from_millis(101)).await;
6204        let result = node.exec_manager.check_inflight_orders();
6205        let status = node
6206            .kernel
6207            .cache
6208            .borrow()
6209            .order(&client_order_id)
6210            .unwrap()
6211            .status();
6212
6213        assert_eq!(status, OrderStatus::Initialized);
6214        assert_eq!(
6215            node.exec_manager.recon_check_retry_count(&client_order_id),
6216            0
6217        );
6218        assert!(result.events.is_empty());
6219        assert!(result.queries.is_empty());
6220    }
6221
6222    #[rstest]
6223    #[cfg_attr(
6224        not(all(feature = "simulation", madsim)),
6225        tokio::test(start_paused = true)
6226    )]
6227    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6228    async fn test_risk_approved_command_registers_inflight() {
6229        let config = LiveNodeConfig {
6230            risk_engine: crate::config::LiveRiskEngineConfig {
6231                bypass: true,
6232                ..Default::default()
6233            },
6234            exec_engine: crate::config::LiveExecutionEngineConfig {
6235                reconciliation: true,
6236                inflight_check_threshold_ms: 100,
6237                inflight_check_retries: 2,
6238                ..Default::default()
6239            },
6240            ..Default::default()
6241        };
6242
6243        let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
6244        msgbus::register_trading_command_endpoint(
6245            MessagingSwitchboard::exec_engine_execute(),
6246            TypedIntoHandler::from(|_: TradingCommand| {}),
6247        );
6248        let instrument = crypto_perpetual_ethusdt();
6249        let instrument_id = instrument.id();
6250        let order = OrderTestBuilder::new(OrderType::Limit)
6251            .trader_id(node.trader_id())
6252            .strategy_id(StrategyId::from("S-RISK-APPROVED"))
6253            .instrument_id(instrument_id)
6254            .side(OrderSide::Buy)
6255            .quantity(Quantity::from("1.000"))
6256            .price(Price::from("100.00"))
6257            .build();
6258        let client_order_id = order.client_order_id();
6259
6260        {
6261            let mut cache = node.kernel.cache.borrow_mut();
6262            cache
6263                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6264                .unwrap();
6265            cache.add_order(order.clone(), None, None, false).unwrap();
6266        }
6267
6268        let submit_order = SubmitOrder::new(
6269            order.trader_id(),
6270            None,
6271            order.strategy_id(),
6272            instrument_id,
6273            client_order_id,
6274            order.init_event().clone(),
6275            None,
6276            None,
6277            None,
6278            UUID4::new(),
6279            UnixNanos::default(),
6280            None,
6281        );
6282        node.process_exec_command(
6283            TradingCommandMessage::new(
6284                MessagingSwitchboard::risk_engine_execute(),
6285                TradingCommand::SubmitOrder(submit_order),
6286            )
6287            .into(),
6288        );
6289
6290        advance_clock(Duration::from_millis(101)).await;
6291        let result = node.exec_manager.check_inflight_orders();
6292
6293        let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6294            panic!("expected one query order command");
6295        };
6296
6297        assert_eq!(query.client_order_id, client_order_id);
6298        assert_eq!(
6299            node.exec_manager.recon_check_retry_count(&client_order_id),
6300            1
6301        );
6302        assert!(result.events.is_empty());
6303    }
6304
6305    #[rstest]
6306    fn test_live_node_builder_clock_factory_drives_kernel_clock() {
6307        let calls = Rc::new(Cell::new(0usize));
6308        let calls_in_factory = calls.clone();
6309        let sentinel = UnixNanos::from(123_456_789_u64);
6310
6311        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6312            .unwrap()
6313            .with_reconciliation(false)
6314            .with_clock_factory(move || {
6315                calls_in_factory.set(calls_in_factory.get() + 1);
6316                let mut clock = VirtualClock::new();
6317                clock.advance_time(sentinel, true);
6318                Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
6319            })
6320            .build()
6321            .unwrap();
6322
6323        assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
6324        assert_eq!(calls.get(), 1);
6325    }
6326
6327    #[derive(Debug)]
6328    struct ReplayKernelEventStore {
6329        fail_restore: bool,
6330    }
6331
6332    impl KernelEventStore for ReplayKernelEventStore {
6333        fn restore_parent_cache(
6334            &mut self,
6335            _instance_id: UUID4,
6336            _cache: &mut Cache,
6337        ) -> anyhow::Result<()> {
6338            if self.fail_restore {
6339                anyhow::bail!("replay restore failed");
6340            }
6341
6342            Ok(())
6343        }
6344
6345        fn open(
6346            &mut self,
6347            _instance_id: UUID4,
6348            _components: &RegisteredComponents,
6349            _environment: Environment,
6350        ) -> anyhow::Result<()> {
6351            Ok(())
6352        }
6353
6354        fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
6355            None
6356        }
6357
6358        fn seal(&mut self, _ts_init: UnixNanos) {}
6359
6360        fn run_id(&self) -> Option<&str> {
6361            Some("replay-child")
6362        }
6363
6364        fn parent_run_id(&self) -> Option<&str> {
6365            Some("seed-run")
6366        }
6367
6368        fn is_event_store_replay_configured(&self) -> bool {
6369            true
6370        }
6371
6372        fn is_halted(&self) -> bool {
6373            false
6374        }
6375    }
6376
6377    #[derive(Debug)]
6378    struct TestStrategy {
6379        core: StrategyCore,
6380    }
6381
6382    impl TestStrategy {
6383        fn new(config: StrategyConfig) -> Self {
6384            Self {
6385                core: StrategyCore::new(config),
6386            }
6387        }
6388    }
6389
6390    impl DataActor for TestStrategy {}
6391
6392    nautilus_strategy!(TestStrategy, {
6393        fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
6394            self.core.config.external_order_instrument_ids.clone()
6395        }
6396    });
6397
6398    fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
6399        // load_state must be true: the kernel rejects event-store replay otherwise,
6400        // and LiveNodeConfig defaults it to false.
6401        let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
6402            .unwrap()
6403            .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
6404                reconciliation: false,
6405                ..Default::default()
6406            })
6407            .with_load_state(true)
6408            .with_name("TestKernel")
6409            .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
6410                Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
6411            });
6412
6413        builder.build().unwrap()
6414    }
6415
6416    #[rstest]
6417    fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
6418        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6419            .unwrap()
6420            .with_reconciliation(false)
6421            .with_delay_post_stop_secs(0)
6422            .with_timeout_connection(1)
6423            .build()
6424            .unwrap();
6425        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6426        let strategy_id = StrategyId::from("CLAIMS-001");
6427
6428        node.add_strategy(TestStrategy::new(StrategyConfig {
6429            strategy_id: Some(strategy_id),
6430            external_order_instrument_ids: Some(vec![instrument_id]),
6431            ..Default::default()
6432        }))
6433        .unwrap();
6434
6435        assert_eq!(
6436            node.exec_manager.get_external_order_claim(&instrument_id),
6437            Some(strategy_id)
6438        );
6439
6440        {
6441            let exec_engine = node.kernel().exec_engine.borrow();
6442            assert_eq!(
6443                exec_engine.get_external_order_claim(&instrument_id),
6444                Some(strategy_id)
6445            );
6446        }
6447    }
6448
6449    #[rstest]
6450    fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
6451        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6452            .unwrap()
6453            .with_reconciliation(false)
6454            .build()
6455            .unwrap();
6456        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6457        let strategy_id = StrategyId::from("CLAIMS-001");
6458
6459        node.register_external_order_claims(strategy_id, &[instrument_id])
6460            .unwrap();
6461
6462        assert_eq!(
6463            node.exec_manager.get_external_order_claim(&instrument_id),
6464            Some(strategy_id)
6465        );
6466        assert_eq!(
6467            node.kernel
6468                .exec_engine
6469                .borrow()
6470                .get_external_order_claim(&instrument_id),
6471            Some(strategy_id)
6472        );
6473    }
6474
6475    #[rstest]
6476    #[tokio::test]
6477    async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
6478        let mut node = live_node_with_replay_store(false);
6479        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6480        let strategy_id = StrategyId::from("CLAIMS-001");
6481
6482        node.start().await.unwrap();
6483        assert_eq!(node.state(), NodeState::Running);
6484
6485        node.register_external_order_claims(strategy_id, &[instrument_id])
6486            .unwrap();
6487
6488        assert_eq!(
6489            node.exec_manager.get_external_order_claim(&instrument_id),
6490            Some(strategy_id)
6491        );
6492        assert_eq!(
6493            node.kernel
6494                .exec_engine
6495                .borrow()
6496                .get_external_order_claim(&instrument_id),
6497            Some(strategy_id)
6498        );
6499    }
6500
6501    #[rstest]
6502    fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
6503        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6504            .unwrap()
6505            .with_reconciliation(false)
6506            .build()
6507            .unwrap();
6508        let existing_instrument = InstrumentId::from("AUDUSD.SIM");
6509        let new_instruments = [
6510            InstrumentId::from("EURUSD.SIM"),
6511            InstrumentId::from("GBPUSD.SIM"),
6512        ];
6513        let existing_strategy_id = StrategyId::from("CLAIMS-001");
6514        let new_strategy_id = StrategyId::from("CLAIMS-002");
6515        node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
6516            .unwrap();
6517
6518        let result = node.register_external_order_claims(
6519            new_strategy_id,
6520            &[new_instruments[0], existing_instrument, new_instruments[1]],
6521        );
6522
6523        assert!(result.is_err());
6524        assert_eq!(
6525            node.exec_manager
6526                .get_external_order_claim(&existing_instrument),
6527            Some(existing_strategy_id)
6528        );
6529
6530        for instrument_id in new_instruments {
6531            assert_eq!(
6532                node.exec_manager.get_external_order_claim(&instrument_id),
6533                None
6534            );
6535            assert_eq!(
6536                node.kernel
6537                    .exec_engine
6538                    .borrow()
6539                    .get_external_order_claim(&instrument_id),
6540                None
6541            );
6542        }
6543    }
6544
6545    #[rstest]
6546    fn test_deregister_external_order_claims_allows_successor_to_claim() {
6547        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6548            .unwrap()
6549            .with_reconciliation(false)
6550            .build()
6551            .unwrap();
6552        let instruments = [
6553            InstrumentId::from("AUDUSD.SIM"),
6554            InstrumentId::from("EURUSD.SIM"),
6555        ];
6556        let first_strategy_id = StrategyId::from("CLAIMS-001");
6557        let successor_strategy_id = StrategyId::from("CLAIMS-002");
6558        node.register_external_order_claims(first_strategy_id, &instruments)
6559            .unwrap();
6560
6561        node.deregister_external_order_claims(first_strategy_id)
6562            .unwrap();
6563        node.register_external_order_claims(successor_strategy_id, &instruments)
6564            .unwrap();
6565
6566        for instrument_id in instruments {
6567            assert_eq!(
6568                node.exec_manager.get_external_order_claim(&instrument_id),
6569                Some(successor_strategy_id)
6570            );
6571            assert_eq!(
6572                node.kernel
6573                    .exec_engine
6574                    .borrow()
6575                    .get_external_order_claim(&instrument_id),
6576                Some(successor_strategy_id)
6577            );
6578        }
6579    }
6580
6581    #[rstest]
6582    fn test_deregister_external_order_claims_without_claims_is_idempotent() {
6583        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6584            .unwrap()
6585            .with_reconciliation(false)
6586            .build()
6587            .unwrap();
6588        let strategy_id = StrategyId::from("CLAIMS-001");
6589
6590        node.deregister_external_order_claims(strategy_id).unwrap();
6591        node.deregister_external_order_claims(strategy_id).unwrap();
6592    }
6593
6594    #[rstest]
6595    fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
6596        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6597            .unwrap()
6598            .with_reconciliation(false)
6599            .with_delay_post_stop_secs(0)
6600            .with_timeout_connection(1)
6601            .build()
6602            .unwrap();
6603        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6604        let strategy_id = StrategyId::from("CLAIMS-001");
6605        let duplicate_strategy_id = StrategyId::from("OTHER-002");
6606
6607        node.add_strategy(TestStrategy::new(StrategyConfig {
6608            strategy_id: Some(strategy_id),
6609            external_order_instrument_ids: Some(vec![instrument_id]),
6610            ..Default::default()
6611        }))
6612        .unwrap();
6613
6614        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
6615            strategy_id: Some(duplicate_strategy_id),
6616            external_order_instrument_ids: Some(vec![instrument_id]),
6617            ..Default::default()
6618        }));
6619
6620        assert!(result.is_err());
6621        assert!(
6622            result
6623                .unwrap_err()
6624                .to_string()
6625                .contains("already exists for CLAIMS-001")
6626        );
6627        assert_eq!(
6628            node.exec_manager.get_external_order_claim(&instrument_id),
6629            Some(strategy_id)
6630        );
6631
6632        {
6633            let exec_engine = node.kernel().exec_engine.borrow();
6634            assert_eq!(
6635                exec_engine.get_external_order_claim(&instrument_id),
6636                Some(strategy_id)
6637            );
6638        }
6639    }
6640
6641    #[rstest]
6642    fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
6643        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6644            .unwrap()
6645            .with_reconciliation(false)
6646            .with_delay_post_stop_secs(0)
6647            .with_timeout_connection(1)
6648            .build()
6649            .unwrap();
6650        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6651        let strategy_id = StrategyId::from("CLAIMS-001");
6652
6653        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
6654            strategy_id: Some(strategy_id),
6655            external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
6656            ..Default::default()
6657        }));
6658
6659        assert!(result.is_err());
6660        assert!(
6661            result
6662                .unwrap_err()
6663                .to_string()
6664                .contains("appears more than once for CLAIMS-001")
6665        );
6666        assert_eq!(
6667            node.exec_manager.get_external_order_claim(&instrument_id),
6668            None
6669        );
6670
6671        {
6672            let exec_engine = node.kernel().exec_engine.borrow();
6673            assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
6674        }
6675    }
6676
6677    #[rstest]
6678    fn test_add_strategy_failure_restores_external_order_claims() {
6679        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6680            .unwrap()
6681            .with_reconciliation(false)
6682            .with_delay_post_stop_secs(0)
6683            .with_timeout_connection(1)
6684            .build()
6685            .unwrap();
6686        let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
6687        let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
6688        let strategy_id = StrategyId::from("CLAIMS-001");
6689        node.register_external_order_claims(strategy_id, &[existing_instrument_id])
6690            .unwrap();
6691
6692        let mut strategy = TestStrategy::new(StrategyConfig {
6693            strategy_id: Some(strategy_id),
6694            external_order_instrument_ids: Some(vec![configured_instrument_id]),
6695            ..Default::default()
6696        });
6697
6698        strategy
6699            .core
6700            .register(
6701                node.trader_id(),
6702                node.kernel.clock(),
6703                node.kernel.cache.clone(),
6704                node.kernel.portfolio.clone(),
6705            )
6706            .unwrap();
6707
6708        let result = node.add_strategy(strategy);
6709
6710        assert!(result.is_err());
6711        assert!(
6712            result
6713                .unwrap_err()
6714                .to_string()
6715                .contains("already registered with trader")
6716        );
6717        assert_eq!(
6718            node.exec_manager
6719                .get_external_order_claim(&existing_instrument_id),
6720            Some(strategy_id)
6721        );
6722        assert_eq!(
6723            node.kernel
6724                .exec_engine
6725                .borrow()
6726                .get_external_order_claim(&existing_instrument_id),
6727            Some(strategy_id)
6728        );
6729        assert_eq!(
6730            node.exec_manager
6731                .get_external_order_claim(&configured_instrument_id),
6732            None
6733        );
6734        assert_eq!(
6735            node.kernel
6736                .exec_engine
6737                .borrow()
6738                .get_external_order_claim(&configured_instrument_id),
6739            None
6740        );
6741    }
6742
6743    #[rstest]
6744    fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
6745        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6746            .unwrap()
6747            .with_reconciliation(false)
6748            .build()
6749            .unwrap();
6750        let exec_engine = node.kernel.exec_engine.clone();
6751        let _engine_borrow = exec_engine.borrow_mut();
6752
6753        node.add_strategy(TestStrategy::new(StrategyConfig {
6754            strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
6755            ..Default::default()
6756        }))
6757        .unwrap();
6758    }
6759
6760    #[rstest]
6761    fn test_add_strategy_registers_configured_hedging_oms_type() {
6762        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6763            .unwrap()
6764            .with_reconciliation(false)
6765            .with_delay_post_stop_secs(0)
6766            .with_timeout_connection(1)
6767            .build()
6768            .unwrap();
6769        let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
6770
6771        node.add_strategy(TestStrategy::new(StrategyConfig {
6772            strategy_id: Some(strategy_id),
6773            oms_type: Some(OmsType::Hedging),
6774            ..Default::default()
6775        }))
6776        .unwrap();
6777
6778        let instrument = crypto_perpetual_ethusdt();
6779        let instrument_id = instrument.id();
6780        let client_id = ClientId::from("STUB");
6781
6782        node.kernel
6783            .cache
6784            .borrow_mut()
6785            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6786            .unwrap();
6787        node.kernel
6788            .exec_engine
6789            .borrow_mut()
6790            .register_client(Box::new(StubExecutionClient::new(
6791                client_id,
6792                AccountId::from("TEST-ACCOUNT"),
6793                instrument_id.venue,
6794                OmsType::Netting,
6795                None,
6796            )))
6797            .unwrap();
6798
6799        let order = OrderTestBuilder::new(OrderType::Market)
6800            .trader_id(node.trader_id())
6801            .strategy_id(strategy_id)
6802            .instrument_id(instrument_id)
6803            .quantity(Quantity::from("1.000"))
6804            .build();
6805        let position_id = PositionId::new("CUSTOM-POSITION-001");
6806
6807        node.kernel
6808            .cache
6809            .borrow_mut()
6810            .add_order(order.clone(), Some(position_id), Some(client_id), true)
6811            .unwrap();
6812
6813        let submit_order = SubmitOrder::new(
6814            order.trader_id(),
6815            Some(client_id),
6816            strategy_id,
6817            instrument_id,
6818            order.client_order_id(),
6819            order.init_event().clone(),
6820            order.exec_algorithm_id(),
6821            Some(position_id),
6822            None,
6823            UUID4::new(),
6824            UnixNanos::default(),
6825            None,
6826        );
6827
6828        node.kernel
6829            .exec_engine
6830            .borrow()
6831            .execute(TradingCommand::SubmitOrder(submit_order));
6832
6833        let exec_engine = node.kernel.exec_engine.borrow();
6834        let cache = exec_engine.cache().borrow();
6835        let cached_order = cache
6836            .order(&order.client_order_id())
6837            .expect("Order should be cached");
6838
6839        assert_eq!(cached_order.status(), OrderStatus::Initialized);
6840    }
6841
6842    #[cfg(all(feature = "simulation", madsim))]
6843    async fn advance_clock(d: Duration) {
6844        madsim::time::advance(d);
6845        madsim::task::yield_now().await;
6846    }
6847
6848    #[cfg(not(all(feature = "simulation", madsim)))]
6849    async fn advance_clock(d: Duration) {
6850        tokio::time::advance(d).await;
6851    }
6852
6853    #[rstest]
6854    #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
6855    #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
6856    #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
6857    #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
6858    #[case::unapplied_cancel_default(
6859        SubmissionRecoveryPolicy::ResolveLocally,
6860        false,
6861        Some(OrderStatus::PendingCancel),
6862        false
6863    )]
6864    #[case::unapplied_modify_default(
6865        SubmissionRecoveryPolicy::ResolveLocally,
6866        false,
6867        Some(OrderStatus::PendingUpdate),
6868        false
6869    )]
6870    #[case::unapplied_cancel_diagnostics(
6871        SubmissionRecoveryPolicy::RetainUnresolved,
6872        false,
6873        Some(OrderStatus::PendingCancel),
6874        false
6875    )]
6876    #[case::unapplied_modify_diagnostics(
6877        SubmissionRecoveryPolicy::RetainUnresolved,
6878        false,
6879        Some(OrderStatus::PendingUpdate),
6880        false
6881    )]
6882    #[case::first_fill_cancel_default(
6883        SubmissionRecoveryPolicy::ResolveLocally,
6884        false,
6885        Some(OrderStatus::PendingCancel),
6886        true
6887    )]
6888    #[case::first_fill_modify_default(
6889        SubmissionRecoveryPolicy::ResolveLocally,
6890        false,
6891        Some(OrderStatus::PendingUpdate),
6892        true
6893    )]
6894    #[case::first_fill_cancel_diagnostics(
6895        SubmissionRecoveryPolicy::RetainUnresolved,
6896        false,
6897        Some(OrderStatus::PendingCancel),
6898        true
6899    )]
6900    #[case::first_fill_modify_diagnostics(
6901        SubmissionRecoveryPolicy::RetainUnresolved,
6902        false,
6903        Some(OrderStatus::PendingUpdate),
6904        true
6905    )]
6906    #[cfg_attr(
6907        not(all(feature = "simulation", madsim)),
6908        tokio::test(start_paused = true)
6909    )]
6910    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6911    async fn test_submission_registry_publishes_after_native_resolution(
6912        #[case] policy: SubmissionRecoveryPolicy,
6913        #[case] order_list: bool,
6914        #[case] pending_status: Option<OrderStatus>,
6915        #[case] first_fill: bool,
6916    ) {
6917        let config = LiveNodeConfig {
6918            exec_engine: crate::config::LiveExecutionEngineConfig {
6919                inflight_check_threshold_ms: 100,
6920                inflight_check_retries: 2,
6921                submission_recovery_policy: policy,
6922                ..Default::default()
6923            },
6924            ..Default::default()
6925        };
6926        let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
6927        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
6928        let client_id = ClientId::from("TEST-SUBMISSION");
6929        let account_id = AccountId::from("TEST-001");
6930        let strategy_id = StrategyId::from("SUBMISSION-001");
6931        let account = AccountAny::Margin(MarginAccount::new(
6932            AccountState::new(
6933                account_id,
6934                AccountType::Margin,
6935                vec![AccountBalance::new(
6936                    Money::from("1000000 USDT"),
6937                    Money::from("0 USDT"),
6938                    Money::from("1000000 USDT"),
6939                )],
6940                Vec::new(),
6941                true,
6942                UUID4::new(),
6943                UnixNanos::default(),
6944                UnixNanos::default(),
6945                Some(Currency::USDT()),
6946            ),
6947            true,
6948        ));
6949        node.kernel.cache.borrow_mut().add_account(account).unwrap();
6950        node.kernel
6951            .cache
6952            .borrow_mut()
6953            .add_instrument(instrument.clone())
6954            .unwrap();
6955        let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
6956            .map(|index| {
6957                OrderTestBuilder::new(OrderType::Limit)
6958                    .trader_id(node.trader_id())
6959                    .strategy_id(strategy_id)
6960                    .client_order_id(ClientOrderId::from(
6961                        format!("O-SUBMISSION-{index}").as_str(),
6962                    ))
6963                    .instrument_id(instrument.id())
6964                    .quantity(Quantity::from("1.000"))
6965                    .price(Price::from("100.00"))
6966                    .build()
6967            })
6968            .collect();
6969        let command = if order_list {
6970            TradingCommand::SubmitOrderList(SubmitOrderList::new(
6971                node.trader_id(),
6972                Some(client_id),
6973                strategy_id,
6974                OrderList::new(
6975                    OrderListId::from("OL-SUBMISSION"),
6976                    instrument.id(),
6977                    strategy_id,
6978                    orders.iter().map(Order::client_order_id).collect(),
6979                    UnixNanos::default(),
6980                ),
6981                orders
6982                    .iter()
6983                    .map(|order| order.init_event().clone())
6984                    .collect(),
6985                None,
6986                None,
6987                None,
6988                UUID4::new(),
6989                UnixNanos::default(),
6990                None,
6991            ))
6992        } else {
6993            TradingCommand::SubmitOrder(SubmitOrder::from_order(
6994                &orders[0],
6995                node.trader_id(),
6996                Some(client_id),
6997                None,
6998                UUID4::new(),
6999                UnixNanos::default(),
7000            ))
7001        };
7002        node.observe_exec_command_before_dispatch(&command);
7003
7004        for order in &orders {
7005            let submitted = TestOrderEventStubs::submitted(order, account_id);
7006            node.kernel
7007                .cache
7008                .borrow_mut()
7009                .add_order(order.clone(), None, Some(client_id), false)
7010                .unwrap();
7011            node.process_exec_event(ExecutionEvent::Order(submitted));
7012        }
7013
7014        let expected_status = match pending_status {
7015            Some(status) if !first_fill => status,
7016            Some(_) => OrderStatus::Canceled,
7017            None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7018            None => OrderStatus::Rejected,
7019        };
7020        let diagnostics = Rc::new(RefCell::new(Vec::new()));
7021        let handler = ShareableMessageHandler::from_typed({
7022            let diagnostics = diagnostics.clone();
7023            let cache = node.kernel.cache.clone();
7024            move |diagnostic: &SubmissionRecoveryExhausted| {
7025                let cache = cache.borrow_mut();
7026                let order = cache.order(&diagnostic.client_order_id).unwrap();
7027                assert_eq!(order.status(), expected_status);
7028                assert!(diagnostic.ts_event >= order.ts_last());
7029                diagnostics.borrow_mut().push(diagnostic.clone());
7030            }
7031        });
7032        let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7033        msgbus::subscribe_any(topic.into(), handler.clone(), None);
7034        let queries = Rc::new(RefCell::new(Vec::new()));
7035        msgbus::register_trading_command_endpoint(
7036            MessagingSwitchboard::exec_engine_execute(),
7037            TypedIntoHandler::from({
7038                let queries = queries.clone();
7039                move |command: TradingCommand| queries.borrow_mut().push(command)
7040            }),
7041        );
7042
7043        let mut last_inflight_check = dst::time::Instant::now();
7044        let mut last_open_check = last_inflight_check;
7045        let mut last_position_check = last_inflight_check;
7046        let mut open_order_report_task = None;
7047        let mut targeted_order_report_task = None;
7048        let mut position_report_task = None;
7049
7050        for check in 1..=3 {
7051            advance_clock(Duration::from_millis(101)).await;
7052            node.run_reconciliation_checks(
7053                dst::time::Instant::now(),
7054                ReconciliationCheckIntervals {
7055                    inflight: Duration::from_millis(100),
7056                    open: Duration::ZERO,
7057                    position: Duration::ZERO,
7058                },
7059                &mut ReconciliationCheckState {
7060                    last_inflight_check: &mut last_inflight_check,
7061                    last_open_check: &mut last_open_check,
7062                    last_position_check: &mut last_position_check,
7063                    open_order_report_task: &mut open_order_report_task,
7064                    targeted_order_report_task: &mut targeted_order_report_task,
7065                    position_report_task: &mut position_report_task,
7066                },
7067            );
7068
7069            if check == 1 {
7070                assert!(diagnostics.borrow().is_empty());
7071
7072                if let Some(pending_status) = pending_status {
7073                    let order = &orders[0];
7074                    let pending = if pending_status == OrderStatus::PendingCancel {
7075                        OrderEventAny::PendingCancel(
7076                            OrderPendingCancelSpec::builder()
7077                                .trader_id(node.trader_id())
7078                                .strategy_id(strategy_id)
7079                                .instrument_id(instrument.id())
7080                                .client_order_id(order.client_order_id())
7081                                .account_id(account_id)
7082                                .build(),
7083                        )
7084                    } else {
7085                        OrderEventAny::PendingUpdate(
7086                            OrderPendingUpdateSpec::builder()
7087                                .trader_id(node.trader_id())
7088                                .strategy_id(strategy_id)
7089                                .instrument_id(instrument.id())
7090                                .client_order_id(order.client_order_id())
7091                                .account_id(account_id)
7092                                .build(),
7093                        )
7094                    };
7095                    node.process_exec_event(ExecutionEvent::Order(pending));
7096                    node.exec_manager.register_inflight(order.client_order_id());
7097                    if first_fill {
7098                        let cached = node
7099                            .kernel
7100                            .cache
7101                            .borrow()
7102                            .order(&order.client_order_id())
7103                            .unwrap()
7104                            .clone();
7105                        let OrderEventAny::Filled(mut fill) =
7106                            OrderFilledTestBuilder::new(&cached, &instrument)
7107                                .last_qty(Quantity::from("0.400"))
7108                                .last_px(Price::from("100.00"))
7109                                .commission(Money::from("0 USDT"))
7110                                .without_position_id()
7111                                .build()
7112                        else {
7113                            unreachable!()
7114                        };
7115                        fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7116                        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7117                    } else {
7118                        let report = OrderStatusReport::new(
7119                            account_id,
7120                            instrument.id(),
7121                            Some(order.client_order_id()),
7122                            VenueOrderId::from("V-SUBMISSION"),
7123                            Some(order.order_side()),
7124                            order.order_type(),
7125                            order.time_in_force(),
7126                            OrderStatus::Submitted,
7127                            order.quantity(),
7128                            Quantity::zero(3),
7129                            UnixNanos::default(),
7130                            UnixNanos::default(),
7131                            UnixNanos::default(),
7132                            None,
7133                        )
7134                        .with_price(Price::from("100.00"));
7135                        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
7136                            Box::new(report),
7137                        )));
7138                    }
7139                    let cache = node.kernel.cache.borrow();
7140                    let cached = cache.order(&order.client_order_id()).unwrap();
7141                    assert_eq!(cached.status(), pending_status);
7142                    assert_eq!(
7143                        cached.filled_qty(),
7144                        if first_fill {
7145                            Quantity::from("0.400")
7146                        } else {
7147                            Quantity::zero(3)
7148                        }
7149                    );
7150                    assert!(
7151                        !cached
7152                            .events()
7153                            .iter()
7154                            .any(|event| matches!(event, OrderEventAny::Accepted(_)))
7155                    );
7156                }
7157            }
7158        }
7159
7160        assert_eq!(
7161            queries.borrow().len(),
7162            orders.len()
7163                + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
7164        );
7165
7166        for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
7167            assert!(matches!(command, TradingCommand::QueryOrder(query)
7168                if query.client_order_id == order.client_order_id()
7169                && query.client_id == Some(client_id)));
7170            assert_eq!(
7171                node.kernel
7172                    .cache
7173                    .borrow()
7174                    .order(&order.client_order_id())
7175                    .unwrap()
7176                    .status(),
7177                if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
7178                    pending_status.unwrap()
7179                } else {
7180                    expected_status
7181                }
7182            );
7183        }
7184        let diagnostics = diagnostics.borrow();
7185        assert_eq!(
7186            diagnostics.len(),
7187            if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
7188                orders.len()
7189            } else {
7190                0
7191            }
7192        );
7193
7194        for (diagnostic, order) in diagnostics.iter().zip(&orders) {
7195            assert_eq!(diagnostic.trader_id, node.trader_id());
7196            assert_eq!(diagnostic.client_id, Some(client_id));
7197            assert_eq!(diagnostic.strategy_id, strategy_id);
7198            assert_eq!(diagnostic.instrument_id, instrument.id());
7199            assert_eq!(diagnostic.client_order_id, order.client_order_id());
7200            assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
7201            assert_eq!(diagnostic.retry_count, 2);
7202        }
7203        msgbus::unsubscribe_any(topic.into(), &handler);
7204        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
7205    }
7206
7207    #[derive(Debug)]
7208    struct SubmissionCallbackStrategy {
7209        core: StrategyCore,
7210        cancel: bool,
7211        batch: bool,
7212        callback: &'static str,
7213        client_id: ClientId,
7214    }
7215
7216    impl DataActor for SubmissionCallbackStrategy {}
7217
7218    impl SubmissionCallbackStrategy {
7219        fn queue_command(&mut self, client_order_id: ClientOrderId) {
7220            match (self.cancel, self.batch) {
7221                (true, true) => {
7222                    self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
7223                        .unwrap();
7224                }
7225                (true, false) => {
7226                    self.cancel_order(client_order_id, Some(self.client_id), None)
7227                        .unwrap();
7228                }
7229                (false, true) => {
7230                    self.modify_orders(
7231                        vec![(client_order_id, None, Some(Price::from("101.00")), None)],
7232                        Some(self.client_id),
7233                        None,
7234                    )
7235                    .unwrap();
7236                }
7237                (false, false) => self
7238                    .modify_order(
7239                        client_order_id,
7240                        None,
7241                        Some(Price::from("101.00")),
7242                        None,
7243                        Some(self.client_id),
7244                        None,
7245                    )
7246                    .unwrap(),
7247            }
7248        }
7249    }
7250
7251    nautilus_strategy!(SubmissionCallbackStrategy, {
7252        fn on_order_accepted(&mut self, event: OrderAccepted) {
7253            if self.callback == "accepted" {
7254                self.queue_command(event.client_order_id);
7255            }
7256        }
7257
7258        fn on_order_updated(&mut self, event: OrderUpdated) {
7259            if self.callback == "updated" {
7260                self.queue_command(event.client_order_id);
7261            }
7262        }
7263    });
7264
7265    #[rstest]
7266    #[case::native(false, "accepted")]
7267    #[case::report(true, "accepted")]
7268    #[case::updated(false, "updated")]
7269    #[cfg_attr(
7270        not(all(feature = "simulation", madsim)),
7271        tokio::test(start_paused = true)
7272    )]
7273    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7274    async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
7275        #[values(
7276            SubmissionRecoveryPolicy::ResolveLocally,
7277            SubmissionRecoveryPolicy::RetainUnresolved
7278        )]
7279        policy: SubmissionRecoveryPolicy,
7280        #[values(false, true)] cancel: bool,
7281        #[values(false, true)] batch: bool,
7282        #[case] report: bool,
7283        #[case] callback: &'static str,
7284    ) {
7285        let config = LiveNodeConfig {
7286            exec_engine: crate::config::LiveExecutionEngineConfig {
7287                inflight_check_threshold_ms: 100,
7288                inflight_check_retries: 2,
7289                submission_recovery_policy: policy,
7290                ..Default::default()
7291            },
7292            ..Default::default()
7293        };
7294        let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
7295        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7296        let account_id = AccountId::from("TEST-001");
7297        let strategy_id = StrategyId::from("CALLBACK-001");
7298        let client_id = ClientId::from("TEST-CALLBACK");
7299        let account = AccountAny::Margin(MarginAccount::new(
7300            AccountState::new(
7301                account_id,
7302                AccountType::Margin,
7303                vec![AccountBalance::new(
7304                    Money::from("1000000 USDT"),
7305                    Money::from("0 USDT"),
7306                    Money::from("1000000 USDT"),
7307                )],
7308                Vec::new(),
7309                true,
7310                UUID4::new(),
7311                UnixNanos::default(),
7312                UnixNanos::default(),
7313                Some(Currency::USDT()),
7314            ),
7315            true,
7316        ));
7317        node.kernel.cache.borrow_mut().add_account(account).unwrap();
7318        node.kernel
7319            .cache
7320            .borrow_mut()
7321            .add_instrument(instrument.clone())
7322            .unwrap();
7323        node.kernel
7324            .exec_engine
7325            .borrow_mut()
7326            .register_client(Box::new(StubExecutionClient::new(
7327                client_id,
7328                account_id,
7329                instrument.id().venue,
7330                OmsType::Netting,
7331                None,
7332            )))
7333            .unwrap();
7334        node.add_strategy(SubmissionCallbackStrategy {
7335            core: StrategyCore::new(StrategyConfig {
7336                strategy_id: Some(strategy_id),
7337                ..Default::default()
7338            }),
7339            cancel,
7340            batch,
7341            callback,
7342            client_id,
7343        })
7344        .unwrap();
7345        node.kernel.trader.borrow_mut().initialize().unwrap();
7346        node.kernel.start_trader().unwrap();
7347        let runner = node.runner.take().unwrap();
7348        runner.bind_senders_for_node(node.handle.clone());
7349        let mut channels = runner.take_channels();
7350        let order = OrderTestBuilder::new(OrderType::Limit)
7351            .trader_id(node.trader_id())
7352            .strategy_id(strategy_id)
7353            .client_order_id(ClientOrderId::from("O-CALLBACK"))
7354            .instrument_id(instrument.id())
7355            .quantity(Quantity::from("1.000"))
7356            .price(Price::from("100.00"))
7357            .build();
7358        let client_order_id = order.client_order_id();
7359        let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
7360            &order,
7361            node.trader_id(),
7362            Some(client_id),
7363            None,
7364            UUID4::new(),
7365            UnixNanos::default(),
7366        ));
7367        node.observe_exec_command_before_dispatch(&submit);
7368        node.kernel
7369            .cache
7370            .borrow_mut()
7371            .add_order(order.clone(), None, Some(client_id), false)
7372            .unwrap();
7373        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
7374            &order, account_id,
7375        )));
7376        advance_clock(Duration::from_millis(101)).await;
7377        let first = node.exec_manager.check_inflight_orders();
7378        assert_eq!(first.queries.len(), 1);
7379        assert!(first.events.is_empty());
7380
7381        advance_clock(Duration::from_millis(101)).await;
7382        let accepted =
7383            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
7384
7385        if callback == "updated" {
7386            let mut update = OrderUpdatedSpec::builder()
7387                .trader_id(node.trader_id())
7388                .strategy_id(strategy_id)
7389                .instrument_id(instrument.id())
7390                .client_order_id(client_order_id)
7391                .account_id(account_id)
7392                .quantity(order.quantity())
7393                .price(order.price().unwrap())
7394                .build();
7395            update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
7396            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
7397        } else if report {
7398            let accepted = OrderStatusReport::new(
7399                account_id,
7400                instrument.id(),
7401                Some(client_order_id),
7402                VenueOrderId::from("V-CALLBACK"),
7403                Some(order.order_side()),
7404                order.order_type(),
7405                order.time_in_force(),
7406                OrderStatus::Accepted,
7407                order.quantity(),
7408                Quantity::zero(3),
7409                UnixNanos::default(),
7410                UnixNanos::default(),
7411                UnixNanos::default(),
7412                None,
7413            )
7414            .with_price(Price::from("100.00"));
7415            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
7416                accepted,
7417            ))));
7418        } else {
7419            node.process_exec_event(ExecutionEvent::Order(accepted));
7420        }
7421
7422        let pending_status = if cancel {
7423            OrderStatus::PendingCancel
7424        } else {
7425            OrderStatus::PendingUpdate
7426        };
7427        assert_eq!(channels.exec_cmd_rx.len(), 1);
7428        assert_eq!(
7429            node.kernel
7430                .cache
7431                .borrow()
7432                .order(&client_order_id)
7433                .unwrap()
7434                .status(),
7435            pending_status
7436        );
7437
7438        // Maintenance may win the runner selection before the queued command dispatches
7439        for _ in 0..3 {
7440            advance_clock(Duration::from_millis(101)).await;
7441            let maintenance = node.exec_manager.check_inflight_orders();
7442            assert!(maintenance.events.is_empty());
7443            assert!(maintenance.queries.is_empty());
7444            assert!(
7445                node.exec_manager
7446                    .take_submission_recovery_exhaustions()
7447                    .is_empty()
7448            );
7449        }
7450
7451        let command = channels.exec_cmd_rx.try_recv().unwrap();
7452        node.process_exec_command(command);
7453        assert!(channels.exec_cmd_rx.is_empty());
7454        let immediate = node.exec_manager.check_inflight_orders();
7455        assert!(immediate.events.is_empty());
7456        assert!(immediate.queries.is_empty());
7457        advance_clock(Duration::from_millis(101)).await;
7458        let first_command_check = node.exec_manager.check_inflight_orders();
7459        assert!(first_command_check.events.is_empty());
7460        assert_eq!(first_command_check.queries.len(), 1);
7461        advance_clock(Duration::from_millis(101)).await;
7462        let exhausted = node.exec_manager.check_inflight_orders();
7463        assert!(exhausted.queries.is_empty());
7464        assert!(
7465            matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
7466            if event.client_order_id == client_order_id)
7467        );
7468        assert!(
7469            node.exec_manager
7470                .take_submission_recovery_exhaustions()
7471                .is_empty()
7472        );
7473        node.process_reconciliation_events(&exhausted.events);
7474        assert_eq!(
7475            node.kernel
7476                .cache
7477                .borrow()
7478                .order(&client_order_id)
7479                .unwrap()
7480                .status(),
7481            OrderStatus::Canceled
7482        );
7483    }
7484
7485    #[cfg_attr(
7486        not(all(feature = "simulation", madsim)),
7487        tokio::test(start_paused = true)
7488    )]
7489    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7490    async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
7491        let last = dst::time::Instant::now();
7492        let interval = Duration::from_millis(100);
7493
7494        assert!(!reconciliation_check_due(last, last, Duration::ZERO));
7495        assert!(!reconciliation_check_due(last, last, interval));
7496
7497        advance_clock(Duration::from_millis(99)).await;
7498        let before_interval = dst::time::Instant::now();
7499        assert!(!reconciliation_check_due(before_interval, last, interval));
7500
7501        advance_clock(Duration::from_millis(1)).await;
7502        let at_interval = dst::time::Instant::now();
7503        assert!(reconciliation_check_due(at_interval, last, interval));
7504
7505        assert!(!reconciliation_check_due(last, at_interval, interval));
7506    }
7507
7508    #[cfg_attr(
7509        not(all(feature = "simulation", madsim)),
7510        tokio::test(start_paused = true)
7511    )]
7512    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7513    async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
7514        let config = LiveNodeConfig {
7515            exec_engine: crate::config::LiveExecutionEngineConfig {
7516                reconciliation: true,
7517                open_check_interval_secs: Some(1.0),
7518                position_check_interval_secs: Some(1.0),
7519                max_single_order_queries_per_cycle: 5,
7520                ..Default::default()
7521            },
7522            ..Default::default()
7523        };
7524
7525        let mut node =
7526            LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
7527        let client_id = ClientId::from("TEST-QUERY");
7528        let account_id = AccountId::from("TEST-QUERY-001");
7529
7530        let trading_commands = Rc::new(RefCell::new(Vec::new()));
7531        msgbus::register_trading_command_endpoint(
7532            MessagingSwitchboard::exec_engine_execute(),
7533            TypedIntoHandler::from({
7534                let trading_commands = trading_commands.clone();
7535                move |command: TradingCommand| {
7536                    trading_commands.borrow_mut().push(command);
7537                }
7538            }),
7539        );
7540
7541        let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
7542        let instrument = crypto_perpetual_ethusdt();
7543        let instrument_id = instrument.id();
7544        let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
7545
7546        node.kernel
7547            .cache
7548            .borrow_mut()
7549            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7550            .unwrap();
7551        insert_accepted_limit_order_in_node(
7552            &node,
7553            account_id,
7554            client_id,
7555            instrument_id,
7556            client_order_id,
7557            venue_order_id,
7558        );
7559
7560        let last = dst::time::Instant::now();
7561        advance_clock(Duration::from_nanos(1)).await;
7562        let now = dst::time::Instant::now();
7563        let mut last_inflight_check = last;
7564        let mut last_open_check = last;
7565        let mut last_position_check = last;
7566        let mut open_order_report_task = None;
7567        let mut targeted_order_report_task = None;
7568        let mut position_report_task = None;
7569
7570        node.run_reconciliation_checks(
7571            now,
7572            ReconciliationCheckIntervals {
7573                inflight: Duration::ZERO,
7574                open: Duration::from_nanos(1),
7575                position: Duration::ZERO,
7576            },
7577            &mut ReconciliationCheckState {
7578                last_inflight_check: &mut last_inflight_check,
7579                last_open_check: &mut last_open_check,
7580                last_position_check: &mut last_position_check,
7581                open_order_report_task: &mut open_order_report_task,
7582                targeted_order_report_task: &mut targeted_order_report_task,
7583                position_report_task: &mut position_report_task,
7584            },
7585        );
7586
7587        let commands = trading_commands.borrow();
7588
7589        assert!(commands.is_empty());
7590        assert!(open_order_report_task.is_none());
7591        assert!(targeted_order_report_task.is_none());
7592        assert!(position_report_task.is_none());
7593
7594        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
7595    }
7596
7597    #[rstest]
7598    #[tokio::test]
7599    async fn test_recurring_report_failure_preserves_client_coverage(
7600        #[values(false, true)] positions: bool,
7601        #[values(false, true)] failure_first: bool,
7602    ) {
7603        let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
7604        let failed_id = ClientId::from("FAILED");
7605        let healthy_id = ClientId::from("HEALTHY");
7606
7607        for (client_id, outcome) in [
7608            (failed_id, FillReportClientOutcome::Failure),
7609            (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
7610        ] {
7611            node.exec_clients
7612                .push(LiveExecutionClient::new(Box::new(FillReportClient {
7613                    client_id,
7614                    account_id: AccountId::from("TEST-001"),
7615                    venue: Venue::from("BINANCE"),
7616                    outcome,
7617                    commands: Rc::new(RefCell::new(Vec::new())),
7618                })));
7619        }
7620
7621        if !failure_first {
7622            node.exec_clients.reverse();
7623        }
7624
7625        let last = dst::time::Instant::now();
7626        let now = last + Duration::from_secs(1);
7627        let mut last_inflight = last;
7628        let mut last_open = last;
7629        let mut last_position = last;
7630        let mut open_task = None;
7631        let mut targeted_task = None;
7632        let mut position_task = None;
7633
7634        node.run_reconciliation_checks(
7635            now,
7636            ReconciliationCheckIntervals {
7637                inflight: Duration::ZERO,
7638                open: if positions {
7639                    Duration::ZERO
7640                } else {
7641                    Duration::from_secs(1)
7642                },
7643                position: if positions {
7644                    Duration::from_secs(1)
7645                } else {
7646                    Duration::ZERO
7647                },
7648            },
7649            &mut ReconciliationCheckState {
7650                last_inflight_check: &mut last_inflight,
7651                last_open_check: &mut last_open,
7652                last_position_check: &mut last_position,
7653                open_order_report_task: &mut open_task,
7654                targeted_order_report_task: &mut targeted_task,
7655                position_report_task: &mut position_task,
7656            },
7657        );
7658
7659        let (queried_clients, failed_clients) = if positions {
7660            let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
7661                position_task.unwrap().future.await
7662            else {
7663                panic!("position collection should complete despite one client failure");
7664            };
7665
7666            assert!(result.reports.is_empty());
7667            assert!(open_task.is_none());
7668            (result.queried_clients, result.failed_clients)
7669        } else {
7670            let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
7671                panic!("order collection should complete despite one client failure");
7672            };
7673
7674            assert!(result.reports.is_empty());
7675            assert!(position_task.is_none());
7676            (result.queried_clients, result.failed_clients)
7677        };
7678
7679        assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
7680        assert_eq!(failed_clients, IndexSet::from([failed_id]));
7681        assert!(targeted_task.is_none());
7682    }
7683
7684    #[rstest]
7685    #[case::inflight(true, false)]
7686    #[case::open(false, false)]
7687    #[case::positions(false, true)]
7688    #[tokio::test]
7689    async fn test_reconciliation_shutdown_preserves_pending_checks(
7690        #[case] inflight: bool,
7691        #[case] positions: bool,
7692    ) {
7693        let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
7694        node.handle.set_shutting_down();
7695        let last = dst::time::Instant::now();
7696        let now = last + Duration::from_secs(1);
7697        let mut last_inflight = last;
7698        let mut last_open = last;
7699        let mut last_position = last;
7700        let mut open_task = None;
7701        let mut targeted_task = None;
7702        let mut position_task = None;
7703
7704        node.run_reconciliation_checks(
7705            now,
7706            ReconciliationCheckIntervals {
7707                inflight: if inflight {
7708                    Duration::from_secs(1)
7709                } else {
7710                    Duration::ZERO
7711                },
7712                open: if !inflight && !positions {
7713                    Duration::from_secs(1)
7714                } else {
7715                    Duration::ZERO
7716                },
7717                position: if positions {
7718                    Duration::from_secs(1)
7719                } else {
7720                    Duration::ZERO
7721                },
7722            },
7723            &mut ReconciliationCheckState {
7724                last_inflight_check: &mut last_inflight,
7725                last_open_check: &mut last_open,
7726                last_position_check: &mut last_position,
7727                open_order_report_task: &mut open_task,
7728                targeted_order_report_task: &mut targeted_task,
7729                position_report_task: &mut position_task,
7730            },
7731        );
7732
7733        assert_eq!(
7734            (last_inflight, last_open, last_position),
7735            (last, last, last)
7736        );
7737        assert!(open_task.is_none());
7738        assert!(targeted_task.is_none());
7739        assert!(position_task.is_none());
7740    }
7741
7742    fn insert_accepted_limit_order_in_node(
7743        node: &LiveNode,
7744        account_id: AccountId,
7745        client_id: ClientId,
7746        instrument_id: InstrumentId,
7747        client_order_id: ClientOrderId,
7748        venue_order_id: VenueOrderId,
7749    ) {
7750        let order = OrderTestBuilder::new(OrderType::Limit)
7751            .client_order_id(client_order_id)
7752            .instrument_id(instrument_id)
7753            .quantity(Quantity::from("10.0"))
7754            .price(Price::from("100.0"))
7755            .build();
7756        let submitted = TestOrderEventStubs::submitted(&order, account_id);
7757        node.kernel
7758            .cache
7759            .borrow_mut()
7760            .add_order(order, None, Some(client_id), false)
7761            .unwrap();
7762        let order = node
7763            .kernel
7764            .cache
7765            .borrow_mut()
7766            .update_order(&submitted)
7767            .unwrap();
7768        let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
7769        node.kernel
7770            .cache
7771            .borrow_mut()
7772            .update_order(&accepted)
7773            .unwrap();
7774    }
7775
7776    fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
7777        let config = LiveNodeConfig {
7778            exec_engine: crate::config::LiveExecutionEngineConfig {
7779                reconciliation: true,
7780                ..Default::default()
7781            },
7782            ..Default::default()
7783        };
7784
7785        let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
7786        let account_id = AccountId::from("TEST-001");
7787        let client_id = ClientId::from("TEST-RECENT-FILL");
7788        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
7789        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
7790        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7791        node.kernel
7792            .cache
7793            .borrow_mut()
7794            .add_instrument(instrument.clone())
7795            .unwrap();
7796        insert_accepted_limit_order_in_node(
7797            &node,
7798            account_id,
7799            client_id,
7800            instrument.id(),
7801            client_order_id,
7802            venue_order_id,
7803        );
7804        let order = node
7805            .kernel
7806            .cache
7807            .borrow()
7808            .order_owned(&client_order_id)
7809            .unwrap();
7810        let fill = TestOrderEventStubs::filled(
7811            &order,
7812            &instrument,
7813            Some(TradeId::from("T-RECENT-FILL")),
7814            None,
7815            Some(Price::from("100.0")),
7816            Some(Quantity::from("1.0")),
7817            Some(LiquiditySide::Taker),
7818            None,
7819            None,
7820            Some(account_id),
7821        );
7822
7823        (node, fill, instrument)
7824    }
7825
7826    fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
7827        let client_order_id = fill_report.client_order_id.unwrap();
7828        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7829        let order = node
7830            .kernel
7831            .cache
7832            .borrow()
7833            .order_owned(&client_order_id)
7834            .unwrap();
7835        let order_report = OrderStatusReport::new(
7836            fill_report.account_id,
7837            fill_report.instrument_id,
7838            Some(client_order_id),
7839            fill_report.venue_order_id,
7840            OrderSide::Buy.into(),
7841            OrderType::Limit,
7842            TimeInForce::Gtc,
7843            OrderStatus::PartiallyFilled,
7844            Quantity::from("10.0"),
7845            Quantity::from("2.0"),
7846            UnixNanos::from(1_000),
7847            UnixNanos::from(1_000),
7848            UnixNanos::from(1_000),
7849            None,
7850        )
7851        .with_avg_px(dec!(100.0));
7852        let inferred = create_inferred_fill_for_qty(
7853            &order,
7854            &order_report,
7855            &fill_report.account_id,
7856            &instrument,
7857            Quantity::from("1.0"),
7858            UnixNanos::from(1_000),
7859            None,
7860        )
7861        .unwrap();
7862
7863        node.process_reconciliation_events(&[inferred]);
7864    }
7865
7866    #[rstest]
7867    #[case::rejected(false, "10.0", "10.0", "1.0")]
7868    #[case::dust(true, "9.004", "10.004", "10.004")]
7869    #[case::price_improvement(true, "12.5", "13.5", "13.5")]
7870    fn test_position_check_overfill_raises_before_fill_when_client_allows(
7871        #[case] allow: bool,
7872        #[case] fill_qty: &str,
7873        #[case] expected_qty: &str,
7874        #[case] expected_filled: &str,
7875    ) {
7876        let (mut node, venue_report, mut fill_report) =
7877            position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
7878        fill_report.commission = Money::from("1.25 USDT");
7879        let key = (fill_report.instrument_id, fill_report.account_id);
7880
7881        let client = StubExecutionClient::new(
7882            ClientId::from("POSITION-FILLS"),
7883            AccountId::from("TEST-001"),
7884            fill_report.instrument_id.venue,
7885            OmsType::Netting,
7886            None,
7887        );
7888
7889        let client = if allow {
7890            client.with_allows_reconciliation_overfill()
7891        } else {
7892            client
7893        };
7894
7895        node.kernel
7896            .exec_engine
7897            .borrow_mut()
7898            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
7899            .unwrap();
7900
7901        let position_result = position_report_result(&node, venue_report);
7902        node.handle_position_fill_report_result(PositionFillReportResult {
7903            position_result,
7904            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
7905            successful_keys: IndexSet::from([key]),
7906        });
7907
7908        let order = node
7909            .kernel
7910            .cache
7911            .borrow()
7912            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
7913            .unwrap();
7914        assert_eq!(order.quantity(), Quantity::from(expected_qty));
7915        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
7916        let applied = order
7917            .trade_ids()
7918            .iter()
7919            .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
7920        assert_eq!(applied, allow);
7921        let revision = node.exec_manager.position_activity_revision(&key);
7922
7923        if !allow {
7924            return;
7925        }
7926
7927        assert_eq!(revision, 2);
7928
7929        {
7930            let cache = node.kernel.cache.borrow();
7931            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
7932            assert_eq!(positions.len(), 1);
7933            assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
7934            assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
7935            assert_eq!(positions[0].trade_ids.len(), 2);
7936        }
7937
7938        let replay_report = position_report_result(
7939            &node,
7940            PositionStatusReport::new(
7941                key.1,
7942                key.0,
7943                PositionSide::Long,
7944                Quantity::from(expected_qty),
7945                UnixNanos::from(2_000),
7946                UnixNanos::from(2_000),
7947                None,
7948                None,
7949                Some(dec!(100.0)),
7950            ),
7951        );
7952        node.handle_position_fill_report_result(PositionFillReportResult {
7953            position_result: replay_report,
7954            reports: IndexMap::from([(key, vec![fill_report])]),
7955            successful_keys: IndexSet::from([key]),
7956        });
7957
7958        let order = node
7959            .kernel
7960            .cache
7961            .borrow()
7962            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
7963            .unwrap();
7964        assert_eq!(order.quantity(), Quantity::from(expected_qty));
7965        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
7966        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
7967    }
7968
7969    #[rstest]
7970    fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
7971        let (mut node, venue_report, mut overfill) =
7972            position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
7973        overfill.commission = Money::from("0.20 USDT");
7974        let key = (overfill.instrument_id, overfill.account_id);
7975        let client = StubExecutionClient::new(
7976            ClientId::from("POSITION-FILLS"),
7977            AccountId::from("TEST-001"),
7978            overfill.instrument_id.venue,
7979            OmsType::Netting,
7980            None,
7981        )
7982        .with_allows_reconciliation_overfill();
7983        node.kernel
7984            .exec_engine
7985            .borrow_mut()
7986            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
7987            .unwrap();
7988
7989        let mut fitting = overfill.clone();
7990        fitting.trade_id = TradeId::from("T-POSITION-FITTING");
7991        fitting.last_qty = Quantity::from("8.0");
7992        fitting.commission = Money::from("0.80 USDT");
7993        let position_result = position_report_result(&node, venue_report);
7994        node.handle_position_fill_report_result(PositionFillReportResult {
7995            position_result,
7996            reports: IndexMap::from([(key, vec![fitting, overfill])]),
7997            successful_keys: IndexSet::from([key]),
7998        });
7999
8000        let order = node
8001            .kernel
8002            .cache
8003            .borrow()
8004            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8005            .unwrap();
8006        assert_eq!(order.quantity(), Quantity::from("11.0"));
8007        assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8008        assert_eq!(order.trade_ids().len(), 3);
8009        assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8010        {
8011            let cache = node.kernel.cache.borrow();
8012            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8013            assert_eq!(positions.len(), 1);
8014            assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8015            assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8016        }
8017    }
8018
8019    #[rstest]
8020    fn test_position_check_second_overfill_does_not_raise_filled_order() {
8021        let (mut node, venue_report, mut first) =
8022            position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8023        first.commission = Money::from("1.20 USDT");
8024        let key = (first.instrument_id, first.account_id);
8025        let client = StubExecutionClient::new(
8026            ClientId::from("POSITION-FILLS"),
8027            AccountId::from("TEST-001"),
8028            first.instrument_id.venue,
8029            OmsType::Netting,
8030            None,
8031        )
8032        .with_allows_reconciliation_overfill();
8033        node.kernel
8034            .exec_engine
8035            .borrow_mut()
8036            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8037            .unwrap();
8038
8039        let mut second = first.clone();
8040        second.trade_id = TradeId::from("T-POSITION-LATER");
8041        second.last_qty = Quantity::from("1.0");
8042        second.commission = Money::from("0.10 USDT");
8043        second.ts_event = UnixNanos::from(1_001);
8044        let position_result = position_report_result(&node, venue_report);
8045        node.handle_position_fill_report_result(PositionFillReportResult {
8046            position_result,
8047            reports: IndexMap::from([(key, vec![first, second])]),
8048            successful_keys: IndexSet::from([key]),
8049        });
8050
8051        let order = node
8052            .kernel
8053            .cache
8054            .borrow()
8055            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8056            .unwrap();
8057        assert_eq!(order.quantity(), Quantity::from("13.0"));
8058        assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8059        assert_eq!(order.status(), OrderStatus::Filled);
8060        assert_eq!(order.trade_ids().len(), 2);
8061        assert!(
8062            order
8063                .trade_ids()
8064                .iter()
8065                .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8066        );
8067        {
8068            let cache = node.kernel.cache.borrow();
8069            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8070            assert_eq!(positions.len(), 1);
8071            assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8072            assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8073        }
8074    }
8075
8076    #[rstest]
8077    fn test_allow_overfills_applies_fill_without_raising_quantity() {
8078        let config = LiveNodeConfig {
8079            exec_engine: crate::config::LiveExecutionEngineConfig {
8080                reconciliation: true,
8081                allow_overfills: true,
8082                ..Default::default()
8083            },
8084            ..Default::default()
8085        };
8086
8087        let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8088        let account_id = AccountId::from("TEST-001");
8089        let client_id = ClientId::from("POSITION-FILLS");
8090        let client_order_id = ClientOrderId::from("O-ALLOW");
8091        let venue_order_id = VenueOrderId::from("V-ALLOW");
8092        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8093        node.kernel
8094            .cache
8095            .borrow_mut()
8096            .add_instrument(instrument.clone())
8097            .unwrap();
8098        insert_accepted_limit_order_in_node(
8099            &node,
8100            account_id,
8101            client_id,
8102            instrument.id(),
8103            client_order_id,
8104            venue_order_id,
8105        );
8106        let client = StubExecutionClient::new(
8107            client_id,
8108            account_id,
8109            instrument.id().venue,
8110            OmsType::Netting,
8111            None,
8112        )
8113        .with_allows_reconciliation_overfill();
8114        node.kernel
8115            .exec_engine
8116            .borrow_mut()
8117            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8118            .unwrap();
8119
8120        let fill = FillReport::new(
8121            account_id,
8122            instrument.id(),
8123            venue_order_id,
8124            TradeId::from("T-ALLOW"),
8125            OrderSide::Buy,
8126            Quantity::from("12.0"),
8127            Price::from("100.0"),
8128            Money::from("1.00 USDT"),
8129            LiquiditySide::Taker,
8130            Some(client_order_id),
8131            None,
8132            UnixNanos::from(1_000),
8133            UnixNanos::from(1_000),
8134            None,
8135        );
8136        node.kernel
8137            .exec_engine
8138            .borrow_mut()
8139            .reconcile_fill_report(&fill);
8140
8141        let order = node
8142            .kernel
8143            .cache
8144            .borrow()
8145            .order_owned(&client_order_id)
8146            .unwrap();
8147        assert_eq!(order.quantity(), Quantity::from("10.0"));
8148        assert_eq!(order.filled_qty(), Quantity::from("12.0"));
8149        assert_eq!(order.trade_ids().len(), 1);
8150    }
8151
8152    fn position_fill_test_fixture(
8153        name: &str,
8154        authoritative_qty: Quantity,
8155    ) -> (LiveNode, PositionStatusReport, FillReport) {
8156        position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
8157    }
8158
8159    fn position_fill_test_fixture_with_threshold(
8160        name: &str,
8161        authoritative_qty: Quantity,
8162        position_check_threshold_ms: u32,
8163    ) -> (LiveNode, PositionStatusReport, FillReport) {
8164        let config = LiveNodeConfig {
8165            exec_engine: crate::config::LiveExecutionEngineConfig {
8166                reconciliation: true,
8167                position_check_threshold_ms,
8168                ..Default::default()
8169            },
8170            ..Default::default()
8171        };
8172
8173        let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8174        let account_id = AccountId::from("TEST-001");
8175        let client_id = ClientId::from("POSITION-FILLS");
8176        let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
8177        let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
8178        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8179        let account = AccountAny::Margin(MarginAccount::new(
8180            AccountState::new(
8181                account_id,
8182                AccountType::Margin,
8183                vec![AccountBalance::new(
8184                    Money::from("1000000 USDT"),
8185                    Money::from("0 USDT"),
8186                    Money::from("1000000 USDT"),
8187                )],
8188                Vec::new(),
8189                true,
8190                UUID4::new(),
8191                UnixNanos::default(),
8192                UnixNanos::default(),
8193                Some(Currency::USDT()),
8194            ),
8195            true,
8196        ));
8197        node.kernel.cache.borrow_mut().add_account(account).unwrap();
8198        node.kernel
8199            .cache
8200            .borrow_mut()
8201            .add_instrument(instrument.clone())
8202            .unwrap();
8203        insert_accepted_limit_order_in_node(
8204            &node,
8205            account_id,
8206            client_id,
8207            instrument.id(),
8208            client_order_id,
8209            venue_order_id,
8210        );
8211        let order = node
8212            .kernel
8213            .cache
8214            .borrow()
8215            .order_owned(&client_order_id)
8216            .unwrap();
8217        let mut initial_fill = TestOrderEventStubs::filled(
8218            &order,
8219            &instrument,
8220            Some(TradeId::from("T-POSITION-INITIAL")),
8221            None,
8222            Some(Price::from("100.0")),
8223            Some(Quantity::from("1.0")),
8224            Some(LiquiditySide::Taker),
8225            None,
8226            None,
8227            Some(account_id),
8228        );
8229
8230        let OrderEventAny::Filled(fill) = &mut initial_fill else {
8231            unreachable!();
8232        };
8233
8234        fill.commission = Some(Money::zero(instrument.quote_currency()));
8235        node.process_reconciliation_events(&[initial_fill]);
8236
8237        let ts_event = UnixNanos::from(1_000);
8238
8239        let fill_report = FillReport::new(
8240            account_id,
8241            instrument.id(),
8242            venue_order_id,
8243            TradeId::from("T-POSITION-AUTHORITATIVE"),
8244            OrderSide::Buy,
8245            authoritative_qty,
8246            Price::from("100.0"),
8247            Money::zero(instrument.quote_currency()),
8248            LiquiditySide::Taker,
8249            Some(client_order_id),
8250            None,
8251            ts_event,
8252            ts_event,
8253            None,
8254        );
8255
8256        let venue_report = PositionStatusReport::new(
8257            account_id,
8258            instrument.id(),
8259            PositionSide::Long,
8260            Quantity::from("2.0"),
8261            ts_event,
8262            ts_event,
8263            None,
8264            None,
8265            Some(dec!(100.0)),
8266        );
8267
8268        (node, venue_report, fill_report)
8269    }
8270
8271    fn position_report_result(
8272        node: &LiveNode,
8273        report: PositionStatusReport,
8274    ) -> PositionReportResult {
8275        let client_id = ClientId::from("POSITION-FILLS");
8276        let key = (report.instrument_id, report.account_id);
8277        let mut check = node
8278            .exec_manager
8279            .prepare_position_report_check(UUID4::new(), &[]);
8280        check.client_coverage.insert(
8281            key,
8282            ReportClientCoverage::Resolved(IndexSet::from([client_id])),
8283        );
8284
8285        PositionReportResult {
8286            check,
8287            reports: vec![report],
8288            queried_clients: IndexSet::from([client_id]),
8289            failed_clients: IndexSet::new(),
8290        }
8291    }
8292
8293    fn position_quantity_after_check(
8294        node: &mut LiveNode,
8295        venue_report: &PositionStatusReport,
8296        reports: &[FillReport],
8297    ) -> Quantity {
8298        let key = (venue_report.instrument_id, venue_report.account_id);
8299        let position_result = position_report_result(node, venue_report.clone());
8300        node.handle_position_fill_report_result(PositionFillReportResult {
8301            position_result,
8302            reports: IndexMap::from([(key, reports.to_vec())]),
8303            successful_keys: IndexSet::from([key]),
8304        });
8305
8306        node.kernel
8307            .cache
8308            .borrow()
8309            .positions_open(None, Some(&key.0), None, Some(&key.1), None)
8310            .iter()
8311            .map(|position| position.quantity)
8312            .sum()
8313    }
8314
8315    fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
8316        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
8317            fill.account_id,
8318            fill.instrument_id,
8319            fill.venue_order_id,
8320            fill.trade_id,
8321            fill.order_side,
8322            fill.last_qty,
8323            fill.last_px,
8324            fill.commission
8325                .unwrap_or_else(|| Money::zero(fill.currency)),
8326            fill.liquidity_side,
8327            Some(fill.client_order_id),
8328            fill.position_id,
8329            fill.ts_event,
8330            fill.ts_init,
8331            None,
8332        ))))
8333    }
8334
8335    fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
8336        node.exec_manager.is_fill_recently_processed(
8337            fill.account_id,
8338            fill.instrument_id,
8339            fill.trade_id,
8340        )
8341    }
8342
8343    #[rstest]
8344    #[case(0, NodeState::Idle)]
8345    #[case(1, NodeState::Starting)]
8346    #[case(2, NodeState::Running)]
8347    #[case(3, NodeState::ShuttingDown)]
8348    #[case(4, NodeState::Stopped)]
8349    fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
8350        assert_eq!(NodeState::from_u8(value), expected);
8351    }
8352
8353    #[rstest]
8354    #[case(5)]
8355    #[case(255)]
8356    #[should_panic(expected = "Invalid NodeState value")]
8357    fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
8358        let _ = NodeState::from_u8(value);
8359    }
8360
8361    #[rstest]
8362    fn test_node_state_roundtrip() {
8363        for state in [
8364            NodeState::Idle,
8365            NodeState::Starting,
8366            NodeState::Running,
8367            NodeState::ShuttingDown,
8368            NodeState::Stopped,
8369        ] {
8370            assert_eq!(NodeState::from_u8(state.as_u8()), state);
8371        }
8372    }
8373
8374    #[rstest]
8375    fn test_node_state_is_running_only_for_running() {
8376        assert!(!NodeState::Idle.is_running());
8377        assert!(!NodeState::Starting.is_running());
8378        assert!(NodeState::Running.is_running());
8379        assert!(!NodeState::ShuttingDown.is_running());
8380        assert!(!NodeState::Stopped.is_running());
8381    }
8382
8383    #[rstest]
8384    #[tokio::test]
8385    async fn test_await_engines_connected_returns_stop_requested() {
8386        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8387        let handle = node.handle();
8388
8389        handle.stop();
8390
8391        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8392        let status = node.await_engines_connected(deadline).await;
8393
8394        assert_eq!(status, EngineConnectionStatus::StopRequested);
8395        assert!(handle.should_stop());
8396    }
8397
8398    #[rstest]
8399    #[tokio::test]
8400    async fn test_await_engines_connected_returns_shutdown_requested() {
8401        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8402
8403        node.kernel().shutdown_flag().set(true);
8404
8405        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8406        let status = node.await_engines_connected(deadline).await;
8407
8408        assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
8409    }
8410
8411    #[rstest]
8412    #[tokio::test]
8413    async fn test_start_stop_request_aborts_startup_without_running() {
8414        let config = LiveNodeConfig {
8415            exec_engine: crate::config::LiveExecutionEngineConfig {
8416                reconciliation: false,
8417                ..Default::default()
8418            },
8419            timeout_disconnection: Duration::from_millis(50),
8420            ..Default::default()
8421        };
8422
8423        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8424        let handle = node.handle();
8425
8426        handle.stop();
8427        node.start().await.unwrap();
8428
8429        assert_eq!(handle.state(), NodeState::Stopped);
8430        assert!(handle.should_stop());
8431        assert!(!handle.is_running());
8432    }
8433
8434    #[rstest]
8435    #[tokio::test(start_paused = true)]
8436    async fn test_stop_processes_residual_exec_event_during_grace_period() {
8437        let config = LiveNodeConfig {
8438            exec_engine: crate::config::LiveExecutionEngineConfig {
8439                reconciliation: false,
8440                ..Default::default()
8441            },
8442            timeout_connection: Duration::ZERO,
8443            timeout_reconciliation: Duration::ZERO,
8444            timeout_portfolio: Duration::ZERO,
8445            timeout_disconnection: Duration::ZERO,
8446            delay_post_stop: Duration::from_millis(20),
8447            timeout_shutdown: Duration::ZERO,
8448            ..Default::default()
8449        };
8450
8451        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8452        let order = OrderTestBuilder::new(OrderType::Market)
8453            .instrument_id(InstrumentId::from("EUR/USD.SIM"))
8454            .quantity(Quantity::from("2"))
8455            .build();
8456        let client_order_id = order.client_order_id();
8457        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
8458
8459        node.kernel
8460            .cache()
8461            .borrow_mut()
8462            .add_order(order, None, None, false)
8463            .unwrap();
8464
8465        node.start().await.unwrap();
8466        let exec_event_sender = get_exec_event_sender();
8467
8468        let send_residual = tokio::spawn(async move {
8469            tokio::time::sleep(Duration::from_millis(1)).await;
8470            exec_event_sender
8471                .send(ExecutionEvent::Order(submitted))
8472                .unwrap();
8473        });
8474
8475        node.stop().await.unwrap();
8476        send_residual.await.unwrap();
8477
8478        assert_eq!(
8479            node.kernel
8480                .cache()
8481                .borrow()
8482                .order(&client_order_id)
8483                .unwrap()
8484                .status(),
8485            OrderStatus::Submitted
8486        );
8487
8488        node.dispose();
8489    }
8490
8491    #[rstest]
8492    #[tokio::test]
8493    async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
8494        let actor_id = ActorId::from("LIVE-STATE-ACTOR");
8495        let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
8496        let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
8497        let strategy_load =
8498            IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
8499        let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
8500        let strategy_save =
8501            IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
8502        let (database, control) = TestCacheDatabaseControl::create();
8503        control.set_actor_state(actor_id, &actor_load);
8504        control.set_strategy_state(strategy_id, &strategy_load);
8505
8506        let config = LiveNodeConfig {
8507            load_state: true,
8508            save_state: true,
8509            exec_engine: crate::config::LiveExecutionEngineConfig {
8510                reconciliation: false,
8511                ..Default::default()
8512            },
8513            timeout_connection: Duration::ZERO,
8514            timeout_reconciliation: Duration::ZERO,
8515            timeout_portfolio: Duration::ZERO,
8516            timeout_disconnection: Duration::ZERO,
8517            delay_post_stop: Duration::ZERO,
8518            timeout_shutdown: Duration::ZERO,
8519            ..Default::default()
8520        };
8521
8522        let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
8523        node.set_cache_database(Box::new(database)).unwrap();
8524        node.add_actor(StateActor::new(
8525            actor_id,
8526            control.clone(),
8527            actor_save.clone(),
8528        ))
8529        .unwrap();
8530        node.add_strategy(StateStrategy::new(
8531            strategy_id,
8532            control.clone(),
8533            strategy_save.clone(),
8534        ))
8535        .unwrap();
8536
8537        node.start().await.unwrap();
8538        node.stop().await.unwrap();
8539        node.dispose();
8540
8541        assert_eq!(
8542            control.events(),
8543            vec![
8544                "actor.load:LIVE-STATE-ACTOR",
8545                "actor.on_load",
8546                "strategy.load:LIVE-STATE-STRATEGY-001",
8547                "strategy.on_load",
8548                "actor.on_start",
8549                "strategy.on_start",
8550                "actor.on_stop",
8551                "strategy.on_stop",
8552                "actor.on_save",
8553                "actor.update:LIVE-STATE-ACTOR",
8554                "strategy.on_save",
8555                "strategy.update:LIVE-STATE-STRATEGY-001",
8556                "database.close",
8557            ]
8558        );
8559        assert_eq!(control.actor_state(&actor_id), Some(actor_save));
8560        assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
8561        assert_eq!(node.state(), NodeState::Stopped);
8562    }
8563
8564    #[rstest]
8565    #[tokio::test]
8566    async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
8567        let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
8568        let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
8569        let (database, control) = TestCacheDatabaseControl::create();
8570
8571        let config = LiveNodeConfig {
8572            save_state: true,
8573            exec_engine: crate::config::LiveExecutionEngineConfig {
8574                reconciliation: false,
8575                ..Default::default()
8576            },
8577            timeout_connection: Duration::ZERO,
8578            timeout_reconciliation: Duration::ZERO,
8579            timeout_portfolio: Duration::ZERO,
8580            timeout_disconnection: Duration::ZERO,
8581            delay_post_stop: Duration::ZERO,
8582            timeout_shutdown: Duration::ZERO,
8583            ..Default::default()
8584        };
8585
8586        let mut node =
8587            LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
8588        node.set_cache_database(Box::new(database)).unwrap();
8589        node.add_actor(
8590            StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
8591        )
8592        .unwrap();
8593        node.add_strategy(
8594            StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
8595        )
8596        .unwrap();
8597
8598        node.start().await.unwrap();
8599        let error = node.stop().await.unwrap_err();
8600        node.dispose();
8601
8602        assert_eq!(
8603            error.to_string(),
8604            "failed while finalizing kernel shutdown: Failed to save component state: actor \
8605             LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
8606             LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
8607        );
8608        assert_eq!(
8609            control.events(),
8610            vec![
8611                "actor.on_start",
8612                "strategy.on_start",
8613                "actor.on_stop",
8614                "strategy.on_stop",
8615                "actor.on_save",
8616                "strategy.on_save",
8617                "database.close",
8618            ]
8619        );
8620        assert_eq!(node.state(), NodeState::Stopped);
8621    }
8622
8623    #[rstest]
8624    #[tokio::test]
8625    async fn test_stop_drains_queued_exec_event_after_zero_grace() {
8626        let config = LiveNodeConfig {
8627            exec_engine: crate::config::LiveExecutionEngineConfig {
8628                reconciliation: false,
8629                ..Default::default()
8630            },
8631            timeout_connection: Duration::ZERO,
8632            timeout_reconciliation: Duration::ZERO,
8633            timeout_portfolio: Duration::ZERO,
8634            timeout_disconnection: Duration::ZERO,
8635            delay_post_stop: Duration::ZERO,
8636            timeout_shutdown: Duration::ZERO,
8637            ..Default::default()
8638        };
8639
8640        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8641        let order = OrderTestBuilder::new(OrderType::Market)
8642            .instrument_id(InstrumentId::from("GBP/USD.SIM"))
8643            .quantity(Quantity::from("3"))
8644            .build();
8645        let client_order_id = order.client_order_id();
8646        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
8647
8648        node.kernel
8649            .cache()
8650            .borrow_mut()
8651            .add_order(order, None, None, false)
8652            .unwrap();
8653
8654        node.start().await.unwrap();
8655        get_exec_event_sender()
8656            .send(ExecutionEvent::Order(submitted))
8657            .unwrap();
8658
8659        node.stop().await.unwrap();
8660
8661        assert_eq!(
8662            node.kernel
8663                .cache()
8664                .borrow()
8665                .order(&client_order_id)
8666                .unwrap()
8667                .status(),
8668            OrderStatus::Submitted
8669        );
8670
8671        node.dispose();
8672    }
8673
8674    #[rstest]
8675    #[tokio::test]
8676    async fn test_start_event_store_replay_skips_live_connections() {
8677        let mut node = live_node_with_replay_store(false);
8678        let handle = node.handle();
8679
8680        node.start().await.unwrap();
8681
8682        assert_eq!(handle.state(), NodeState::Running);
8683        assert!(handle.is_running());
8684        assert!(node.kernel.is_event_store_replay());
8685        assert!(node.runner.is_some());
8686    }
8687
8688    #[rstest]
8689    #[tokio::test]
8690    async fn test_start_event_store_replay_preserves_stop_request() {
8691        let mut node = live_node_with_replay_store(false);
8692        let handle = node.handle();
8693        handle.stop();
8694
8695        node.start().await.unwrap();
8696
8697        assert_eq!(handle.state(), NodeState::Stopped);
8698        assert!(handle.should_stop());
8699        assert!(node.kernel.is_event_store_replay());
8700        assert!(node.runner.is_some());
8701    }
8702
8703    #[rstest]
8704    #[tokio::test]
8705    async fn test_start_event_store_replay_config_failure_aborts_startup() {
8706        let mut node = live_node_with_replay_store(true);
8707        let handle = node.handle();
8708
8709        node.start().await.unwrap();
8710
8711        assert_eq!(handle.state(), NodeState::Stopped);
8712        assert!(!handle.is_running());
8713        assert!(node.kernel.is_event_store_replay_configured());
8714        assert!(!node.kernel.is_event_store_replay());
8715        assert!(node.runner.is_some());
8716    }
8717
8718    #[rstest]
8719    #[tokio::test]
8720    async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
8721        let mut node = live_node_with_replay_store(false);
8722        let handle = node.handle();
8723
8724        node.run().await.unwrap();
8725
8726        assert_eq!(handle.state(), NodeState::Running);
8727        assert!(handle.is_running());
8728        assert!(node.kernel.is_event_store_replay());
8729        assert!(node.runner.is_none());
8730    }
8731
8732    #[rstest]
8733    #[tokio::test]
8734    async fn test_run_event_store_replay_preserves_stop_request() {
8735        let mut node = live_node_with_replay_store(false);
8736        let handle = node.handle();
8737        handle.stop();
8738
8739        node.run().await.unwrap();
8740
8741        assert_eq!(handle.state(), NodeState::Stopped);
8742        assert!(handle.should_stop());
8743        assert!(node.kernel.is_event_store_replay());
8744        assert!(node.runner.is_none());
8745    }
8746
8747    #[rstest]
8748    #[tokio::test]
8749    async fn test_run_event_store_replay_config_failure_aborts_startup() {
8750        let mut node = live_node_with_replay_store(true);
8751        let handle = node.handle();
8752
8753        node.run().await.unwrap();
8754
8755        assert_eq!(handle.state(), NodeState::Stopped);
8756        assert!(!handle.is_running());
8757        assert!(node.kernel.is_event_store_replay_configured());
8758        assert!(!node.kernel.is_event_store_replay());
8759        assert!(node.runner.is_none());
8760    }
8761
8762    #[rstest]
8763    fn test_build_rejects_event_store_config_without_factory() {
8764        let config = LiveNodeConfig {
8765            event_store: Some(EventStoreConfig::default()),
8766            exec_engine: crate::config::LiveExecutionEngineConfig {
8767                reconciliation: false,
8768                ..Default::default()
8769            },
8770            ..Default::default()
8771        };
8772
8773        let err = LiveNodeBuilder::from_config(config)
8774            .expect("builder")
8775            .build()
8776            .expect_err("should reject event_store config without factory");
8777
8778        assert!(
8779            err.to_string().contains("with_event_store"),
8780            "error message should mention with_event_store, was: {err}"
8781        );
8782    }
8783
8784    #[rstest]
8785    fn test_direct_build_rejects_event_store_config() {
8786        let config = LiveNodeConfig {
8787            event_store: Some(EventStoreConfig::default()),
8788            exec_engine: crate::config::LiveExecutionEngineConfig {
8789                reconciliation: false,
8790                ..Default::default()
8791            },
8792            ..Default::default()
8793        };
8794
8795        let err = LiveNode::build("TestNode".to_string(), Some(config))
8796            .expect_err("LiveNode::build should reject event_store config");
8797
8798        assert!(
8799            err.to_string().contains("with_event_store"),
8800            "error message should mention with_event_store, was: {err}"
8801        );
8802    }
8803
8804    #[rstest]
8805    fn test_dispose_before_start_is_idempotent() {
8806        let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
8807        node.add_strategy(TestStrategy::new(StrategyConfig {
8808            strategy_id: Some(StrategyId::from("DISPOSAL-001")),
8809            ..Default::default()
8810        }))
8811        .unwrap();
8812
8813        node.dispose();
8814        node.dispose();
8815
8816        assert!(node.kernel.trader().borrow().is_disposed());
8817        assert_eq!(node.kernel.trader().borrow().component_count(), 0);
8818        assert_eq!(node.state(), NodeState::Stopped);
8819    }
8820
8821    #[rstest]
8822    fn test_dispose_releases_retained_callback_roots(
8823        #[values(false, true)] fatal: bool,
8824        #[values(false, true)] external: bool,
8825    ) {
8826        actor::clear_callbacks().unwrap();
8827        let directory =
8828            std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
8829
8830        let config = LiveNodeConfig {
8831            logging: LoggerConfig {
8832                fileout_level: LevelFilter::Info,
8833                file_config: Some(FileWriterConfig {
8834                    directory: Some(directory.to_str().unwrap().to_string()),
8835                    file_name: Some("disposal".to_string()),
8836                    ..Default::default()
8837                }),
8838                ..Default::default()
8839            },
8840            ..Default::default()
8841        };
8842
8843        let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
8844        let received = Rc::new(RefCell::new(Vec::new()));
8845        let observed = received.clone();
8846
8847        let retained = DispatchMessage::from(()).dispatch(|()| {
8848            nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
8849                TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
8850                TimeEventCallback::RustLocal(Rc::new(move |_| {
8851                    observed.borrow_mut().push("delivered");
8852                })),
8853            ));
8854
8855            external.then(|| DispatchMessage::new((), std::thread::current().id()))
8856        });
8857
8858        if let Some(retained) = &retained {
8859            assert!(retained.is_rooted());
8860        }
8861
8862        assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
8863
8864        if fatal {
8865            crate::dispatch::tests::latch_callback_failure();
8866        }
8867
8868        node.dispose();
8869
8870        assert!(node.runner.is_none());
8871        assert!(received.borrow().is_empty());
8872        assert_eq!(Rc::strong_count(&received), 1);
8873        assert_eq!(
8874            actor::callback_failure(),
8875            (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
8876        );
8877        assert_eq!(
8878            actor::clear_callbacks(),
8879            if external {
8880                Err(CallbackDispatchError::Active)
8881            } else {
8882                Ok(())
8883            }
8884        );
8885
8886        logging_sync_to_disk().unwrap();
8887        let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
8888        drop(retained);
8889        node.dispose();
8890
8891        assert_eq!(actor::callback_failure(), None);
8892        assert_eq!(actor::clear_callbacks(), Ok(()));
8893        drop(node);
8894        std::fs::remove_dir_all(directory).unwrap();
8895
8896        let errors: Vec<_> = output
8897            .lines()
8898            .filter(|line| line.contains("[ERROR]"))
8899            .filter_map(|line| {
8900                line.split_once(".nautilus_live::node: ")
8901                    .map(|(_, text)| text)
8902            })
8903            .collect();
8904
8905        let mut expected = Vec::new();
8906
8907        if fatal {
8908            expected.push(
8909                "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
8910            );
8911        }
8912
8913        if external {
8914            expected.push(
8915                "Failed to clear callback dispatch during disposal: Callback work or access is still active",
8916            );
8917        }
8918
8919        assert_eq!(errors, expected);
8920    }
8921
8922    #[tokio::test]
8923    async fn test_dispose_releases_stop_generated_callback_roots() {
8924        actor::clear_callbacks().unwrap();
8925
8926        let config = LiveNodeConfig {
8927            exec_engine: crate::config::LiveExecutionEngineConfig {
8928                reconciliation: false,
8929                ..Default::default()
8930            },
8931            timeout_connection: Duration::ZERO,
8932            timeout_reconciliation: Duration::ZERO,
8933            timeout_portfolio: Duration::ZERO,
8934            timeout_shutdown: Duration::ZERO,
8935            ..Default::default()
8936        };
8937
8938        let mut node =
8939            LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
8940        let received = Rc::new(RefCell::new(Vec::new()));
8941        node.add_actor(StopCallbackActor {
8942            core: DataActorCore::new(DataActorConfig {
8943                actor_id: Some(ActorId::from("STOP-CALLBACK")),
8944                ..Default::default()
8945            }),
8946            received: received.clone(),
8947        })
8948        .unwrap();
8949
8950        node.start().await.unwrap();
8951
8952        assert!(node.kernel.trader().borrow().is_running());
8953        assert_eq!(actor::clear_callbacks(), Ok(()));
8954
8955        node.dispose();
8956
8957        assert_eq!(*received.borrow(), ["stop", "queued"]);
8958        assert_eq!(Rc::strong_count(&received), 1);
8959        assert!(node.runner.is_none());
8960        assert!(node.kernel.trader().borrow().is_disposed());
8961        assert_eq!(node.state(), NodeState::Stopped);
8962        assert_eq!(actor::clear_callbacks(), Ok(()));
8963    }
8964
8965    #[derive(Debug)]
8966    struct StopCallbackActor {
8967        core: DataActorCore,
8968        received: Rc<RefCell<Vec<&'static str>>>,
8969    }
8970
8971    nautilus_actor!(StopCallbackActor);
8972
8973    impl DataActor for StopCallbackActor {
8974        fn on_stop(&mut self) -> anyhow::Result<()> {
8975            self.received.borrow_mut().push("stop");
8976            let received = self.received.clone();
8977            DispatchMessage::from(()).dispatch(|()| {
8978                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
8979                    TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
8980                    TimeEventCallback::RustLocal(Rc::new(move |_| {
8981                        received.borrow_mut().push("delivered");
8982                    })),
8983                ));
8984            });
8985
8986            let clear_result = actor::clear_callbacks();
8987            anyhow::ensure!(
8988                clear_result == Err(CallbackDispatchError::Active),
8989                "Expected active callback roots during stop, received {clear_result:?}"
8990            );
8991            self.received.borrow_mut().push("queued");
8992            Ok(())
8993        }
8994    }
8995
8996    #[rstest]
8997    fn test_handle_initial_state() {
8998        let handle = LiveNodeHandle::new();
8999
9000        assert_eq!(handle.state(), NodeState::Idle);
9001        assert!(!handle.should_stop());
9002        assert!(!handle.is_running());
9003    }
9004
9005    #[rstest]
9006    fn test_handle_initial_metrics_snapshot_is_zero() {
9007        let handle = LiveNodeHandle::new();
9008
9009        assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9010    }
9011
9012    #[rstest]
9013    fn test_record_runner_dispatch_updates_selected_channel() {
9014        let metrics = RunnerMetrics::default();
9015        let dispatch_start = dst::time::Instant::now();
9016        let metrics_start = dispatch_start
9017            .checked_sub(Duration::from_micros(1))
9018            .expect("test instant should support a one-microsecond lookback");
9019
9020        record_runner_dispatch(
9021            &metrics,
9022            SystemChannel::DataCommands,
9023            dispatch_start,
9024            metrics_start,
9025        );
9026        let snapshot = metrics.snapshot();
9027
9028        assert_eq!(snapshot.time_events.dispatched, 0);
9029        assert_eq!(snapshot.exec_events.dispatched, 0);
9030        assert_eq!(snapshot.exec_commands.dispatched, 0);
9031        assert_eq!(snapshot.data_events.dispatched, 0);
9032        assert_eq!(snapshot.data_commands.dispatched, 1);
9033        assert_eq!(
9034            snapshot.data_commands.last_dispatch_at_ns,
9035            snapshot.elapsed_ns
9036        );
9037        assert_eq!(
9038            snapshot.data_commands.dispatch_busy_ns,
9039            snapshot.dispatch_busy_ns
9040        );
9041        assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9042    }
9043
9044    #[rstest]
9045    fn test_handle_stop_sets_flag() {
9046        let handle = LiveNodeHandle::new();
9047
9048        handle.stop();
9049
9050        assert!(handle.should_stop());
9051    }
9052
9053    #[rstest]
9054    fn test_handle_stop_blocks_running_transition() {
9055        let handle = LiveNodeHandle::new();
9056        handle.set_starting();
9057        handle.stop();
9058
9059        let transition = handle.try_set_running();
9060
9061        assert_eq!(transition, RunningTransition::StopRequested);
9062        assert_eq!(handle.state(), NodeState::Starting);
9063        assert!(handle.should_stop());
9064        assert!(!handle.is_running());
9065    }
9066
9067    #[rstest]
9068    fn test_handle_stop_after_running_transition_remains_pending() {
9069        let handle = LiveNodeHandle::new();
9070        handle.set_starting();
9071
9072        let transition = handle.try_set_running();
9073        handle.stop();
9074
9075        assert_eq!(transition, RunningTransition::Entered);
9076        assert_eq!(handle.state(), NodeState::Running);
9077        assert!(handle.should_stop());
9078        assert!(handle.is_running());
9079    }
9080
9081    #[rstest]
9082    fn test_handle_node_state_transitions() {
9083        let handle = LiveNodeHandle::new();
9084        assert_eq!(handle.state(), NodeState::Idle);
9085
9086        handle.set_starting();
9087        assert_eq!(handle.state(), NodeState::Starting);
9088        assert!(!handle.is_running());
9089
9090        assert_eq!(handle.try_set_running(), RunningTransition::Entered);
9091        assert_eq!(handle.state(), NodeState::Running);
9092        assert!(handle.is_running());
9093
9094        handle.set_shutting_down();
9095        assert_eq!(handle.state(), NodeState::ShuttingDown);
9096        assert!(!handle.is_running());
9097
9098        handle.set_stopped();
9099        assert_eq!(handle.state(), NodeState::Stopped);
9100        assert!(!handle.is_running());
9101    }
9102
9103    #[rstest]
9104    fn test_handle_clone_shares_state_bidirectionally() {
9105        let handle1 = LiveNodeHandle::new();
9106        let handle2 = handle1.clone();
9107
9108        handle1.set_starting();
9109        let transition = handle2.try_set_running();
9110        handle1.stop();
9111
9112        assert_eq!(transition, RunningTransition::Entered);
9113        assert_eq!(handle1.state(), NodeState::Running);
9114        assert!(handle2.should_stop());
9115    }
9116
9117    #[rstest]
9118    fn test_handle_stop_flag_survives_non_running_state_changes() {
9119        let handle = LiveNodeHandle::new();
9120
9121        handle.set_starting();
9122        handle.stop();
9123        handle.set_shutting_down();
9124        handle.set_stopped();
9125
9126        assert_eq!(handle.state(), NodeState::Stopped);
9127        assert!(handle.should_stop());
9128    }
9129
9130    #[rstest]
9131    fn test_builder_creation() {
9132        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9133
9134        assert!(result.is_ok());
9135    }
9136
9137    #[rstest]
9138    fn test_stream_processor_receives_unregistered_typed_payload() {
9139        let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9140            .unwrap()
9141            .build()
9142            .unwrap();
9143        let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
9144            InstrumentId::from("AUD/USD.SIM"),
9145            Some(ClientId::from("EXTERNAL")),
9146            Some(Venue::from("SIM")),
9147            UUID4::from("00000000-0000-4000-8000-000000000001"),
9148            UnixNanos::from(1),
9149            None,
9150            None,
9151        ));
9152        let expected = serde_json::to_value(&command).unwrap();
9153        let received = Rc::new(RefCell::new(Vec::new()));
9154        let steps = Rc::new(RefCell::new(Vec::new()));
9155        let received_processor = received.clone();
9156        let first_steps = steps.clone();
9157        node.add_stream_processor(move |message| {
9158            let command = message
9159                .downcast_ref::<SubscribeCommand>()
9160                .expect("processor must receive the decoded concrete command");
9161            received_processor
9162                .borrow_mut()
9163                .push(serde_json::to_value(command).unwrap());
9164            first_steps.borrow_mut().push(1);
9165        });
9166
9167        let second_steps = steps.clone();
9168        node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
9169        let republished = Rc::new(RefCell::new(Vec::new()));
9170        let republished_handler = republished.clone();
9171        let subscriber_steps = steps.clone();
9172        msgbus::subscribe_any(
9173            "external.test".into(),
9174            ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
9175                republished_handler
9176                    .borrow_mut()
9177                    .push(serde_json::to_value(command).unwrap());
9178                subscriber_steps.borrow_mut().push(3);
9179            }),
9180            None,
9181        );
9182
9183        let message = BusMessage::with_str_topic(
9184            "external.test",
9185            BusPayloadType::SubscribeCommand,
9186            Bytes::from(serde_json::to_vec(&command).unwrap()),
9187            SerializationEncoding::Json,
9188        );
9189
9190        assert!(
9191            !msgbus::get_message_bus()
9192                .borrow()
9193                .is_streaming_type(BusPayloadType::SubscribeCommand)
9194        );
9195        node.process_external_msgbus_message(&message);
9196
9197        assert_eq!(*received.borrow(), vec![expected.clone()]);
9198        assert_eq!(*steps.borrow(), vec![1, 2]);
9199        assert!(republished.borrow().is_empty());
9200
9201        received.borrow_mut().clear();
9202        steps.borrow_mut().clear();
9203        msgbus::get_message_bus()
9204            .borrow_mut()
9205            .add_streaming_type(BusPayloadType::SubscribeCommand);
9206        node.process_external_msgbus_message(&message);
9207
9208        assert_eq!(*received.borrow(), vec![expected.clone()]);
9209        assert_eq!(*republished.borrow(), vec![expected]);
9210        assert_eq!(*steps.borrow(), vec![1, 2, 3]);
9211        msgbus::get_message_bus().borrow_mut().dispose();
9212    }
9213
9214    #[rstest]
9215    fn test_builder_rejects_backtest() {
9216        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
9217
9218        assert!(result.is_err());
9219        assert!(result.unwrap_err().to_string().contains("Backtest"));
9220    }
9221
9222    #[rstest]
9223    fn test_builder_accepts_live_environment() {
9224        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
9225
9226        assert!(result.is_ok());
9227    }
9228
9229    #[rstest]
9230    fn test_builder_accepts_sandbox_environment() {
9231        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9232
9233        assert!(result.is_ok());
9234    }
9235
9236    #[rstest]
9237    fn test_builder_fluent_api_chaining() {
9238        let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
9239            .unwrap()
9240            .with_name("TestNode")
9241            .with_instance_id(UUID4::new())
9242            .with_load_state(false)
9243            .with_save_state(true)
9244            .with_timeout_connection(30)
9245            .with_timeout_reconciliation(60)
9246            .with_reconciliation(true)
9247            .with_reconciliation_lookback_mins(120)
9248            .with_timeout_portfolio(10)
9249            .with_timeout_disconnection_secs(5)
9250            .with_delay_post_stop_secs(3)
9251            .with_delay_shutdown_secs(10);
9252
9253        assert_eq!(builder.name(), "TestNode");
9254    }
9255
9256    #[rstest]
9257    fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
9258        let (external_egress, publications, closed) = CapturingExternalEgress::new();
9259
9260        let msgbus_config = MessageBusConfig {
9261            encoding: SerializationEncoding::Json,
9262            ..Default::default()
9263        };
9264
9265        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9266            .unwrap()
9267            .with_msgbus_config(msgbus_config)
9268            .with_external_msgbus_egress(Box::new(external_egress))
9269            .build()
9270            .expect("node builds with external message bus egress");
9271        let quote = QuoteTick::default();
9272
9273        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
9274
9275        let publications = publications.borrow();
9276        assert_eq!(publications.len(), 1);
9277        assert_eq!(publications[0].topic, "data.quotes.TEST");
9278        assert_eq!(
9279            serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9280                .expect("JSON payload must decode as QuoteTick"),
9281            quote
9282        );
9283        drop(publications);
9284
9285        msgbus::get_message_bus().borrow_mut().dispose();
9286        assert!(closed.get());
9287        drop(node);
9288    }
9289
9290    #[rstest]
9291    #[tokio::test(flavor = "current_thread")]
9292    async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
9293        let quote = QuoteTick::default();
9294        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9295        let publications = Arc::new(Mutex::new(Vec::new()));
9296        let closed = Arc::new(AtomicBool::new(false));
9297        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
9298
9299        let msgbus_config = MessageBusConfig {
9300            external_streams: Some(vec!["stream".to_string()]),
9301            ..Default::default()
9302        };
9303
9304        let config = LiveNodeConfig {
9305            environment: Environment::Sandbox,
9306            msgbus: Some(msgbus_config),
9307            exec_engine: crate::config::LiveExecutionEngineConfig {
9308                reconciliation: false,
9309                ..Default::default()
9310            },
9311            delay_post_stop: Duration::ZERO,
9312            timeout_connection: Duration::from_millis(500),
9313            timeout_disconnection: Duration::from_millis(500),
9314            ..Default::default()
9315        };
9316
9317        let mut node = LiveNodeBuilder::from_config(config)
9318            .unwrap()
9319            .with_external_msgbus_factory(Box::new(factory))
9320            .build()
9321            .expect("node builds with external message bus factory");
9322
9323        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
9324        {
9325            let publications = publications.lock();
9326            assert_eq!(publications.len(), 1);
9327            assert_eq!(publications[0].topic, "data.quotes.TEST");
9328            assert_eq!(
9329                serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9330                    .expect("JSON payload must decode as QuoteTick"),
9331                quote
9332            );
9333        }
9334
9335        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9336        let handle = node.handle();
9337
9338        // Stopping from `drive` rather than here, so the node cannot finish `run` before the
9339        // republished quote is observed.
9340        let handler = TypedHandler::from({
9341            let received = received.clone();
9342            move |quote: &QuoteTick| {
9343                received.borrow_mut().push(*quote);
9344            }
9345        });
9346
9347        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9348        msgbus::get_message_bus()
9349            .borrow_mut()
9350            .add_streaming_type(BusPayloadType::QuoteTick);
9351
9352        let payload =
9353            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
9354        let message = BusMessage::with_str_topic(
9355            "data.quotes.TEST",
9356            BusPayloadType::QuoteTick,
9357            payload,
9358            SerializationEncoding::Json,
9359        );
9360
9361        tokio::time::timeout(Duration::from_secs(30), async {
9362            let run = node.run();
9363            tokio::pin!(run);
9364
9365            let drive = async {
9366                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9367
9368                tx.send(message)
9369                    .await
9370                    .expect("external ingress receiver should be open");
9371
9372                wait_until_async(
9373                    || async { received.borrow().len() == 1 },
9374                    Duration::from_secs(10),
9375                )
9376                .await;
9377                assert_eq!(*received.borrow(), vec![quote]);
9378                handle.stop();
9379            };
9380
9381            tokio::select! {
9382                biased;
9383
9384                () = drive => {}
9385                result = &mut run => {
9386                    panic!("node stopped before factory ingress was republished: {result:?}");
9387                }
9388            }
9389
9390            run.await.expect("node should stop cleanly");
9391        })
9392        .await
9393        .expect("live node should republish factory ingress and stop before timeout");
9394
9395        assert_eq!(handle.state(), NodeState::Stopped);
9396        assert!(closed.load(Ordering::Relaxed));
9397        msgbus::get_message_bus().borrow_mut().dispose();
9398    }
9399
9400    #[rstest]
9401    #[tokio::test(flavor = "current_thread")]
9402    async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
9403        let quote = QuoteTick::default();
9404        let publications = Arc::new(Mutex::new(Vec::new()));
9405        let closed = Arc::new(AtomicBool::new(false));
9406        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
9407
9408        let config = LiveNodeConfig {
9409            environment: Environment::Sandbox,
9410            msgbus: Some(MessageBusConfig::default()),
9411            exec_engine: crate::config::LiveExecutionEngineConfig {
9412                reconciliation: false,
9413                ..Default::default()
9414            },
9415            delay_post_stop: Duration::ZERO,
9416            timeout_connection: Duration::from_millis(500),
9417            timeout_disconnection: Duration::from_millis(500),
9418            ..Default::default()
9419        };
9420
9421        let mut node = LiveNodeBuilder::from_config(config)
9422            .unwrap()
9423            .with_external_msgbus_factory(Box::new(factory))
9424            .build()
9425            .expect("node builds with egress-only message bus factory");
9426        let handle = node.handle();
9427
9428        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
9429        {
9430            let publications = publications.lock();
9431            assert_eq!(publications.len(), 1);
9432            assert_eq!(publications[0].topic, "data.quotes.TEST");
9433        }
9434
9435        tokio::time::timeout(Duration::from_secs(30), async {
9436            let run = node.run();
9437            tokio::pin!(run);
9438
9439            let drive = async {
9440                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9441                handle.stop();
9442            };
9443
9444            tokio::select! {
9445                biased;
9446
9447                () = drive => {}
9448                result = &mut run => {
9449                    panic!("node stopped before egress-only factory run was observed: {result:?}");
9450                }
9451            }
9452
9453            run.await.expect("node should stop cleanly");
9454        })
9455        .await
9456        .expect("live node should run without external ingress before timeout");
9457
9458        assert_eq!(handle.state(), NodeState::Stopped);
9459        msgbus::get_message_bus().borrow_mut().dispose();
9460        assert!(closed.load(Ordering::Relaxed));
9461    }
9462
9463    #[rstest]
9464    fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
9465        let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
9466
9467        let egress_factory = CapturingBackingFactory::new(
9468            Arc::new(Mutex::new(Vec::new())),
9469            Arc::new(AtomicBool::new(false)),
9470            None,
9471        );
9472        let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9473            .unwrap()
9474            .with_external_msgbus_factory(Box::new(egress_factory))
9475            .with_external_msgbus_egress(Box::new(external_egress))
9476            .build()
9477            .expect_err("builder should reject factory plus injected egress");
9478
9479        assert!(
9480            egress_error
9481                .to_string()
9482                .contains("cannot be combined with injected egress or ingress")
9483        );
9484
9485        let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9486
9487        let ingress_factory = CapturingBackingFactory::new(
9488            Arc::new(Mutex::new(Vec::new())),
9489            Arc::new(AtomicBool::new(false)),
9490            None,
9491        );
9492        let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
9493        let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9494            .unwrap()
9495            .with_external_msgbus_factory(Box::new(ingress_factory))
9496            .with_external_ingress(Box::new(ingress))
9497            .build()
9498            .expect_err("builder should reject factory plus injected ingress");
9499
9500        assert!(
9501            ingress_error
9502                .to_string()
9503                .contains("cannot be combined with injected egress or ingress")
9504        );
9505    }
9506
9507    #[rstest]
9508    #[tokio::test(flavor = "current_thread")]
9509    async fn test_run_republishes_external_ingress_on_local_msgbus() {
9510        let quote = QuoteTick::default();
9511        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9512        let payload =
9513            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
9514        let message = BusMessage::with_str_topic(
9515            "data.quotes.TEST",
9516            BusPayloadType::QuoteTick,
9517            payload,
9518            SerializationEncoding::Json,
9519        );
9520        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9521        let closed = Rc::new(Cell::new(false));
9522        let ingress = CapturingExternalIngress::new(rx, closed.clone());
9523
9524        let config = LiveNodeConfig {
9525            environment: Environment::Sandbox,
9526            exec_engine: crate::config::LiveExecutionEngineConfig {
9527                reconciliation: false,
9528                ..Default::default()
9529            },
9530            delay_post_stop: Duration::ZERO,
9531            timeout_connection: Duration::from_millis(500),
9532            timeout_disconnection: Duration::from_millis(500),
9533            ..Default::default()
9534        };
9535
9536        let mut node = LiveNodeBuilder::from_config(config)
9537            .unwrap()
9538            .with_external_ingress(Box::new(ingress))
9539            .build()
9540            .expect("node builds with external message bus ingress");
9541        let handle = node.handle();
9542
9543        let handler = TypedHandler::from({
9544            let received = received.clone();
9545            move |quote: &QuoteTick| {
9546                received.borrow_mut().push(*quote);
9547            }
9548        });
9549
9550        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9551        msgbus::get_message_bus()
9552            .borrow_mut()
9553            .add_streaming_type(BusPayloadType::QuoteTick);
9554
9555        tokio::time::timeout(Duration::from_secs(30), async {
9556            let run = node.run();
9557            tokio::pin!(run);
9558
9559            let drive = async {
9560                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9561
9562                tx.send(message)
9563                    .await
9564                    .expect("external ingress receiver should be open");
9565
9566                wait_until_async(
9567                    || async { received.borrow().len() == 1 },
9568                    Duration::from_secs(10),
9569                )
9570                .await;
9571                assert_eq!(*received.borrow(), vec![quote]);
9572                handle.stop();
9573            };
9574
9575            tokio::select! {
9576                biased;
9577
9578                () = drive => {}
9579                result = &mut run => {
9580                    panic!("node stopped before external message was republished: {result:?}");
9581                }
9582            }
9583
9584            run.await.expect("node should stop cleanly");
9585        })
9586        .await
9587        .expect("live node should republish ingress and stop before timeout");
9588
9589        assert_eq!(handle.state(), NodeState::Stopped);
9590        assert!(closed.get());
9591        msgbus::get_message_bus().borrow_mut().dispose();
9592    }
9593
9594    #[rstest]
9595    #[tokio::test(flavor = "current_thread")]
9596    async fn test_run_closes_external_ingress_when_receiver_closes() {
9597        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9598        let closed = Rc::new(Cell::new(false));
9599        let ingress = CapturingExternalIngress::new(rx, closed.clone());
9600
9601        let config = LiveNodeConfig {
9602            environment: Environment::Sandbox,
9603            exec_engine: crate::config::LiveExecutionEngineConfig {
9604                reconciliation: false,
9605                ..Default::default()
9606            },
9607            delay_post_stop: Duration::ZERO,
9608            timeout_connection: Duration::from_millis(500),
9609            timeout_disconnection: Duration::from_millis(500),
9610            ..Default::default()
9611        };
9612
9613        let mut node = LiveNodeBuilder::from_config(config)
9614            .unwrap()
9615            .with_external_ingress(Box::new(ingress))
9616            .build()
9617            .expect("node builds with external message bus ingress");
9618        let handle = node.handle();
9619
9620        tokio::time::timeout(Duration::from_secs(30), async {
9621            let run = node.run();
9622            tokio::pin!(run);
9623
9624            let drive = async {
9625                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9626
9627                drop(tx);
9628
9629                wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
9630                assert!(
9631                    handle.is_running(),
9632                    "node should keep running after ingress closes"
9633                );
9634                handle.stop();
9635            };
9636
9637            tokio::select! {
9638                biased;
9639
9640                () = drive => {}
9641                result = &mut run => {
9642                    panic!("node stopped before ingress close was observed: {result:?}");
9643                }
9644            }
9645
9646            run.await.expect("node should stop cleanly");
9647        })
9648        .await
9649        .expect("live node should close ingress and stop before timeout");
9650
9651        assert_eq!(handle.state(), NodeState::Stopped);
9652    }
9653
9654    #[rstest]
9655    #[tokio::test(flavor = "current_thread")]
9656    async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
9657        let closed = Rc::new(Cell::new(false));
9658        let ingress = FailingExternalIngress::new(closed.clone());
9659
9660        let config = LiveNodeConfig {
9661            environment: Environment::Sandbox,
9662            exec_engine: crate::config::LiveExecutionEngineConfig {
9663                reconciliation: false,
9664                ..Default::default()
9665            },
9666            delay_post_stop: Duration::ZERO,
9667            timeout_connection: Duration::from_millis(500),
9668            timeout_disconnection: Duration::from_millis(500),
9669            ..Default::default()
9670        };
9671
9672        let mut node = LiveNodeBuilder::from_config(config)
9673            .unwrap()
9674            .with_external_ingress(Box::new(ingress))
9675            .build()
9676            .expect("node builds with external message bus ingress");
9677        let handle = node.handle();
9678
9679        let err = node.run().await.expect_err("run should fail");
9680
9681        assert!(
9682            err.to_string()
9683                .contains("external ingress receiver unavailable")
9684        );
9685        assert_eq!(handle.state(), NodeState::Stopped);
9686        assert!(closed.get());
9687    }
9688
9689    #[rstest]
9690    fn test_node_build_rejects_existing_node_preserving_account_delivery(
9691        #[values(false, true)] first_builder: bool,
9692        #[values(false, true)] second_builder: bool,
9693    ) {
9694        let config = LiveNodeConfig {
9695            trader_id: TraderId::from("PROBE-001"),
9696            ..Default::default()
9697        };
9698
9699        let mut node = if first_builder {
9700            LiveNodeBuilder::from_config(config)
9701                .unwrap()
9702                .build()
9703                .unwrap()
9704        } else {
9705            LiveNode::build("Original".to_string(), Some(config)).unwrap()
9706        };
9707
9708        let bus = msgbus::get_message_bus();
9709
9710        let ExecutionEvent::Account(before) = stub_account_event() else {
9711            unreachable!()
9712        };
9713
9714        get_exec_event_sender()
9715            .send(ExecutionEvent::Account(before.clone()))
9716            .unwrap();
9717        assert_eq!(node.drain_runner_pending(), 1);
9718        assert_eq!(
9719            node.kernel
9720                .cache
9721                .borrow()
9722                .account(&before.account_id)
9723                .unwrap()
9724                .last_event(),
9725            Some(before)
9726        );
9727
9728        let result = if second_builder {
9729            LiveNodeBuilder::from_config(LiveNodeConfig::default())
9730                .unwrap()
9731                .build()
9732        } else {
9733            LiveNode::build("Extra".to_string(), None)
9734        };
9735
9736        assert_eq!(
9737            result.unwrap_err().to_string(),
9738            "A LiveNode already exists or is being built on this thread; drop it before building another"
9739        );
9740
9741        let ExecutionEvent::Account(after) = stub_account_event() else {
9742            unreachable!()
9743        };
9744
9745        get_exec_event_sender()
9746            .send(ExecutionEvent::Account(after.clone()))
9747            .unwrap();
9748        assert_eq!(node.drain_runner_pending(), 1);
9749        assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
9750        assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
9751        assert_eq!(
9752            node.kernel
9753                .cache
9754                .borrow()
9755                .account(&after.account_id)
9756                .unwrap()
9757                .last_event(),
9758            Some(after)
9759        );
9760    }
9761
9762    #[rstest]
9763    fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
9764        let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
9765            .unwrap()
9766            .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
9767                let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
9768                assert!(
9769                    error
9770                        .to_string()
9771                        .contains("already exists or is being built")
9772                );
9773                anyhow::bail!("Event store construction failed")
9774            })
9775            .build();
9776
9777        assert_eq!(
9778            result.unwrap_err().to_string(),
9779            "Event store construction failed"
9780        );
9781
9782        let node = LiveNode::build("Retry".to_string(), None).unwrap();
9783        assert_eq!(node.state(), NodeState::Idle);
9784    }
9785
9786    #[rstest]
9787    fn test_node_build_allows_separate_threads() {
9788        let node = LiveNode::build("First".to_string(), None).unwrap();
9789
9790        let other_state = std::thread::spawn(|| {
9791            let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
9792            other.state()
9793        })
9794        .join()
9795        .unwrap();
9796
9797        assert_eq!(node.state(), NodeState::Idle);
9798        assert_eq!(other_state, NodeState::Idle);
9799    }
9800
9801    #[rstest]
9802    fn test_node_build_releases_thread_after_drop() {
9803        let mut node = LiveNode::build("First".to_string(), None).unwrap();
9804        node.dispose();
9805        assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
9806        drop(node);
9807
9808        let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
9809        assert_eq!(replacement.state(), NodeState::Idle);
9810    }
9811
9812    #[rstest]
9813    fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
9814        let config = LiveNodeConfig {
9815            event_store: Some(EventStoreConfig::default()),
9816            ..Default::default()
9817        };
9818
9819        let result = if builder {
9820            LiveNodeBuilder::from_config(config).unwrap().build()
9821        } else {
9822            LiveNode::build("Failure".to_string(), Some(config))
9823        };
9824
9825        assert!(result.unwrap_err().to_string().contains("factory"));
9826
9827        let node = LiveNode::build("Retry".to_string(), None).unwrap();
9828        assert_eq!(node.state(), NodeState::Idle);
9829    }
9830
9831    #[cfg(feature = "python")]
9832    #[rstest]
9833    fn test_node_build_and_initial_state() {
9834        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9835            .unwrap()
9836            .with_name("TestNode")
9837            .build()
9838            .unwrap();
9839
9840        assert_eq!(node.state(), NodeState::Idle);
9841        assert!(!node.is_running());
9842        assert_eq!(node.environment(), Environment::Sandbox);
9843        assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
9844    }
9845
9846    #[cfg(feature = "python")]
9847    #[rstest]
9848    fn test_node_build_replaces_stale_runner_senders() {
9849        replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
9850        replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
9851
9852        let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9853            .unwrap()
9854            .with_name("FirstNode")
9855            .build()
9856            .unwrap();
9857
9858        assert_eq!(first.state(), NodeState::Idle);
9859        drop(first);
9860
9861        let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9862            .unwrap()
9863            .with_name("SecondNode")
9864            .build()
9865            .unwrap();
9866
9867        assert_eq!(second.state(), NodeState::Idle);
9868        assert!(!second.is_running());
9869    }
9870
9871    #[cfg(feature = "python")]
9872    #[rstest]
9873    fn test_node_handle_reflects_node_state() {
9874        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9875            .unwrap()
9876            .with_name("TestNode")
9877            .build()
9878            .unwrap();
9879
9880        let handle = node.handle();
9881
9882        assert_eq!(handle.state(), NodeState::Idle);
9883        assert!(!handle.is_running());
9884    }
9885
9886    #[rstest]
9887    fn test_pending_drain_data_returns_false_when_empty() {
9888        let mut pending = PendingEvents::default();
9889
9890        assert!(!pending.drain_data());
9891    }
9892
9893    #[rstest]
9894    fn test_pending_drain_data_returns_true_when_non_empty() {
9895        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
9896
9897        let mut pending = PendingEvents::default();
9898        pending.data_evts.push(
9899            DataEvent::Instrument(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()))
9900                .into(),
9901        );
9902
9903        assert!(pending.drain_data());
9904        assert!(pending.data_evts.is_empty());
9905    }
9906
9907    fn stub_data_event() -> DataEvent {
9908        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
9909
9910        DataEvent::Instrument(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()))
9911    }
9912
9913    fn stub_data_command() -> DataCommand {
9914        use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
9915        use nautilus_core::{UUID4, UnixNanos};
9916        use nautilus_model::identifiers::Venue;
9917
9918        DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
9919            None,
9920            Venue::from("TEST"),
9921            UUID4::new(),
9922            UnixNanos::default(),
9923            None,
9924            None,
9925        )))
9926    }
9927
9928    fn stub_system_command() -> SystemCommand {
9929        SystemCommand::ReconnectSocket(ReconnectSocket::new(
9930            TraderId::from("TRADER-001"),
9931            ClientId::from("POLYMARKET"),
9932            Ustr::from("polymarket-market-streams"),
9933            UnixNanos::default(),
9934        ))
9935    }
9936
9937    #[rstest]
9938    fn test_flush_pending_data_drains_events_and_commands() {
9939        let (evt_tx, mut evt_rx) =
9940            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
9941        let (cmd_tx, mut cmd_rx) =
9942            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
9943
9944        let mut pending = PendingEvents::default();
9945
9946        // Pre-load pending (items captured by the select loop)
9947        pending.data_evts.push((stub_data_event()).into());
9948        pending.data_cmds.push(stub_data_command().into());
9949
9950        // Pre-load channels (items missed by the select loop)
9951        evt_tx.send((stub_data_event()).into()).unwrap();
9952        cmd_tx.send(stub_data_command().into()).unwrap();
9953
9954        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
9955
9956        assert!(pending.data_evts.is_empty());
9957        assert!(pending.data_cmds.is_empty());
9958        assert!(evt_rx.try_recv().is_err());
9959        assert!(cmd_rx.try_recv().is_err());
9960    }
9961
9962    #[rstest]
9963    fn test_flush_pending_data_drains_mixed_sources() {
9964        let (evt_tx, mut evt_rx) =
9965            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
9966        let (cmd_tx, mut cmd_rx) =
9967            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
9968
9969        let mut pending = PendingEvents::default();
9970
9971        // First pass: pending has an event, channel has a command
9972        pending.data_evts.push((stub_data_event()).into());
9973        cmd_tx.send(stub_data_command().into()).unwrap();
9974
9975        // Second pass: channel has items that simulate arrival during first drain
9976        evt_tx.send((stub_data_event()).into()).unwrap();
9977        evt_tx.send((stub_data_event()).into()).unwrap();
9978        cmd_tx.send(stub_data_command().into()).unwrap();
9979
9980        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
9981
9982        assert!(pending.data_evts.is_empty());
9983        assert!(pending.data_cmds.is_empty());
9984        assert!(evt_rx.try_recv().is_err());
9985        assert!(cmd_rx.try_recv().is_err());
9986    }
9987
9988    #[rstest]
9989    fn test_pending_system_events_stay_separate_from_data() {
9990        let mut pending = PendingEvents::default();
9991
9992        let change = SocketStateChange::new(
9993            ClientId::from("BINANCE"),
9994            Some(Venue::from("BINANCE")),
9995            ustr::Ustr::from("binance-futures-market-streams"),
9996            SocketState::Connected,
9997        );
9998
9999        pending
10000            .system_events
10001            .push(SystemEvent::SocketState(change).into());
10002        let system_events = pending
10003            .take_system_events()
10004            .into_iter()
10005            .map(|message| message.dispatch(|value| value))
10006            .collect::<Vec<_>>();
10007
10008        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10009        assert!(pending.is_empty());
10010    }
10011
10012    #[rstest]
10013    fn test_pending_system_commands_stay_separate_from_data() {
10014        let mut pending = PendingEvents::default();
10015        let command = stub_system_command();
10016
10017        pending.system_commands.push(command.into());
10018        let system_commands = pending
10019            .take_system_commands()
10020            .into_iter()
10021            .map(|message| message.dispatch(|value| value))
10022            .collect::<Vec<_>>();
10023
10024        assert_eq!(system_commands, vec![command]);
10025        assert!(pending.is_empty());
10026    }
10027
10028    fn stub_time_event_handler() -> TimeEventMessage {
10029        use std::rc::Rc;
10030
10031        use nautilus_common::{
10032            runner::TimeEventMessage,
10033            timer::{TimeEvent, TimeEventCallback},
10034        };
10035        use nautilus_core::{UUID4, UnixNanos};
10036        use ustr::Ustr;
10037
10038        TimeEventMessage::new(
10039            TimeEvent::new(
10040                Ustr::from("test-timer"),
10041                UUID4::new(),
10042                UnixNanos::default(),
10043                UnixNanos::default(),
10044            ),
10045            TimeEventCallback::RustLocal(Rc::new(|_| {})),
10046        )
10047    }
10048
10049    fn stub_trading_command_message() -> TradingCommandMessage {
10050        use nautilus_common::messages::execution::query::QueryAccount;
10051        use nautilus_core::{UUID4, UnixNanos};
10052        use nautilus_model::identifiers::AccountId;
10053
10054        TradingCommandMessage::new(
10055            MessagingSwitchboard::exec_engine_execute(),
10056            TradingCommand::QueryAccount(QueryAccount::new(
10057                TraderId::from("TESTER-001"),
10058                None,
10059                AccountId::from("TEST-001"),
10060                UUID4::new(),
10061                UnixNanos::default(),
10062                None,
10063                None, // correlation_id
10064            )),
10065        )
10066    }
10067
10068    fn stub_exec_event() -> ExecutionEvent {
10069        use nautilus_model::{
10070            enums::{LiquiditySide, OrderSide},
10071            identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
10072            reports::FillReport,
10073            types::{Money, Price, Quantity},
10074        };
10075
10076        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
10077            AccountId::from("TEST-001"),
10078            InstrumentId::from("TEST.VENUE"),
10079            VenueOrderId::from("V-001"),
10080            TradeId::from("T-001"),
10081            OrderSide::Buy,
10082            Quantity::from("1.0"),
10083            Price::from("100.0"),
10084            Money::from("0.01 USD"),
10085            LiquiditySide::Maker,
10086            None,
10087            None,
10088            nautilus_core::UnixNanos::default(),
10089            nautilus_core::UnixNanos::default(),
10090            None,
10091        ))))
10092    }
10093
10094    #[rstest]
10095    fn test_flush_all_pending_drains_buffered_channels() {
10096        let (time_tx, mut time_rx) =
10097            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10098        let (system_evt_tx, mut system_evt_rx) =
10099            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10100        let (system_cmd_tx, mut system_cmd_rx) =
10101            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10102        let (data_evt_tx, mut data_evt_rx) =
10103            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10104        let (data_cmd_tx, mut data_cmd_rx) =
10105            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10106        let (exec_evt_tx, mut exec_evt_rx) =
10107            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10108        let (exec_cmd_tx, mut exec_cmd_rx) =
10109            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10110
10111        let mut pending = PendingEvents::default();
10112
10113        // Pre-load pending with data items
10114        pending.data_evts.push((stub_data_event()).into());
10115        pending.data_cmds.push(stub_data_command().into());
10116
10117        // Pre-load all channel types
10118        time_tx.send((stub_time_event_handler()).into()).unwrap();
10119
10120        let change = SocketStateChange::new(
10121            ClientId::from("BINANCE"),
10122            Some(Venue::from("BINANCE")),
10123            Ustr::from("binance-futures-market-streams"),
10124            SocketState::Connected,
10125        );
10126        system_evt_tx
10127            .send((SystemEvent::SocketState(change)).into())
10128            .unwrap();
10129        system_cmd_tx.send((stub_system_command()).into()).unwrap();
10130        data_evt_tx.send((stub_data_event()).into()).unwrap();
10131        data_cmd_tx.send(stub_data_command().into()).unwrap();
10132        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10133        exec_cmd_tx
10134            .send(stub_trading_command_message().into())
10135            .unwrap();
10136
10137        flush_all_pending(
10138            &mut pending,
10139            &mut time_rx,
10140            &mut system_evt_rx,
10141            &mut system_cmd_rx,
10142            &mut exec_evt_rx,
10143            &mut exec_cmd_rx,
10144            &mut data_evt_rx,
10145            &mut data_cmd_rx,
10146        );
10147
10148        let system_events = pending
10149            .take_system_events()
10150            .into_iter()
10151            .map(|message| message.dispatch(|value| value))
10152            .collect::<Vec<_>>();
10153        let system_commands = pending
10154            .take_system_commands()
10155            .into_iter()
10156            .map(|message| message.dispatch(|value| value))
10157            .collect::<Vec<_>>();
10158        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10159        assert_eq!(system_commands, vec![stub_system_command()]);
10160        assert!(pending.data_evts.is_empty());
10161        assert!(pending.data_cmds.is_empty());
10162        assert!(pending.exec_reports.is_empty());
10163        assert!(pending.exec_cmds.is_empty());
10164        assert!(pending.order_evts.is_empty());
10165        assert!(time_rx.try_recv().is_err());
10166        assert!(system_evt_rx.try_recv().is_err());
10167        assert!(system_cmd_rx.try_recv().is_err());
10168        assert!(data_evt_rx.try_recv().is_err());
10169        assert!(data_cmd_rx.try_recv().is_err());
10170        assert!(exec_evt_rx.try_recv().is_err());
10171        assert!(exec_cmd_rx.try_recv().is_err());
10172    }
10173
10174    fn stub_order_event() -> ExecutionEvent {
10175        use nautilus_model::events::order::spec::OrderSubmittedSpec;
10176
10177        ExecutionEvent::Order(OrderEventAny::Submitted(
10178            OrderSubmittedSpec::builder().build(),
10179        ))
10180    }
10181
10182    fn stub_account_event() -> ExecutionEvent {
10183        use nautilus_core::{UUID4, UnixNanos};
10184        use nautilus_model::{
10185            enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
10186        };
10187
10188        ExecutionEvent::Account(AccountState::new(
10189            AccountId::from("TEST-001"),
10190            AccountType::Cash,
10191            vec![],
10192            vec![],
10193            true,
10194            UUID4::new(),
10195            UnixNanos::default(),
10196            UnixNanos::default(),
10197            None,
10198        ))
10199    }
10200
10201    #[rstest]
10202    fn test_flush_all_pending_routes_order_event_to_order_evts() {
10203        let (_time_tx, mut time_rx) =
10204            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10205        let (_system_evt_tx, mut system_evt_rx) =
10206            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10207        let (_system_cmd_tx, mut system_cmd_rx) =
10208            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10209        let (_data_evt_tx, mut data_evt_rx) =
10210            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10211        let (_data_cmd_tx, mut data_cmd_rx) =
10212            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10213        let (exec_evt_tx, mut exec_evt_rx) =
10214            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10215        let (_exec_cmd_tx, mut exec_cmd_rx) =
10216            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10217
10218        let mut pending = PendingEvents::default();
10219
10220        exec_evt_tx.send((stub_order_event()).into()).unwrap();
10221        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10222
10223        flush_all_pending(
10224            &mut pending,
10225            &mut time_rx,
10226            &mut system_evt_rx,
10227            &mut system_cmd_rx,
10228            &mut exec_evt_rx,
10229            &mut exec_cmd_rx,
10230            &mut data_evt_rx,
10231            &mut data_cmd_rx,
10232        );
10233
10234        // Both order and report events are drained by pending.drain()
10235        assert!(pending.order_evts.is_empty());
10236        assert!(pending.exec_reports.is_empty());
10237        assert!(exec_evt_rx.try_recv().is_err());
10238    }
10239
10240    #[rstest]
10241    fn test_flush_all_pending_routes_account_event_immediately() {
10242        let (_time_tx, mut time_rx) =
10243            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10244        let (_system_evt_tx, mut system_evt_rx) =
10245            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10246        let (_system_cmd_tx, mut system_cmd_rx) =
10247            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10248        let (_data_evt_tx, mut data_evt_rx) =
10249            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10250        let (_data_cmd_tx, mut data_cmd_rx) =
10251            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10252        let (exec_evt_tx, mut exec_evt_rx) =
10253            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10254        let (_exec_cmd_tx, mut exec_cmd_rx) =
10255            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10256
10257        let mut pending = PendingEvents::default();
10258
10259        exec_evt_tx.send((stub_account_event()).into()).unwrap();
10260
10261        flush_all_pending(
10262            &mut pending,
10263            &mut time_rx,
10264            &mut system_evt_rx,
10265            &mut system_cmd_rx,
10266            &mut exec_evt_rx,
10267            &mut exec_cmd_rx,
10268            &mut data_evt_rx,
10269            &mut data_cmd_rx,
10270        );
10271
10272        // Account events are forwarded immediately, never buffered in pending
10273        assert!(pending.exec_reports.is_empty());
10274        assert!(pending.order_evts.is_empty());
10275        assert!(pending.exec_cmds.is_empty());
10276        assert!(exec_evt_rx.try_recv().is_err());
10277    }
10278
10279    #[rstest]
10280    fn test_pending_is_empty_when_default() {
10281        let pending = PendingEvents::default();
10282
10283        assert!(pending.is_empty());
10284    }
10285
10286    #[rstest]
10287    fn test_pending_is_empty_false_with_data_evt() {
10288        let mut pending = PendingEvents::default();
10289        pending.data_evts.push((stub_data_event()).into());
10290
10291        assert!(!pending.is_empty());
10292    }
10293
10294    #[rstest]
10295    fn test_pending_is_empty_false_with_data_cmd() {
10296        let mut pending = PendingEvents::default();
10297        pending.data_cmds.push(stub_data_command().into());
10298
10299        assert!(!pending.is_empty());
10300    }
10301
10302    #[rstest]
10303    fn test_pending_is_empty_false_with_exec_cmd() {
10304        let mut pending = PendingEvents::default();
10305        pending
10306            .exec_cmds
10307            .push(stub_trading_command_message().into());
10308
10309        assert!(!pending.is_empty());
10310    }
10311
10312    #[rstest]
10313    fn test_pending_drain_preserves_trading_command_target() {
10314        std::thread::spawn(|| {
10315            msgbus::get_message_bus().borrow_mut().dispose();
10316            let risk_commands = Rc::new(RefCell::new(Vec::new()));
10317            let exec_commands = Rc::new(RefCell::new(Vec::new()));
10318
10319            let risk_commands_handler = risk_commands.clone();
10320            msgbus::register_trading_command_endpoint(
10321                MessagingSwitchboard::risk_engine_execute(),
10322                TypedIntoHandler::from(move |command: TradingCommand| {
10323                    risk_commands_handler.borrow_mut().push(command);
10324                }),
10325            );
10326
10327            let exec_commands_handler = exec_commands.clone();
10328            msgbus::register_trading_command_endpoint(
10329                MessagingSwitchboard::exec_engine_execute(),
10330                TypedIntoHandler::from(move |command: TradingCommand| {
10331                    exec_commands_handler.borrow_mut().push(command);
10332                }),
10333            );
10334
10335            let mut pending = PendingEvents::default();
10336            pending.exec_cmds.push(
10337                TradingCommandMessage::new(
10338                    MessagingSwitchboard::risk_engine_execute(),
10339                    TradingCommand::QueryAccount(QueryAccount::new(
10340                        TraderId::from("TESTER-001"),
10341                        None,
10342                        AccountId::from("TEST-001"),
10343                        UUID4::new(),
10344                        UnixNanos::default(),
10345                        None,
10346                        None,
10347                    )),
10348                )
10349                .into(),
10350            );
10351
10352            pending.drain();
10353
10354            assert!(pending.is_empty());
10355            assert_eq!(risk_commands.borrow().len(), 1);
10356            assert!(matches!(
10357                &risk_commands.borrow()[0],
10358                TradingCommand::QueryAccount(_)
10359            ));
10360            assert_eq!(exec_commands.borrow().as_slice(), &[]);
10361        })
10362        .join()
10363        .unwrap();
10364    }
10365
10366    #[rstest]
10367    fn test_pending_is_empty_false_with_exec_report() {
10368        let mut pending = PendingEvents::default();
10369
10370        if let ExecutionEvent::Report(report) = stub_exec_event() {
10371            pending.exec_reports.push((report).into());
10372        }
10373
10374        assert!(!pending.is_empty());
10375    }
10376
10377    #[rstest]
10378    fn test_pending_is_empty_false_with_order_evt() {
10379        let mut pending = PendingEvents::default();
10380
10381        if let ExecutionEvent::Order(order_evt) = stub_order_event() {
10382            pending.order_evts.push((order_evt).into());
10383        }
10384
10385        assert!(!pending.is_empty());
10386    }
10387
10388    fn stub_submitted_batch_event() -> ExecutionEvent {
10389        use nautilus_model::{
10390            events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
10391            identifiers::ClientOrderId,
10392        };
10393
10394        let events = vec![
10395            OrderSubmittedSpec::builder()
10396                .client_order_id(ClientOrderId::from("O-001"))
10397                .build(),
10398            OrderSubmittedSpec::builder()
10399                .client_order_id(ClientOrderId::from("O-002"))
10400                .build(),
10401        ];
10402
10403        ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
10404    }
10405
10406    fn stub_canceled_batch_event() -> ExecutionEvent {
10407        use nautilus_model::{
10408            events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
10409            identifiers::ClientOrderId,
10410        };
10411
10412        let events = vec![
10413            OrderCanceledSpec::builder()
10414                .client_order_id(ClientOrderId::from("O-001"))
10415                .build(),
10416            OrderCanceledSpec::builder()
10417                .client_order_id(ClientOrderId::from("O-002"))
10418                .build(),
10419        ];
10420
10421        ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
10422    }
10423
10424    #[rstest]
10425    fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
10426        let (_time_tx, mut time_rx) =
10427            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10428        let (_system_evt_tx, mut system_evt_rx) =
10429            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10430        let (_system_cmd_tx, mut system_cmd_rx) =
10431            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10432        let (_data_evt_tx, mut data_evt_rx) =
10433            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10434        let (_data_cmd_tx, mut data_cmd_rx) =
10435            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10436        let (exec_evt_tx, mut exec_evt_rx) =
10437            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10438        let (_exec_cmd_tx, mut exec_cmd_rx) =
10439            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10440
10441        let mut pending = PendingEvents::default();
10442
10443        exec_evt_tx
10444            .send((stub_submitted_batch_event()).into())
10445            .unwrap();
10446
10447        flush_all_pending(
10448            &mut pending,
10449            &mut time_rx,
10450            &mut system_evt_rx,
10451            &mut system_cmd_rx,
10452            &mut exec_evt_rx,
10453            &mut exec_cmd_rx,
10454            &mut data_evt_rx,
10455            &mut data_cmd_rx,
10456        );
10457
10458        // Batch should be unpacked into individual Submitted events then drained
10459        assert!(pending.order_evts.is_empty());
10460        assert!(exec_evt_rx.try_recv().is_err());
10461    }
10462
10463    #[rstest]
10464    fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
10465        let (_time_tx, mut time_rx) =
10466            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10467        let (_system_evt_tx, mut system_evt_rx) =
10468            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10469        let (_system_cmd_tx, mut system_cmd_rx) =
10470            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10471        let (_data_evt_tx, mut data_evt_rx) =
10472            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10473        let (_data_cmd_tx, mut data_cmd_rx) =
10474            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10475        let (exec_evt_tx, mut exec_evt_rx) =
10476            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10477        let (_exec_cmd_tx, mut exec_cmd_rx) =
10478            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10479
10480        let mut pending = PendingEvents::default();
10481
10482        exec_evt_tx
10483            .send((stub_canceled_batch_event()).into())
10484            .unwrap();
10485
10486        flush_all_pending(
10487            &mut pending,
10488            &mut time_rx,
10489            &mut system_evt_rx,
10490            &mut system_cmd_rx,
10491            &mut exec_evt_rx,
10492            &mut exec_cmd_rx,
10493            &mut data_evt_rx,
10494            &mut data_cmd_rx,
10495        );
10496
10497        // Batch should be unpacked into individual Canceled events then drained
10498        assert!(pending.order_evts.is_empty());
10499        assert!(exec_evt_rx.try_recv().is_err());
10500    }
10501
10502    #[rstest]
10503    fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
10504        use nautilus_model::identifiers::ClientOrderId;
10505
10506        let (exec_evt_tx, mut exec_evt_rx) =
10507            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10508
10509        exec_evt_tx
10510            .send((stub_canceled_batch_event()).into())
10511            .unwrap();
10512
10513        let mut pending = PendingEvents::default();
10514
10515        while let Ok(evt) = exec_evt_rx.try_recv() {
10516            pending.push_exec_event(evt);
10517        }
10518
10519        let events: Vec<_> = pending
10520            .order_evts
10521            .drain(..)
10522            .map(|event| event.dispatch(|event| event))
10523            .collect();
10524        assert_eq!(events.len(), 2);
10525        assert!(
10526            matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
10527        );
10528        assert!(
10529            matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
10530        );
10531    }
10532
10533    #[rstest]
10534    #[cfg_attr(
10535        not(all(feature = "simulation", madsim)),
10536        tokio::test(start_paused = true)
10537    )]
10538    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10539    async fn test_submission_confirmation_prevents_duplicate_tracking(
10540        #[values(false, true)] report_confirmation: bool,
10541        #[values(false, true)] exhausted: bool,
10542    ) {
10543        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
10544        let client_order_id = order.client_order_id();
10545        let account_id = AccountId::from("TEST-001");
10546        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10547        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10548            OrderPendingUpdateSpec::builder()
10549                .trader_id(order.trader_id())
10550                .strategy_id(order.strategy_id())
10551                .instrument_id(order.instrument_id())
10552                .client_order_id(client_order_id)
10553                .account_id(account_id)
10554                .build(),
10555        )));
10556        advance_clock(Duration::from_millis(101)).await;
10557        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10558
10559        if exhausted {
10560            advance_clock(Duration::from_millis(101)).await;
10561            let result = node.exec_manager.check_inflight_orders();
10562            assert!(result.events.is_empty());
10563            assert!(result.queries.is_empty());
10564            assert_eq!(
10565                node.exec_manager
10566                    .take_submission_recovery_exhaustions()
10567                    .len(),
10568                1
10569            );
10570        }
10571
10572        if report_confirmation {
10573            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
10574                OrderStatusReport::new(
10575                    account_id,
10576                    order.instrument_id(),
10577                    Some(client_order_id),
10578                    venue_order_id,
10579                    Some(order.order_side()),
10580                    order.order_type(),
10581                    order.time_in_force(),
10582                    OrderStatus::Submitted,
10583                    order.quantity(),
10584                    Quantity::zero(3),
10585                    UnixNanos::default(),
10586                    UnixNanos::default(),
10587                    UnixNanos::default(),
10588                    None,
10589                )
10590                .with_price(order.price().unwrap()),
10591            ))));
10592            node.exec_manager
10593                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
10594            advance_clock(Duration::from_millis(101)).await;
10595            let result = node.exec_manager.check_inflight_orders();
10596            assert!(result.queries.is_empty());
10597            assert!(result.events.is_empty());
10598        }
10599
10600        let mut update = OrderUpdatedSpec::builder()
10601            .trader_id(order.trader_id())
10602            .strategy_id(order.strategy_id())
10603            .instrument_id(order.instrument_id())
10604            .client_order_id(client_order_id)
10605            .account_id(account_id)
10606            .quantity(order.quantity())
10607            .price(order.price().unwrap())
10608            .build();
10609
10610        if !report_confirmation {
10611            update.venue_order_id = Some(venue_order_id);
10612        }
10613        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
10614        assert_eq!(
10615            node.kernel
10616                .cache
10617                .borrow()
10618                .order(&client_order_id)
10619                .unwrap()
10620                .status(),
10621            OrderStatus::Submitted
10622        );
10623
10624        for _ in 0..3 {
10625            node.exec_manager
10626                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
10627            advance_clock(Duration::from_millis(101)).await;
10628            let result = node.exec_manager.check_inflight_orders();
10629            assert!(result.queries.is_empty());
10630            assert!(result.events.is_empty());
10631            assert!(
10632                node.exec_manager
10633                    .take_submission_recovery_exhaustions()
10634                    .is_empty()
10635            );
10636        }
10637    }
10638
10639    #[rstest]
10640    #[cfg_attr(
10641        not(all(feature = "simulation", madsim)),
10642        tokio::test(start_paused = true)
10643    )]
10644    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10645    async fn test_submission_report_confirmation_preserves_later_command_budget(
10646        #[values(false, true)] cancel: bool,
10647        #[values(false, true)] later_fill: bool,
10648        #[values(false, true)] repeated_report: bool,
10649        #[values(
10650            SubmissionRecoveryPolicy::ResolveLocally,
10651            SubmissionRecoveryPolicy::RetainUnresolved
10652        )]
10653        policy: SubmissionRecoveryPolicy,
10654    ) {
10655        let (mut node, order, instrument) = venue_evidence_node(policy);
10656        let client_order_id = order.client_order_id();
10657        let account_id = AccountId::from("TEST-001");
10658        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10659        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10660            OrderPendingUpdateSpec::builder()
10661                .trader_id(order.trader_id())
10662                .strategy_id(order.strategy_id())
10663                .instrument_id(order.instrument_id())
10664                .client_order_id(client_order_id)
10665                .account_id(account_id)
10666                .build(),
10667        )));
10668        node.exec_manager.register_inflight(client_order_id);
10669        advance_clock(Duration::from_millis(101)).await;
10670        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10671        let report = ExecutionReport::Order(Box::new(
10672            OrderStatusReport::new(
10673                account_id,
10674                order.instrument_id(),
10675                Some(client_order_id),
10676                venue_order_id,
10677                Some(order.order_side()),
10678                order.order_type(),
10679                order.time_in_force(),
10680                OrderStatus::Submitted,
10681                order.quantity(),
10682                Quantity::zero(3),
10683                UnixNanos::default(),
10684                UnixNanos::default(),
10685                UnixNanos::default(),
10686                None,
10687            )
10688            .with_price(order.price().unwrap()),
10689        ));
10690        node.process_exec_event(ExecutionEvent::Report(report.clone()));
10691        advance_clock(Duration::from_millis(101)).await;
10692        let result = node.exec_manager.check_inflight_orders();
10693        assert!(result.events.is_empty());
10694        assert!(result.queries.is_empty());
10695
10696        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
10697            OrderUpdatedSpec::builder()
10698                .trader_id(order.trader_id())
10699                .strategy_id(order.strategy_id())
10700                .instrument_id(order.instrument_id())
10701                .client_order_id(client_order_id)
10702                .account_id(account_id)
10703                .quantity(order.quantity())
10704                .price(order.price().unwrap())
10705                .build(),
10706        )));
10707        let pending = if cancel {
10708            OrderEventAny::PendingCancel(
10709                OrderPendingCancelSpec::builder()
10710                    .trader_id(order.trader_id())
10711                    .strategy_id(order.strategy_id())
10712                    .instrument_id(order.instrument_id())
10713                    .client_order_id(client_order_id)
10714                    .account_id(account_id)
10715                    .build(),
10716            )
10717        } else {
10718            OrderEventAny::PendingUpdate(
10719                OrderPendingUpdateSpec::builder()
10720                    .trader_id(order.trader_id())
10721                    .strategy_id(order.strategy_id())
10722                    .instrument_id(order.instrument_id())
10723                    .client_order_id(client_order_id)
10724                    .account_id(account_id)
10725                    .build(),
10726            )
10727        };
10728        node.process_exec_event(ExecutionEvent::Order(pending));
10729        node.exec_manager.register_inflight(client_order_id);
10730
10731        if repeated_report {
10732            node.process_exec_event(ExecutionEvent::Report(report));
10733        }
10734        advance_clock(Duration::from_millis(50)).await;
10735
10736        if later_fill {
10737            let cached = node
10738                .kernel
10739                .cache
10740                .borrow()
10741                .order_owned(&client_order_id)
10742                .unwrap();
10743            let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
10744                .last_qty(Quantity::from("0.400"))
10745                .last_px(order.price().unwrap())
10746                .commission(Money::from("0 USDT"))
10747                .without_position_id()
10748                .build()
10749            else {
10750                unreachable!()
10751            };
10752            fill.venue_order_id = venue_order_id;
10753            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
10754        }
10755        let early = node.exec_manager.check_inflight_orders();
10756        assert!(early.events.is_empty());
10757        assert!(early.queries.is_empty());
10758        advance_clock(Duration::from_millis(51)).await;
10759        let first = node.exec_manager.check_inflight_orders();
10760        assert_eq!(first.queries.len(), usize::from(!repeated_report));
10761        assert!(first.events.is_empty());
10762        advance_clock(Duration::from_millis(101)).await;
10763        let second = node.exec_manager.check_inflight_orders();
10764        assert!(second.queries.is_empty());
10765        if repeated_report {
10766            assert!(second.events.is_empty());
10767        } else {
10768            assert!(
10769                matches!(&second.events[..], [OrderEventAny::Canceled(event)] if event.client_order_id == client_order_id)
10770            );
10771        }
10772        assert!(
10773            node.exec_manager
10774                .take_submission_recovery_exhaustions()
10775                .is_empty()
10776        );
10777    }
10778
10779    #[rstest]
10780    #[cfg_attr(
10781        not(all(feature = "simulation", madsim)),
10782        tokio::test(start_paused = true)
10783    )]
10784    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10785    async fn test_confirmed_submission_command_dispatch_starts_own_budget(
10786        #[values(false, true)] report_confirmation: bool,
10787        #[values(false, true)] cancel: bool,
10788        #[values(false, true)] batch: bool,
10789        #[values(false, true)] pending_before_dispatch: bool,
10790        #[values(false, true)] missing_order_check: bool,
10791        #[values(
10792            SubmissionRecoveryPolicy::ResolveLocally,
10793            SubmissionRecoveryPolicy::RetainUnresolved
10794        )]
10795        policy: SubmissionRecoveryPolicy,
10796    ) {
10797        let (mut node, order, instrument) = venue_evidence_node_with_config(
10798            crate::config::LiveExecutionEngineConfig {
10799                reconciliation: true,
10800                inflight_check_threshold_ms: 100,
10801                inflight_check_retries: 2,
10802                open_check_threshold_ms: 5_000,
10803                open_check_open_only: !missing_order_check,
10804                open_check_lookback_mins: None,
10805                open_check_missing_retries: 5,
10806                single_order_query_delay_ms: 0,
10807                submission_recovery_policy: policy,
10808                ..Default::default()
10809            },
10810            OrderType::Limit,
10811            true,
10812        );
10813        let client_order_id = order.client_order_id();
10814        let client_id = ClientId::from("TEST-EVIDENCE");
10815        let account_id = AccountId::from("TEST-001");
10816        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10817        node.kernel
10818            .exec_engine
10819            .borrow_mut()
10820            .register_client(Box::new(StubExecutionClient::new(
10821                client_id,
10822                account_id,
10823                instrument.id().venue,
10824                OmsType::Netting,
10825                None,
10826            )))
10827            .unwrap();
10828
10829        // Consume part of the original submission budget before venue confirmation
10830        advance_clock(Duration::from_millis(101)).await;
10831        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10832
10833        if report_confirmation {
10834            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10835                OrderPendingUpdateSpec::builder()
10836                    .trader_id(order.trader_id())
10837                    .strategy_id(order.strategy_id())
10838                    .instrument_id(order.instrument_id())
10839                    .client_order_id(client_order_id)
10840                    .account_id(account_id)
10841                    .build(),
10842            )));
10843            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
10844                OrderStatusReport::new(
10845                    account_id,
10846                    order.instrument_id(),
10847                    Some(client_order_id),
10848                    venue_order_id,
10849                    Some(order.order_side()),
10850                    order.order_type(),
10851                    order.time_in_force(),
10852                    OrderStatus::Submitted,
10853                    order.quantity(),
10854                    Quantity::zero(3),
10855                    UnixNanos::default(),
10856                    UnixNanos::default(),
10857                    UnixNanos::default(),
10858                    None,
10859                )
10860                .with_price(order.price().unwrap()),
10861            ))));
10862        }
10863
10864        let mut update = OrderUpdatedSpec::builder()
10865            .trader_id(order.trader_id())
10866            .strategy_id(order.strategy_id())
10867            .instrument_id(order.instrument_id())
10868            .client_order_id(client_order_id)
10869            .account_id(account_id)
10870            .quantity(order.quantity())
10871            .price(order.price().unwrap())
10872            .build();
10873
10874        if !report_confirmation {
10875            update.venue_order_id = Some(venue_order_id);
10876        }
10877        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
10878        assert_eq!(
10879            node.kernel
10880                .cache
10881                .borrow()
10882                .order(&client_order_id)
10883                .unwrap()
10884                .status(),
10885            OrderStatus::Submitted,
10886        );
10887        advance_clock(Duration::from_millis(101)).await;
10888        let confirmed = node.exec_manager.check_inflight_orders();
10889        assert!(confirmed.queries.is_empty());
10890        assert!(confirmed.events.is_empty());
10891
10892        let pending = if cancel {
10893            OrderEventAny::PendingCancel(
10894                OrderPendingCancelSpec::builder()
10895                    .trader_id(order.trader_id())
10896                    .strategy_id(order.strategy_id())
10897                    .instrument_id(order.instrument_id())
10898                    .client_order_id(client_order_id)
10899                    .account_id(account_id)
10900                    .build(),
10901            )
10902        } else {
10903            OrderEventAny::PendingUpdate(
10904                OrderPendingUpdateSpec::builder()
10905                    .trader_id(order.trader_id())
10906                    .strategy_id(order.strategy_id())
10907                    .instrument_id(order.instrument_id())
10908                    .client_order_id(client_order_id)
10909                    .account_id(account_id)
10910                    .build(),
10911            )
10912        };
10913        let command = venue_evidence_command(&order, cancel, batch);
10914
10915        if pending_before_dispatch {
10916            node.process_exec_event(ExecutionEvent::Order(pending.clone()));
10917        }
10918        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
10919            MessagingSwitchboard::exec_engine_execute(),
10920            command,
10921        )));
10922        advance_clock(Duration::from_millis(50)).await;
10923
10924        if missing_order_check {
10925            let engine = node.kernel.exec_engine.borrow();
10926            let clients = engine.get_all_clients();
10927            let check = node
10928                .exec_manager
10929                .prepare_open_order_report_check(UUID4::new(), &clients);
10930            assert_eq!(check.filtered_orders.len(), 1);
10931            let missing = node.exec_manager.reconcile_open_order_reports(
10932                &check,
10933                Vec::new(),
10934                &IndexSet::from([client_id]),
10935                &IndexSet::new(),
10936                &clients,
10937            );
10938            assert!(missing.events.is_empty());
10939            assert!(missing.targeted_queries.is_empty());
10940            assert_eq!(
10941                node.exec_manager.recon_check_retry_count(&client_order_id),
10942                1
10943            );
10944        }
10945        let early = node.exec_manager.check_inflight_orders();
10946        assert!(early.queries.is_empty());
10947        assert!(early.events.is_empty());
10948
10949        if !pending_before_dispatch {
10950            // The queued pending event must not be needed to register the command,
10951            // nor shift its timeout start from dispatch to event application.
10952            assert_eq!(
10953                node.kernel
10954                    .cache
10955                    .borrow()
10956                    .order(&client_order_id)
10957                    .unwrap()
10958                    .status(),
10959                OrderStatus::Submitted,
10960            );
10961            node.process_exec_event(ExecutionEvent::Order(pending));
10962        }
10963
10964        if policy == SubmissionRecoveryPolicy::RetainUnresolved {
10965            node.exec_manager
10966                .register_submission(order.init_event(), Some(client_id));
10967        }
10968        advance_clock(Duration::from_millis(51)).await;
10969        let first = node.exec_manager.check_inflight_orders();
10970        assert_eq!(first.queries.len(), 1);
10971        assert!(
10972            matches!(&first.queries[0], TradingCommand::QueryOrder(query)
10973            if query.client_order_id == client_order_id)
10974        );
10975        assert!(first.events.is_empty());
10976        advance_clock(Duration::from_millis(101)).await;
10977        let exhausted = node.exec_manager.check_inflight_orders();
10978        assert!(exhausted.queries.is_empty());
10979        assert!(
10980            matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
10981                if event.client_order_id == client_order_id)
10982        );
10983        assert!(
10984            node.exec_manager
10985                .take_submission_recovery_exhaustions()
10986                .is_empty()
10987        );
10988        node.process_reconciliation_events(&exhausted.events);
10989        assert_eq!(
10990            node.kernel
10991                .cache
10992                .borrow()
10993                .order(&client_order_id)
10994                .unwrap()
10995                .status(),
10996            OrderStatus::Canceled,
10997        );
10998    }
10999
11000    #[rstest]
11001    #[cfg_attr(
11002        not(all(feature = "simulation", madsim)),
11003        tokio::test(start_paused = true)
11004    )]
11005    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11006    async fn test_submission_command_report_cleanup_matches_default(
11007        #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11008        #[values(
11009            SubmissionRecoveryPolicy::ResolveLocally,
11010            SubmissionRecoveryPolicy::RetainUnresolved
11011        )]
11012        policy: SubmissionRecoveryPolicy,
11013    ) {
11014        let (mut node, order, instrument) = venue_evidence_node(policy);
11015        let client_order_id = order.client_order_id();
11016        let client_id = ClientId::from("TEST-EVIDENCE");
11017        let account_id = AccountId::from("TEST-001");
11018        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11019        node.kernel
11020            .exec_engine
11021            .borrow_mut()
11022            .register_client(Box::new(StubExecutionClient::new(
11023                client_id,
11024                account_id,
11025                instrument.id().venue,
11026                OmsType::Netting,
11027                None,
11028            )))
11029            .unwrap();
11030        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11031            &order,
11032            account_id,
11033            venue_order_id,
11034        )));
11035        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11036            MessagingSwitchboard::exec_engine_execute(),
11037            venue_evidence_command(&order, true, false),
11038        )));
11039        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11040            OrderPendingCancelSpec::builder()
11041                .trader_id(order.trader_id())
11042                .strategy_id(order.strategy_id())
11043                .instrument_id(order.instrument_id())
11044                .client_order_id(client_order_id)
11045                .account_id(account_id)
11046                .venue_order_id(venue_order_id)
11047                .build(),
11048        )));
11049        advance_clock(Duration::from_millis(101)).await;
11050        let first = node.exec_manager.check_inflight_orders();
11051        assert_eq!(first.queries.len(), 1);
11052        assert!(first.events.is_empty());
11053
11054        let filled_qty = if report_status == OrderStatus::PartiallyFilled {
11055            Quantity::from("0.400")
11056        } else {
11057            Quantity::zero(3)
11058        };
11059
11060        let report = OrderStatusReport::new(
11061            account_id,
11062            order.instrument_id(),
11063            Some(client_order_id),
11064            venue_order_id,
11065            Some(order.order_side()),
11066            order.order_type(),
11067            order.time_in_force(),
11068            report_status,
11069            order.quantity(),
11070            filled_qty,
11071            UnixNanos::default(),
11072            UnixNanos::default(),
11073            UnixNanos::default(),
11074            None,
11075        )
11076        .with_price(order.price().unwrap())
11077        .with_avg_px(dec!(100.00));
11078        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11079            report,
11080        ))));
11081        advance_clock(Duration::from_millis(101)).await;
11082        let exhausted = node.exec_manager.check_inflight_orders();
11083
11084        assert!(exhausted.queries.is_empty());
11085        assert!(exhausted.events.is_empty());
11086        assert!(
11087            node.exec_manager
11088                .take_submission_recovery_exhaustions()
11089                .is_empty()
11090        );
11091        let cache = node.kernel.cache.borrow();
11092        let cached = cache.order_ref(&client_order_id).unwrap();
11093        assert_eq!(cached.status(), OrderStatus::PendingCancel);
11094        assert_eq!(cached.filled_qty(), filled_qty);
11095    }
11096
11097    #[rstest]
11098    #[case::accepted_modify(false, "accepted")]
11099    #[case::accepted_cancel(true, "accepted")]
11100    #[case::accepted_batch_modify(false, "accepted_batch")]
11101    #[case::accepted_batch_cancel(true, "accepted_batch")]
11102    #[case::updated_cancel(true, "updated")]
11103    #[case::triggered_cancel(true, "triggered")]
11104    #[cfg_attr(
11105        not(all(feature = "simulation", madsim)),
11106        tokio::test(start_paused = true)
11107    )]
11108    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11109    async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
11110        #[case] cancel: bool,
11111        #[case] evidence: &str,
11112        #[values(false, true)] batch: bool,
11113        #[values(false, true)] report_first: bool,
11114        #[values(
11115            SubmissionRecoveryPolicy::ResolveLocally,
11116            SubmissionRecoveryPolicy::RetainUnresolved
11117        )]
11118        policy: SubmissionRecoveryPolicy,
11119    ) {
11120        use nautilus_model::events::order::spec::OrderTriggeredSpec;
11121
11122        let triggered = evidence == "triggered";
11123
11124        let order_type = if triggered {
11125            OrderType::StopLimit
11126        } else {
11127            OrderType::Limit
11128        };
11129        let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
11130        let client_order_id = order.client_order_id();
11131        let account_id = AccountId::from("TEST-001");
11132        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11133        node.kernel
11134            .exec_engine
11135            .borrow_mut()
11136            .register_client(Box::new(StubExecutionClient::new(
11137                ClientId::from("TEST-EVIDENCE"),
11138                account_id,
11139                instrument.id().venue,
11140                OmsType::Netting,
11141                None,
11142            )))
11143            .unwrap();
11144        advance_clock(Duration::from_millis(101)).await;
11145        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11146
11147        if triggered || report_first {
11148            // Evidence can overtake a different, queued command intent
11149            let previous_pending = if cancel {
11150                OrderEventAny::PendingUpdate(
11151                    OrderPendingUpdateSpec::builder()
11152                        .trader_id(order.trader_id())
11153                        .strategy_id(order.strategy_id())
11154                        .instrument_id(order.instrument_id())
11155                        .client_order_id(client_order_id)
11156                        .account_id(account_id)
11157                        .build(),
11158                )
11159            } else {
11160                OrderEventAny::PendingCancel(
11161                    OrderPendingCancelSpec::builder()
11162                        .trader_id(order.trader_id())
11163                        .strategy_id(order.strategy_id())
11164                        .instrument_id(order.instrument_id())
11165                        .client_order_id(client_order_id)
11166                        .account_id(account_id)
11167                        .build(),
11168                )
11169            };
11170            node.process_exec_event(ExecutionEvent::Order(previous_pending));
11171        }
11172        advance_clock(Duration::from_millis(10)).await;
11173        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11174            MessagingSwitchboard::exec_engine_execute(),
11175            venue_evidence_command(&order, cancel, batch),
11176        )));
11177
11178        if report_first {
11179            let report = OrderStatusReport::new(
11180                account_id,
11181                instrument.id(),
11182                Some(client_order_id),
11183                venue_order_id,
11184                Some(order.order_side()),
11185                order.order_type(),
11186                order.time_in_force(),
11187                OrderStatus::Submitted,
11188                order.quantity(),
11189                Quantity::zero(3),
11190                UnixNanos::default(),
11191                UnixNanos::default(),
11192                UnixNanos::default(),
11193                None,
11194            )
11195            .with_price(order.price().unwrap());
11196            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11197                report,
11198            ))));
11199            assert!(
11200                node.kernel
11201                    .cache
11202                    .borrow()
11203                    .order(&client_order_id)
11204                    .unwrap()
11205                    .venue_order_id()
11206                    .is_none()
11207            );
11208        }
11209        advance_clock(Duration::from_millis(40)).await;
11210        let acknowledgement = if triggered {
11211            OrderEventAny::Triggered(
11212                OrderTriggeredSpec::builder()
11213                    .trader_id(order.trader_id())
11214                    .strategy_id(order.strategy_id())
11215                    .instrument_id(order.instrument_id())
11216                    .client_order_id(client_order_id)
11217                    .account_id(account_id)
11218                    .venue_order_id(venue_order_id)
11219                    .build(),
11220            )
11221        } else if evidence == "updated" {
11222            OrderEventAny::Updated(
11223                OrderUpdatedSpec::builder()
11224                    .trader_id(order.trader_id())
11225                    .strategy_id(order.strategy_id())
11226                    .instrument_id(order.instrument_id())
11227                    .client_order_id(client_order_id)
11228                    .account_id(account_id)
11229                    .venue_order_id(venue_order_id)
11230                    .quantity(order.quantity())
11231                    .price(order.price().unwrap())
11232                    .build(),
11233            )
11234        } else {
11235            TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
11236        };
11237
11238        if evidence == "accepted_batch" {
11239            let OrderEventAny::Accepted(accepted) = acknowledgement else {
11240                unreachable!()
11241            };
11242            node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
11243                vec![accepted],
11244            )));
11245        } else {
11246            node.process_exec_event(ExecutionEvent::Order(acknowledgement));
11247        }
11248        assert_eq!(
11249            node.kernel
11250                .cache
11251                .borrow()
11252                .order(&client_order_id)
11253                .unwrap()
11254                .status(),
11255            match evidence {
11256                "triggered" => OrderStatus::Triggered,
11257                "updated" => OrderStatus::Submitted,
11258                _ => OrderStatus::Accepted,
11259            }
11260        );
11261        advance_clock(Duration::from_millis(20)).await;
11262        let pending = if cancel {
11263            OrderEventAny::PendingCancel(
11264                OrderPendingCancelSpec::builder()
11265                    .trader_id(order.trader_id())
11266                    .strategy_id(order.strategy_id())
11267                    .instrument_id(order.instrument_id())
11268                    .client_order_id(client_order_id)
11269                    .account_id(account_id)
11270                    .build(),
11271            )
11272        } else {
11273            OrderEventAny::PendingUpdate(
11274                OrderPendingUpdateSpec::builder()
11275                    .trader_id(order.trader_id())
11276                    .strategy_id(order.strategy_id())
11277                    .instrument_id(order.instrument_id())
11278                    .client_order_id(client_order_id)
11279                    .account_id(account_id)
11280                    .build(),
11281            )
11282        };
11283        node.process_exec_event(ExecutionEvent::Order(pending));
11284        let early = node.exec_manager.check_inflight_orders();
11285        assert!(early.queries.is_empty());
11286        assert!(early.events.is_empty());
11287        advance_clock(Duration::from_millis(41)).await;
11288        let first = node.exec_manager.check_inflight_orders();
11289        // The default does not clear inflight tracking on Triggered events.
11290        let recover_command =
11291            triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
11292        assert_eq!(first.queries.len(), usize::from(recover_command));
11293        assert!(first.events.is_empty());
11294        advance_clock(Duration::from_millis(101)).await;
11295        let exhausted = node.exec_manager.check_inflight_orders();
11296        assert!(exhausted.queries.is_empty());
11297        assert!(
11298            node.exec_manager
11299                .take_submission_recovery_exhaustions()
11300                .is_empty()
11301        );
11302
11303        if recover_command {
11304            assert!(
11305                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11306                if event.client_order_id == client_order_id)
11307            );
11308        } else {
11309            assert!(exhausted.events.is_empty());
11310        }
11311    }
11312
11313    #[rstest]
11314    #[case::submission_only(None, false)]
11315    #[case::modify(Some(false), false)]
11316    #[case::cancel(Some(true), false)]
11317    #[case::batch_modify(Some(false), true)]
11318    #[case::batch_cancel(Some(true), true)]
11319    #[cfg_attr(
11320        not(all(feature = "simulation", madsim)),
11321        tokio::test(start_paused = true)
11322    )]
11323    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11324    async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
11325        #[case] cancel: Option<bool>,
11326        #[case] batch: bool,
11327        #[values(false, true)] missing_order_check: bool,
11328        #[values(
11329            SubmissionRecoveryPolicy::ResolveLocally,
11330            SubmissionRecoveryPolicy::RetainUnresolved
11331        )]
11332        policy: SubmissionRecoveryPolicy,
11333    ) {
11334        let (mut node, order, instrument) = venue_evidence_node_with_config(
11335            crate::config::LiveExecutionEngineConfig {
11336                reconciliation: true,
11337                inflight_check_threshold_ms: 100,
11338                inflight_check_retries: 2,
11339                open_check_threshold_ms: 1,
11340                open_check_open_only: false,
11341                open_check_lookback_mins: None,
11342                open_check_missing_retries: 5,
11343                single_order_query_delay_ms: 0,
11344                submission_recovery_policy: policy,
11345                ..Default::default()
11346            },
11347            OrderType::Limit,
11348            false,
11349        );
11350        let client_order_id = order.client_order_id();
11351        let client_id = ClientId::from("TEST-EVIDENCE");
11352        let account_id = AccountId::from("TEST-001");
11353        node.kernel
11354            .exec_engine
11355            .borrow_mut()
11356            .register_client(Box::new(StubExecutionClient::new(
11357                client_id,
11358                account_id,
11359                instrument.id().venue,
11360                OmsType::Netting,
11361                None,
11362            )))
11363            .unwrap();
11364        let queries = Rc::new(RefCell::new(Vec::new()));
11365        let client = OrderReportClient {
11366            client_id,
11367            report: OrderStatusReport::new(
11368                account_id,
11369                instrument.id(),
11370                Some(client_order_id),
11371                VenueOrderId::from("V-VENUE-EVIDENCE"),
11372                Some(order.order_side()),
11373                order.order_type(),
11374                order.time_in_force(),
11375                OrderStatus::Submitted,
11376                order.quantity(),
11377                Quantity::zero(3),
11378                UnixNanos::default(),
11379                UnixNanos::default(),
11380                UnixNanos::default(),
11381                None,
11382            )
11383            .with_price(order.price().unwrap()),
11384            targeted: false,
11385            queries: queries.clone(),
11386        };
11387        advance_clock(Duration::from_millis(2)).await;
11388        let events = node.exec_manager.check_open_orders(&[&client]).await;
11389        assert!(events.is_empty());
11390        assert_eq!(*queries.borrow(), vec!["bulk"]);
11391        assert_eq!(
11392            node.kernel
11393                .cache
11394                .borrow()
11395                .order(&client_order_id)
11396                .unwrap()
11397                .status(),
11398            OrderStatus::Submitted
11399        );
11400        advance_clock(Duration::from_millis(2)).await;
11401
11402        if missing_order_check {
11403            let engine = node.kernel.exec_engine.borrow();
11404            let clients = engine.get_all_clients();
11405            let check = node
11406                .exec_manager
11407                .prepare_open_order_report_check(UUID4::new(), &clients);
11408            assert_eq!(check.filtered_orders.len(), 1);
11409            let missing = node.exec_manager.reconcile_open_order_reports(
11410                &check,
11411                Vec::new(),
11412                &IndexSet::from([client_id]),
11413                &IndexSet::new(),
11414                &clients,
11415            );
11416            assert!(missing.events.is_empty());
11417            assert!(missing.targeted_queries.is_empty());
11418            assert_eq!(
11419                node.exec_manager.recon_check_retry_count(&client_order_id),
11420                1
11421            );
11422        }
11423        node.exec_manager
11424            .register_submission(order.init_event(), Some(client_id));
11425        advance_clock(Duration::from_millis(101)).await;
11426        let duplicate = node.exec_manager.check_inflight_orders();
11427        assert!(duplicate.events.is_empty());
11428        assert_eq!(
11429            duplicate.queries.len(),
11430            usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
11431        );
11432
11433        if let Some(cancel) = cancel {
11434            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11435                MessagingSwitchboard::exec_engine_execute(),
11436                venue_evidence_command(&order, cancel, batch),
11437            )));
11438            let pending = if cancel {
11439                OrderEventAny::PendingCancel(
11440                    OrderPendingCancelSpec::builder()
11441                        .trader_id(order.trader_id())
11442                        .strategy_id(order.strategy_id())
11443                        .instrument_id(order.instrument_id())
11444                        .client_order_id(client_order_id)
11445                        .account_id(account_id)
11446                        .build(),
11447                )
11448            } else {
11449                OrderEventAny::PendingUpdate(
11450                    OrderPendingUpdateSpec::builder()
11451                        .trader_id(order.trader_id())
11452                        .strategy_id(order.strategy_id())
11453                        .instrument_id(order.instrument_id())
11454                        .client_order_id(client_order_id)
11455                        .account_id(account_id)
11456                        .build(),
11457                )
11458            };
11459            node.process_exec_event(ExecutionEvent::Order(pending));
11460        }
11461        advance_clock(Duration::from_millis(101)).await;
11462        let first = node.exec_manager.check_inflight_orders();
11463        assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
11464
11465        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
11466            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
11467                if event.client_order_id == client_order_id));
11468        } else {
11469            assert!(first.events.is_empty());
11470        }
11471        node.process_reconciliation_events(&first.events);
11472        advance_clock(Duration::from_millis(101)).await;
11473        let exhausted = node.exec_manager.check_inflight_orders();
11474        assert!(exhausted.queries.is_empty());
11475
11476        if cancel.is_some() {
11477            assert!(
11478                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11479                if event.client_order_id == client_order_id)
11480            );
11481        } else {
11482            assert!(exhausted.events.is_empty());
11483        }
11484        assert!(
11485            node.exec_manager
11486                .take_submission_recovery_exhaustions()
11487                .is_empty()
11488        );
11489    }
11490
11491    #[rstest]
11492    #[case::submission_only(None, false)]
11493    #[case::modify(Some(false), false)]
11494    #[case::cancel(Some(true), false)]
11495    #[case::batch_modify(Some(false), true)]
11496    #[case::batch_cancel(Some(true), true)]
11497    #[cfg_attr(
11498        not(all(feature = "simulation", madsim)),
11499        tokio::test(start_paused = true)
11500    )]
11501    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11502    async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
11503        #[case] cancel: Option<bool>,
11504        #[case] batch: bool,
11505        #[values(false, true)] targeted: bool,
11506        #[values(
11507            SubmissionRecoveryPolicy::ResolveLocally,
11508            SubmissionRecoveryPolicy::RetainUnresolved
11509        )]
11510        policy: SubmissionRecoveryPolicy,
11511    ) {
11512        let (mut node, order, instrument) = venue_evidence_node_with_config(
11513            crate::config::LiveExecutionEngineConfig {
11514                reconciliation: true,
11515                inflight_check_threshold_ms: 100,
11516                inflight_check_retries: 2,
11517                open_check_threshold_ms: 1,
11518                open_check_open_only: false,
11519                open_check_lookback_mins: None,
11520                open_check_missing_retries: 1,
11521                single_order_query_delay_ms: 0,
11522                submission_recovery_policy: policy,
11523                ..Default::default()
11524            },
11525            OrderType::Limit,
11526            true,
11527        );
11528        let client_order_id = order.client_order_id();
11529        let client_id = ClientId::from("TEST-EVIDENCE");
11530        let account_id = AccountId::from("TEST-001");
11531        node.kernel
11532            .exec_engine
11533            .borrow_mut()
11534            .register_client(Box::new(StubExecutionClient::new(
11535                client_id,
11536                account_id,
11537                instrument.id().venue,
11538                OmsType::Netting,
11539                None,
11540            )))
11541            .unwrap();
11542        advance_clock(Duration::from_millis(101)).await;
11543        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11544        advance_clock(Duration::from_millis(10)).await;
11545
11546        if let Some(cancel) = cancel {
11547            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11548                MessagingSwitchboard::exec_engine_execute(),
11549                venue_evidence_command(&order, cancel, batch),
11550            )));
11551            let pending = if cancel {
11552                OrderEventAny::PendingCancel(
11553                    OrderPendingCancelSpec::builder()
11554                        .trader_id(order.trader_id())
11555                        .strategy_id(order.strategy_id())
11556                        .instrument_id(order.instrument_id())
11557                        .client_order_id(client_order_id)
11558                        .account_id(account_id)
11559                        .build(),
11560                )
11561            } else {
11562                OrderEventAny::PendingUpdate(
11563                    OrderPendingUpdateSpec::builder()
11564                        .trader_id(order.trader_id())
11565                        .strategy_id(order.strategy_id())
11566                        .instrument_id(order.instrument_id())
11567                        .client_order_id(client_order_id)
11568                        .account_id(account_id)
11569                        .build(),
11570                )
11571            };
11572            node.process_exec_event(ExecutionEvent::Order(pending));
11573        }
11574        let queries = Rc::new(RefCell::new(Vec::new()));
11575        let client = OrderReportClient {
11576            client_id,
11577            report: OrderStatusReport::new(
11578                account_id,
11579                order.instrument_id(),
11580                Some(client_order_id),
11581                VenueOrderId::from("V-VENUE-EVIDENCE"),
11582                Some(order.order_side()),
11583                order.order_type(),
11584                order.time_in_force(),
11585                OrderStatus::Submitted,
11586                order.quantity(),
11587                Quantity::zero(3),
11588                UnixNanos::default(),
11589                UnixNanos::default(),
11590                UnixNanos::default(),
11591                None,
11592            )
11593            .with_price(order.price().unwrap()),
11594            targeted,
11595            queries: queries.clone(),
11596        };
11597        advance_clock(Duration::from_millis(40)).await;
11598        let events = node.exec_manager.check_open_orders(&[&client]).await;
11599        assert!(events.is_empty());
11600        assert_eq!(
11601            *queries.borrow(),
11602            if targeted {
11603                vec!["bulk", "targeted"]
11604            } else {
11605                vec!["bulk"]
11606            }
11607        );
11608        node.process_reconciliation_events(&events);
11609        advance_clock(Duration::from_millis(61)).await;
11610        let first = node.exec_manager.check_inflight_orders();
11611        let recover_command =
11612            cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
11613        assert_eq!(first.queries.len(), usize::from(recover_command));
11614
11615        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
11616            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
11617                if event.client_order_id == client_order_id));
11618        } else {
11619            assert!(first.events.is_empty());
11620        }
11621        node.process_reconciliation_events(&first.events);
11622        advance_clock(Duration::from_millis(101)).await;
11623        let exhausted = node.exec_manager.check_inflight_orders();
11624        assert!(exhausted.queries.is_empty());
11625
11626        if recover_command {
11627            assert!(
11628                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11629                if event.client_order_id == client_order_id)
11630            );
11631        } else {
11632            assert!(exhausted.events.is_empty());
11633        }
11634        assert!(
11635            node.exec_manager
11636                .take_submission_recovery_exhaustions()
11637                .is_empty()
11638        );
11639    }
11640
11641    #[rstest]
11642    #[case::local_update_before_pending("updated")]
11643    #[case::modify_rejected_after_pending("modify_rejected")]
11644    #[case::cancel_rejected_after_pending("cancel_rejected")]
11645    #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
11646    #[cfg_attr(
11647        not(all(feature = "simulation", madsim)),
11648        tokio::test(start_paused = true)
11649    )]
11650    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11651    async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
11652        #[case] completion: &str,
11653        #[values(
11654            SubmissionRecoveryPolicy::ResolveLocally,
11655            SubmissionRecoveryPolicy::RetainUnresolved
11656        )]
11657        policy: SubmissionRecoveryPolicy,
11658    ) {
11659        use nautilus_model::events::order::spec::{
11660            OrderCancelRejectedSpec, OrderModifyRejectedSpec,
11661        };
11662
11663        let (mut node, order, instrument) = venue_evidence_node(policy);
11664        let client_order_id = order.client_order_id();
11665        let client_id = ClientId::from("TEST-EVIDENCE");
11666        let account_id = AccountId::from("TEST-001");
11667        let cancel = completion == "cancel_rejected";
11668        let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
11669        node.kernel
11670            .exec_engine
11671            .borrow_mut()
11672            .register_client(Box::new(StubExecutionClient::new(
11673                client_id,
11674                account_id,
11675                instrument.id().venue,
11676                OmsType::Netting,
11677                None,
11678            )))
11679            .unwrap();
11680        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11681            MessagingSwitchboard::exec_engine_execute(),
11682            venue_evidence_command(&order, cancel, false),
11683        )));
11684        advance_clock(Duration::from_millis(10)).await;
11685
11686        if completion != "updated" {
11687            let pending = if cancel {
11688                OrderEventAny::PendingCancel(
11689                    OrderPendingCancelSpec::builder()
11690                        .trader_id(order.trader_id())
11691                        .strategy_id(order.strategy_id())
11692                        .instrument_id(order.instrument_id())
11693                        .client_order_id(client_order_id)
11694                        .account_id(account_id)
11695                        .build(),
11696                )
11697            } else {
11698                OrderEventAny::PendingUpdate(
11699                    OrderPendingUpdateSpec::builder()
11700                        .trader_id(order.trader_id())
11701                        .strategy_id(order.strategy_id())
11702                        .instrument_id(order.instrument_id())
11703                        .client_order_id(client_order_id)
11704                        .account_id(account_id)
11705                        .build(),
11706                )
11707            };
11708            node.process_exec_event(ExecutionEvent::Order(pending));
11709        }
11710
11711        if later_cancel_pending {
11712            // The cancel is pending in native state, but has not dispatched
11713            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11714                OrderPendingCancelSpec::builder()
11715                    .trader_id(order.trader_id())
11716                    .strategy_id(order.strategy_id())
11717                    .instrument_id(order.instrument_id())
11718                    .client_order_id(client_order_id)
11719                    .account_id(account_id)
11720                    .build(),
11721            )));
11722        }
11723        advance_clock(Duration::from_millis(10)).await;
11724        let completed = match completion {
11725            "updated" => OrderEventAny::Updated(
11726                OrderUpdatedSpec::builder()
11727                    .trader_id(order.trader_id())
11728                    .strategy_id(order.strategy_id())
11729                    .instrument_id(order.instrument_id())
11730                    .client_order_id(client_order_id)
11731                    .account_id(account_id)
11732                    .quantity(order.quantity())
11733                    .price(Price::from("101.00"))
11734                    .build(),
11735            ),
11736            "modify_rejected" | "modify_rejected_after_cancel_pending" => {
11737                OrderEventAny::ModifyRejected(
11738                    OrderModifyRejectedSpec::builder()
11739                        .trader_id(order.trader_id())
11740                        .strategy_id(order.strategy_id())
11741                        .instrument_id(order.instrument_id())
11742                        .client_order_id(client_order_id)
11743                        .account_id(account_id)
11744                        .build(),
11745                )
11746            }
11747            "cancel_rejected" => OrderEventAny::CancelRejected(
11748                OrderCancelRejectedSpec::builder()
11749                    .trader_id(order.trader_id())
11750                    .strategy_id(order.strategy_id())
11751                    .instrument_id(order.instrument_id())
11752                    .client_order_id(client_order_id)
11753                    .account_id(account_id)
11754                    .build(),
11755            ),
11756            _ => unreachable!(),
11757        };
11758        node.process_exec_event(ExecutionEvent::Order(completed));
11759        assert_eq!(
11760            node.kernel
11761                .cache
11762                .borrow()
11763                .order(&client_order_id)
11764                .unwrap()
11765                .status(),
11766            if later_cancel_pending {
11767                OrderStatus::PendingCancel
11768            } else {
11769                OrderStatus::Submitted
11770            }
11771        );
11772
11773        // A newer pending event is queued, but that command has not dispatched
11774        advance_clock(Duration::from_millis(10)).await;
11775
11776        if !later_cancel_pending {
11777            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11778                OrderPendingUpdateSpec::builder()
11779                    .trader_id(order.trader_id())
11780                    .strategy_id(order.strategy_id())
11781                    .instrument_id(order.instrument_id())
11782                    .client_order_id(client_order_id)
11783                    .account_id(account_id)
11784                    .build(),
11785            )));
11786        }
11787        advance_clock(Duration::from_millis(10)).await;
11788        let cached = node
11789            .kernel
11790            .cache
11791            .borrow()
11792            .order_owned(&client_order_id)
11793            .unwrap();
11794        let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11795            .last_qty(Quantity::from("0.400"))
11796            .last_px(order.price().unwrap())
11797            .commission(Money::from("0 USDT"))
11798            .without_position_id()
11799            .build()
11800        else {
11801            unreachable!()
11802        };
11803        fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11804        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11805
11806        for _ in 0..3 {
11807            advance_clock(Duration::from_millis(101)).await;
11808            let result = node.exec_manager.check_inflight_orders();
11809            assert!(result.queries.is_empty());
11810            assert!(result.events.is_empty());
11811            assert!(
11812                node.exec_manager
11813                    .take_submission_recovery_exhaustions()
11814                    .is_empty()
11815            );
11816        }
11817        assert_eq!(
11818            node.kernel
11819                .cache
11820                .borrow()
11821                .order(&client_order_id)
11822                .unwrap()
11823                .status(),
11824            if later_cancel_pending {
11825                OrderStatus::PendingCancel
11826            } else {
11827                OrderStatus::PendingUpdate
11828            }
11829        );
11830    }
11831
11832    #[rstest]
11833    #[case::wrong_account("account")]
11834    #[case::wrong_instrument("instrument")]
11835    #[case::conflicting_report_venue("report_venue")]
11836    #[case::unapplied_venue_update("update_venue")]
11837    #[case::local_update("local_update")]
11838    #[cfg_attr(
11839        not(all(feature = "simulation", madsim)),
11840        tokio::test(start_paused = true)
11841    )]
11842    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11843    async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
11844        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
11845        let client_order_id = order.client_order_id();
11846        let account_id = AccountId::from("TEST-001");
11847        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11848
11849        if invalid.ends_with("venue") {
11850            insert_accepted_limit_order_in_node(
11851                &node,
11852                account_id,
11853                ClientId::from("TEST-EVIDENCE"),
11854                order.instrument_id(),
11855                ClientOrderId::from("O-OTHER"),
11856                venue_order_id,
11857            );
11858        }
11859        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11860            OrderPendingCancelSpec::builder()
11861                .trader_id(order.trader_id())
11862                .strategy_id(order.strategy_id())
11863                .instrument_id(order.instrument_id())
11864                .client_order_id(client_order_id)
11865                .account_id(account_id)
11866                .build(),
11867        )));
11868        advance_clock(Duration::from_millis(101)).await;
11869        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11870
11871        if invalid == "update_venue" || invalid == "local_update" {
11872            let mut updated = OrderUpdatedSpec::builder()
11873                .trader_id(order.trader_id())
11874                .strategy_id(order.strategy_id())
11875                .instrument_id(order.instrument_id())
11876                .client_order_id(client_order_id)
11877                .account_id(account_id)
11878                .quantity(order.quantity())
11879                .price(order.price().unwrap())
11880                .build();
11881
11882            if invalid == "update_venue" {
11883                updated.venue_order_id = Some(venue_order_id);
11884            }
11885            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
11886        } else {
11887            let report = OrderStatusReport::new(
11888                if invalid == "account" {
11889                    AccountId::from("OTHER-001")
11890                } else {
11891                    account_id
11892                },
11893                if invalid == "instrument" {
11894                    InstrumentId::from("OTHER.BINANCE")
11895                } else {
11896                    order.instrument_id()
11897                },
11898                Some(client_order_id),
11899                venue_order_id,
11900                Some(order.order_side()),
11901                order.order_type(),
11902                order.time_in_force(),
11903                OrderStatus::Submitted,
11904                order.quantity(),
11905                Quantity::zero(3),
11906                UnixNanos::default(),
11907                UnixNanos::default(),
11908                UnixNanos::default(),
11909                None,
11910            )
11911            .with_price(order.price().unwrap());
11912            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11913                report,
11914            ))));
11915        }
11916        assert!(
11917            node.kernel
11918                .cache
11919                .borrow()
11920                .order(&client_order_id)
11921                .unwrap()
11922                .venue_order_id()
11923                .is_none()
11924        );
11925        advance_clock(Duration::from_millis(101)).await;
11926        let result = node.exec_manager.check_inflight_orders();
11927        assert!(result.events.is_empty());
11928        assert!(result.queries.is_empty());
11929        let cached = node
11930            .kernel
11931            .cache
11932            .borrow()
11933            .order_owned(&client_order_id)
11934            .unwrap();
11935        assert_eq!(
11936            cached.status(),
11937            if invalid == "local_update" {
11938                OrderStatus::Submitted
11939            } else {
11940                OrderStatus::PendingCancel
11941            }
11942        );
11943        let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
11944        assert_eq!(diagnostics.len(), 1);
11945        assert_eq!(diagnostics[0].client_order_id, client_order_id);
11946    }
11947
11948    fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
11949        use nautilus_common::messages::execution::{
11950            BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
11951        };
11952
11953        let client_id = ClientId::from("TEST-EVIDENCE");
11954        let client_order_id = order.client_order_id();
11955        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11956
11957        if cancel {
11958            let cancel = CancelOrder::new(
11959                order.trader_id(),
11960                Some(client_id),
11961                order.strategy_id(),
11962                order.instrument_id(),
11963                client_order_id,
11964                Some(venue_order_id),
11965                UUID4::new(),
11966                UnixNanos::default(),
11967                None,
11968                None,
11969            );
11970
11971            if batch {
11972                TradingCommand::CancelOrders(BatchCancelOrders::new(
11973                    order.trader_id(),
11974                    Some(client_id),
11975                    order.strategy_id(),
11976                    order.instrument_id(),
11977                    vec![cancel],
11978                    UUID4::new(),
11979                    UnixNanos::default(),
11980                    None,
11981                    None,
11982                ))
11983            } else {
11984                TradingCommand::CancelOrder(cancel)
11985            }
11986        } else {
11987            let modify = ModifyOrder::new(
11988                order.trader_id(),
11989                Some(client_id),
11990                order.strategy_id(),
11991                order.instrument_id(),
11992                client_order_id,
11993                Some(venue_order_id),
11994                None,
11995                Some(Price::from("101.00")),
11996                None,
11997                UUID4::new(),
11998                UnixNanos::default(),
11999                None,
12000                None,
12001            );
12002
12003            if batch {
12004                TradingCommand::ModifyOrders(BatchModifyOrders::new(
12005                    order.trader_id(),
12006                    Some(client_id),
12007                    order.strategy_id(),
12008                    order.instrument_id(),
12009                    vec![modify],
12010                    UUID4::new(),
12011                    UnixNanos::default(),
12012                    None,
12013                    None,
12014                ))
12015            } else {
12016                TradingCommand::ModifyOrder(modify)
12017            }
12018        }
12019    }
12020
12021    fn venue_evidence_node(
12022        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12023    ) -> (LiveNode, OrderAny, InstrumentAny) {
12024        venue_evidence_node_with_order_type(policy, OrderType::Limit)
12025    }
12026
12027    fn venue_evidence_node_with_order_type(
12028        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12029        order_type: OrderType,
12030    ) -> (LiveNode, OrderAny, InstrumentAny) {
12031        venue_evidence_node_with_config(
12032            crate::config::LiveExecutionEngineConfig {
12033                reconciliation: true,
12034                inflight_check_threshold_ms: 100,
12035                inflight_check_retries: 2,
12036                open_check_threshold_ms: 5_000,
12037                single_order_query_delay_ms: 0,
12038                submission_recovery_policy: policy,
12039                ..Default::default()
12040            },
12041            order_type,
12042            true,
12043        )
12044    }
12045
12046    fn venue_evidence_node_with_config(
12047        exec_engine: crate::config::LiveExecutionEngineConfig,
12048        order_type: OrderType,
12049        register_submission: bool,
12050    ) -> (LiveNode, OrderAny, InstrumentAny) {
12051        let config = LiveNodeConfig {
12052            exec_engine,
12053            ..Default::default()
12054        };
12055        let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
12056        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
12057        let account_id = AccountId::from("TEST-001");
12058        let client_id = ClientId::from("TEST-EVIDENCE");
12059        let account = AccountAny::Margin(MarginAccount::new(
12060            AccountState::new(
12061                account_id,
12062                AccountType::Margin,
12063                vec![AccountBalance::new(
12064                    Money::from("1000000 USDT"),
12065                    Money::from("0 USDT"),
12066                    Money::from("1000000 USDT"),
12067                )],
12068                Vec::new(),
12069                true,
12070                UUID4::new(),
12071                UnixNanos::default(),
12072                UnixNanos::default(),
12073                Some(Currency::USDT()),
12074            ),
12075            true,
12076        ));
12077        node.kernel.cache.borrow_mut().add_account(account).unwrap();
12078        node.kernel
12079            .cache
12080            .borrow_mut()
12081            .add_instrument(instrument.clone())
12082            .unwrap();
12083        let mut builder = OrderTestBuilder::new(order_type);
12084        builder
12085            .trader_id(node.trader_id())
12086            .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
12087            .instrument_id(instrument.id())
12088            .quantity(Quantity::from("1.000"))
12089            .price(Price::from("100.00"));
12090
12091        if order_type == OrderType::StopLimit {
12092            builder.trigger_price(Price::from("110.00"));
12093        }
12094        let order = builder.build();
12095        let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
12096            &order,
12097            node.trader_id(),
12098            Some(client_id),
12099            None,
12100            UUID4::new(),
12101            UnixNanos::default(),
12102        ));
12103
12104        if register_submission {
12105            node.observe_exec_command_before_dispatch(&command);
12106        }
12107        node.kernel
12108            .cache
12109            .borrow_mut()
12110            .add_order(order.clone(), None, Some(client_id), false)
12111            .unwrap();
12112        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
12113            &order, account_id,
12114        )));
12115        (node, order, instrument)
12116    }
12117
12118    #[rstest]
12119    #[case::partial("0.400", OrderStatus::PartiallyFilled)]
12120    #[case::full("1.000", OrderStatus::Filled)]
12121    #[tokio::test(start_paused = true)]
12122    async fn test_retained_submission_resolves_late_fill(
12123        #[case] quantity: &str,
12124        #[case] expected_status: OrderStatus,
12125    ) {
12126        let (mut node, order, instrument) =
12127            venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12128        let client_order_id = order.client_order_id();
12129
12130        for _ in 0..2 {
12131            advance_clock(Duration::from_millis(101)).await;
12132            assert!(node.exec_manager.check_inflight_orders().events.is_empty());
12133        }
12134
12135        assert_eq!(
12136            node.exec_manager
12137                .take_submission_recovery_exhaustions()
12138                .len(),
12139            1
12140        );
12141        let fill = TestOrderEventStubs::filled(
12142            &order,
12143            &instrument,
12144            Some(TradeId::from("T-LATE-RETAINED")),
12145            None,
12146            Some(Price::from("100.00")),
12147            Some(Quantity::from(quantity)),
12148            Some(LiquiditySide::Taker),
12149            None,
12150            None,
12151            Some(AccountId::from("TEST-001")),
12152        );
12153        node.process_exec_event(ExecutionEvent::Order(fill));
12154        advance_clock(Duration::from_millis(101)).await;
12155        let result = node.exec_manager.check_inflight_orders();
12156        let cache = node.kernel.cache.borrow();
12157        let cached = cache.order_ref(&client_order_id).unwrap();
12158        assert_eq!(cached.status(), expected_status);
12159        assert_eq!(cached.filled_qty(), Quantity::from(quantity));
12160        assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
12161        assert!(result.events.is_empty());
12162        assert!(result.queries.is_empty());
12163        assert!(
12164            node.exec_manager
12165                .take_submission_recovery_exhaustions()
12166                .is_empty()
12167        );
12168    }
12169
12170    #[rstest]
12171    #[case::submitted_report("order")]
12172    #[case::submitted_report_with_fills("order_with_fills")]
12173    #[case::venue_update("updated")]
12174    #[cfg_attr(
12175        not(all(feature = "simulation", madsim)),
12176        tokio::test(start_paused = true)
12177    )]
12178    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12179    async fn test_venue_evidence_stops_submission_timeout(
12180        #[case] evidence: &str,
12181        #[values(
12182            OrderStatus::Submitted,
12183            OrderStatus::PendingCancel,
12184            OrderStatus::PendingUpdate
12185        )]
12186        status: OrderStatus,
12187        #[values(
12188            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12189            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12190        )]
12191        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12192    ) {
12193        let (mut node, order, _) = venue_evidence_node(policy);
12194        let client_order_id = order.client_order_id();
12195        let account_id = AccountId::from("TEST-001");
12196        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12197        let pending = match status {
12198            OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
12199                OrderPendingCancelSpec::builder()
12200                    .trader_id(order.trader_id())
12201                    .strategy_id(order.strategy_id())
12202                    .instrument_id(order.instrument_id())
12203                    .client_order_id(client_order_id)
12204                    .account_id(account_id)
12205                    .build(),
12206            )),
12207            OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
12208                OrderPendingUpdateSpec::builder()
12209                    .trader_id(order.trader_id())
12210                    .strategy_id(order.strategy_id())
12211                    .instrument_id(order.instrument_id())
12212                    .client_order_id(client_order_id)
12213                    .account_id(account_id)
12214                    .build(),
12215            )),
12216            _ => None,
12217        };
12218
12219        if let Some(pending) = pending {
12220            node.process_exec_event(ExecutionEvent::Order(pending));
12221        }
12222        advance_clock(Duration::from_millis(101)).await;
12223        let first = node.exec_manager.check_inflight_orders();
12224        assert_eq!(first.queries.len(), 1);
12225        assert!(first.events.is_empty());
12226        advance_clock(Duration::from_millis(94)).await;
12227
12228        let event = if evidence == "updated" {
12229            ExecutionEvent::Order(OrderEventAny::Updated(
12230                OrderUpdatedSpec::builder()
12231                    .trader_id(order.trader_id())
12232                    .strategy_id(order.strategy_id())
12233                    .instrument_id(order.instrument_id())
12234                    .client_order_id(client_order_id)
12235                    .account_id(account_id)
12236                    .venue_order_id(venue_order_id)
12237                    .quantity(order.quantity())
12238                    .price(Price::from("101.00"))
12239                    .build(),
12240            ))
12241        } else {
12242            let report = OrderStatusReport::new(
12243                account_id,
12244                order.instrument_id(),
12245                Some(client_order_id),
12246                venue_order_id,
12247                Some(order.order_side()),
12248                order.order_type(),
12249                order.time_in_force(),
12250                OrderStatus::Submitted,
12251                order.quantity(),
12252                Quantity::zero(3),
12253                UnixNanos::default(),
12254                UnixNanos::default(),
12255                UnixNanos::default(),
12256                None,
12257            )
12258            .with_price(Price::from("100.00"));
12259            ExecutionEvent::Report(if evidence == "order" {
12260                ExecutionReport::Order(Box::new(report))
12261            } else {
12262                ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
12263            })
12264        };
12265        node.process_exec_event(event);
12266        let expected_status = if evidence == "updated" {
12267            OrderStatus::Submitted
12268        } else if status == OrderStatus::Submitted {
12269            OrderStatus::Accepted
12270        } else {
12271            status
12272        };
12273        assert_eq!(
12274            node.kernel
12275                .cache
12276                .borrow()
12277                .order(&client_order_id)
12278                .unwrap()
12279                .status(),
12280            expected_status,
12281        );
12282
12283        for _ in 0..3 {
12284            advance_clock(Duration::from_millis(101)).await;
12285            let result = node.exec_manager.check_inflight_orders();
12286            assert!(result.queries.is_empty());
12287            assert!(result.events.is_empty());
12288        }
12289        assert_eq!(
12290            node.exec_manager.recon_check_retry_count(&client_order_id),
12291            0
12292        );
12293        assert!(node.exec_manager.check_open_order_queries().is_empty());
12294        advance_clock(Duration::from_secs(5)).await;
12295        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12296    }
12297
12298    #[rstest]
12299    #[case::fill("fill", false)]
12300    #[case::terminal_order("order", true)]
12301    #[case::terminal_order_with_fills("order_with_fills", true)]
12302    #[case::working_order_for_closed_cache("order", false)]
12303    #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
12304    #[cfg_attr(
12305        not(all(feature = "simulation", madsim)),
12306        tokio::test(start_paused = true)
12307    )]
12308    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12309    async fn test_report_cleanup_preserves_original_settling_predicate(
12310        #[case] report_kind: &str,
12311        #[case] terminal_report: bool,
12312        #[values(false, true)] explicit_client_id: bool,
12313        #[values(
12314            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12315            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12316        )]
12317        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12318    ) {
12319        let (mut node, order, instrument) = venue_evidence_node(policy);
12320        let client_order_id = order.client_order_id();
12321        let account_id = AccountId::from("TEST-001");
12322        let client_id = ClientId::from("TEST-EVIDENCE");
12323        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12324        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
12325            &order,
12326            account_id,
12327            venue_order_id,
12328        )));
12329        let accepted = node
12330            .kernel
12331            .cache
12332            .borrow()
12333            .order_owned(&client_order_id)
12334            .unwrap();
12335        let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
12336            .last_qty(accepted.quantity())
12337            .last_px(Price::from("100.00"))
12338            .commission(Money::from("0 USDT"))
12339            .without_position_id()
12340            .build()
12341        else {
12342            unreachable!()
12343        };
12344        let mut fill_report = FillReport::new(
12345            account_id,
12346            instrument.id(),
12347            venue_order_id,
12348            fill.trade_id,
12349            fill.order_side,
12350            fill.last_qty,
12351            fill.last_px,
12352            fill.commission.unwrap(),
12353            fill.liquidity_side,
12354            Some(client_order_id),
12355            None,
12356            fill.ts_event,
12357            fill.ts_init,
12358            None,
12359        );
12360
12361        if !explicit_client_id {
12362            fill_report.client_order_id = None;
12363        }
12364        let event = if report_kind == "fill" {
12365            ExecutionReport::Fill(Box::new(fill_report))
12366        } else {
12367            if !terminal_report {
12368                node.kernel
12369                    .cache
12370                    .borrow_mut()
12371                    .update_order(&OrderEventAny::Filled(fill))
12372                    .unwrap();
12373            }
12374            let report = OrderStatusReport::new(
12375                account_id,
12376                instrument.id(),
12377                explicit_client_id.then_some(client_order_id),
12378                venue_order_id,
12379                Some(order.order_side()),
12380                order.order_type(),
12381                order.time_in_force(),
12382                if terminal_report {
12383                    OrderStatus::Filled
12384                } else {
12385                    OrderStatus::Accepted
12386                },
12387                order.quantity(),
12388                order.quantity(),
12389                UnixNanos::default(),
12390                UnixNanos::default(),
12391                UnixNanos::default(),
12392                None,
12393            )
12394            .with_price(Price::from("100.00"))
12395            .with_avg_px(dec!(100.00));
12396
12397            if report_kind == "order" {
12398                ExecutionReport::Order(Box::new(report))
12399            } else {
12400                ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
12401            }
12402        };
12403        node.process_exec_event(ExecutionEvent::Report(event));
12404        assert_eq!(
12405            node.kernel
12406                .cache
12407                .borrow()
12408                .order(&client_order_id)
12409                .unwrap()
12410                .status(),
12411            OrderStatus::Filled
12412        );
12413
12414        // Replace only cached state, without an observer stamp, to expose the existing grace gate
12415        node.kernel
12416            .cache
12417            .borrow_mut()
12418            .add_order(order.clone(), None, Some(client_id), true)
12419            .unwrap();
12420        node.kernel
12421            .cache
12422            .borrow_mut()
12423            .update_order(&TestOrderEventStubs::submitted(&order, account_id))
12424            .unwrap();
12425        node.kernel
12426            .cache
12427            .borrow_mut()
12428            .update_order(&TestOrderEventStubs::accepted(
12429                &order,
12430                account_id,
12431                venue_order_id,
12432            ))
12433            .unwrap();
12434        let should_clear = terminal_report && explicit_client_id;
12435        assert_eq!(
12436            node.exec_manager.check_open_order_queries().len(),
12437            usize::from(should_clear)
12438        );
12439
12440        if !should_clear {
12441            advance_clock(Duration::from_secs(5)).await;
12442            assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12443        }
12444    }
12445
12446    #[rstest]
12447    #[case(stub_order_event, 1)]
12448    #[case(stub_submitted_batch_event, 2)]
12449    #[case(stub_accepted_batch_event, 2)]
12450    #[case(stub_canceled_batch_event, 2)]
12451    fn test_pending_events_preserve_roots_when_splitting_batches(
12452        #[values(false, true)] rooted: bool,
12453        #[case] make_event: fn() -> ExecutionEvent,
12454        #[case] orders: usize,
12455    ) {
12456        let runner = AsyncRunner::new();
12457        runner.bind_senders();
12458        let mut channels = runner.take_channels();
12459
12460        let send = move || {
12461            get_data_event_sender().send(stub_data_event()).unwrap();
12462            get_exec_event_sender().send(stub_exec_event()).unwrap();
12463            get_exec_event_sender().send(make_event()).unwrap();
12464        };
12465
12466        if rooted {
12467            msgbus::register_trading_command_endpoint(
12468                MessagingSwitchboard::risk_engine_execute(),
12469                TypedIntoHandler::from(move |_| send()),
12470            );
12471            TradingCommandSender::execute(
12472                &SyncTradingCommandSender,
12473                TradingCommandMessage::new(
12474                    MessagingSwitchboard::risk_engine_execute(),
12475                    TradingCommand::QueryAccount(QueryAccount::new(
12476                        "TRADER-001".into(),
12477                        None,
12478                        "SIM-001".into(),
12479                        UUID4::new(),
12480                        37.into(),
12481                        None,
12482                        None,
12483                    )),
12484                ),
12485            );
12486            nautilus_common::runner::drain_trading_cmd_queue();
12487        } else {
12488            send();
12489        }
12490
12491        let mut pending = PendingEvents::default();
12492        pending
12493            .data_evts
12494            .push(channels.data_evt_rx.try_recv().unwrap());
12495        while let Ok(event) = channels.exec_evt_rx.try_recv() {
12496            pending.push_exec_event(event);
12497        }
12498
12499        assert_eq!(pending.data_evts.len(), 1);
12500        assert_eq!(pending.exec_reports.len(), 1);
12501        assert_eq!(pending.order_evts.len(), orders);
12502        assert_eq!(pending.data_evts[0].is_rooted(), rooted);
12503        assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
12504        assert!(
12505            pending
12506                .order_evts
12507                .iter()
12508                .all(|event| event.is_rooted() == rooted)
12509        );
12510
12511        let results = std::thread::spawn(move || {
12512            let mut results = Vec::new();
12513
12514            for event in pending.data_evts {
12515                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12516                    || event.dispatch(drop),
12517                )));
12518            }
12519
12520            for event in pending.exec_reports {
12521                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12522                    || event.dispatch(drop),
12523                )));
12524            }
12525
12526            for event in pending.order_evts {
12527                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12528                    || event.dispatch(drop),
12529                )));
12530            }
12531
12532            results
12533        })
12534        .join()
12535        .unwrap();
12536
12537        assert_eq!(results.len(), orders + 2);
12538
12539        for result in results {
12540            if rooted {
12541                let error = result.unwrap_err();
12542                assert!(
12543                    error
12544                        .downcast_ref::<String>()
12545                        .unwrap()
12546                        .contains("command context dispatched outside its owner thread")
12547                );
12548            } else {
12549                result.unwrap();
12550            }
12551        }
12552    }
12553
12554    fn stub_accepted_batch_event() -> ExecutionEvent {
12555        ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
12556            OrderAcceptedSpec::builder()
12557                .client_order_id(ClientOrderId::from("O-017"))
12558                .build(),
12559            OrderAcceptedSpec::builder()
12560                .client_order_id(ClientOrderId::from("O-023"))
12561                .build(),
12562        ]))
12563    }
12564
12565    #[derive(Debug)]
12566    struct CapturedEgressMessage {
12567        topic: String,
12568        payload: Bytes,
12569    }
12570
12571    type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
12572    type SharedClosed = Rc<Cell<bool>>;
12573
12574    #[derive(Debug)]
12575    struct CapturingExternalIngress {
12576        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
12577        closed: SharedClosed,
12578    }
12579
12580    impl CapturingExternalIngress {
12581        fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
12582            Self {
12583                rx: Some(rx),
12584                closed,
12585            }
12586        }
12587    }
12588
12589    impl MessageBusExternalIngress for CapturingExternalIngress {
12590        fn is_closed(&self) -> bool {
12591            self.closed.get()
12592        }
12593
12594        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
12595            self.rx
12596                .take()
12597                .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
12598        }
12599
12600        fn close(&mut self) {
12601            self.closed.set(true);
12602        }
12603    }
12604
12605    #[derive(Debug)]
12606    struct FailingExternalIngress {
12607        closed: SharedClosed,
12608    }
12609
12610    impl FailingExternalIngress {
12611        fn new(closed: SharedClosed) -> Self {
12612            Self { closed }
12613        }
12614    }
12615
12616    impl MessageBusExternalIngress for FailingExternalIngress {
12617        fn is_closed(&self) -> bool {
12618            self.closed.get()
12619        }
12620
12621        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
12622            anyhow::bail!("external ingress receiver unavailable")
12623        }
12624
12625        fn close(&mut self) {
12626            self.closed.set(true);
12627        }
12628    }
12629
12630    struct CapturingExternalEgress {
12631        publications: CapturedEgressMessages,
12632        closed: SharedClosed,
12633    }
12634
12635    impl CapturingExternalEgress {
12636        fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
12637            let publications = Rc::new(RefCell::new(Vec::new()));
12638            let closed = Rc::new(Cell::new(false));
12639            (
12640                Self {
12641                    publications: publications.clone(),
12642                    closed: closed.clone(),
12643                },
12644                publications,
12645                closed,
12646            )
12647        }
12648    }
12649
12650    impl MessageBusExternalEgress for CapturingExternalEgress {
12651        fn is_closed(&self) -> bool {
12652            self.closed.get()
12653        }
12654
12655        fn publish(&self, message: BusMessage) {
12656            self.publications.borrow_mut().push(CapturedEgressMessage {
12657                topic: message.topic.to_string(),
12658                payload: message.payload,
12659            });
12660        }
12661
12662        fn close(&mut self) {
12663            self.closed.set(true);
12664        }
12665    }
12666
12667    struct CapturingBackingFactory {
12668        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
12669        closed: Arc<AtomicBool>,
12670        rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
12671    }
12672
12673    impl CapturingBackingFactory {
12674        fn new(
12675            publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
12676            closed: Arc<AtomicBool>,
12677            rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
12678        ) -> Self {
12679            Self {
12680                publications,
12681                closed,
12682                rx: Mutex::new(rx),
12683            }
12684        }
12685    }
12686
12687    impl Debug for CapturingBackingFactory {
12688        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
12689            f.debug_struct(stringify!(CapturingBackingFactory))
12690                .finish_non_exhaustive()
12691        }
12692    }
12693
12694    impl MessageBusBackingFactory for CapturingBackingFactory {
12695        fn create(
12696            &self,
12697            _trader_id: TraderId,
12698            _instance_id: UUID4,
12699            _config: MessageBusConfig,
12700        ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
12701            let rx = self.rx.lock().take();
12702            Ok(Box::new(CapturingBacking {
12703                publications: self.publications.clone(),
12704                closed: self.closed.clone(),
12705                rx,
12706            }))
12707        }
12708    }
12709
12710    struct CapturingBacking {
12711        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
12712        closed: Arc<AtomicBool>,
12713        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
12714    }
12715
12716    impl MessageBusBacking for CapturingBacking {
12717        fn is_closed(&self) -> bool {
12718            self.closed.load(Ordering::Relaxed)
12719        }
12720
12721        fn publish(&self, message: BusMessage) {
12722            self.publications.lock().push(CapturedEgressMessage {
12723                topic: message.topic.to_string(),
12724                payload: message.payload,
12725            });
12726        }
12727
12728        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
12729            self.rx
12730                .take()
12731                .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
12732        }
12733
12734        fn close(&mut self) {
12735            self.closed.store(true, Ordering::Relaxed);
12736        }
12737    }
12738}